feat(import): import Carnet/QuickNote (.sqd) archives in-app

Adds an 'Import 3rd Party Notes' button to Settings > Backup & Restore that reads a Carnet/QuickNote .zip export (a single archive bundling one .sqd per note) and imports every note directly on-device, no pre-conversion step. Each .sqd is itself a zip (index.html + metadata.json + optional data/ attachments); the importer does a nested-zip read. Rich-text bodies are flattened to plain text, colours/keywords/links carry over, to-do lists become native checklists, and image attachments are imported (auto-generated preview thumbnails skipped). Additive import with per-note fault isolation.
This commit is contained in:
Incido Developer committed 2026-09-18 16:04:53 +12:00
1 parent 7e335d14ef
commit cbe015839b
4 files changed
+451

No files matched your search

@@ -0,0 +1,405 @@
package org.incido.compose.data.backup
import android.content.Context
import android.net.Uri
import java.io.ByteArrayInputStream
import java.util.zip.ZipInputStream
import kotlinx.coroutines.Dispatchers
import kotlinx.coroutines.withContext
import kotlinx.serialization.SerialName
import kotlinx.serialization.Serializable
import kotlinx.serialization.json.Json
import kotlinx.serialization.json.JsonElement
import org.incido.domain.model.Hashtag
import org.incido.domain.model.Note
import org.incido.domain.model.NoteColor
import org.incido.domain.model.NoteType
import org.incido.domain.repository.NoteRepository
/**
* v0.3.9.0 — imports notes from a **Carnet / QuickNote** archive.
*
* The source is a `.zip` whose top-level entries are `*.sqd` files (plus
* some metadata dirs we ignore). Each `.sqd` is *itself* a zip containing:
* * `index.html` — the note body inside `<div class="edit-zone">…</div>`
* * `metadata.json` — `creation_date`, `last_modification_date`,
* `keywords[]`, `rating`, `color`, `urls{}`, `todolists[]`
* * `data/<file>` — optional attachments (images etc). A companion
* `data/preview_` thumbnail is skipped so we don't
* import a low-res duplicate.
*
* Incido stores note bodies as **plain text**, so the rich-text HTML is
* flattened to text (block tags → newlines, entities decoded, tags stripped).
* The `.sqd` filename becomes the note's title line, which also gives
* image-only notes (no HTML body) a readable title.
*
* Import is **additive** — matching the existing backup-import behaviour:
* existing notes are preserved, every imported note gets a fresh id, and
* attachments are stored via [NoteRepository.addAttachment] (which applies
* EXIF normalisation + at-rest encryption transparently).
*
* Both archive shapes are accepted: a bundle of `*.sqd` files, or a single
* `.sqd` file picked directly (its zip entries are the note's own
* `index.html` / `metadata.json` / files under `data/`).
*/
class CarnetImporter(
private val context: Context,
private val repository: NoteRepository,
) {
private val json = Json {
ignoreUnknownKeys = true
isLenient = true
}
/**
* Read the archive at [uri] and import every Carnet note it contains.
* Returns the number of notes imported.
*/
suspend fun import(
uri: Uri,
onProgress: (current: Int, total: Int) -> Unit = { _, _ -> },
): Int = withContext(Dispatchers.IO) {
val archiveBytes = context.contentResolver.openInputStream(uri)?.use { it.readBytes() }
?: throw java.io.IOException("Could not open input stream for $uri")
val topLevel = readZip(archiveBytes)
// The picked file is a single archive that bundles every note as a
// nested `.sqd` entry (each `.sqd` is itself a zip). Collect those.
val sqdNotes = topLevel.filter { (name, _) -> name.endsWith(".sqd", ignoreCase = true) }
.toList()
// Candidates as (title-file-name, raw-bytes). If no `.sqd` entries
// were found, the picked file is treated as a single `.sqd` (its own
// zip entries are index.html / metadata.json / data/…).
val candidates: List<Pair<String, ByteArray>> = if (sqdNotes.isNotEmpty()) {
sqdNotes
} else {
listOf((queryDisplayName(uri) ?: "Imported note.sqd") to archiveBytes)
}
// Parse phase — fault-isolated so one corrupt `.sqd` can't abort the
// rest of the archive.
val notes = candidates.mapNotNull { (name, bytes) ->
runCatching { parseSqd(name.substringAfterLast('/'), bytes) }.getOrNull()
}.filter { it.hasContent() }
// Persist phase — also per-note, so a single bad note doesn't lose the
// ones already written (the repository commits each note independently).
val total = notes.size
var imported = 0
notes.forEach { parsed ->
val ok = runCatching { persist(parsed) }.isSuccess
if (ok) imported++
onProgress(imported, total)
}
imported
}
// --- Persistence ---------------------------------------------------------
private suspend fun persist(parsed: ParsedSqd) {
val hashtags = parsed.keywords
.flatMap { raw ->
// Prefer the whole keyword as one tag; if it isn't a valid
// hashtag (Incido allows only [a-z0-9._-], 1–31 chars), split
// on whitespace/commas and keep the valid sub-tokens.
val one = Hashtag.normalise(raw)
if (Hashtag.parseOrNull(one) != null) {
listOf(one)
} else {
raw.split(Regex("[\\s,]+"))
.map { Hashtag.normalise(it) }
.filter { Hashtag.parseOrNull(it) != null }
}
}
.filter { it.isNotEmpty() }
.toSet()
val saved = repository.saveNoteWithHashtags(
note = Note(
id = 0L, // force a fresh id
content = parsed.content,
createdAt = parsed.createdAt,
updatedAt = parsed.updatedAt,
isFavorite = parsed.isFavorite,
color = parsed.color,
isTrashed = false,
type = parsed.type,
),
hashtagsToAdd = hashtags,
hashtagsToRemove = emptySet(),
)
for (att in parsed.attachments) {
repository.addAttachment(
noteId = saved.id,
bytes = att.bytes,
mimeType = sniffMimeType(att.bytes),
displayName = att.displayName,
)
}
}
// --- .sqd parsing --------------------------------------------------------
private fun parseSqd(fileName: String, sqdBytes: ByteArray): ParsedSqd {
val files = readZip(sqdBytes)
val title = fileName.substringBeforeLast('.')
.ifBlank { "Untitled" }
val html = files["index.html"]?.toString(Charsets.UTF_8).orEmpty()
val meta = files["metadata.json"]
?.let {
runCatching {
json.decodeFromString(CarnetMetadata.serializer(), it.toString(Charsets.UTF_8))
}.getOrNull()
}
?: CarnetMetadata()
val body = htmlToText(html)
// Attachments: anything under data/ that is a real file (not a dir
// entry) and not an auto-generated preview thumbnail.
val attachments = files
.filter { (name, bytes) ->
name.startsWith("data/") &&
!name.endsWith("/") &&
bytes.isNotEmpty() &&
!name.substringAfterLast('/').startsWith("preview_")
}
.map { (name, bytes) -> Attachment(name.substringAfterLast('/'), bytes) }
val type = if (meta.todolists.isNotEmpty()) NoteType.TASK_LIST else NoteType.TEXT
return ParsedSqd(
content = buildContent(title, body, meta),
createdAt = meta.creationDate.takeIf { it > 0 } ?: System.currentTimeMillis(),
updatedAt = meta.lastModificationDate.takeIf { it > 0 } ?: System.currentTimeMillis(),
isFavorite = meta.rating >= 5,
color = mapColor(meta.color),
type = type,
keywords = meta.keywords,
attachments = attachments,
)
}
/**
* Assemble the final plain-text body. The filename is used as the title
* (first non-blank line in Incido's title/body split); then the HTML
* body; then any URLs from the metadata; then any to-do lists rendered as
* GFM checkbox lines (`- [ ]` / `- [x]`) so [NoteType.TASK_LIST] renders.
*/
private fun buildContent(title: String, body: String, meta: CarnetMetadata): String {
val blocks = mutableListOf<String>()
blocks += title
if (body.isNotBlank()) blocks += body
if (meta.urls.isNotEmpty()) {
blocks += meta.urls.keys.filter { it.isNotBlank() }
}
for (todo in meta.todolists) {
val lines = buildList {
todo.done.forEach { if (it.isNotBlank()) add("- [x] ${it.trim()}") }
todo.todo.forEach { if (it.isNotBlank()) add("- [ ] ${it.trim()}") }
}
if (lines.isNotEmpty()) blocks += lines.joinToString("\n")
}
return blocks.filter { it.isNotBlank() }.joinToString("\n\n").trim()
}
// --- Helpers -------------------------------------------------------------
/** Read every file entry (name → bytes) from a zip, skipping dirs. */
private fun readZip(bytes: ByteArray): LinkedHashMap<String, ByteArray> {
val out = LinkedHashMap<String, ByteArray>()
ZipInputStream(ByteArrayInputStream(bytes)).use { zis ->
while (true) {
val entry = zis.nextEntry ?: break
if (entry.isDirectory) { zis.closeEntry(); continue }
val data = zis.readBytes()
out[entry.name] = data
zis.closeEntry()
}
}
return out
}
/** Query the user-visible name of a SAF [uri] (falls back to null). */
private fun queryDisplayName(uri: Uri): String? {
return try {
context.contentResolver.query(uri, null, null, null, null)?.use { cursor ->
val idx = cursor.getColumnIndex(android.provider.OpenableColumns.DISPLAY_NAME)
if (idx >= 0 && cursor.moveToFirst()) cursor.getString(idx) else null
}
} catch (_: Exception) {
null
}
}
companion object {
/** MIME type detected from magic bytes, falling back to a default. */
fun sniffMimeType(bytes: ByteArray): String {
fun b(i: Int) = if (i < bytes.size) bytes[i].toInt() and 0xFF else -1
if (b(0) == 0xFF && b(1) == 0xD8 && b(2) == 0xFF) return "image/jpeg"
if (b(0) == 0x89 && b(1) == 0x50 && b(2) == 0x4E && b(3) == 0x47) return "image/png"
if (b(0) == 0x47 && b(1) == 0x49 && b(2) == 0x46 && b(3) == 0x38) return "image/gif"
if (b(0) == 0x52 && b(1) == 0x49 && b(2) == 0x46 && b(3) == 0x46 &&
b(8) == 0x57 && b(9) == 0x45 && b(10) == 0x42 && b(11) == 0x50
) return "image/webp"
if (b(0) == 0x42 && b(1) == 0x4D) return "image/bmp"
return "application/octet-stream"
}
/** Carnet color string → Incido [NoteColor]. Unknown values → NONE. */
fun mapColor(raw: String): NoteColor = when (raw.trim().lowercase()) {
"red" -> NoteColor.RED
"green" -> NoteColor.GREEN
"blue" -> NoteColor.BLUE
"yellow", "amber" -> NoteColor.AMBER
"purple" -> NoteColor.PURPLE
else -> NoteColor.NONE
}
/**
* Flatten a Carnet `index.html` note body to plain text:
* strip comments, isolate the `.edit-zone` content, turn block
* boundaries into newlines, decode entities (incl. `&nbsp;`), then
* drop the remaining tags.
*/
fun htmlToText(html: String): String {
var s = html.replace(Regex("<!--.*?-->", RegexOption.DOT_MATCHES_ALL), " ")
// Keep only the edit-zone body (everything after its opening tag,
// up to the sibling `<div id="floating">` that follows it).
val zoneMatch = Regex("class=\"edit-zone\"[^>]*>").find(s)
if (zoneMatch != null) {
s = s.substring(zoneMatch.range.last + 1)
val floatIdx = s.indexOf("<div id=\"floating\"")
if (floatIdx >= 0) s = s.substring(0, floatIdx)
}
s = s.replace(Regex("<br\\s*/?>", RegexOption.IGNORE_CASE), "\n")
s = s.replace(Regex("</(div|p|li|ul|ol|tr|h[1-6])>", RegexOption.IGNORE_CASE), "\n")
s = s.replace(Regex("<[^>]+>"), "")
s = decodeEntities(s)
// Normalise whitespace: drop trailing spaces per line, collapse
// runs of 2+ blank lines to a single blank line.
val lines = s.replace("\r", "").split('\n').map { it.trimEnd() }
val collapsed = StringBuilder()
var blanks = 0
for (line in lines) {
if (line.isEmpty()) {
blanks++
if (blanks <= 1) collapsed.append('\n')
} else {
blanks = 0
collapsed.append(line).append('\n')
}
}
return collapsed.toString().trim()
}
/** Decode the named + numeric HTML entities a Carnet body can contain. */
private fun decodeEntities(input: String): String {
if (!input.contains('&')) return input
val sb = StringBuilder(input.length)
var i = 0
while (i < input.length) {
val c = input[i]
if (c == '&') {
val semi = input.indexOf(';', i + 1)
if (semi in (i + 1)..(i + 12)) {
val entity = input.substring(i, semi + 1)
val decoded = NAMED_ENTITIES[entity] ?: numericEntity(entity)
if (decoded != null) {
sb.append(decoded)
i = semi + 1
continue
}
}
}
sb.append(c)
i++
}
return sb.toString()
}
private fun numericEntity(entity: String): String? {
if (!entity.startsWith("&#") || !entity.endsWith(";")) return null
val body = entity.substring(2, entity.length - 1)
return try {
val code = if (body.startsWith("x") || body.startsWith("X")) {
body.substring(1).toLong(16)
} else {
body.toLong()
}
if (code in 0..0x10FFFF) String(Character.toChars(code.toInt())) else null
} catch (_: NumberFormatException) {
null
}
}
private val NAMED_ENTITIES = mapOf(
"&nbsp;" to " ",
"&amp;" to "&",
"&lt;" to "<",
"&gt;" to ">",
"&quot;" to "\"",
"&#39;" to "'",
"&apos;" to "'",
"&copy;" to "\u00A9",
"&reg;" to "\u00AE",
"&trade;" to "\u2122",
"&hellip;" to "\u2026",
"&mdash;" to "\u2014",
"&ndash;" to "\u2013",
"&rsquo;" to "\u2019",
"&lsquo;" to "\u2018",
"&rdquo;" to "\u201D",
"&ldquo;" to "\u201C",
)
}
// --- Data ----------------------------------------------------------------
/** A parsed, ready-to-persist Carnet note. */
data class ParsedSqd(
val content: String,
val createdAt: Long,
val updatedAt: Long,
val isFavorite: Boolean,
val color: NoteColor,
val type: NoteType,
val keywords: List<String>,
val attachments: List<Attachment>,
) {
fun hasContent(): Boolean = content.isNotBlank() || attachments.isNotEmpty()
}
/** A single attachment (file name + raw bytes) inside a `.sqd`'s `data/`. */
data class Attachment(val displayName: String, val bytes: ByteArray)
@Serializable
private data class CarnetMetadata(
@SerialName("creation_date") val creationDate: Long = 0,
@SerialName("last_modification_date") val lastModificationDate: Long = 0,
val keywords: List<String> = emptyList(),
val rating: Int = -1,
val color: String = "none",
val urls: Map<String, JsonElement> = emptyMap(),
val todolists: List<CarnetTodo> = emptyList(),
)
@Serializable
private data class CarnetTodo(
val id: String = "",
val todo: List<String> = emptyList(),
val done: List<String> = emptyList(),
)
}
@@ -11,6 +11,7 @@ import org.incido.data.crypto.NoteCryptoService
import org.incido.data.repository.NoteRepositoryImpl
import org.incido.domain.repository.NoteRepository
import org.incido.compose.data.backup.BackupManager
import org.incido.compose.data.backup.CarnetImporter
import org.incido.compose.data.security.AesGcmFieldCipher
import org.incido.compose.data.security.EncryptionVerifier
import org.incido.compose.data.security.ObjectBoxKeyProvider
@@ -78,6 +79,10 @@ class AppContainer(context: Context) {
val backupManager: BackupManager = BackupManager(appContext, noteRepository)
/** v0.3.9.0 — Carnet / QuickNote `.sqd` archive importer. */
val carnetImporter: CarnetImporter = CarnetImporter(appContext, noteRepository)
/** Migration sweep + startup integrity checker for the note store. */
val noteCryptoService = NoteCryptoService(boxStore, cipherHolder)
@@ -122,6 +122,15 @@ fun SettingsScreen(
var showRecoveryImportWarning by remember { mutableStateOf(false) }
// SAF launchers
// v0.3.9.0 — Carnet / QuickNote archive importer. Picks a .zip (or a
// lone .sqd) and imports it directly; no passphrase, additive.
val carnetImportLauncher = rememberLauncherForActivityResult(
contract = ActivityResultContracts.OpenDocument(),
) { uri ->
if (uri == null) return@rememberLauncherForActivityResult
scope.launch { viewModel.importCarnet(uri) }
}
val exportLauncher = rememberLauncherForActivityResult(
contract = ActivityResultContracts.CreateDocument("application/json"),
) { uri ->
@@ -296,6 +305,12 @@ fun SettingsScreen(
securityManager?.beginExternalActivity()
importLauncher.launch(arrayOf("application/json", "*/*"))
},
onImportCarnet = {
securityManager?.beginExternalActivity()
carnetImportLauncher.launch(
arrayOf("application/zip", "application/x-zip-compressed", "application/octet-stream", "*/*"),
)
},
)
AboutSection(
version = SettingsViewModel.APP_VERSION,
@@ -825,6 +840,7 @@ private fun SecuritySection(
private fun BackupSection(
onExport: () -> Unit,
onImport: () -> Unit,
onImportCarnet: () -> Unit,
) {
SettingsSection(title = "Backup & Restore") {
Column(
@@ -846,6 +862,9 @@ private fun BackupSection(
OutlinedButton(onClick = onImport, modifier = Modifier.fillMaxWidth()) {
Text("Import from file…")
}
OutlinedButton(onClick = onImportCarnet, modifier = Modifier.fillMaxWidth()) {
Text("Import 3rd Party Notes")
}
}
}
}
@@ -14,6 +14,7 @@ import kotlinx.coroutines.flow.stateIn
import kotlinx.coroutines.launch
import org.incido.compose.IncidoApplication
import org.incido.compose.data.backup.BackupManager
import org.incido.compose.data.backup.CarnetImporter
import org.incido.compose.data.security.AesGcmFieldCipher
import org.incido.compose.data.security.EncryptionVerifier
import org.incido.compose.data.security.ObjectBoxKeyProvider
@@ -46,6 +47,7 @@ class SettingsViewModel(
private val security: SecurityManager,
private val objectBoxKeyProvider: ObjectBoxKeyProvider,
private val backupManager: BackupManager,
private val carnetImporter: CarnetImporter,
private val noteCryptoService: NoteCryptoService,
private val cipherHolder: CipherHolder,
private val encryptionVerifier: EncryptionVerifier,
@@ -245,6 +247,25 @@ class SettingsViewModel(
_snackbar.value = "Imported $count note(s)."
}
/**
* v0.3.9.0 — import a Carnet / QuickNote archive (`.zip` of `*.sqd`, or a
* single `.sqd`) picked via SAF. Additive — existing notes are kept.
* Failures are caught and surfaced via the snackbar so a bad file never
* crashes the app.
*/
suspend fun importCarnet(uri: android.net.Uri) {
try {
val count = carnetImporter.import(uri)
_snackbar.value = if (count > 0) {
"Imported $count Carnet note(s)."
} else {
"No Carnet notes found in that file."
}
} catch (e: Exception) {
_snackbar.value = "Carnet import failed: ${e.message ?: "unreadable file"}"
}
}
// --- v0.3.6.0 recovery key ---
/**
@@ -344,6 +365,7 @@ class SettingsViewModel(
security = c.securityManager,
objectBoxKeyProvider = c.objectBoxKeyProvider,
backupManager = c.backupManager,
carnetImporter = c.carnetImporter,
noteCryptoService = c.noteCryptoService,
cipherHolder = c.cipherHolder,
encryptionVerifier = c.encryptionVerifier,