From cbe015839b1fa0d109bfef4174e9439dcea9a9ce Mon Sep 17 00:00:00 2001 From: Incido Developer Date: Fri, 18 Sep 2026 16:04:53 +1200 Subject: [PATCH] feat(import): import Carnet/QuickNote (.sqd) archives in-app Adds an 'Import 3rd Party Notes' button to Settings > Backup & Restore that reads a Carnet/QuickNote .zip export (a single archive bundling one .sqd per note) and imports every note directly on-device, no pre-conversion step. Each .sqd is itself a zip (index.html + metadata.json + optional data/ attachments); the importer does a nested-zip read. Rich-text bodies are flattened to plain text, colours/keywords/links carry over, to-do lists become native checklists, and image attachments are imported (auto-generated preview thumbnails skipped). Additive import with per-note fault isolation. --- .../compose/data/backup/CarnetImporter.kt | 405 ++++++++++++++++++ .../org/incido/compose/di/AppContainer.kt | 5 + .../compose/ui/settings/SettingsScreen.kt | 19 + .../compose/ui/settings/SettingsViewModel.kt | 22 + 4 files changed, 451 insertions(+) create mode 100644 composeApp/src/main/kotlin/org/incido/compose/data/backup/CarnetImporter.kt diff --git a/composeApp/src/main/kotlin/org/incido/compose/data/backup/CarnetImporter.kt b/composeApp/src/main/kotlin/org/incido/compose/data/backup/CarnetImporter.kt new file mode 100644 index 0000000..575fe77 --- /dev/null +++ b/composeApp/src/main/kotlin/org/incido/compose/data/backup/CarnetImporter.kt @@ -0,0 +1,405 @@ +package org.incido.compose.data.backup + +import android.content.Context +import android.net.Uri +import java.io.ByteArrayInputStream +import java.util.zip.ZipInputStream +import kotlinx.coroutines.Dispatchers +import kotlinx.coroutines.withContext +import kotlinx.serialization.SerialName +import kotlinx.serialization.Serializable +import kotlinx.serialization.json.Json +import kotlinx.serialization.json.JsonElement +import org.incido.domain.model.Hashtag +import org.incido.domain.model.Note +import org.incido.domain.model.NoteColor +import org.incido.domain.model.NoteType +import org.incido.domain.repository.NoteRepository + +/** + * v0.3.9.0 — imports notes from a **Carnet / QuickNote** archive. + * + * The source is a `.zip` whose top-level entries are `*.sqd` files (plus + * some metadata dirs we ignore). Each `.sqd` is *itself* a zip containing: + * * `index.html` — the note body inside `
…
` + * * `metadata.json` — `creation_date`, `last_modification_date`, + * `keywords[]`, `rating`, `color`, `urls{}`, `todolists[]` + * * `data/` — optional attachments (images etc). A companion + * `data/preview_` thumbnail is skipped so we don't + * import a low-res duplicate. + * + * Incido stores note bodies as **plain text**, so the rich-text HTML is + * flattened to text (block tags → newlines, entities decoded, tags stripped). + * The `.sqd` filename becomes the note's title line, which also gives + * image-only notes (no HTML body) a readable title. + * + * Import is **additive** — matching the existing backup-import behaviour: + * existing notes are preserved, every imported note gets a fresh id, and + * attachments are stored via [NoteRepository.addAttachment] (which applies + * EXIF normalisation + at-rest encryption transparently). + * + * Both archive shapes are accepted: a bundle of `*.sqd` files, or a single + * `.sqd` file picked directly (its zip entries are the note's own + * `index.html` / `metadata.json` / files under `data/`). + */ +class CarnetImporter( + private val context: Context, + private val repository: NoteRepository, +) { + + private val json = Json { + ignoreUnknownKeys = true + isLenient = true + } + + /** + * Read the archive at [uri] and import every Carnet note it contains. + * Returns the number of notes imported. + */ + suspend fun import( + uri: Uri, + onProgress: (current: Int, total: Int) -> Unit = { _, _ -> }, + ): Int = withContext(Dispatchers.IO) { + val archiveBytes = context.contentResolver.openInputStream(uri)?.use { it.readBytes() } + ?: throw java.io.IOException("Could not open input stream for $uri") + + val topLevel = readZip(archiveBytes) + + // The picked file is a single archive that bundles every note as a + // nested `.sqd` entry (each `.sqd` is itself a zip). Collect those. + val sqdNotes = topLevel.filter { (name, _) -> name.endsWith(".sqd", ignoreCase = true) } + .toList() + + // Candidates as (title-file-name, raw-bytes). If no `.sqd` entries + // were found, the picked file is treated as a single `.sqd` (its own + // zip entries are index.html / metadata.json / data/…). + val candidates: List> = if (sqdNotes.isNotEmpty()) { + sqdNotes + } else { + listOf((queryDisplayName(uri) ?: "Imported note.sqd") to archiveBytes) + } + + // Parse phase — fault-isolated so one corrupt `.sqd` can't abort the + // rest of the archive. + val notes = candidates.mapNotNull { (name, bytes) -> + runCatching { parseSqd(name.substringAfterLast('/'), bytes) }.getOrNull() + }.filter { it.hasContent() } + + // Persist phase — also per-note, so a single bad note doesn't lose the + // ones already written (the repository commits each note independently). + val total = notes.size + var imported = 0 + notes.forEach { parsed -> + val ok = runCatching { persist(parsed) }.isSuccess + if (ok) imported++ + onProgress(imported, total) + } + imported + } + + // --- Persistence --------------------------------------------------------- + + private suspend fun persist(parsed: ParsedSqd) { + val hashtags = parsed.keywords + .flatMap { raw -> + // Prefer the whole keyword as one tag; if it isn't a valid + // hashtag (Incido allows only [a-z0-9._-], 1–31 chars), split + // on whitespace/commas and keep the valid sub-tokens. + val one = Hashtag.normalise(raw) + if (Hashtag.parseOrNull(one) != null) { + listOf(one) + } else { + raw.split(Regex("[\\s,]+")) + .map { Hashtag.normalise(it) } + .filter { Hashtag.parseOrNull(it) != null } + } + } + .filter { it.isNotEmpty() } + .toSet() + + val saved = repository.saveNoteWithHashtags( + note = Note( + id = 0L, // force a fresh id + content = parsed.content, + createdAt = parsed.createdAt, + updatedAt = parsed.updatedAt, + isFavorite = parsed.isFavorite, + color = parsed.color, + isTrashed = false, + type = parsed.type, + ), + hashtagsToAdd = hashtags, + hashtagsToRemove = emptySet(), + ) + + for (att in parsed.attachments) { + repository.addAttachment( + noteId = saved.id, + bytes = att.bytes, + mimeType = sniffMimeType(att.bytes), + displayName = att.displayName, + ) + } + } + + // --- .sqd parsing -------------------------------------------------------- + + private fun parseSqd(fileName: String, sqdBytes: ByteArray): ParsedSqd { + val files = readZip(sqdBytes) + val title = fileName.substringBeforeLast('.') + .ifBlank { "Untitled" } + + val html = files["index.html"]?.toString(Charsets.UTF_8).orEmpty() + val meta = files["metadata.json"] + ?.let { + runCatching { + json.decodeFromString(CarnetMetadata.serializer(), it.toString(Charsets.UTF_8)) + }.getOrNull() + } + ?: CarnetMetadata() + + val body = htmlToText(html) + + // Attachments: anything under data/ that is a real file (not a dir + // entry) and not an auto-generated preview thumbnail. + val attachments = files + .filter { (name, bytes) -> + name.startsWith("data/") && + !name.endsWith("/") && + bytes.isNotEmpty() && + !name.substringAfterLast('/').startsWith("preview_") + } + .map { (name, bytes) -> Attachment(name.substringAfterLast('/'), bytes) } + + val type = if (meta.todolists.isNotEmpty()) NoteType.TASK_LIST else NoteType.TEXT + + return ParsedSqd( + content = buildContent(title, body, meta), + createdAt = meta.creationDate.takeIf { it > 0 } ?: System.currentTimeMillis(), + updatedAt = meta.lastModificationDate.takeIf { it > 0 } ?: System.currentTimeMillis(), + isFavorite = meta.rating >= 5, + color = mapColor(meta.color), + type = type, + keywords = meta.keywords, + attachments = attachments, + ) + } + + /** + * Assemble the final plain-text body. The filename is used as the title + * (first non-blank line in Incido's title/body split); then the HTML + * body; then any URLs from the metadata; then any to-do lists rendered as + * GFM checkbox lines (`- [ ]` / `- [x]`) so [NoteType.TASK_LIST] renders. + */ + private fun buildContent(title: String, body: String, meta: CarnetMetadata): String { + val blocks = mutableListOf() + blocks += title + if (body.isNotBlank()) blocks += body + if (meta.urls.isNotEmpty()) { + blocks += meta.urls.keys.filter { it.isNotBlank() } + } + for (todo in meta.todolists) { + val lines = buildList { + todo.done.forEach { if (it.isNotBlank()) add("- [x] ${it.trim()}") } + todo.todo.forEach { if (it.isNotBlank()) add("- [ ] ${it.trim()}") } + } + if (lines.isNotEmpty()) blocks += lines.joinToString("\n") + } + return blocks.filter { it.isNotBlank() }.joinToString("\n\n").trim() + } + + // --- Helpers ------------------------------------------------------------- + + /** Read every file entry (name → bytes) from a zip, skipping dirs. */ + private fun readZip(bytes: ByteArray): LinkedHashMap { + val out = LinkedHashMap() + ZipInputStream(ByteArrayInputStream(bytes)).use { zis -> + while (true) { + val entry = zis.nextEntry ?: break + if (entry.isDirectory) { zis.closeEntry(); continue } + val data = zis.readBytes() + out[entry.name] = data + zis.closeEntry() + } + } + return out + } + + /** Query the user-visible name of a SAF [uri] (falls back to null). */ + private fun queryDisplayName(uri: Uri): String? { + return try { + context.contentResolver.query(uri, null, null, null, null)?.use { cursor -> + val idx = cursor.getColumnIndex(android.provider.OpenableColumns.DISPLAY_NAME) + if (idx >= 0 && cursor.moveToFirst()) cursor.getString(idx) else null + } + } catch (_: Exception) { + null + } + } + + companion object { + /** MIME type detected from magic bytes, falling back to a default. */ + fun sniffMimeType(bytes: ByteArray): String { + fun b(i: Int) = if (i < bytes.size) bytes[i].toInt() and 0xFF else -1 + if (b(0) == 0xFF && b(1) == 0xD8 && b(2) == 0xFF) return "image/jpeg" + if (b(0) == 0x89 && b(1) == 0x50 && b(2) == 0x4E && b(3) == 0x47) return "image/png" + if (b(0) == 0x47 && b(1) == 0x49 && b(2) == 0x46 && b(3) == 0x38) return "image/gif" + if (b(0) == 0x52 && b(1) == 0x49 && b(2) == 0x46 && b(3) == 0x46 && + b(8) == 0x57 && b(9) == 0x45 && b(10) == 0x42 && b(11) == 0x50 + ) return "image/webp" + if (b(0) == 0x42 && b(1) == 0x4D) return "image/bmp" + return "application/octet-stream" + } + + /** Carnet color string → Incido [NoteColor]. Unknown values → NONE. */ + fun mapColor(raw: String): NoteColor = when (raw.trim().lowercase()) { + "red" -> NoteColor.RED + "green" -> NoteColor.GREEN + "blue" -> NoteColor.BLUE + "yellow", "amber" -> NoteColor.AMBER + "purple" -> NoteColor.PURPLE + else -> NoteColor.NONE + } + + /** + * Flatten a Carnet `index.html` note body to plain text: + * strip comments, isolate the `.edit-zone` content, turn block + * boundaries into newlines, decode entities (incl. ` `), then + * drop the remaining tags. + */ + fun htmlToText(html: String): String { + var s = html.replace(Regex("", RegexOption.DOT_MATCHES_ALL), " ") + + // Keep only the edit-zone body (everything after its opening tag, + // up to the sibling `
` that follows it). + val zoneMatch = Regex("class=\"edit-zone\"[^>]*>").find(s) + if (zoneMatch != null) { + s = s.substring(zoneMatch.range.last + 1) + val floatIdx = s.indexOf("
= 0) s = s.substring(0, floatIdx) + } + + s = s.replace(Regex("", RegexOption.IGNORE_CASE), "\n") + s = s.replace(Regex("", RegexOption.IGNORE_CASE), "\n") + s = s.replace(Regex("<[^>]+>"), "") + s = decodeEntities(s) + + // Normalise whitespace: drop trailing spaces per line, collapse + // runs of 2+ blank lines to a single blank line. + val lines = s.replace("\r", "").split('\n').map { it.trimEnd() } + val collapsed = StringBuilder() + var blanks = 0 + for (line in lines) { + if (line.isEmpty()) { + blanks++ + if (blanks <= 1) collapsed.append('\n') + } else { + blanks = 0 + collapsed.append(line).append('\n') + } + } + return collapsed.toString().trim() + } + + /** Decode the named + numeric HTML entities a Carnet body can contain. */ + private fun decodeEntities(input: String): String { + if (!input.contains('&')) return input + val sb = StringBuilder(input.length) + var i = 0 + while (i < input.length) { + val c = input[i] + if (c == '&') { + val semi = input.indexOf(';', i + 1) + if (semi in (i + 1)..(i + 12)) { + val entity = input.substring(i, semi + 1) + val decoded = NAMED_ENTITIES[entity] ?: numericEntity(entity) + if (decoded != null) { + sb.append(decoded) + i = semi + 1 + continue + } + } + } + sb.append(c) + i++ + } + return sb.toString() + } + + private fun numericEntity(entity: String): String? { + if (!entity.startsWith("&#") || !entity.endsWith(";")) return null + val body = entity.substring(2, entity.length - 1) + return try { + val code = if (body.startsWith("x") || body.startsWith("X")) { + body.substring(1).toLong(16) + } else { + body.toLong() + } + if (code in 0..0x10FFFF) String(Character.toChars(code.toInt())) else null + } catch (_: NumberFormatException) { + null + } + } + + private val NAMED_ENTITIES = mapOf( + " " to " ", + "&" to "&", + "<" to "<", + ">" to ">", + """ to "\"", + "'" to "'", + "'" to "'", + "©" to "\u00A9", + "®" to "\u00AE", + "™" to "\u2122", + "…" to "\u2026", + "—" to "\u2014", + "–" to "\u2013", + "’" to "\u2019", + "‘" to "\u2018", + "”" to "\u201D", + "“" to "\u201C", + ) + + } + + // --- Data ---------------------------------------------------------------- + + /** A parsed, ready-to-persist Carnet note. */ + data class ParsedSqd( + val content: String, + val createdAt: Long, + val updatedAt: Long, + val isFavorite: Boolean, + val color: NoteColor, + val type: NoteType, + val keywords: List, + val attachments: List, + ) { + fun hasContent(): Boolean = content.isNotBlank() || attachments.isNotEmpty() + } + + /** A single attachment (file name + raw bytes) inside a `.sqd`'s `data/`. */ + data class Attachment(val displayName: String, val bytes: ByteArray) + + @Serializable + private data class CarnetMetadata( + @SerialName("creation_date") val creationDate: Long = 0, + @SerialName("last_modification_date") val lastModificationDate: Long = 0, + val keywords: List = emptyList(), + val rating: Int = -1, + val color: String = "none", + val urls: Map = emptyMap(), + val todolists: List = emptyList(), + ) + + @Serializable + private data class CarnetTodo( + val id: String = "", + val todo: List = emptyList(), + val done: List = emptyList(), + ) +} + + + diff --git a/composeApp/src/main/kotlin/org/incido/compose/di/AppContainer.kt b/composeApp/src/main/kotlin/org/incido/compose/di/AppContainer.kt index cb7d6f2..55e68e4 100644 --- a/composeApp/src/main/kotlin/org/incido/compose/di/AppContainer.kt +++ b/composeApp/src/main/kotlin/org/incido/compose/di/AppContainer.kt @@ -11,6 +11,7 @@ import org.incido.data.crypto.NoteCryptoService import org.incido.data.repository.NoteRepositoryImpl import org.incido.domain.repository.NoteRepository import org.incido.compose.data.backup.BackupManager +import org.incido.compose.data.backup.CarnetImporter import org.incido.compose.data.security.AesGcmFieldCipher import org.incido.compose.data.security.EncryptionVerifier import org.incido.compose.data.security.ObjectBoxKeyProvider @@ -78,6 +79,10 @@ class AppContainer(context: Context) { val backupManager: BackupManager = BackupManager(appContext, noteRepository) + /** v0.3.9.0 — Carnet / QuickNote `.sqd` archive importer. */ + val carnetImporter: CarnetImporter = CarnetImporter(appContext, noteRepository) + + /** Migration sweep + startup integrity checker for the note store. */ val noteCryptoService = NoteCryptoService(boxStore, cipherHolder) diff --git a/composeApp/src/main/kotlin/org/incido/compose/ui/settings/SettingsScreen.kt b/composeApp/src/main/kotlin/org/incido/compose/ui/settings/SettingsScreen.kt index f379111..b12974e 100644 --- a/composeApp/src/main/kotlin/org/incido/compose/ui/settings/SettingsScreen.kt +++ b/composeApp/src/main/kotlin/org/incido/compose/ui/settings/SettingsScreen.kt @@ -122,6 +122,15 @@ fun SettingsScreen( var showRecoveryImportWarning by remember { mutableStateOf(false) } // SAF launchers + // v0.3.9.0 — Carnet / QuickNote archive importer. Picks a .zip (or a + // lone .sqd) and imports it directly; no passphrase, additive. + val carnetImportLauncher = rememberLauncherForActivityResult( + contract = ActivityResultContracts.OpenDocument(), + ) { uri -> + if (uri == null) return@rememberLauncherForActivityResult + scope.launch { viewModel.importCarnet(uri) } + } + val exportLauncher = rememberLauncherForActivityResult( contract = ActivityResultContracts.CreateDocument("application/json"), ) { uri -> @@ -296,6 +305,12 @@ fun SettingsScreen( securityManager?.beginExternalActivity() importLauncher.launch(arrayOf("application/json", "*/*")) }, + onImportCarnet = { + securityManager?.beginExternalActivity() + carnetImportLauncher.launch( + arrayOf("application/zip", "application/x-zip-compressed", "application/octet-stream", "*/*"), + ) + }, ) AboutSection( version = SettingsViewModel.APP_VERSION, @@ -825,6 +840,7 @@ private fun SecuritySection( private fun BackupSection( onExport: () -> Unit, onImport: () -> Unit, + onImportCarnet: () -> Unit, ) { SettingsSection(title = "Backup & Restore") { Column( @@ -846,6 +862,9 @@ private fun BackupSection( OutlinedButton(onClick = onImport, modifier = Modifier.fillMaxWidth()) { Text("Import from file…") } + OutlinedButton(onClick = onImportCarnet, modifier = Modifier.fillMaxWidth()) { + Text("Import 3rd Party Notes") + } } } } diff --git a/composeApp/src/main/kotlin/org/incido/compose/ui/settings/SettingsViewModel.kt b/composeApp/src/main/kotlin/org/incido/compose/ui/settings/SettingsViewModel.kt index 2502570..b6f80f3 100644 --- a/composeApp/src/main/kotlin/org/incido/compose/ui/settings/SettingsViewModel.kt +++ b/composeApp/src/main/kotlin/org/incido/compose/ui/settings/SettingsViewModel.kt @@ -14,6 +14,7 @@ import kotlinx.coroutines.flow.stateIn import kotlinx.coroutines.launch import org.incido.compose.IncidoApplication import org.incido.compose.data.backup.BackupManager +import org.incido.compose.data.backup.CarnetImporter import org.incido.compose.data.security.AesGcmFieldCipher import org.incido.compose.data.security.EncryptionVerifier import org.incido.compose.data.security.ObjectBoxKeyProvider @@ -46,6 +47,7 @@ class SettingsViewModel( private val security: SecurityManager, private val objectBoxKeyProvider: ObjectBoxKeyProvider, private val backupManager: BackupManager, + private val carnetImporter: CarnetImporter, private val noteCryptoService: NoteCryptoService, private val cipherHolder: CipherHolder, private val encryptionVerifier: EncryptionVerifier, @@ -245,6 +247,25 @@ class SettingsViewModel( _snackbar.value = "Imported $count note(s)." } + /** + * v0.3.9.0 — import a Carnet / QuickNote archive (`.zip` of `*.sqd`, or a + * single `.sqd`) picked via SAF. Additive — existing notes are kept. + * Failures are caught and surfaced via the snackbar so a bad file never + * crashes the app. + */ + suspend fun importCarnet(uri: android.net.Uri) { + try { + val count = carnetImporter.import(uri) + _snackbar.value = if (count > 0) { + "Imported $count Carnet note(s)." + } else { + "No Carnet notes found in that file." + } + } catch (e: Exception) { + _snackbar.value = "Carnet import failed: ${e.message ?: "unreadable file"}" + } + } + // --- v0.3.6.0 recovery key --- /** @@ -344,6 +365,7 @@ class SettingsViewModel( security = c.securityManager, objectBoxKeyProvider = c.objectBoxKeyProvider, backupManager = c.backupManager, + carnetImporter = c.carnetImporter, noteCryptoService = c.noteCryptoService, cipherHolder = c.cipherHolder, encryptionVerifier = c.encryptionVerifier,