perf(phase-1/m2.5): export memory measured on synthesized worst-case libraries

Phase 1 M2 task 5 (tools/DESKTOP_PHASE_PLAN.md §4): measure & record,
no redesign. New :core:data-jvm:backupMemory probe (2g heap) samples
heap-used at 1 ms during BackupBridge.exportPlain — the phone's own
export path — across four fresh libraries:

  24 MB data (3 att)  → peak 230 MB   |  2 MB text x2000 notes → 140 MB
  80 MB data (10 att) → peak 777 MB   | 320 MB data (40 att)  → OOM at -Xmx2g

Attachments drive memory (~9–10x allocation pressure: store load →
base64 String → UTF-8 bytes → JsonElement tree); note count alone is
trivial. The plan's 250 MB gate trips at ~25–30 MB of attachments, and
the phone (no largeHeap) shares the ceiling — a pre-existing Android
limitation, unchanged by the bridge extraction. The real library
exports at 366 KB ≈ 1000x headroom. Bonus: a ~1.6 GB library dies in
objectbox-java's DbFullException before the bridge ever sees it.

Verdict: record, don't redesign. Streaming formatVersion=2 stays parked
for Phase 2, required only if attachment libraries approach ~25 MB.
This commit is contained in:
Incido Developer committed 2026-10-05 19:48:02 +13:00
1 parent 52ccceb732
commit 9cb7fdbf58
3 files changed
+195

No files matched your search

+15
View File
@@ -103,4 +103,19 @@ tasks.register<JavaExec>("backupSmoke") {
description = "Round-trips a real Android backup export through BackupBridge on the JVM."
classpath = sourceSets["main"].runtimeClasspath
mainClass.set("org.incido.data.jvm.BackupRoundTripKt")
}
/**
* `./gradlew :core:data-jvm:backupMemory` — synthesizes worst-case
* libraries and samples heap peak during BackupBridge.exportPlain
* (Phase 1, M2 task 5: measure & record, no redesign). Needs a bigger
* heap than the other probes on purpose — the "watch tier" is ~320 MB of
* attachments and the bridge holds the whole JSON in memory by design.
*/
tasks.register<JavaExec>("backupMemory") {
group = "verification"
description = "Measures export heap peaks on synthesized worst-case libraries (M2 t5)."
classpath = sourceSets["main"].runtimeClasspath
mainClass.set("org.incido.data.jvm.BackupMemoryProbeKt")
maxHeapSize = "2g"
}
@@ -0,0 +1,145 @@
package org.incido.data.jvm
import java.io.File
import java.io.FileOutputStream
import kotlin.random.Random
import kotlin.system.measureTimeMillis
import java.lang.management.ManagementFactory
import kotlinx.coroutines.runBlocking
import org.incido.compose.data.backup.BackupBridge
import org.incido.data.ImageOrientationNormalizer
import org.incido.data.ObjectBoxStore
import org.incido.data.crypto.CipherHolder
import org.incido.data.repository.NoteRepositoryImpl
import org.incido.domain.model.Note
import org.incido.domain.repository.NoteRepository
/**
* Phase 1 (M2 task 5): MEASURE the export path's memory on synthesized
* worst-case libraries — do not redesign it. The plan's gate: ~10–20
* notes with attachments must be trivial, 200+ with large attachments is
* "watch", and only a >250 MB peak forces the streaming-redesign
* discussion (which would be a formatVersion=2 change, Phase 2).
*
* The probe drives `BackupBridge.exportPlain` — the exact code the phone
* runs inside its `Uri → stream` shim — against three fresh ObjectBox
* libraries, sampling heap-used at 1 ms while the export runs.
*
* Run with `./gradlew :core:data-jvm:backupMemory` (2 GB ceiling set in
* the task; a tier that outruns it is itself the finding).
*/
private const val MB = 1024 * 1024
private class Tier(
val label: String,
val dirName: String,
val notes: Int,
/** One attachment each on the first N notes; 0 = no attachments. */
val attachmentsTotal: Int,
val attachmentMb: Int,
) {
val payloadMb: Long get() = notes * 1300L / MB + attachmentsTotal * attachmentMb.toLong()
}
/** 1 ms heap sampler; peak is the highest heap-used seen while running. */
private class HeapSampler : Runnable {
@Volatile var stop = false
@Volatile var peak = 0L
private val bean = ManagementFactory.getMemoryMXBean()
override fun run() {
while (!stop) {
val used = bean.heapMemoryUsage.used
if (used > peak) peak = used
try {
Thread.sleep(1)
} catch (_: InterruptedException) {
return
}
}
}
}
fun main() = runBlocking {
println("=== Incido Phase 1 M2 task 5: backup export memory (measure, don't redesign) ===")
println("heap ceiling : ${Runtime.getRuntime().maxMemory() / MB} MB (-Xmx of the backupMemory task)")
val libRoot = File("build/backup-memory").apply { deleteRecursively(); mkdirs() }
val tiers = listOf(
// Cheapest first so a failure in a later tier can't eat the findings.
Tier("phone-at-gate (20 notes, 3 att x 8 MB)", "t1-gate", 20, 3, 8),
Tier("many notes (2000 notes, no att)", "t2-count", 2000, 0, 0),
Tier("phone-heavy (100 notes, 10 att x 8 MB)", "t3-heavy", 100, 10, 8),
Tier("watch tier (200 notes, 40 att x 8 MB)", "t4-watch", 200, 40, 8),
)
for (tier in tiers) {
val dir = File(libRoot, tier.dirName).apply { mkdirs() }
val out = File(libRoot, "${tier.dirName}.out.json")
runCatching {
ObjectBoxStore.reset()
ObjectBoxStore.init(dir)
val repo: NoteRepository =
NoteRepositoryImpl(ObjectBoxStore.get(), CipherHolder(), ImageOrientationNormalizer.NoOp)
// Synthesize: fixed seed for repeatability; ~1.3 KB text per note.
val rnd = Random(42)
val ids = ArrayList<Long>(tier.notes)
repeat(tier.notes) { i ->
val content = (1..24).joinToString("\n") { "Note $i line $it — synthesized filler for the memory probe." }
val now = 1_700_000_000_000L + i * 1000L
val saved = repo.saveNoteWithHashtags(
note = Note(id = 0L, content = content, createdAt = now, updatedAt = now),
hashtagsToAdd = emptySet(),
hashtagsToRemove = emptySet(),
)
ids += saved.id
}
if (tier.attachmentMb > 0) {
val blob = ByteArray(tier.attachmentMb * MB)
repeat(tier.attachmentsTotal) { j ->
rnd.nextBytes(blob)
repo.addAttachment(
noteId = ids[j % ids.size],
bytes = blob,
mimeType = "image/jpeg",
displayName = "att-$j.jpg",
)
}
}
System.gc()
Thread.sleep(100)
val baseline = ManagementFactory.getMemoryMXBean().heapMemoryUsage.used
val sampler = HeapSampler()
val samplerThread = Thread(sampler).apply { isDaemon = true; start() }
val ms = measureTimeMillis {
BackupBridge.exportPlain(repo, FileOutputStream(out), "m2-memory-probe")
}
sampler.stop = true
samplerThread.join()
println(
"%s | data≈%d MB → export %d ms | heap peak %d MB (+%d MB alloc pressure) | file %d MB"
.format(
tier.label, tier.payloadMb, ms,
sampler.peak / MB, (sampler.peak - baseline) / MB, out.length() / MB,
),
)
}.onFailure {
println(
"%s | data≈%d MB → FAILED: %s: %s"
.format(tier.label, tier.payloadMb, it::class.java.simpleName, it.message),
)
}
out.delete()
ObjectBoxStore.reset()
dir.deleteRecursively()
}
val hwm = runCatching {
File("/proc/self/status").readLines().first { it.startsWith("VmHWM:") }
.filter(Char::isDigit).toLong() / 1024
}.getOrNull()
println("process peak RSS (VmHWM, incl. ObjectBox native): ${hwm ?: -1} MB")
println("verdict reference: plan gate = >250 MB forces streaming-redesign talk (formatVersion 2, Phase 2)")
println("=== backupMemory done ===")
}