diff --git a/app/src/main/kotlin/com/looker/droidify/database/Database.kt b/app/src/main/kotlin/com/looker/droidify/database/Database.kt index 657575ccb..eef88872a 100644 --- a/app/src/main/kotlin/com/looker/droidify/database/Database.kt +++ b/app/src/main/kotlin/com/looker/droidify/database/Database.kt @@ -18,6 +18,8 @@ import com.looker.droidify.model.Repository import com.looker.droidify.utility.common.extension.Json import com.looker.droidify.utility.common.extension.asSequence import com.looker.droidify.utility.common.extension.firstOrNull +import com.looker.droidify.utility.common.extension.gunzipIfNeeded +import com.looker.droidify.utility.common.extension.gzip import com.looker.droidify.utility.common.extension.parseDictionary import com.looker.droidify.utility.common.extension.writeDictionary import com.looker.droidify.utility.serialization.product @@ -598,6 +600,7 @@ object Database { private fun transform(cursor: Cursor): Product { return cursor.getBlob(cursor.getColumnIndexOrThrow(Schema.Product.ROW_DATA)) + .gunzipIfNeeded() .jsonParse { it.product().apply { this.repositoryId = cursor @@ -796,7 +799,7 @@ object Database { put(Schema.Product.ROW_VERSION_CODE, product.versionCode) put(Schema.Product.ROW_SIGNATURES, signatures) put(Schema.Product.ROW_COMPATIBLE, if (product.compatible) 1 else 0) - put(Schema.Product.ROW_DATA, jsonGenerate(product::serialize)) + put(Schema.Product.ROW_DATA, jsonGenerate(product::serialize).gzip()) put( Schema.Product.ROW_DATA_ITEM, jsonGenerate(product.item()::serialize), diff --git a/app/src/main/kotlin/com/looker/droidify/index/IndexMerger.kt b/app/src/main/kotlin/com/looker/droidify/index/IndexMerger.kt index 0e8a11ae0..27f62f14c 100644 --- a/app/src/main/kotlin/com/looker/droidify/index/IndexMerger.kt +++ b/app/src/main/kotlin/com/looker/droidify/index/IndexMerger.kt @@ -8,6 +8,8 @@ import com.looker.droidify.model.Release import com.looker.droidify.utility.common.extension.Json import com.looker.droidify.utility.common.extension.asSequence import com.looker.droidify.utility.common.extension.collectNotNull +import com.looker.droidify.utility.common.extension.gunzipIfNeeded +import com.looker.droidify.utility.common.extension.gzip import com.looker.droidify.utility.common.extension.writeDictionary import com.looker.droidify.utility.serialization.product import com.looker.droidify.utility.serialization.release @@ -43,7 +45,7 @@ class IndexMerger(file: File) : Closeable { ContentValues().apply { put("package_name", product.packageName) put("description", product.description) - put("data", outputStream.toByteArray()) + put("data", outputStream.toByteArray().gzip()) }, ) } @@ -65,7 +67,7 @@ class IndexMerger(file: File) : Closeable { null, ContentValues().apply { put("package_name", packageName) - put("data", outputStream.toByteArray()) + put("data", outputStream.toByteArray().gzip()) }, ) } @@ -87,7 +89,7 @@ class IndexMerger(file: File) : Closeable { ).use { cursor -> cursor.asSequence().map { currentCursor -> val description = currentCursor.getString(0) - val product = Json.factory.createParser(currentCursor.getBlob(1)).use { + val product = Json.factory.createParser(currentCursor.getBlob(1).gunzipIfNeeded()).use { it.nextToken() it.product().apply { this.repositoryId = repositoryId @@ -95,7 +97,7 @@ class IndexMerger(file: File) : Closeable { } } val releases = currentCursor.getBlob(2)?.let { bytes -> - Json.factory.createParser(bytes).use { + Json.factory.createParser(bytes.gunzipIfNeeded()).use { it.nextToken() it.collectNotNull( JsonToken.START_OBJECT, diff --git a/app/src/main/kotlin/com/looker/droidify/utility/common/extension/Gzip.kt b/app/src/main/kotlin/com/looker/droidify/utility/common/extension/Gzip.kt new file mode 100644 index 000000000..723226bca --- /dev/null +++ b/app/src/main/kotlin/com/looker/droidify/utility/common/extension/Gzip.kt @@ -0,0 +1,38 @@ +package com.looker.droidify.utility.common.extension + +import java.io.ByteArrayOutputStream +import java.util.zip.GZIPInputStream +import java.util.zip.GZIPOutputStream + +private const val GZIP_MAGIC_FIRST = 0x1f +private const val GZIP_MAGIC_SECOND = 0x8b +private const val BYTE_MASK = 0xff +private const val GZIP_HEADER_SIZE = 2 + +/** + * GZIP-compresses [this]. + * + * Large serialized JSON blobs (e.g. an app that ships hundreds of releases, like the Brave repos) + * can exceed SQLite's ~2 MB per-row `CursorWindow` limit, which makes the whole row unreadable and + * throws `SQLiteBlobTooBigException`. Compressing the blob before storing it keeps the row small + * enough to be read back. JSON typically shrinks ~5-10x, so this raises the practical ceiling well + * beyond any realistic single entry. + */ +fun ByteArray.gzip(): ByteArray { + val output = ByteArrayOutputStream() + GZIPOutputStream(output).use { it.write(this) } + return output.toByteArray() +} + +/** + * Reverses [gzip]. If [this] is not GZIP data (detected via the magic header), it is returned + * unchanged, so reads remain backward compatible with rows written before compression was + * introduced — no database migration or re-sync is required. + */ +fun ByteArray.gunzipIfNeeded(): ByteArray { + val isGzip = size >= GZIP_HEADER_SIZE && + this[0].toInt() and BYTE_MASK == GZIP_MAGIC_FIRST && + this[1].toInt() and BYTE_MASK == GZIP_MAGIC_SECOND + if (!isGzip) return this + return GZIPInputStream(inputStream()).use { it.readBytes() } +} diff --git a/app/src/test/kotlin/com/looker/droidify/utility/common/extension/GzipTest.kt b/app/src/test/kotlin/com/looker/droidify/utility/common/extension/GzipTest.kt new file mode 100644 index 000000000..3bfbc518b --- /dev/null +++ b/app/src/test/kotlin/com/looker/droidify/utility/common/extension/GzipTest.kt @@ -0,0 +1,43 @@ +package com.looker.droidify.utility.common.extension + +import org.junit.jupiter.api.Assertions.assertArrayEquals +import org.junit.jupiter.api.Assertions.assertTrue +import org.junit.jupiter.api.Test + +class GzipTest { + + @Test + fun `gzip then gunzip round-trips the original bytes`() { + val original = """{"packageName":"com.example","releases":[1,2,3]}""".toByteArray() + val restored = original.gzip().gunzipIfNeeded() + assertArrayEquals(original, restored) + } + + @Test + fun `round-trips a large payload that would overflow a CursorWindow`() { + // ~5 MB of JSON-ish data, larger than SQLite's ~2 MB per-row window. + val original = ("""{"v":"1.0","sig":"abc"},""".repeat(220_000)).toByteArray() + val restored = original.gzip().gunzipIfNeeded() + assertArrayEquals(original, restored) + } + + @Test + fun `gunzipIfNeeded leaves uncompressed JSON untouched for backward compatibility`() { + // Rows written before compression are plain JSON (start with '{'); they must still read. + val legacy = """{"packageName":"com.legacy"}""".toByteArray() + assertArrayEquals(legacy, legacy.gunzipIfNeeded()) + } + + @Test + fun `gunzipIfNeeded leaves short or empty arrays untouched`() { + assertArrayEquals(ByteArray(0), ByteArray(0).gunzipIfNeeded()) + val oneByte = byteArrayOf(0x1f) + assertArrayEquals(oneByte, oneByte.gunzipIfNeeded()) + } + + @Test + fun `gzip actually shrinks compressible data`() { + val repetitive = "a".repeat(100_000).toByteArray() + assertTrue(repetitive.gzip().size < repetitive.size) + } +}