Skip to content

Commit 8aeac82

Browse files
committed
Compress large text changes in ChangeHistory
1 parent c7628eb commit 8aeac82

7 files changed

Lines changed: 328 additions & 4 deletions

File tree

app/build.gradle.kts

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -269,6 +269,7 @@ dependencies {
269269
}
270270
implementation("org.commonmark:commonmark:0.27.0")
271271
implementation("org.commonmark:commonmark-ext-gfm-strikethrough:0.27.0")
272+
implementation("com.github.luben:zstd-jni:1.5.7-6@aar")
272273

273274
androidTestImplementation("androidx.room:room-testing:$roomVersion")
274275
androidTestImplementation("androidx.work:work-testing:2.9.1")
@@ -282,4 +283,5 @@ dependencies {
282283
testImplementation("org.mockito.kotlin:mockito-kotlin:5.4.0")
283284
testImplementation("org.mockito:mockito-core:5.13.0")
284285
testImplementation("org.robolectric:robolectric:4.15.1")
286+
testImplementation("com.github.luben:zstd-jni:1.5.7-6")
285287
}

app/src/main/java/com/philkes/notallyx/presentation/UiExtensions.kt

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -349,7 +349,7 @@ fun StylableEditTextWithHistory.createTextWatcherWithHistory(
349349

350350
override fun afterTextChanged(s: Editable?) {
351351
val textAfter = requireNotNull(s, { "afterTextChanged: Editable is null" }).clone()
352-
if (textAfter.hasNotChanged(stateBefore.text)) {
352+
if (textAfter.hasNotChanged(stateBefore.getEditableText())) {
353353
return
354354
}
355355
updateModel.invoke(textAfter)

app/src/main/java/com/philkes/notallyx/presentation/view/note/listitem/ListManager.kt

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -287,7 +287,7 @@ class ListManager(
287287
endSearch?.invoke()
288288
// }
289289
val item = items[position]
290-
item.body = value.text.toString()
290+
item.body = value.getEditableText().toString()
291291
if (pushChange) {
292292
changeHistory.push(ListEditTextChange(stateBefore, getState(), this))
293293
// TODO: fix focus change
Lines changed: 86 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,86 @@
1+
package com.philkes.notallyx.utils
2+
3+
import android.util.Base64
4+
import android.util.Log
5+
import com.github.luben.zstd.Zstd
6+
import com.philkes.notallyx.data.model.Converters
7+
import com.philkes.notallyx.data.model.SpanRepresentation
8+
import java.io.ByteArrayInputStream
9+
import java.io.ByteArrayOutputStream
10+
import java.util.zip.GZIPInputStream
11+
import java.util.zip.GZIPOutputStream
12+
import org.json.JSONObject
13+
14+
/**
15+
* Shared compression utilities for large text payloads to decrease memory and storage usage.
16+
* - For EditText state (text + spans), we store a GZIP-compressed JSON as ByteArray.
17+
*/
18+
object CompressUtility {
19+
20+
// Threshold in characters for when to compress text (approximately 7KB)
21+
const val COMPRESSION_THRESHOLD: Int = 7_000
22+
23+
// Prefix to mark a String as compressed (so we can store in a TEXT column).
24+
private const val PREFIX: String = "GZ:"
25+
26+
// region Text + Spans (ByteArray)
27+
28+
/** Compresses text and spans using GZIP compression into a ByteArray. */
29+
fun compressTextAndSpans(text: String, spans: List<SpanRepresentation>): ByteArray {
30+
val jsonObject = JSONObject()
31+
jsonObject.put("text", text)
32+
jsonObject.put("spans", Converters.spansToJSONArray(spans))
33+
val bytes = jsonObject.toString().toByteArray(Charsets.UTF_8)
34+
return Zstd.compress(bytes, 4)
35+
}
36+
37+
/** Decompresses text and spans that were compressed with GZIP. */
38+
fun decompressTextAndSpans(compressedData: ByteArray): Pair<String, List<SpanRepresentation>> {
39+
val decompressedSize = Zstd.decompressedSize(compressedData)
40+
val result = ByteArray(decompressedSize.toInt())
41+
Zstd.decompress(result, compressedData)
42+
val jsonString = result.toString(Charsets.UTF_8)
43+
// val bis = ByteArrayInputStream(compressedData)
44+
// val jsonString = GZIPInputStream(bis).use { gzipIS ->
45+
// gzipIS.readBytes().toString(Charsets.UTF_8)
46+
// }
47+
val jsonObject = JSONObject(jsonString)
48+
val text = jsonObject.getString("text")
49+
val spansArray = jsonObject.getJSONArray("spans")
50+
val spans = Converters.jsonToSpans(spansArray)
51+
return Pair(text, spans)
52+
}
53+
54+
// endregion
55+
56+
// region String-only (BaseNote.body)
57+
58+
/** Compress String if above threshold. Returns original if already small. */
59+
fun compressIfNeeded(text: String): String {
60+
if (text.length <= COMPRESSION_THRESHOLD) return text
61+
return try {
62+
val bos = ByteArrayOutputStream()
63+
GZIPOutputStream(bos).use { it.write(text.toByteArray(Charsets.UTF_8)) }
64+
val b64 = Base64.encodeToString(bos.toByteArray(), Base64.NO_WRAP)
65+
PREFIX + b64
66+
} catch (e: Exception) {
67+
Log.w("CompressUtility", "Failed to compress, returning original", e)
68+
text
69+
}
70+
}
71+
72+
/** Decompress String if it was previously compressed with [compressIfNeeded]. */
73+
fun decompressIfNeeded(text: String): String {
74+
if (!text.startsWith(PREFIX)) return text
75+
val b64 = text.removePrefix(PREFIX)
76+
return try {
77+
val bytes = Base64.decode(b64, Base64.NO_WRAP)
78+
val bis = ByteArrayInputStream(bytes)
79+
GZIPInputStream(bis).use { it.readBytes().toString(Charsets.UTF_8) }
80+
} catch (e: Exception) {
81+
Log.w("CompressUtility", "Failed to decompress, returning original", e)
82+
text
83+
}
84+
}
85+
// endregion
86+
}

app/src/main/java/com/philkes/notallyx/utils/changehistory/EditTextWithHistoryChange.kt

Lines changed: 105 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,10 +1,20 @@
11
package com.philkes.notallyx.utils.changehistory
22

3+
import android.graphics.Typeface
34
import android.text.Editable
5+
import android.text.SpannableStringBuilder
6+
import android.text.style.CharacterStyle
7+
import android.text.style.StrikethroughSpan
8+
import android.text.style.StyleSpan
9+
import android.text.style.TypefaceSpan
10+
import android.text.style.URLSpan
411
import androidx.core.text.getSpans
12+
import com.philkes.notallyx.data.model.SpanRepresentation
13+
import com.philkes.notallyx.presentation.applySpans
514
import com.philkes.notallyx.presentation.clone
615
import com.philkes.notallyx.presentation.view.misc.StylableEditTextWithHistory
716
import com.philkes.notallyx.presentation.view.misc.highlightableview.HighlightSpan
17+
import com.philkes.notallyx.utils.CompressUtility
818

919
class EditTextWithHistoryChange(
1020
private val editText: StylableEditTextWithHistory,
@@ -15,7 +25,7 @@ class EditTextWithHistoryChange(
1525

1626
override fun update(value: EditTextState, isUndo: Boolean) {
1727
editText.applyWithoutTextWatcher {
18-
val text = value.text.withoutSpans<HighlightSpan>()
28+
val text = value.getEditableText().withoutSpans<HighlightSpan>()
1929
setText(text)
2030
updateModel.invoke(text)
2131
requestFocus()
@@ -24,7 +34,100 @@ class EditTextWithHistoryChange(
2434
}
2535
}
2636

27-
data class EditTextState(val text: Editable, val cursorPos: Int)
37+
/**
38+
* Represents the state of an EditText, storing either the full text or a compressed version for
39+
* large text to reduce memory usage.
40+
*/
41+
class EditTextState(text: Editable, val cursorPos: Int) {
42+
companion object {}
43+
44+
// Either Editable (for small text) or ByteArray (compressed, for large text and spans)
45+
private val textContent: Any
46+
47+
init {
48+
// Extract spans from the Editable
49+
// Compress text and spans together
50+
this.textContent =
51+
if (text.length > CompressUtility.COMPRESSION_THRESHOLD) {
52+
// Extract spans from the Editable
53+
val spans = extractSpansFromEditable(text)
54+
// Compress text and spans together
55+
CompressUtility.compressTextAndSpans(
56+
text.toString(),
57+
spans as List<SpanRepresentation>,
58+
)
59+
} else {
60+
text
61+
}
62+
}
63+
64+
/** Extracts spans from an Editable and converts them to SpanRepresentation objects. */
65+
private fun extractSpansFromEditable(text: Editable): List<SpanRepresentation> {
66+
val representations = mutableListOf<SpanRepresentation>()
67+
68+
text.getSpans(0, text.length, CharacterStyle::class.java).forEach { span ->
69+
val end = text.getSpanEnd(span)
70+
val start = text.getSpanStart(span)
71+
72+
// Skip invalid spans
73+
if (start < 0 || end < 0 || start >= text.length || end > text.length) {
74+
return@forEach
75+
}
76+
77+
val representation =
78+
SpanRepresentation(
79+
start = start,
80+
end = end,
81+
bold = false,
82+
link = false,
83+
linkData = null,
84+
italic = false,
85+
monospace = false,
86+
strikethrough = false,
87+
)
88+
89+
when (span) {
90+
is StyleSpan -> {
91+
if (span.style == Typeface.BOLD) {
92+
representation.bold = true
93+
} else if (span.style == Typeface.ITALIC) {
94+
representation.italic = true
95+
}
96+
}
97+
is URLSpan -> {
98+
representation.link = true
99+
representation.linkData = span.url
100+
}
101+
is TypefaceSpan -> {
102+
if (span.family == "monospace") {
103+
representation.monospace = true
104+
}
105+
}
106+
is StrikethroughSpan -> {
107+
representation.strikethrough = true
108+
}
109+
}
110+
111+
if (representation.isNotUseless()) {
112+
representations.add(representation)
113+
}
114+
}
115+
116+
return representations
117+
}
118+
119+
/** Returns the Editable text, decompressing it if necessary and applying spans. */
120+
fun getEditableText(): Editable {
121+
return when (textContent) {
122+
is Editable -> textContent
123+
is ByteArray -> {
124+
val (text, spans) = CompressUtility.decompressTextAndSpans(textContent)
125+
text.applySpans(spans)
126+
}
127+
else -> SpannableStringBuilder()
128+
}
129+
}
130+
}
28131

29132
inline fun <reified T : Any> Editable.withoutSpans(): Editable =
30133
clone().apply { this.getSpans<T>().forEach { removeSpan(it) } }

app/src/test/kotlin/com/philkes/notallyx/test/TestUtils.kt

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -30,6 +30,8 @@ fun mockAndroidLog() {
3030
every { Log.v(any(), any()) } returns 0
3131
every { Log.d(any(), any()) } returns 0
3232
every { Log.i(any(), any()) } returns 0
33+
every { Log.w(any<String>(), any<String>()) } returns 0
34+
every { Log.w(any<String>(), any<String>(), any<Throwable>()) } returns 0
3335
every { Log.e(any(), any()) } returns 0
3436
}
3537

Lines changed: 131 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,131 @@
1+
package com.philkes.notallyx.utils
2+
3+
import android.graphics.Typeface
4+
import android.text.Editable
5+
import android.text.Spanned
6+
import android.text.style.StrikethroughSpan
7+
import android.text.style.StyleSpan
8+
import android.text.style.TypefaceSpan
9+
import android.text.style.URLSpan
10+
import com.philkes.notallyx.test.mockAndroidLog
11+
import com.philkes.notallyx.utils.changehistory.EditTextState
12+
import org.junit.Assert.assertEquals
13+
import org.junit.Assert.assertFalse
14+
import org.junit.Assert.assertTrue
15+
import org.junit.Before
16+
import org.junit.Test
17+
import org.junit.runner.RunWith
18+
import org.robolectric.RobolectricTestRunner
19+
import org.robolectric.annotation.Config
20+
21+
@RunWith(RobolectricTestRunner::class)
22+
@Config(manifest = Config.NONE, sdk = [35])
23+
class EditTextCompressionTest {
24+
25+
@Before
26+
fun setUp() {
27+
// Silence Android Log calls in unit tests
28+
mockAndroidLog()
29+
}
30+
31+
@Test
32+
fun `compressIfNeeded does not compress at threshold and compresses above`() {
33+
val threshold = CompressUtility.COMPRESSION_THRESHOLD
34+
val base = "x".repeat(threshold)
35+
val atThreshold = CompressUtility.compressIfNeeded(base)
36+
// exact threshold should not be compressed
37+
assertFalse(atThreshold.startsWith("GZ:"))
38+
assertEquals(base, CompressUtility.decompressIfNeeded(atThreshold))
39+
40+
val above = base + "y"
41+
val compressed = CompressUtility.compressIfNeeded(above)
42+
// above threshold should be compressed and round-trip correctly
43+
assertTrue(compressed.startsWith("GZ:"))
44+
val roundTrip = CompressUtility.decompressIfNeeded(compressed)
45+
assertEquals(above, roundTrip)
46+
}
47+
48+
@Test
49+
fun `EditTextState stores uncompressed Editable when at or below threshold`() {
50+
val text = "a".repeat(CompressUtility.COMPRESSION_THRESHOLD)
51+
val editable = Editable.Factory.getInstance().newEditable(text)
52+
val state = EditTextState(editable, cursorPos = 0)
53+
54+
// Reflect to access the private field textContent and assert it's an Editable
55+
val field =
56+
EditTextState::class.java.getDeclaredField("textContent").apply { isAccessible = true }
57+
val content = field.get(state)
58+
assertTrue("Expected Editable for small text", content is Editable)
59+
60+
// getEditableText should return the same text
61+
val out = state.getEditableText()
62+
assertEquals(text, out.toString())
63+
}
64+
65+
@Test
66+
fun `EditTextState compresses large text and round-trips text and spans`() {
67+
// Create text larger than threshold with distinct regions for spans
68+
val threshold = CompressUtility.COMPRESSION_THRESHOLD
69+
val prefix = "p".repeat(10)
70+
val body = "b".repeat(threshold + 100)
71+
val suffix = "s".repeat(10)
72+
val longText = prefix + body + suffix
73+
74+
val editable = Editable.Factory.getInstance().newEditable(longText)
75+
76+
// Apply multiple span types on defined ranges
77+
// bold on [5, 25)
78+
editable.setSpan(StyleSpan(Typeface.BOLD), 5, 25, Spanned.SPAN_EXCLUSIVE_EXCLUSIVE)
79+
// italic on [30, 60)
80+
editable.setSpan(StyleSpan(Typeface.ITALIC), 30, 60, Spanned.SPAN_EXCLUSIVE_EXCLUSIVE)
81+
// monospace on [100, 130)
82+
editable.setSpan(TypefaceSpan("monospace"), 100, 130, Spanned.SPAN_EXCLUSIVE_EXCLUSIVE)
83+
// strikethrough on [200, 240)
84+
editable.setSpan(StrikethroughSpan(), 200, 240, Spanned.SPAN_EXCLUSIVE_EXCLUSIVE)
85+
// link on [300, 330)
86+
editable.setSpan(URLSpan("https://example.com"), 300, 330, Spanned.SPAN_EXCLUSIVE_EXCLUSIVE)
87+
88+
val state = EditTextState(editable, cursorPos = 42)
89+
90+
// Verify compressed backing storage via reflection
91+
val field =
92+
EditTextState::class.java.getDeclaredField("textContent").apply { isAccessible = true }
93+
val content = field.get(state)
94+
assertTrue("Expected ByteArray backing for large text", content is ByteArray)
95+
96+
// Decompress and verify text and spans are preserved
97+
val out = state.getEditableText()
98+
assertEquals(longText, out.toString())
99+
100+
// Validate spans exist at the same ranges
101+
fun hasSpan(
102+
rangeStart: Int,
103+
rangeEnd: Int,
104+
clazz: Class<*>,
105+
extra: (Any) -> Boolean = { true },
106+
): Boolean {
107+
val spans = out.getSpans(rangeStart, rangeEnd, clazz)
108+
return spans.any { span ->
109+
val s = out.getSpanStart(span)
110+
val e = out.getSpanEnd(span)
111+
s == rangeStart && e == rangeEnd && extra(span)
112+
}
113+
}
114+
115+
assertTrue(
116+
hasSpan(5, 25, StyleSpan::class.java) { (it as StyleSpan).style == Typeface.BOLD }
117+
)
118+
assertTrue(
119+
hasSpan(30, 60, StyleSpan::class.java) { (it as StyleSpan).style == Typeface.ITALIC }
120+
)
121+
assertTrue(
122+
hasSpan(100, 130, TypefaceSpan::class.java) {
123+
(it as TypefaceSpan).family == "monospace"
124+
}
125+
)
126+
assertTrue(hasSpan(200, 240, StrikethroughSpan::class.java))
127+
assertTrue(
128+
hasSpan(300, 330, URLSpan::class.java) { (it as URLSpan).url == "https://example.com" }
129+
)
130+
}
131+
}

0 commit comments

Comments
 (0)