From a9b8ed2ecb1ea89402e805df93570eeb926212eb Mon Sep 17 00:00:00 2001 From: HexaField <10372036+HexaField@users.noreply.github.com> Date: Thu, 4 Jun 2026 00:01:00 +1000 Subject: [PATCH 01/19] feat(executor): short-circuit literal language in resolve_property_value (V1) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Part of the Channel V / Channel E separation. See ~/.sovereign/membranes/coasys/research/literal-encoding-and-sparql-pushdown-2026-06-03.md §9.1 for the audit. When resolve_language == "literal", encode the value directly via literal_encode and return a deterministic plain URI (literal:string:X / :number: / :boolean: / :json:) instead of going through LanguageController.expression_create. Provenance for property values lives on the link reifier — the signed-envelope shape is for Channel E (expression.create) callers, not for property storage. For all other resolve_language values (real language controllers — note, image, etc.) the existing expression_create flow is unchanged. --- .../src/perspectives/perspective_instance.rs | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/rust-executor/src/perspectives/perspective_instance.rs b/rust-executor/src/perspectives/perspective_instance.rs index 030d98674..bb9fcb6fd 100644 --- a/rust-executor/src/perspectives/perspective_instance.rs +++ b/rust-executor/src/perspectives/perspective_instance.rs @@ -4020,7 +4020,21 @@ impl PerspectiveInstance { .await?; if let Some(resolve_language) = resolve_language { - // Create an expression for the value + // For the built-in "literal" language, encode the value directly into a + // deterministic plain URI (`literal:string:X` / `:number:` / `:boolean:` / `:json:`). + // The link reifier is the canonical source of provenance — the signed-envelope + // shape is for Channel E (`expression.create(value, "literal")`) callers, not + // for property storage. Skipping the language controller here is the Channel V + // short-circuit; see + // ~/.sovereign/membranes/coasys/research/literal-encoding-and-sparql-pushdown-2026-06-03.md + // §9.1 (V1) for the audit. + if resolve_language == "literal" { + let encoded = crate::languages::literal_encode(value); + return Ok(format!("literal:{}", encoded)); + } + + // Real language controllers (note, image, etc.) — Channel E. Keep the + // signed-expression path so the produced URL carries its own provenance. let controller = crate::languages::LanguageController::global_instance(); let agent_context = context.clone(); match controller From b93ae1fe0c047c00188ada3bc935669c4eee3e61 Mon Sep 17 00:00:00 2001 From: HexaField <10372036+HexaField@users.noreply.github.com> Date: Thu, 4 Jun 2026 00:01:06 +1000 Subject: [PATCH 02/19] refactor(literal): stop wrapping primitives in synthetic envelope (V6) literal_decode used to wrap non-object primitives in a fake {author: "", timestamp: "", data: value, proof: {}} envelope. That conflated Channel V (link-property values; provenance lives on the link reifier) with Channel E (signed expressions). Now primitives decode to their raw JSON value and objects pass through unchanged. Channel-E callers that legitimately want envelope shape go through create_signed_expression, not literal_decode. Unit tests updated to assert raw round-trip; the JSON-object test is unchanged. --- rust-executor/src/languages/literal.rs | 46 +++++++++++--------------- 1 file changed, 19 insertions(+), 27 deletions(-) diff --git a/rust-executor/src/languages/literal.rs b/rust-executor/src/languages/literal.rs index 6f5ff8e1c..cb5d16ffd 100644 --- a/rust-executor/src/languages/literal.rs +++ b/rust-executor/src/languages/literal.rs @@ -32,8 +32,16 @@ pub fn literal_encode(value: &JsonValue) -> String { /// Decode a literal URL expression part back into a JSON value. /// /// Mirrors the TypeScript `Literal.fromUrl("literal://").get()` behavior. -/// If the decoded value is not an object (i.e., is a primitive), wraps it in a standard -/// expression envelope with `author`, `timestamp`, `data`, and `proof` fields. +/// Primitives (`string:` / `number:` / `boolean:`) decode to their raw JSON value. +/// Objects decoded from `json:` payloads pass through unchanged — if they happen to +/// be a Channel-E signed-expression envelope (`{author, timestamp, data, proof}`) +/// the caller is responsible for interpreting that shape. +/// +/// Note: this used to wrap primitives in a synthetic `{author: "", …}` +/// envelope. That conflated Channel V (link-property values; provenance lives on +/// the link reifier) with Channel E (signed expressions). See +/// ~/.sovereign/membranes/coasys/research/literal-encoding-and-sparql-pushdown-2026-06-03.md +/// §9.1 (V6) for the audit. pub fn literal_decode(expression_part: &str) -> Result { let value = if let Some(rest) = expression_part.strip_prefix("string:") { let decoded = percent_decode_str(rest).decode_utf8().map_err(|e| { @@ -86,27 +94,11 @@ pub fn literal_decode(expression_part: &str) -> Result serde_json::from_str(&decoded).unwrap_or(JsonValue::String(decoded.to_string())) }; - // If the value is already an object (e.g., a full expression), return it as-is. - // Otherwise, wrap it in a standard expression envelope. - if value.is_object() { - Ok(value) - } else { - let mut envelope = serde_json::Map::new(); - envelope.insert( - "author".to_string(), - JsonValue::String("".to_string()), - ); - envelope.insert( - "timestamp".to_string(), - JsonValue::String("".to_string()), - ); - envelope.insert("data".to_string(), value); - envelope.insert( - "proof".to_string(), - JsonValue::Object(serde_json::Map::new()), - ); - Ok(JsonValue::Object(envelope)) - } + // Return the decoded value as-is. Objects (which may be legitimate signed-expression + // envelopes from Channel E callers) pass through unchanged; primitives are returned + // raw — the synthetic-envelope wrapper that used to live here was removed as part of + // the Channel V / Channel E separation. + Ok(value) } #[cfg(test)] @@ -119,8 +111,8 @@ mod tests { let encoded = literal_encode(&value); assert!(encoded.starts_with("string:")); let decoded = literal_decode(&encoded).unwrap(); - // Primitives get wrapped in an envelope - assert_eq!(decoded["data"], value); + // Primitives now round-trip as raw values (no synthetic envelope). + assert_eq!(decoded, value); } #[test] @@ -129,7 +121,7 @@ mod tests { let encoded = literal_encode(&value); assert!(encoded.starts_with("number:")); let decoded = literal_decode(&encoded).unwrap(); - assert_eq!(decoded["data"], value); + assert_eq!(decoded, value); } #[test] @@ -138,7 +130,7 @@ mod tests { let encoded = literal_encode(&value); assert!(encoded.starts_with("boolean:")); let decoded = literal_decode(&encoded).unwrap(); - assert_eq!(decoded["data"], value); + assert_eq!(decoded, value); } #[test] From d15a6cb6a6ac90fd8ccfd63ebe16650929a6e4b7 Mon Sep 17 00:00:00 2001 From: HexaField <10372036+HexaField@users.noreply.github.com> Date: Thu, 4 Jun 2026 00:01:12 +1000 Subject: [PATCH 03/19] refactor(sdk): simplify parseLit; Channel V values are plain (V11) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit After the Channel V refactor, Literal.fromUrl(val).get() for property values returns the plain primitive directly. The only objects it can return now are legitimate JSON objects from literal:json: payloads, which we JSON-stringify for display. Drops the .data extraction branch that used to unwrap signed-envelope literals — those no longer exist for new property writes. --- core/src/perspectives/SparqlBindings.ts | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/core/src/perspectives/SparqlBindings.ts b/core/src/perspectives/SparqlBindings.ts index 0de6d2ac0..8964ea8f9 100644 --- a/core/src/perspectives/SparqlBindings.ts +++ b/core/src/perspectives/SparqlBindings.ts @@ -50,14 +50,17 @@ export function parseSparqlCount(result: CountBinding[] | undefined | null): num * Mirrors Flux's local `parseLit` helper. Returns `''` for `undefined`/empty * input. Falls through to the raw value if decoding fails (defensive — the * binding may already be a plain string for unwrapped properties). + * + * Channel V values (link-property literals) are stored as plain + * `literal:string:` / `:number:` / `:boolean:` / `:json:` URIs and decode + * straight to their primitive form. Only `literal:json:` payloads return + * objects, which we JSON-stringify for display. */ export function parseLit(val: string | undefined | null): string { if (val === undefined || val === null || val === '') return ''; try { const result = Literal.fromUrl(val).get(); - if (result && typeof result === 'object') { - return (result as { data?: string }).data ?? JSON.stringify(result); - } + if (typeof result === 'object' && result !== null) return JSON.stringify(result); return String(result); } catch { return val; From d494d7c4a029a82778d52c6b9cfe1ad5c0f154f9 Mon Sep 17 00:00:00 2001 From: HexaField <10372036+HexaField@users.noreply.github.com> Date: Thu, 4 Jun 2026 00:02:19 +1000 Subject: [PATCH 04/19] test: assert plain-value round-trip for resolveLanguage=literal (T1-T3) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit After the Channel V refactor, properties with resolveLanguage: "literal" store their values as plain literal:string:X / :number: / :boolean: / :json: URIs — no signed envelope. Tests now assert the value rather than the envelope shape. Renames the long-value test's local `expression` to `literal` for consistency with the other two cases. --- tests/js/tests/prolog-and-literals.test.ts | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/tests/js/tests/prolog-and-literals.test.ts b/tests/js/tests/prolog-and-literals.test.ts index 0732e4931..30437ec23 100644 --- a/tests/js/tests/prolog-and-literals.test.ts +++ b/tests/js/tests/prolog-and-literals.test.ts @@ -299,7 +299,7 @@ describe("Prolog + Literals", () => { let links = await perspective!.get(new LinkQuery({source: todo.id, predicate: "todo://has_title"})) expect(links.length).to.equal(1) let literal = Literal.fromUrl(links[0].data.target).get() - expect(literal.data).to.equal("new title") + expect(literal).to.equal("new title") }) it("can easily be initialized with PerspectiveProxy.ensureSDNASubjectClass()", async () => { @@ -646,7 +646,7 @@ describe("Prolog + Literals", () => { let links = await perspective!.get(new LinkQuery({source: root, predicate: "recipe://resolve"})) expect(links.length).to.equal(1) let literal = Literal.fromUrl(links[0].data.target).get() - expect(literal.data).to.equal(recipe.resolve) + expect(literal).to.equal(recipe.resolve) const recipe3 = new Recipe(perspective!, root); await recipe3.get(); @@ -707,8 +707,8 @@ describe("Prolog + Literals", () => { let linksResolve = await perspective!.get(new LinkQuery({source: root, predicate: "recipe://resolve"})) expect(linksResolve.length).to.equal(1) - let expression = Literal.fromUrl(linksResolve[0].data.target).get() - expect(expression.data).to.equal(longName) + let literal = Literal.fromUrl(linksResolve[0].data.target).get() + expect(literal).to.equal(longName) const recipe2 = new Recipe(perspective!, root) await recipe2.get() From 760db9a57a9ce1ebf99af92911473d07514c09d5 Mon Sep 17 00:00:00 2001 From: HexaField <10372036+HexaField@users.noreply.github.com> Date: Thu, 4 Jun 2026 00:02:29 +1000 Subject: [PATCH 05/19] feat(executor): restore signed-envelope to plain-literal migration (back-compat read path retained) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Restores migrate_signed_envelopes_to_plain_literals (originally added in a9e98ccd, removed by 353a7ad7). Walks every reifier, finds targets of the form literal:json: whose JSON has data+author+proof, extracts .data, re-encodes as the appropriate plain literal type, and rebuilds both the direct triple and the reifier (the reifier IRI hash includes the target, so it must be recomputed). Removes the old direct triple only if no other reifiers still reference it. Migration is idempotent — short-circuits when migration_version() >= 3 — and runs during perspective initialization right after the named-graphs migration (v2 -> v3). Also annotates the back-compat envelope-unwrap branches in parse_literal_fn (sparql_store.rs) and parse_literal_value (model_query/utils.rs) with a note that they exist for pre-migration data; new writes use plain literal: forms. See §9.1 / §9.6 of literal-encoding-and-sparql-pushdown-2026-06-03.md. --- rust-executor/src/perspectives/mod.rs | 20 ++ .../src/perspectives/model_query/utils.rs | 5 +- .../src/perspectives/sparql_store.rs | 297 +++++++++++++++++- 3 files changed, 319 insertions(+), 3 deletions(-) diff --git a/rust-executor/src/perspectives/mod.rs b/rust-executor/src/perspectives/mod.rs index 65182116c..6862c9108 100644 --- a/rust-executor/src/perspectives/mod.rs +++ b/rust-executor/src/perspectives/mod.rs @@ -119,6 +119,26 @@ pub fn initialize_from_db() { Err(e) => log::warn!("Reifier migration for {}: {}", handle_clone.uuid, e), } + // Run signed-envelope → plain-literal migration (Channel V refactor, idempotent) + match p + .sparql_store + .migrate_signed_envelopes_to_plain_literals() + { + Ok(count) if count > 0 => { + log::info!( + "🔄 Signed-envelope migration for {}: {} envelopes converted", + handle_clone.uuid, + count + ); + } + Ok(_) => {} // Already migrated or nothing to migrate + Err(e) => log::warn!( + "Signed-envelope migration for {}: {}", + handle_clone.uuid, + e + ), + } + // Rebuild SPARQL index from existing links // Skip SPARQL rebuild if persistent store already has data if p.sparql_store.has_data() { diff --git a/rust-executor/src/perspectives/model_query/utils.rs b/rust-executor/src/perspectives/model_query/utils.rs index 73b3c09f3..bc7471a97 100644 --- a/rust-executor/src/perspectives/model_query/utils.rs +++ b/rust-executor/src/perspectives/model_query/utils.rs @@ -90,7 +90,10 @@ pub(super) fn parse_literal_value(uri: &str) -> Value { } else if let Some(rest) = body.strip_prefix("json:") { let decoded = urlencoding::decode(rest).unwrap_or_else(|_| rest.into()); if let Ok(json_val) = serde_json::from_str::(&decoded) { - // For signed expression envelopes, extract .data + // Back-compat: legacy data may still contain signed-envelope literals + // from before the Channel V refactor (Jun 2026). New writes use plain + // literal: forms — see resolve_property_value. For envelopes that look + // like signed expressions, extract .data so callers see the inner value. if let Some(data) = json_val.get("data") { if json_val.get("author").is_some() && json_val.get("proof").is_some() { return data.clone(); diff --git a/rust-executor/src/perspectives/sparql_store.rs b/rust-executor/src/perspectives/sparql_store.rs index d5cc2a9ba..6914fe9f0 100644 --- a/rust-executor/src/perspectives/sparql_store.rs +++ b/rust-executor/src/perspectives/sparql_store.rs @@ -82,8 +82,10 @@ fn parse_literal_fn(args: &[Term]) -> Option { Some(Literal::new_simple_literal(rest).into()) } else if let Some(rest) = body.strip_prefix("json:") { let decoded = urlencoding::decode(rest).unwrap_or_else(|_| rest.into()); - // For JSON literals that are signed expressions (contain "data" field), - // extract just the data field value for content matching. + // Back-compat: legacy data may still contain signed-envelope literals from + // before the Channel V refactor (Jun 2026). New writes use plain literal: + // forms — see resolve_property_value. For JSON literals that look like a + // signed envelope (have a "data" field), extract it for content matching. if let Ok(json_val) = serde_json::from_str::(&decoded) { if let Some(data) = json_val.get("data") { let data_str = match data { @@ -1118,6 +1120,297 @@ impl SparqlStore { ); Ok(count) } + + /// Migrate signed expression envelopes to plain literal values. + /// + /// Old literal-language writes stored values as `literal:json:` + /// where the envelope was `{author, timestamp, data, proof}`. The provenance is + /// redundant with the RDF 1.2 reifier metadata — the Channel V refactor (Jun 2026) + /// stops emitting envelopes for property writes. This migration extracts the + /// `.data` field from any existing envelope and stores it as a plain + /// `literal:string:X`, `literal:number:X`, `literal:boolean:X`, or + /// `literal:json:X` value. + /// + /// Since the target IRI changes, we must rebuild both the direct triple and + /// the reifier (whose IRI is a hash of source+pred+target+author+ts). + pub fn migrate_signed_envelopes_to_plain_literals(&self) -> Result { + if self.migration_version() >= 3 { + return Ok(0); + } + + log::info!("Channel V refactor: migrating signed-envelope literals to plain form"); + + use percent_encoding::{percent_decode_str, utf8_percent_encode, NON_ALPHANUMERIC}; + + let rdf_reifies = NamedNodeRef::new_unchecked(RDF_REIFIES); + let ont_author = NamedNodeRef::new_unchecked(ONT_AUTHOR); + let ont_timestamp = NamedNodeRef::new_unchecked(ONT_TIMESTAMP); + let ont_proof_key = NamedNodeRef::new_unchecked(ONT_PROOF_KEY); + let ont_proof_sig = NamedNodeRef::new_unchecked(ONT_PROOF_SIG); + let ont_proof_valid = NamedNodeRef::new_unchecked(ONT_PROOF_VALID); + let ont_status = NamedNodeRef::new_unchecked(ONT_STATUS); + + // Collect all reifier quads with their triple terms + let reifier_quads: Vec = self + .store + .quads_for_pattern( + None, + Some(rdf_reifies), + None, + Some(GraphNameRef::DefaultGraph), + ) + .filter_map(|r| r.ok()) + .collect(); + + struct LinkToMigrate { + reifier_iri: NamedNode, + source: String, + predicate: String, + old_target: String, + new_target: String, + author: String, + timestamp: String, + proof_key: String, + proof_sig: String, + proof_valid: String, + status: String, + } + + let mut links_to_migrate: Vec = Vec::new(); + + for quad in &reifier_quads { + // Extract the triple term from the reifier + let (source, predicate, old_target) = match &quad.object { + Term::Triple(t) => { + let s = match &t.subject { + NamedOrBlankNode::NamedNode(n) => n.as_str().to_string(), + _ => continue, + }; + let p = t.predicate.as_str().to_string(); + let o = match &t.object { + Term::NamedNode(n) => n.as_str().to_string(), + _ => continue, + }; + (s, p, o) + } + _ => continue, + }; + + // Check if the target is a signed envelope literal:json: + if !old_target.starts_with("literal:json:") { + continue; + } + + // Try to decode and check for signed expression envelope + let json_part = &old_target["literal:json:".len()..]; + let decoded = match percent_decode_str(json_part).decode_utf8() { + Ok(d) => d.to_string(), + Err(_) => continue, + }; + + let json_val: serde_json::Value = match serde_json::from_str(&decoded) { + Ok(v) => v, + Err(_) => continue, + }; + + // Only migrate if it has the signed expression envelope shape + let data = match json_val.get("data") { + Some(d) if json_val.get("author").is_some() && json_val.get("proof").is_some() => { + d.clone() + } + _ => continue, // Not a signed envelope, leave as-is + }; + + // Encode the data field as a plain literal + let new_target = match &data { + serde_json::Value::String(s) => { + let encoded = utf8_percent_encode(s, NON_ALPHANUMERIC).to_string(); + format!("literal:string:{}", encoded) + } + serde_json::Value::Number(n) => { + format!("literal:number:{}", n) + } + serde_json::Value::Bool(b) => { + format!("literal:boolean:{}", b) + } + _ => { + let json_str = serde_json::to_string(&data).unwrap_or_default(); + let encoded = utf8_percent_encode(&json_str, NON_ALPHANUMERIC).to_string(); + format!("literal:json:{}", encoded) + } + }; + + if new_target == old_target { + continue; // No change needed + } + + // Read reifier metadata + let reifier_iri = match &quad.subject { + NamedOrBlankNode::NamedNode(n) => n.clone(), + _ => continue, + }; + + let get_meta = |pred: NamedNodeRef| -> String { + self.store + .quads_for_pattern( + Some(reifier_iri.as_ref().into()), + Some(pred), + None, + Some(GraphNameRef::DefaultGraph), + ) + .filter_map(|r| r.ok()) + .next() + .map(|q| match &q.object { + Term::Literal(l) => l.value().to_string(), + Term::NamedNode(n) => n.as_str().to_string(), + _ => String::new(), + }) + .unwrap_or_default() + }; + + let author = get_meta(ont_author); + let timestamp = get_meta(ont_timestamp); + let proof_key = get_meta(ont_proof_key); + let proof_sig = get_meta(ont_proof_sig); + let proof_valid = get_meta(ont_proof_valid); + let status = get_meta(ont_status); + + links_to_migrate.push(LinkToMigrate { + reifier_iri, + source, + predicate, + old_target, + new_target, + author, + timestamp, + proof_key, + proof_sig, + proof_valid, + status, + }); + } + + let count = links_to_migrate.len(); + if count == 0 { + self.set_migration_version(3)?; + return Ok(0); + } + + log::info!( + "Migrating {} signed expression envelopes to plain literals...", + count + ); + + for link in &links_to_migrate { + let source_iri = NamedNode::new_unchecked(&link.source); + let predicate_iri = NamedNode::new_unchecked(&link.predicate); + let old_target_iri = NamedNode::new_unchecked(&link.old_target); + let new_target_iri = NamedNode::new_unchecked(&link.new_target); + + // 1. Insert new direct triple + self.store.insert(QuadRef::new( + source_iri.as_ref(), + predicate_iri.as_ref(), + TermRef::NamedNode(new_target_iri.as_ref()), + GraphNameRef::DefaultGraph, + ))?; + + // 2. Build new reifier IRI (hash changes because target changed) + let new_reifier_iri = { + let mut hasher = Sha256::new(); + hasher.update(link.author.as_bytes()); + hasher.update(link.source.as_bytes()); + hasher.update(link.predicate.as_bytes()); + hasher.update(link.new_target.as_bytes()); + hasher.update(link.timestamp.as_bytes()); + let hash = hex::encode(hasher.finalize()); + NamedNode::new_unchecked(format!("link:{}", &hash[..32])) + }; + + // 3. Insert new reifier triple + let new_triple = Triple::new( + source_iri.clone(), + predicate_iri.clone(), + new_target_iri.clone(), + ); + self.store.insert(QuadRef::new( + new_reifier_iri.as_ref(), + rdf_reifies, + TermRef::Triple(&new_triple), + GraphNameRef::DefaultGraph, + ))?; + + // 4. Insert metadata on new reifier + let annotations: &[(&str, &str)] = &[ + (ONT_AUTHOR, &link.author), + (ONT_TIMESTAMP, &link.timestamp), + (ONT_PROOF_KEY, &link.proof_key), + (ONT_PROOF_SIG, &link.proof_sig), + (ONT_PROOF_VALID, &link.proof_valid), + (ONT_STATUS, &link.status), + ]; + for (pred_uri, value) in annotations { + if value.is_empty() { + continue; + } + let pred = NamedNodeRef::new_unchecked(pred_uri); + let lit = literal(value); + self.store.insert(QuadRef::new( + new_reifier_iri.as_ref(), + pred, + TermRef::Literal(lit.as_ref()), + GraphNameRef::DefaultGraph, + ))?; + } + + // 5. Remove old reifier metadata + let old_meta: Vec = self + .store + .quads_for_pattern( + Some(link.reifier_iri.as_ref().into()), + None, + None, + Some(GraphNameRef::DefaultGraph), + ) + .filter_map(|r| r.ok()) + .collect(); + for q in &old_meta { + self.store.remove(q)?; + } + + // 6. Remove old direct triple (only if no other reifier references it) + let old_triple = Triple::new( + source_iri.clone(), + predicate_iri.clone(), + old_target_iri.clone(), + ); + let other_reifiers = self + .store + .quads_for_pattern( + None, + Some(rdf_reifies), + Some(TermRef::Triple(&old_triple)), + Some(GraphNameRef::DefaultGraph), + ) + .any(|r| r.is_ok()); + if !other_reifiers { + let _ = self.store.remove(&Quad::new( + source_iri, + predicate_iri, + old_target_iri, + GraphName::DefaultGraph, + )); + } + } + + self.set_migration_version(3)?; + + log::info!( + "Migration complete: {} signed envelopes converted to plain literals", + count + ); + Ok(count) + } } #[cfg(test)] From ce5f738562e6c23b52fdd585081bef84cf7020c8 Mon Sep 17 00:00:00 2001 From: HexaField <10372036+HexaField@users.noreply.github.com> Date: Thu, 4 Jun 2026 00:20:13 +1000 Subject: [PATCH 06/19] perf(executor): index-friendly WHERE for literal properties (V4) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Equality WHERE filters on `resolveLanguage: literal` properties now emit direct IRI matches against the deterministic encoding produced by `resolve_property_value` (Phase 1): ?source . ?source . ?source . Previously each row was checked via `STR(fn/parse_literal(?v)) = "val"` inside a FILTER, which forced Oxigraph to scan every triple bound to the predicate. Switching to a direct IRI lets the POS index probe straight to the matching row. The `fn/parse_literal` BIND + FILTER path is preserved for: - `WhereCondition::Ops` (gt/lt/gte/lte/between/contains/not) — typed comparison still needs the unwrapped value. - Properties without `resolveLanguage: literal` — back-compat for raw string storage. - Legacy envelope-form data that hasn't yet been migrated by `migrate_signed_envelopes_to_plain_literals` (Phase 2) — `parse_literal` also unwraps `.data` from signed envelopes. Where a where-value also looks like an absolute IRI (scheme + colon), a UNION fallback matches the raw-IRI form too — covers the case where a constructor's initial value was stored as a raw URI on a property whose shape declares `resolveLanguage: literal` (enum-like state defaults). Values are validated before injection: strings are `NON_ALPHANUMERIC`-percent-encoded (matching `literal_encode`), numbers must be finite, booleans must be `true|false`. Non-finite numeric filters short-circuit to `FILTER(false)`. Refs: research/literal-encoding-and-sparql-pushdown-2026-06-03.md (V4) Co-Authored-By: Claude Opus 4.7 --- .../model_query/sparql_builder.rs | 167 ++++++++++++++---- .../src/perspectives/model_query/utils.rs | 42 ++++- 2 files changed, 169 insertions(+), 40 deletions(-) diff --git a/rust-executor/src/perspectives/model_query/sparql_builder.rs b/rust-executor/src/perspectives/model_query/sparql_builder.rs index ac3d80725..bb87108fa 100644 --- a/rust-executor/src/perspectives/model_query/sparql_builder.rs +++ b/rust-executor/src/perspectives/model_query/sparql_builder.rs @@ -19,7 +19,10 @@ use super::types::{ InstanceQueryPlan, ModelQueryInput, ModelShape, OrderDirection, ParentScope, SortKey, SparqlPagination, WhereCondition, }; -use super::utils::{escape_sparql_string, validate_iri}; +use super::utils::{ + escape_sparql_string, format_literal_number, literal_percent_encode, looks_like_absolute_iri, + validate_iri, +}; /// Build a targeted reifier timestamp probe for the pagination sub-query. /// @@ -510,13 +513,28 @@ pub(super) fn build_query_patterns( match condition { WhereCondition::String(val) => { if is_literal_prop { - let var = format!("?_pw_{safe_name}"); - where_patterns - .push(format!(" ?source <{}> {var} .", prop.predicate)); - where_patterns.push(format!( - " FILTER(STR(({var})) = \"{}\")", - escape_sparql_string(val) - )); + // V4: Equality on a literal-encoded property compares against the + // deterministic `literal:string:` IRI emitted by + // `resolve_property_value`. Emitting a direct IRI match lets + // Oxigraph use the POS index instead of evaluating + // `fn/parse_literal` per row. + // + // If the where-value is also a valid raw IRI, a UNION covers the + // case where the constructor stored a raw URI on a property whose + // shape declares `resolveLanguage: literal` (e.g. enum-like state + // initial values). + let encoded = literal_percent_encode(val); + if looks_like_absolute_iri(val) { + where_patterns.push(format!( + " {{ ?source <{0}> . }} UNION {{ ?source <{0}> <{val}> . }}", + prop.predicate + )); + } else { + where_patterns.push(format!( + " ?source <{}> .", + prop.predicate + )); + } } else if validate_iri(val).is_ok() { where_patterns .push(format!(" ?source <{}> <{val}> .", prop.predicate)); @@ -531,42 +549,115 @@ pub(super) fn build_query_patterns( } } WhereCondition::Number(n) => { - let var = format!("?_pw_{safe_name}"); - where_patterns.push(format!(" ?source <{}> {var} .", prop.predicate)); - where_patterns.push(format!( - " FILTER(STR(({var})) = \"{n}\")" - )); + if is_literal_prop { + // V4: direct IRI match against `literal:number:` for + // index-friendly equality. Non-finite values are dropped + // (the FILTER(false) path matches no rows). + if let Some(num_str) = format_literal_number(*n) { + where_patterns.push(format!( + " ?source <{}> .", + prop.predicate + )); + } else { + where_patterns.push(" FILTER(false)".to_string()); + } + } else { + let var = format!("?_pw_{safe_name}"); + where_patterns + .push(format!(" ?source <{}> {var} .", prop.predicate)); + where_patterns.push(format!( + " FILTER(STR(({var})) = \"{n}\")" + )); + } } WhereCondition::Bool(b) => { - let var = format!("?_pw_{safe_name}"); - where_patterns.push(format!(" ?source <{}> {var} .", prop.predicate)); - where_patterns.push(format!( - " FILTER(STR(({var})) = \"{b}\")" - )); + if is_literal_prop { + // V4: direct IRI match against `literal:boolean:true|false`. + where_patterns.push(format!( + " ?source <{}> .", + prop.predicate + )); + } else { + let var = format!("?_pw_{safe_name}"); + where_patterns + .push(format!(" ?source <{}> {var} .", prop.predicate)); + where_patterns.push(format!( + " FILTER(STR(({var})) = \"{b}\")" + )); + } } WhereCondition::StringArray(vals) => { - let values_list = vals - .iter() - .map(|v| format!("\"{}\"", escape_sparql_string(v))) - .collect::>() - .join(", "); - let var = format!("?_pw_{safe_name}"); - where_patterns.push(format!(" ?source <{}> {var} .", prop.predicate)); - where_patterns.push(format!( - " FILTER(STR(({var})) IN ({values_list}))" - )); + if is_literal_prop { + // V4: VALUES of `literal:string:` IRIs → POS probe per IRI. + // Also include raw-IRI forms for values that parse as valid IRIs + // (mirrors the single-String UNION above). + let mut iris: Vec = Vec::with_capacity(vals.len() * 2); + for v in vals { + iris.push(format!( + "", + literal_percent_encode(v) + )); + if looks_like_absolute_iri(v) { + iris.push(format!("<{v}>")); + } + } + let iv_var = format!("?_iv_{safe_name}"); + where_patterns.push(format!( + " VALUES {iv_var} {{ {} }}", + iris.join(" ") + )); + where_patterns.push(format!( + " ?source <{}> {iv_var} .", + prop.predicate + )); + } else { + let values_list = vals + .iter() + .map(|v| format!("\"{}\"", escape_sparql_string(v))) + .collect::>() + .join(", "); + let var = format!("?_pw_{safe_name}"); + where_patterns + .push(format!(" ?source <{}> {var} .", prop.predicate)); + where_patterns.push(format!( + " FILTER(STR(({var})) IN ({values_list}))" + )); + } } WhereCondition::NumberArray(vals) => { - let values_list = vals - .iter() - .map(|n| format!("\"{n}\"")) - .collect::>() - .join(", "); - let var = format!("?_pw_{safe_name}"); - where_patterns.push(format!(" ?source <{}> {var} .", prop.predicate)); - where_patterns.push(format!( - " FILTER(STR(({var})) IN ({values_list}))" - )); + if is_literal_prop { + // V4: VALUES of `literal:number:` IRIs. Non-finite values are dropped. + let iris = vals + .iter() + .filter_map(|n| { + format_literal_number(*n) + .map(|s| format!("")) + }) + .collect::>() + .join(" "); + if iris.is_empty() { + where_patterns.push(" FILTER(false)".to_string()); + } else { + let iv_var = format!("?_iv_{safe_name}"); + where_patterns.push(format!(" VALUES {iv_var} {{ {iris} }}")); + where_patterns.push(format!( + " ?source <{}> {iv_var} .", + prop.predicate + )); + } + } else { + let values_list = vals + .iter() + .map(|n| format!("\"{n}\"")) + .collect::>() + .join(", "); + let var = format!("?_pw_{safe_name}"); + where_patterns + .push(format!(" ?source <{}> {var} .", prop.predicate)); + where_patterns.push(format!( + " FILTER(STR(({var})) IN ({values_list}))" + )); + } } WhereCondition::Ops(ops) => { let var = format!("?_pw_{safe_name}"); diff --git a/rust-executor/src/perspectives/model_query/utils.rs b/rust-executor/src/perspectives/model_query/utils.rs index bc7471a97..67078ff2a 100644 --- a/rust-executor/src/perspectives/model_query/utils.rs +++ b/rust-executor/src/perspectives/model_query/utils.rs @@ -6,7 +6,6 @@ //! coercion helpers used by the filtering engine. use deno_core::anyhow::{anyhow, Error}; -#[cfg(test)] use percent_encoding::{utf8_percent_encode, NON_ALPHANUMERIC}; use serde_json::Value; @@ -19,11 +18,27 @@ use serde_json::Value; /// Uses `NON_ALPHANUMERIC` percent-encoding, matching `literal_encode` in /// `languages/literal.rs`. `urlencoding::encode` uses RFC 3986 unreserved /// chars (keeps `.-_~`), which diverges from the storage encoding. -#[cfg(test)] pub(super) fn literal_percent_encode(s: &str) -> String { utf8_percent_encode(s, NON_ALPHANUMERIC).to_string() } +/// Format a finite f64 for the `literal:number:` IRI tail, mirroring +/// `literal_encode` in `languages/literal.rs`: integers render without a +/// fractional part (e.g. `42`), floats use `{}` formatting (e.g. `3.14`). +/// +/// Returns `None` if the value is non-finite (NaN or +/- infinity) — these +/// are rejected so we never emit a malformed IRI into the SPARQL. +pub(super) fn format_literal_number(n: f64) -> Option { + if !n.is_finite() { + return None; + } + if n.fract() == 0.0 && n.abs() < (i64::MAX as f64) { + Some(format!("{}", n as i64)) + } else { + Some(format!("{n}")) + } +} + /// Escape a string value for use inside a SPARQL string literal (double-quoted). pub(super) fn escape_sparql_string(s: &str) -> String { s.replace('\\', "\\\\") @@ -33,6 +48,29 @@ pub(super) fn escape_sparql_string(s: &str) -> String { .replace('\t', "\\t") } +/// Cheap check: is this value plausibly an absolute IRI? Used by the +/// index-friendly WHERE builders (V4/V5) to decide whether to add a +/// raw-IRI fallback alongside the `literal:string:` direct probe. +/// +/// Requires (a) `validate_iri` passes (no chars that would break the +/// SPARQL IRIREF token), and (b) the value contains a `:` and starts with +/// an ASCII letter — a minimal proxy for "has a URI scheme". This rejects +/// bare strings like `"active"` that would otherwise produce the +/// un-parseable IRIREF ``. +pub(super) fn looks_like_absolute_iri(s: &str) -> bool { + if validate_iri(s).is_err() { + return false; + } + let Some(colon_idx) = s.find(':') else { + return false; + }; + if colon_idx == 0 { + return false; + } + let first = s.as_bytes()[0]; + first.is_ascii_alphabetic() +} + /// Validate a value for use inside an IRI `<…>`. Rejects characters that /// would break or inject into a SPARQL IRI token. pub(super) fn validate_iri(s: &str) -> Result<&str, Error> { From b68ebe035fcc1273d258940449b30335851c4a5e Mon Sep 17 00:00:00 2001 From: HexaField <10372036+HexaField@users.noreply.github.com> Date: Thu, 4 Jun 2026 00:20:21 +1000 Subject: [PATCH 07/19] perf(executor): index-friendly WHERE in projection (V5) Mirror the V4 transformation in the projection where-clause builder (`build_projection_where_patterns`). Equality on a target-shape property declared `resolveLanguage: literal` now emits a direct IRI match against `` / `` / `` instead of `STR(fn/parse_literal(?v)) = "X"`. The pred_lookup now carries `(predicate, is_literal_prop)` so the projection layer can distinguish properties whose targets are stored as literal IRIs (use POS-index probe) from those backed by raw URIs or unknown storage (keep the fn/parse_literal fallback). String/StringArray inputs that themselves look like absolute IRIs also include the raw-IRI form via VALUES, matching V4's UNION fallback semantics. Refs: research/literal-encoding-and-sparql-pushdown-2026-06-03.md (V5) Co-Authored-By: Claude Opus 4.7 --- .../perspectives/model_query/projection.rs | 148 ++++++++++++++---- 1 file changed, 116 insertions(+), 32 deletions(-) diff --git a/rust-executor/src/perspectives/model_query/projection.rs b/rust-executor/src/perspectives/model_query/projection.rs index 3e8848c10..f76bc0d29 100644 --- a/rust-executor/src/perspectives/model_query/projection.rs +++ b/rust-executor/src/perspectives/model_query/projection.rs @@ -20,7 +20,10 @@ use std::collections::{BTreeMap, HashMap}; use super::types::{ ModelQueryInput, ModelShape, OrderDirection, ProjectionInput, ShapeResolver, WhereCondition, }; -use super::utils::{escape_sparql_string, validate_iri}; +use super::utils::{ + escape_sparql_string, format_literal_number, literal_percent_encode, looks_like_absolute_iri, + validate_iri, +}; use crate::perspectives::sparql_store::SparqlStore; /// Resolve all projections for a set of parent instances. @@ -286,19 +289,24 @@ pub(super) fn build_projection_where_patterns( // Resolve the target class's shape through the perspective cache so we // can translate property names in the projection's where-clause into the - // predicate IRIs they map to in the store. + // predicate IRIs they map to in the store. We also track whether each + // property is literal-encoded (`resolve_language: literal`) so V5 can + // emit direct IRI matches against the deterministic `literal:*:X` form. let (pred_lookup, resolution_failed) = if let Some(ref target_name) = proj.target_class_name { match resolver.get_shape(target_name) { Ok(target_shape) => { - let mut map = HashMap::new(); + let mut map: HashMap = HashMap::new(); for p in &target_shape.properties { if !p.predicate.is_empty() { - map.insert(p.name.clone(), p.predicate.clone()); + map.insert( + p.name.clone(), + (p.predicate.clone(), p.resolve_language.is_some()), + ); } } for r in &target_shape.include_relations { if !r.predicate.is_empty() { - map.insert(r.name.clone(), r.predicate.clone()); + map.insert(r.name.clone(), (r.predicate.clone(), false)); } } (map, false) @@ -307,11 +315,11 @@ pub(super) fn build_projection_where_patterns( log::warn!( "Projection where-clause resolution failed for target '{target_name}': {e}" ); - (HashMap::::new(), true) + (HashMap::::new(), true) } } } else { - (HashMap::::new(), false) + (HashMap::::new(), false) }; // Fail closed when the projection has non-system property filters but @@ -354,8 +362,8 @@ pub(super) fn build_projection_where_patterns( continue; } - let pred = match pred_lookup.get(prop_name) { - Some(p) => p.clone(), + let (pred, is_literal_prop) = match pred_lookup.get(prop_name) { + Some(v) => (v.0.clone(), v.1), None => continue, }; @@ -368,35 +376,111 @@ pub(super) fn build_projection_where_patterns( match condition { WhereCondition::String(val) => { - let escaped = escape_sparql_string(val); - patterns.push(format!(" ?t <{pred}> ?{var} .\n")); - patterns.push(format!( - " FILTER(STR((?{var})) = \"{escaped}\")\n", - )); + if is_literal_prop { + // V5: direct IRI match against the deterministic + // `literal:string:` IRI — POS-index friendly. + // Use a VALUES set so we can also include the raw-IRI form for + // properties whose constructor stored a raw URI on a shape that + // declares `resolveLanguage: literal` (mirrors V4 behavior). + let encoded = literal_percent_encode(val); + let mut iris = vec![format!("")]; + if looks_like_absolute_iri(val) { + iris.push(format!("<{val}>")); + } + patterns.push(format!( + " VALUES ?{var} {{ {} }}\n", + iris.join(" ") + )); + patterns.push(format!(" ?t <{pred}> ?{var} .\n")); + } else { + let escaped = escape_sparql_string(val); + patterns.push(format!(" ?t <{pred}> ?{var} .\n")); + patterns.push(format!( + " FILTER(STR((?{var})) = \"{escaped}\")\n", + )); + } } WhereCondition::Bool(b) => { - let bval = if *b { "true" } else { "false" }; - patterns.push(format!(" ?t <{pred}> ?{var} .\n")); - patterns.push(format!( - " FILTER(STR((?{var})) = \"{bval}\")\n", - )); + if is_literal_prop { + patterns.push(format!( + " ?t <{pred}> .\n" + )); + } else { + let bval = if *b { "true" } else { "false" }; + patterns.push(format!(" ?t <{pred}> ?{var} .\n")); + patterns.push(format!( + " FILTER(STR((?{var})) = \"{bval}\")\n", + )); + } } WhereCondition::Number(n) => { - patterns.push(format!(" ?t <{pred}> ?{var} .\n")); - patterns.push(format!( - " FILTER(STR((?{var})) = \"{n}\")\n", - )); + if is_literal_prop { + if let Some(num_str) = format_literal_number(*n) { + patterns.push(format!( + " ?t <{pred}> .\n" + )); + } else { + patterns.push(" FILTER(false)\n".to_string()); + } + } else { + patterns.push(format!(" ?t <{pred}> ?{var} .\n")); + patterns.push(format!( + " FILTER(STR((?{var})) = \"{n}\")\n", + )); + } } WhereCondition::StringArray(vals) => { - let list = vals - .iter() - .map(|v| format!("\"{}\"", escape_sparql_string(v))) - .collect::>() - .join(", "); - patterns.push(format!(" ?t <{pred}> ?{var} .\n")); - patterns.push(format!( - " FILTER(STR((?{var})) IN ({list}))\n", - )); + if is_literal_prop { + let mut iris: Vec = Vec::with_capacity(vals.len() * 2); + for v in vals { + iris.push(format!("", literal_percent_encode(v))); + if looks_like_absolute_iri(v) { + iris.push(format!("<{v}>")); + } + } + patterns.push(format!( + " VALUES ?{var} {{ {} }}\n", + iris.join(" ") + )); + patterns.push(format!(" ?t <{pred}> ?{var} .\n")); + } else { + let list = vals + .iter() + .map(|v| format!("\"{}\"", escape_sparql_string(v))) + .collect::>() + .join(", "); + patterns.push(format!(" ?t <{pred}> ?{var} .\n")); + patterns.push(format!( + " FILTER(STR((?{var})) IN ({list}))\n", + )); + } + } + WhereCondition::NumberArray(vals) => { + if is_literal_prop { + let iris = vals + .iter() + .filter_map(|n| { + format_literal_number(*n).map(|s| format!("")) + }) + .collect::>() + .join(" "); + if iris.is_empty() { + patterns.push(" FILTER(false)\n".to_string()); + } else { + patterns.push(format!(" VALUES ?{var} {{ {iris} }}\n")); + patterns.push(format!(" ?t <{pred}> ?{var} .\n")); + } + } else { + let list = vals + .iter() + .map(|n| format!("\"{n}\"")) + .collect::>() + .join(", "); + patterns.push(format!(" ?t <{pred}> ?{var} .\n")); + patterns.push(format!( + " FILTER(STR((?{var})) IN ({list}))\n", + )); + } } _ => {} } From 8b317cddd58c81df4fbda398e7fc9587d643b4f0 Mon Sep 17 00:00:00 2001 From: HexaField <10372036+HexaField@users.noreply.github.com> Date: Thu, 4 Jun 2026 00:20:32 +1000 Subject: [PATCH 08/19] test: split legacy-envelope WHERE tests from plain-literal WHERE tests (T4) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Restructure the WHERE-clause regression tests around the Phase 1/2/3 storage contract: - Rename helper `signed_envelope_literal` -> `legacy_envelope_literal`. The envelope form models pre-migration data shape only; it is not the current write path. - `test_signed_envelope_where_paginate_count` -> `test_legacy_envelope_migrated_then_paginate_count`. Now seeds envelope-form data, runs `migrate_signed_envelopes_to_plain_literals`, then asserts the V4 direct-IRI WHERE probe finds the migrated rows. - `test_mixed_plain_and_signed_envelope_where` -> `test_legacy_mixed_migrated_then_contains`. Same migration-then-query pattern; mixes pre-existing plain literals with envelope rows to verify the migration is idempotent on plain data. - Add `test_plain_literal_where_paginate_count` — parallel coverage using plain literals from the start (no migration call). Asserts the V4 POS-index path returns identical results to the legacy migrated test. - Add `test_plain_literal_contains_works_on_fn_parse_literal_path` — exercises `WhereOps::contains`, which still routes through `fn/parse_literal`, against plain `literal:string:X` storage. Confirms the fallback path still produces correct substring semantics. `test_resolve_projections_where_filter_via_target_shape_property` is unchanged — it already stored plain `literal:string:like_type_id123` and continues to pass because V5 emits a direct IRI probe matching the stored form. Co-Authored-By: Claude Opus 4.7 --- .../model_query/integration_tests.rs | 244 ++++++++++++++++-- 1 file changed, 222 insertions(+), 22 deletions(-) diff --git a/rust-executor/src/perspectives/model_query/integration_tests.rs b/rust-executor/src/perspectives/model_query/integration_tests.rs index c3e3ef175..314f14506 100644 --- a/rust-executor/src/perspectives/model_query/integration_tests.rs +++ b/rust-executor/src/perspectives/model_query/integration_tests.rs @@ -3352,9 +3352,12 @@ async fn test_full_model_query_ops_contains_with_pagination() { ); } -/// Helper: create a signed-envelope literal IRI (mimics what expression.create("literal", value) -/// produces in production). The signed envelope is JSON with {author, timestamp, data, proof}. -fn signed_envelope_literal(value: &str) -> String { +/// Helper: create a legacy signed-envelope literal IRI — models the pre-migration +/// data shape that older databases still contain on disk. New writes use plain +/// `literal:string:X` form; the migration step converts envelopes → plain form +/// on first boot. Tests that exercise the migration path use this helper to +/// seed envelope-shaped data before calling `migrate_signed_envelopes_to_plain_literals`. +fn legacy_envelope_literal(value: &str) -> String { let envelope = serde_json::json!({ "author": "did:key:zQ3shTestAgent", "timestamp": "2024-01-01T00:00:00.000Z", @@ -3370,12 +3373,13 @@ fn signed_envelope_literal(value: &str) -> String { format!("literal:json:{}", literal_percent_encode(&json_str)) } -/// Regression test for signed-envelope literals with fn/parse_literal WHERE clauses. -/// Exercises the exact pattern used by paginateSubscribe: model query with WHERE -/// filtering on a literal property, pagination (limit/offset), and count=true, -/// where stored values are signed expression envelopes (literal:json:{signed}). +/// Regression test for legacy signed-envelope literals: insert envelope-form data, +/// run the migration, then verify the model query (WHERE + pagination + count) +/// returns the correct results against the migrated plain-literal form. This +/// guards the back-compat path for stores that haven't yet migrated when the +/// new executor boots. #[tokio::test] -async fn test_signed_envelope_where_paginate_count() { +async fn test_legacy_envelope_migrated_then_paginate_count() { let store = SparqlStore::new(None).unwrap(); let ts_base = 1700000000000i64; @@ -3395,7 +3399,7 @@ async fn test_signed_envelope_where_paginate_count() { .add_link(&make_link( uri, "ns://status", - &signed_envelope_literal(status), + &legacy_envelope_literal(status), &ts, )) .unwrap(); @@ -3403,12 +3407,22 @@ async fn test_signed_envelope_where_paginate_count() { .add_link(&make_link( uri, "ns://name", - &signed_envelope_literal(name), + &legacy_envelope_literal(name), &ts, )) .unwrap(); } + // T4: simulate first-boot migration converting envelope-form data to plain literals. + // After migration the new index-friendly WHERE (V4) can probe the POS index. + let migrated = store + .migrate_signed_envelopes_to_plain_literals() + .expect("migration should succeed"); + assert!( + migrated > 0, + "expected migration to rewrite at least one envelope, got {migrated}" + ); + let shape_json = r#"{ "className": "Task", "properties": { @@ -3455,11 +3469,12 @@ async fn test_signed_envelope_where_paginate_count() { "Second item by timestamp" ); - // Verify hydration: name should be the unwrapped data, not the full signed envelope + // Verify hydration: after migration the stored target is `literal:string:active`, + // and parse_literal_value decodes it back to the plain "active" string. assert_eq!( result.instances[0]["status"].as_str().unwrap(), "active", - "Status should be unwrapped from signed envelope" + "Status should be plain literal post-migration" ); // Page 2: offset 2 @@ -3492,14 +3507,16 @@ async fn test_signed_envelope_where_paginate_count() { ); } -/// Regression: mixed literal formats (plain + signed envelope) coexist in the same query. -/// This can happen during migration or when different code paths create links. +/// Regression: mixed legacy data (some plain literals already written, some +/// still in envelope form) all get rewritten to plain form by the migration, +/// after which the new index-friendly WHERE (V4) finds everything via direct +/// IRI match. Also exercises `contains` which routes through `fn/parse_literal`. #[tokio::test] -async fn test_mixed_plain_and_signed_envelope_where() { +async fn test_legacy_mixed_migrated_then_contains() { let store = SparqlStore::new(None).unwrap(); let ts_base = 1700000000000i64; - // Item 1: plain literal (old format) + // Item 1: plain literal (already in new form on disk) store .add_link(&make_link( "test://old", @@ -3517,7 +3534,7 @@ async fn test_mixed_plain_and_signed_envelope_where() { )) .unwrap(); - // Item 2: signed envelope (new format) + // Item 2: legacy signed envelope (pre-migration) store .add_link(&make_link( "test://new", @@ -3530,11 +3547,17 @@ async fn test_mixed_plain_and_signed_envelope_where() { .add_link(&make_link( "test://new", "ns://body", - &signed_envelope_literal("hello signed"), + &legacy_envelope_literal("hello signed"), &format!("{}", ts_base + 1), )) .unwrap(); + // Run migration: envelope-form rows become plain literals; pre-existing + // plain literals are untouched. + store + .migrate_signed_envelopes_to_plain_literals() + .expect("migration should succeed"); + let shape_json = r#"{ "className": "Msg", "properties": { @@ -3544,7 +3567,8 @@ async fn test_mixed_plain_and_signed_envelope_where() { "relations": {} }"#; - // Query with contains "hello" — should match both formats + // Query with contains "hello" — `contains` still uses fn/parse_literal, so + // it works against the plain-literal form post-migration. let mut wc = BTreeMap::new(); wc.insert( "body".to_string(), @@ -3566,14 +3590,14 @@ async fn test_mixed_plain_and_signed_envelope_where() { .await .unwrap(); - assert_eq!(result.instances.len(), 2, "Both formats should match"); + assert_eq!(result.instances.len(), 2, "Both items should match"); assert_eq!(result.instances[0]["body"].as_str().unwrap(), "hello plain"); assert_eq!( result.instances[1]["body"].as_str().unwrap(), "hello signed" ); - // Exact match on signed envelope value + // Exact equality on the migrated value — V4 emits a direct IRI probe. let mut wc2 = BTreeMap::new(); wc2.insert( "body".to_string(), @@ -3591,13 +3615,189 @@ async fn test_mixed_plain_and_signed_envelope_where() { .await .unwrap(); - assert_eq!(result2.instances.len(), 1, "Exact match on signed envelope"); + assert_eq!( + result2.instances.len(), + 1, + "Exact match on migrated literal" + ); assert_eq!( result2.instances[0]["body"].as_str().unwrap(), "hello signed" ); } +/// Parallel to `test_legacy_envelope_migrated_then_paginate_count` — same data +/// shape, but inserted as plain `literal:string:X` from the start (no migration). +/// Verifies V4's direct IRI match finds the data via POS-index probe without +/// going through `fn/parse_literal`. +#[tokio::test] +async fn test_plain_literal_where_paginate_count() { + let store = SparqlStore::new(None).unwrap(); + let ts_base = 1700000000000i64; + + let items = vec![ + ("test://item-1", "active", "Alpha"), + ("test://item-2", "active", "Beta"), + ("test://item-3", "inactive", "Gamma"), + ("test://item-4", "active", "Delta"), + ]; + for (i, (uri, status, name)) in items.iter().enumerate() { + let ts = format!("{}", ts_base + i as i64); + store + .add_link(&make_link(uri, "ns://type", "ns://task", &ts)) + .unwrap(); + store + .add_link(&make_link( + uri, + "ns://status", + &signed_literal(status), + &ts, + )) + .unwrap(); + store + .add_link(&make_link(uri, "ns://name", &signed_literal(name), &ts)) + .unwrap(); + } + + let shape_json = r#"{ + "className": "Task", + "properties": { + "type": { "predicate": "ns://type", "required": true, "flag": true, "initial": "ns://task" }, + "status": { "predicate": "ns://status", "required": false, "resolveLanguage": "literal" }, + "name": { "predicate": "ns://name", "required": false, "resolveLanguage": "literal" } + }, + "relations": {} + }"#; + + let mut wc = BTreeMap::new(); + wc.insert( + "status".to_string(), + WhereCondition::String("active".to_string()), + ); + let result = execute_model_query_from_json( + &store, + "Task", + &ModelQueryInput { + where_clause: Some(wc.clone()), + limit: Some(2), + offset: Some(0), + order: Some(vec![("timestamp".to_string(), OrderDirection::ASC)]), + count: Some(true), + ..Default::default() + }, + shape_json, + ) + .await + .unwrap(); + + assert_eq!(result.instances.len(), 2, "Page should have 2 items"); + assert_eq!(result.total_count, 3, "Total active items should be 3"); + assert_eq!(result.instances[0]["name"].as_str().unwrap(), "Alpha"); + assert_eq!(result.instances[1]["name"].as_str().unwrap(), "Beta"); + assert_eq!(result.instances[0]["status"].as_str().unwrap(), "active"); + + let result2 = execute_model_query_from_json( + &store, + "Task", + &ModelQueryInput { + where_clause: Some(wc), + limit: Some(2), + offset: Some(2), + order: Some(vec![("timestamp".to_string(), OrderDirection::ASC)]), + count: Some(true), + ..Default::default() + }, + shape_json, + ) + .await + .unwrap(); + + assert_eq!(result2.instances.len(), 1); + assert_eq!(result2.total_count, 3); + assert_eq!(result2.instances[0]["name"].as_str().unwrap(), "Delta"); +} + +/// Parallel to `test_legacy_mixed_migrated_then_contains` for the plain-literal +/// storage path. Confirms `contains` (which still routes through +/// `fn/parse_literal` for substring semantics) works correctly on plain +/// `literal:string:X` IRIs. +#[tokio::test] +async fn test_plain_literal_contains_works_on_fn_parse_literal_path() { + let store = SparqlStore::new(None).unwrap(); + let ts_base = 1700000000000i64; + + store + .add_link(&make_link( + "test://a", + "ns://type", + "ns://msg", + &format!("{ts_base}"), + )) + .unwrap(); + store + .add_link(&make_link( + "test://a", + "ns://body", + &signed_literal("hello world"), + &format!("{ts_base}"), + )) + .unwrap(); + + store + .add_link(&make_link( + "test://b", + "ns://type", + "ns://msg", + &format!("{}", ts_base + 1), + )) + .unwrap(); + store + .add_link(&make_link( + "test://b", + "ns://body", + &signed_literal("goodbye world"), + &format!("{}", ts_base + 1), + )) + .unwrap(); + + let shape_json = r#"{ + "className": "Msg", + "properties": { + "type": { "predicate": "ns://type", "required": true, "flag": true, "initial": "ns://msg" }, + "body": { "predicate": "ns://body", "required": false, "resolveLanguage": "literal" } + }, + "relations": {} + }"#; + + let mut wc = BTreeMap::new(); + wc.insert( + "body".to_string(), + WhereCondition::Ops(WhereOps { + contains: Some(Value::String("hello".to_string())), + ..Default::default() + }), + ); + let result = execute_model_query_from_json( + &store, + "Msg", + &ModelQueryInput { + where_clause: Some(wc), + order: Some(vec![("timestamp".to_string(), OrderDirection::ASC)]), + ..Default::default() + }, + shape_json, + ) + .await + .unwrap(); + + assert_eq!( + result.instances.len(), + 1, + "contains 'hello' should match only one row" + ); + assert_eq!(result.instances[0]["body"].as_str().unwrap(), "hello world"); +} + // ----------------------------------------------------------------------- // Performance / scale tests // ----------------------------------------------------------------------- From f886c8c1550e413d06d7ebd1d8e95dd6e8d36dc8 Mon Sep 17 00:00:00 2001 From: HexaField <10372036+HexaField@users.noreply.github.com> Date: Thu, 4 Jun 2026 00:24:37 +1000 Subject: [PATCH 09/19] ci: re-trigger (initial workflow failed to start) From c28d64da3cbe41e02abd145ea3f6b152733e1919 Mon Sep 17 00:00:00 2001 From: HexaField <10372036+HexaField@users.noreply.github.com> Date: Thu, 4 Jun 2026 09:57:41 +1000 Subject: [PATCH 10/19] =?UTF-8?q?docs:=20tighten=20comments=20=E2=80=94=20?= =?UTF-8?q?drop=20process=20labels,=20explain=20non-obvious=20intent?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The previous pass left a number of comments referencing internal phase labels (V1/V4/V5), a 'Channel V/E' design vocabulary, and a 2026-06-03 research doc path. Replace those with comments that describe what the code does and why a reader can't infer it from the code itself. --- core/src/perspectives/SparqlBindings.ts | 12 ++--- rust-executor/src/languages/literal.rs | 21 ++------- rust-executor/src/perspectives/mod.rs | 1 - .../model_query/integration_tests.rs | 47 ++++++++----------- .../perspectives/model_query/projection.rs | 16 +++---- .../model_query/sparql_builder.rs | 30 +++++------- .../src/perspectives/model_query/utils.rs | 29 ++++-------- .../src/perspectives/perspective_instance.rs | 17 +++---- .../src/perspectives/sparql_store.rs | 29 +++++------- 9 files changed, 79 insertions(+), 123 deletions(-) diff --git a/core/src/perspectives/SparqlBindings.ts b/core/src/perspectives/SparqlBindings.ts index 8964ea8f9..5bcab8103 100644 --- a/core/src/perspectives/SparqlBindings.ts +++ b/core/src/perspectives/SparqlBindings.ts @@ -47,14 +47,10 @@ export function parseSparqlCount(result: CountBinding[] | undefined | null): num /** * Decode a `Literal`-encoded SPARQL binding back to a plain string. * - * Mirrors Flux's local `parseLit` helper. Returns `''` for `undefined`/empty - * input. Falls through to the raw value if decoding fails (defensive — the - * binding may already be a plain string for unwrapped properties). - * - * Channel V values (link-property literals) are stored as plain - * `literal:string:` / `:number:` / `:boolean:` / `:json:` URIs and decode - * straight to their primitive form. Only `literal:json:` payloads return - * objects, which we JSON-stringify for display. + * Returns `''` for `undefined`/empty input. Falls through to the raw value + * if decoding fails — bindings against properties without `resolveLanguage` + * are already raw URIs and should pass through unchanged. `literal:json:` + * objects are JSON-stringified for display. */ export function parseLit(val: string | undefined | null): string { if (val === undefined || val === null || val === '') return ''; diff --git a/rust-executor/src/languages/literal.rs b/rust-executor/src/languages/literal.rs index cb5d16ffd..7e22221fd 100644 --- a/rust-executor/src/languages/literal.rs +++ b/rust-executor/src/languages/literal.rs @@ -31,17 +31,11 @@ pub fn literal_encode(value: &JsonValue) -> String { /// Decode a literal URL expression part back into a JSON value. /// -/// Mirrors the TypeScript `Literal.fromUrl("literal://").get()` behavior. -/// Primitives (`string:` / `number:` / `boolean:`) decode to their raw JSON value. -/// Objects decoded from `json:` payloads pass through unchanged — if they happen to -/// be a Channel-E signed-expression envelope (`{author, timestamp, data, proof}`) -/// the caller is responsible for interpreting that shape. -/// -/// Note: this used to wrap primitives in a synthetic `{author: "", …}` -/// envelope. That conflated Channel V (link-property values; provenance lives on -/// the link reifier) with Channel E (signed expressions). See -/// ~/.sovereign/membranes/coasys/research/literal-encoding-and-sparql-pushdown-2026-06-03.md -/// §9.1 (V6) for the audit. +/// Mirrors the TypeScript `Literal.fromUrl("literal:").get()`. +/// `string:` / `number:` / `boolean:` decode to their typed primitive. `json:` +/// payloads are returned as the decoded JSON value verbatim — if the payload +/// happens to be a signed-expression envelope (`{author, timestamp, data, proof}`), +/// it is the caller's job to interpret that shape. pub fn literal_decode(expression_part: &str) -> Result { let value = if let Some(rest) = expression_part.strip_prefix("string:") { let decoded = percent_decode_str(rest).decode_utf8().map_err(|e| { @@ -94,10 +88,6 @@ pub fn literal_decode(expression_part: &str) -> Result serde_json::from_str(&decoded).unwrap_or(JsonValue::String(decoded.to_string())) }; - // Return the decoded value as-is. Objects (which may be legitimate signed-expression - // envelopes from Channel E callers) pass through unchanged; primitives are returned - // raw — the synthetic-envelope wrapper that used to live here was removed as part of - // the Channel V / Channel E separation. Ok(value) } @@ -111,7 +101,6 @@ mod tests { let encoded = literal_encode(&value); assert!(encoded.starts_with("string:")); let decoded = literal_decode(&encoded).unwrap(); - // Primitives now round-trip as raw values (no synthetic envelope). assert_eq!(decoded, value); } diff --git a/rust-executor/src/perspectives/mod.rs b/rust-executor/src/perspectives/mod.rs index 6862c9108..b3733b92d 100644 --- a/rust-executor/src/perspectives/mod.rs +++ b/rust-executor/src/perspectives/mod.rs @@ -119,7 +119,6 @@ pub fn initialize_from_db() { Err(e) => log::warn!("Reifier migration for {}: {}", handle_clone.uuid, e), } - // Run signed-envelope → plain-literal migration (Channel V refactor, idempotent) match p .sparql_store .migrate_signed_envelopes_to_plain_literals() diff --git a/rust-executor/src/perspectives/model_query/integration_tests.rs b/rust-executor/src/perspectives/model_query/integration_tests.rs index 314f14506..20662864c 100644 --- a/rust-executor/src/perspectives/model_query/integration_tests.rs +++ b/rust-executor/src/perspectives/model_query/integration_tests.rs @@ -3352,11 +3352,10 @@ async fn test_full_model_query_ops_contains_with_pagination() { ); } -/// Helper: create a legacy signed-envelope literal IRI — models the pre-migration -/// data shape that older databases still contain on disk. New writes use plain -/// `literal:string:X` form; the migration step converts envelopes → plain form -/// on first boot. Tests that exercise the migration path use this helper to -/// seed envelope-shaped data before calling `migrate_signed_envelopes_to_plain_literals`. +/// Build a `literal:json:` IRI — the shape produced by +/// `expression.create("literal", value)` and the shape stored on disk by +/// older databases that pre-date plain-literal writes. Used by tests that +/// seed envelope-form data and then exercise the migration path. fn legacy_envelope_literal(value: &str) -> String { let envelope = serde_json::json!({ "author": "did:key:zQ3shTestAgent", @@ -3373,17 +3372,15 @@ fn legacy_envelope_literal(value: &str) -> String { format!("literal:json:{}", literal_percent_encode(&json_str)) } -/// Regression test for legacy signed-envelope literals: insert envelope-form data, -/// run the migration, then verify the model query (WHERE + pagination + count) -/// returns the correct results against the migrated plain-literal form. This -/// guards the back-compat path for stores that haven't yet migrated when the -/// new executor boots. +/// Seed envelope-form data, run the migration, and verify model queries +/// (WHERE + pagination + count) succeed against the rewritten plain-literal +/// targets. Guards the boot-time upgrade path for stores that still hold +/// envelope-shaped targets from older writers. #[tokio::test] async fn test_legacy_envelope_migrated_then_paginate_count() { let store = SparqlStore::new(None).unwrap(); let ts_base = 1700000000000i64; - // Insert 4 items: 3 active, 1 inactive — all using signed envelope format let items = vec![ ("test://item-1", "active", "Alpha"), ("test://item-2", "active", "Beta"), @@ -3507,16 +3504,15 @@ async fn test_legacy_envelope_migrated_then_paginate_count() { ); } -/// Regression: mixed legacy data (some plain literals already written, some -/// still in envelope form) all get rewritten to plain form by the migration, -/// after which the new index-friendly WHERE (V4) finds everything via direct -/// IRI match. Also exercises `contains` which routes through `fn/parse_literal`. +/// Mixed envelope-form and plain-form rows in the same store all become +/// queryable after one migration pass, including via the `contains` filter +/// (which still routes through `fn/parse_literal`). #[tokio::test] async fn test_legacy_mixed_migrated_then_contains() { let store = SparqlStore::new(None).unwrap(); let ts_base = 1700000000000i64; - // Item 1: plain literal (already in new form on disk) + // Item 1: target is already a plain literal. store .add_link(&make_link( "test://old", @@ -3534,7 +3530,7 @@ async fn test_legacy_mixed_migrated_then_contains() { )) .unwrap(); - // Item 2: legacy signed envelope (pre-migration) + // Item 2: target is an envelope-form literal. store .add_link(&make_link( "test://new", @@ -3552,8 +3548,6 @@ async fn test_legacy_mixed_migrated_then_contains() { )) .unwrap(); - // Run migration: envelope-form rows become plain literals; pre-existing - // plain literals are untouched. store .migrate_signed_envelopes_to_plain_literals() .expect("migration should succeed"); @@ -3626,10 +3620,10 @@ async fn test_legacy_mixed_migrated_then_contains() { ); } -/// Parallel to `test_legacy_envelope_migrated_then_paginate_count` — same data -/// shape, but inserted as plain `literal:string:X` from the start (no migration). -/// Verifies V4's direct IRI match finds the data via POS-index probe without -/// going through `fn/parse_literal`. +/// Same workload as `test_legacy_envelope_migrated_then_paginate_count` but +/// with plain `literal:string:` targets from the start. Confirms model +/// queries reach the rows through the indexed direct-IRI WHERE form alone, +/// without any envelope unwrap step. #[tokio::test] async fn test_plain_literal_where_paginate_count() { let store = SparqlStore::new(None).unwrap(); @@ -3717,10 +3711,9 @@ async fn test_plain_literal_where_paginate_count() { assert_eq!(result2.instances[0]["name"].as_str().unwrap(), "Delta"); } -/// Parallel to `test_legacy_mixed_migrated_then_contains` for the plain-literal -/// storage path. Confirms `contains` (which still routes through -/// `fn/parse_literal` for substring semantics) works correctly on plain -/// `literal:string:X` IRIs. +/// Guards that the `contains` filter — which can't reduce to a direct IRI +/// equality and so still goes through `fn/parse_literal` for substring +/// semantics — keeps matching plain `literal:string:` targets correctly. #[tokio::test] async fn test_plain_literal_contains_works_on_fn_parse_literal_path() { let store = SparqlStore::new(None).unwrap(); diff --git a/rust-executor/src/perspectives/model_query/projection.rs b/rust-executor/src/perspectives/model_query/projection.rs index f76bc0d29..8e155ce9c 100644 --- a/rust-executor/src/perspectives/model_query/projection.rs +++ b/rust-executor/src/perspectives/model_query/projection.rs @@ -289,9 +289,10 @@ pub(super) fn build_projection_where_patterns( // Resolve the target class's shape through the perspective cache so we // can translate property names in the projection's where-clause into the - // predicate IRIs they map to in the store. We also track whether each - // property is literal-encoded (`resolve_language: literal`) so V5 can - // emit direct IRI matches against the deterministic `literal:*:X` form. + // predicate IRIs they map to in the store. The second tuple element + // records whether each property carries `resolve_language` so the + // condition branches below know to compare against the encoded + // `literal:*:` IRI form instead of going through `fn/parse_literal`. let (pred_lookup, resolution_failed) = if let Some(ref target_name) = proj.target_class_name { match resolver.get_shape(target_name) { Ok(target_shape) => { @@ -377,11 +378,10 @@ pub(super) fn build_projection_where_patterns( match condition { WhereCondition::String(val) => { if is_literal_prop { - // V5: direct IRI match against the deterministic - // `literal:string:` IRI — POS-index friendly. - // Use a VALUES set so we can also include the raw-IRI form for - // properties whose constructor stored a raw URI on a shape that - // declares `resolveLanguage: literal` (mirrors V4 behavior). + // Match the encoded `literal:string:` form, plus the raw IRI + // form when the value is itself a valid absolute IRI — same + // dual-shape that the model-query where-clause emits for + // constructor-seeded raw URIs on literal properties. let encoded = literal_percent_encode(val); let mut iris = vec![format!("")]; if looks_like_absolute_iri(val) { diff --git a/rust-executor/src/perspectives/model_query/sparql_builder.rs b/rust-executor/src/perspectives/model_query/sparql_builder.rs index bb87108fa..d354ecb0f 100644 --- a/rust-executor/src/perspectives/model_query/sparql_builder.rs +++ b/rust-executor/src/perspectives/model_query/sparql_builder.rs @@ -513,16 +513,12 @@ pub(super) fn build_query_patterns( match condition { WhereCondition::String(val) => { if is_literal_prop { - // V4: Equality on a literal-encoded property compares against the - // deterministic `literal:string:` IRI emitted by - // `resolve_property_value`. Emitting a direct IRI match lets - // Oxigraph use the POS index instead of evaluating - // `fn/parse_literal` per row. - // - // If the where-value is also a valid raw IRI, a UNION covers the - // case where the constructor stored a raw URI on a property whose - // shape declares `resolveLanguage: literal` (e.g. enum-like state - // initial values). + // Match the deterministic `literal:string:` form of the + // value directly against the indexed object position. UNION the + // raw-IRI form when the where-value itself parses as an absolute + // IRI — constructors can seed a literal-resolveLanguage property + // with a raw URI (enum-style initial values) which the storage + // layer preserves as-is. let encoded = literal_percent_encode(val); if looks_like_absolute_iri(val) { where_patterns.push(format!( @@ -550,9 +546,8 @@ pub(super) fn build_query_patterns( } WhereCondition::Number(n) => { if is_literal_prop { - // V4: direct IRI match against `literal:number:` for - // index-friendly equality. Non-finite values are dropped - // (the FILTER(false) path matches no rows). + // Non-finite filter values cannot be stored as a `literal:number:` + // target, so emit a never-matching pattern instead of a malformed IRI. if let Some(num_str) = format_literal_number(*n) { where_patterns.push(format!( " ?source <{}> .", @@ -572,7 +567,6 @@ pub(super) fn build_query_patterns( } WhereCondition::Bool(b) => { if is_literal_prop { - // V4: direct IRI match against `literal:boolean:true|false`. where_patterns.push(format!( " ?source <{}> .", prop.predicate @@ -588,9 +582,8 @@ pub(super) fn build_query_patterns( } WhereCondition::StringArray(vals) => { if is_literal_prop { - // V4: VALUES of `literal:string:` IRIs → POS probe per IRI. - // Also include raw-IRI forms for values that parse as valid IRIs - // (mirrors the single-String UNION above). + // Same shape as the single-value String branch above, expanded + // into a VALUES set: one or two IRIs per input value. let mut iris: Vec = Vec::with_capacity(vals.len() * 2); for v in vals { iris.push(format!( @@ -626,7 +619,8 @@ pub(super) fn build_query_patterns( } WhereCondition::NumberArray(vals) => { if is_literal_prop { - // V4: VALUES of `literal:number:` IRIs. Non-finite values are dropped. + // Non-finite values are silently dropped from the VALUES set; if + // none remain, fall through to a never-matching pattern. let iris = vals .iter() .filter_map(|n| { diff --git a/rust-executor/src/perspectives/model_query/utils.rs b/rust-executor/src/perspectives/model_query/utils.rs index 67078ff2a..8592acd0b 100644 --- a/rust-executor/src/perspectives/model_query/utils.rs +++ b/rust-executor/src/perspectives/model_query/utils.rs @@ -48,15 +48,10 @@ pub(super) fn escape_sparql_string(s: &str) -> String { .replace('\t', "\\t") } -/// Cheap check: is this value plausibly an absolute IRI? Used by the -/// index-friendly WHERE builders (V4/V5) to decide whether to add a -/// raw-IRI fallback alongside the `literal:string:` direct probe. -/// -/// Requires (a) `validate_iri` passes (no chars that would break the -/// SPARQL IRIREF token), and (b) the value contains a `:` and starts with -/// an ASCII letter — a minimal proxy for "has a URI scheme". This rejects -/// bare strings like `"active"` that would otherwise produce the -/// un-parseable IRIREF ``. +/// Cheap heuristic for "this string is plausibly an absolute IRI" — passes +/// `validate_iri`, has a `:`, and starts with an ASCII letter. Used to decide +/// whether a where-value is safe to emit as a `<…>` IRIREF; rejects bare +/// strings like `"active"` that would produce un-parseable SPARQL. pub(super) fn looks_like_absolute_iri(s: &str) -> bool { if validate_iri(s).is_err() { return false; @@ -93,12 +88,8 @@ pub(super) const MAX_INCLUDE_DEPTH: u8 = 8; // literal: URI parsing (typed) // --------------------------------------------------------------------------- -/// Parse a `literal:` URI into a typed JSON value. -/// Returns the raw string as Value::String if not a literal: URI. -/// -/// Since the signed-envelope migration (v3), all literal values are stored -/// as plain `literal:string:X`, `literal:number:X`, `literal:boolean:X`, -/// or `literal:json:X` (for non-envelope JSON objects/arrays). +/// Parse a `literal:` URI into a typed JSON value, or return the input as a +/// string when it is not a literal URI. pub(super) fn parse_literal_value(uri: &str) -> Value { let body = if let Some(rest) = uri.strip_prefix("literal:") { rest @@ -128,10 +119,10 @@ pub(super) fn parse_literal_value(uri: &str) -> Value { } else if let Some(rest) = body.strip_prefix("json:") { let decoded = urlencoding::decode(rest).unwrap_or_else(|_| rest.into()); if let Ok(json_val) = serde_json::from_str::(&decoded) { - // Back-compat: legacy data may still contain signed-envelope literals - // from before the Channel V refactor (Jun 2026). New writes use plain - // literal: forms — see resolve_property_value. For envelopes that look - // like signed expressions, extract .data so callers see the inner value. + // Unwrap signed-expression envelopes (`{author, timestamp, data, proof}`) + // to the inner `.data` so consumers always see the underlying value + // rather than the wrapper, regardless of whether the target was + // written as a plain literal or as a signed expression. if let Some(data) = json_val.get("data") { if json_val.get("author").is_some() && json_val.get("proof").is_some() { return data.clone(); diff --git a/rust-executor/src/perspectives/perspective_instance.rs b/rust-executor/src/perspectives/perspective_instance.rs index bb9fcb6fd..9f535da92 100644 --- a/rust-executor/src/perspectives/perspective_instance.rs +++ b/rust-executor/src/perspectives/perspective_instance.rs @@ -4020,21 +4020,18 @@ impl PerspectiveInstance { .await?; if let Some(resolve_language) = resolve_language { - // For the built-in "literal" language, encode the value directly into a - // deterministic plain URI (`literal:string:X` / `:number:` / `:boolean:` / `:json:`). - // The link reifier is the canonical source of provenance — the signed-envelope - // shape is for Channel E (`expression.create(value, "literal")`) callers, not - // for property storage. Skipping the language controller here is the Channel V - // short-circuit; see - // ~/.sovereign/membranes/coasys/research/literal-encoding-and-sparql-pushdown-2026-06-03.md - // §9.1 (V1) for the audit. + // The built-in "literal" language produces a deterministic plain URI + // (`literal:string:X` / `:number:` / `:boolean:` / `:json:`) directly + // from the value. Routing through `expression_create` would wrap the + // value in a signed envelope whose IRI depends on author+timestamp, + // making property values non-deterministic and breaking exact-match + // SPARQL WHERE filters. Provenance for the link as a whole already + // lives on the reifier; the literal payload doesn't need its own. if resolve_language == "literal" { let encoded = crate::languages::literal_encode(value); return Ok(format!("literal:{}", encoded)); } - // Real language controllers (note, image, etc.) — Channel E. Keep the - // signed-expression path so the produced URL carries its own provenance. let controller = crate::languages::LanguageController::global_instance(); let agent_context = context.clone(); match controller diff --git a/rust-executor/src/perspectives/sparql_store.rs b/rust-executor/src/perspectives/sparql_store.rs index 6914fe9f0..7483c870a 100644 --- a/rust-executor/src/perspectives/sparql_store.rs +++ b/rust-executor/src/perspectives/sparql_store.rs @@ -82,10 +82,10 @@ fn parse_literal_fn(args: &[Term]) -> Option { Some(Literal::new_simple_literal(rest).into()) } else if let Some(rest) = body.strip_prefix("json:") { let decoded = urlencoding::decode(rest).unwrap_or_else(|_| rest.into()); - // Back-compat: legacy data may still contain signed-envelope literals from - // before the Channel V refactor (Jun 2026). New writes use plain literal: - // forms — see resolve_property_value. For JSON literals that look like a - // signed envelope (have a "data" field), extract it for content matching. + // Unwrap signed-expression envelopes (`{author, timestamp, data, proof}`) + // so WHERE filters can compare against the inner content. Required for + // pre-migration link stores and for the small set of expressions that + // are themselves stored as `literal:json:` (e.g. entanglement proofs). if let Ok(json_val) = serde_json::from_str::(&decoded) { if let Some(data) = json_val.get("data") { let data_str = match data { @@ -1121,24 +1121,21 @@ impl SparqlStore { Ok(count) } - /// Migrate signed expression envelopes to plain literal values. + /// Rewrite link targets shaped like `literal:json:` to the + /// plain typed form of their inner `.data` value (`literal:string:` / + /// `:number:` / `:boolean:` / `:json:`). /// - /// Old literal-language writes stored values as `literal:json:` - /// where the envelope was `{author, timestamp, data, proof}`. The provenance is - /// redundant with the RDF 1.2 reifier metadata — the Channel V refactor (Jun 2026) - /// stops emitting envelopes for property writes. This migration extracts the - /// `.data` field from any existing envelope and stores it as a plain - /// `literal:string:X`, `literal:number:X`, `literal:boolean:X`, or - /// `literal:json:X` value. - /// - /// Since the target IRI changes, we must rebuild both the direct triple and - /// the reifier (whose IRI is a hash of source+pred+target+author+ts). + /// Per-link provenance lives on the RDF 1.2 reifier; the envelope-on-target + /// form duplicates that and produces non-deterministic IRIs (the envelope + /// signature varies per write) which exact-match WHERE filters can't index. + /// The reifier IRI hashes the target, so rewriting the target requires + /// rebuilding both the direct triple and the reifier with all its metadata. pub fn migrate_signed_envelopes_to_plain_literals(&self) -> Result { if self.migration_version() >= 3 { return Ok(0); } - log::info!("Channel V refactor: migrating signed-envelope literals to plain form"); + log::info!("Migrating signed-envelope literal targets to plain literal form"); use percent_encoding::{percent_decode_str, utf8_percent_encode, NON_ALPHANUMERIC}; From f4273de9ac78cfd8b4d7eb7ccbb01a4916754261 Mon Sep 17 00:00:00 2001 From: HexaField <10372036+HexaField@users.noreply.github.com> Date: Thu, 4 Jun 2026 10:08:41 +1000 Subject: [PATCH 11/19] test: indexed-IRI vs fn/parse_literal microbench MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Compares two equivalent SPARQL queries against the same data — the indexed direct-IRI probe that the WHERE builders now emit, and the fn/parse_literal-wrapped FILTER they used to emit. Gated to release builds via cfg!(debug_assertions); scale tunable via WT_BENCH_LINKS. Indexed stays flat at ~12-40us across 1k-50k links (POS index probe). Filter scales linearly because every row materialises through the custom function. --- .../model_query/integration_tests.rs | 92 +++++++++++++++++++ 1 file changed, 92 insertions(+) diff --git a/rust-executor/src/perspectives/model_query/integration_tests.rs b/rust-executor/src/perspectives/model_query/integration_tests.rs index 20662864c..d0f44350b 100644 --- a/rust-executor/src/perspectives/model_query/integration_tests.rs +++ b/rust-executor/src/perspectives/model_query/integration_tests.rs @@ -4392,3 +4392,95 @@ async fn test_resolve_projections_where_filter_via_target_shape_property() { "list with limit:1 should return the hydrated like signal id, got {got}" ); } + +// ----------------------------------------------------------------------------- +// Indexed-WHERE benchmark +// ----------------------------------------------------------------------------- +// +// `cargo test --release --lib perspectives::model_query::integration_tests::bench` +// +// Compares two equivalent SPARQL queries against the same `literal:string:` +// data: an indexed direct-IRI probe vs. a `fn/parse_literal`-wrapped FILTER. +// The former is what the WHERE builders now emit; the latter is the shape +// they emitted before. Both queries find the same rows; the difference is +// whether Oxigraph's planner can use the POS index. + +#[test] +fn bench_indexed_iri_vs_fn_parse_literal_filter() { + use std::time::Instant; + + // Skip in debug builds — comparing per-row function call to an index probe + // is meaningless without optimisations. + if cfg!(debug_assertions) { + eprintln!("(bench skipped — run with --release)"); + return; + } + + // Toggle scale with WT_BENCH_LINKS; 10k by default. + let n_links: usize = std::env::var("WT_BENCH_LINKS") + .ok() + .and_then(|s| s.parse().ok()) + .unwrap_or(10_000); + + let store = SparqlStore::new(None).unwrap(); + let pred = "ns://body"; + let target_value = "needle"; + let stored_target = format!( + "literal:string:{}", + literal_percent_encode(target_value) + ); + + // Seed N rows; only the last carries the matching target. + let needle_idx = n_links - 1; + for i in 0..n_links { + let source = format!("test://row/{i}"); + store + .add_link(&make_link(&source, "ns://type", "ns://row", &format!("{}", 1_700_000_000_000_i64 + i as i64))) + .unwrap(); + let target = if i == needle_idx { + stored_target.clone() + } else { + format!("literal:string:{}", literal_percent_encode(&format!("row-{i}"))) + }; + store + .add_link(&make_link(&source, pred, &target, &format!("{}", 1_700_000_000_000_i64 + i as i64))) + .unwrap(); + } + + let indexed = format!( + "SELECT ?source WHERE {{ ?source <{pred}> <{stored_target}> . }}" + ); + let filtered = format!( + "SELECT ?source WHERE {{ \ + ?source <{pred}> ?t . \ + FILTER(STR((?t)) = \"{target_value}\") \ + }}" + ); + + // Warm-up — touch every triple under both query plans before timing. + let _ = store.query(&indexed).unwrap(); + let _ = store.query(&filtered).unwrap(); + + let runs = 5; + let mut indexed_total = std::time::Duration::ZERO; + let mut filtered_total = std::time::Duration::ZERO; + + for _ in 0..runs { + let start = Instant::now(); + let r = store.query(&indexed).unwrap(); + indexed_total += start.elapsed(); + assert!(r.contains(&format!("test://row/{needle_idx}"))); + + let start = Instant::now(); + let r = store.query(&filtered).unwrap(); + filtered_total += start.elapsed(); + assert!(r.contains(&format!("test://row/{needle_idx}"))); + } + + let indexed_us = (indexed_total.as_secs_f64() * 1_000_000.0) / runs as f64; + let filtered_us = (filtered_total.as_secs_f64() * 1_000_000.0) / runs as f64; + let speedup = filtered_us / indexed_us; + eprintln!( + "[bench] n={n_links} indexed={indexed_us:.1}µs fn_parse_literal_filter={filtered_us:.1}µs speedup={speedup:.1}x" + ); +} From 08240a07a7a9c30db2afebe7293d0069337c4e92 Mon Sep 17 00:00:00 2001 From: HexaField <10372036+HexaField@users.noreply.github.com> Date: Thu, 4 Jun 2026 13:35:08 +1000 Subject: [PATCH 12/19] ci: re-trigger after dev merge From f4fbdb3f4434c39adcf2ea7e19edfb74380f0fa2 Mon Sep 17 00:00:00 2001 From: HexaField <10372036+HexaField@users.noreply.github.com> Date: Mon, 8 Jun 2026 21:51:42 +1000 Subject: [PATCH 13/19] ci: re-trigger after dev merge Force CircleCI to re-evaluate; previous webhook delivery failed with `why: github, start_time: None` (no runner accepted the job). This is a no-op commit on the branch tip purely to nudge the queue. From df05f468eada8888c4d24b74c9c7ed69d0c1713e Mon Sep 17 00:00:00 2001 From: HexaField <10372036+HexaField@users.noreply.github.com> Date: Mon, 8 Jun 2026 21:57:35 +1000 Subject: [PATCH 14/19] style: cargo fmt --- rust-executor/src/perspectives/mod.rs | 11 ++---- .../model_query/integration_tests.rs | 35 ++++++++++--------- .../perspectives/model_query/projection.rs | 18 +++------- .../model_query/sparql_builder.rs | 18 ++++------ 4 files changed, 31 insertions(+), 51 deletions(-) diff --git a/rust-executor/src/perspectives/mod.rs b/rust-executor/src/perspectives/mod.rs index 95a145bbe..be3de4ff4 100644 --- a/rust-executor/src/perspectives/mod.rs +++ b/rust-executor/src/perspectives/mod.rs @@ -119,10 +119,7 @@ pub fn initialize_from_db() { Err(e) => log::warn!("Reifier migration for {}: {}", handle_clone.uuid, e), } - match p - .sparql_store - .migrate_signed_envelopes_to_plain_literals() - { + match p.sparql_store.migrate_signed_envelopes_to_plain_literals() { Ok(count) if count > 0 => { log::info!( "🔄 Signed-envelope migration for {}: {} envelopes converted", @@ -131,11 +128,7 @@ pub fn initialize_from_db() { ); } Ok(_) => {} // Already migrated or nothing to migrate - Err(e) => log::warn!( - "Signed-envelope migration for {}: {}", - handle_clone.uuid, - e - ), + Err(e) => log::warn!("Signed-envelope migration for {}: {}", handle_clone.uuid, e), } // Rebuild SPARQL index from existing links diff --git a/rust-executor/src/perspectives/model_query/integration_tests.rs b/rust-executor/src/perspectives/model_query/integration_tests.rs index d0f44350b..a30780661 100644 --- a/rust-executor/src/perspectives/model_query/integration_tests.rs +++ b/rust-executor/src/perspectives/model_query/integration_tests.rs @@ -3641,12 +3641,7 @@ async fn test_plain_literal_where_paginate_count() { .add_link(&make_link(uri, "ns://type", "ns://task", &ts)) .unwrap(); store - .add_link(&make_link( - uri, - "ns://status", - &signed_literal(status), - &ts, - )) + .add_link(&make_link(uri, "ns://status", &signed_literal(status), &ts)) .unwrap(); store .add_link(&make_link(uri, "ns://name", &signed_literal(name), &ts)) @@ -4425,31 +4420,39 @@ fn bench_indexed_iri_vs_fn_parse_literal_filter() { let store = SparqlStore::new(None).unwrap(); let pred = "ns://body"; let target_value = "needle"; - let stored_target = format!( - "literal:string:{}", - literal_percent_encode(target_value) - ); + let stored_target = format!("literal:string:{}", literal_percent_encode(target_value)); // Seed N rows; only the last carries the matching target. let needle_idx = n_links - 1; for i in 0..n_links { let source = format!("test://row/{i}"); store - .add_link(&make_link(&source, "ns://type", "ns://row", &format!("{}", 1_700_000_000_000_i64 + i as i64))) + .add_link(&make_link( + &source, + "ns://type", + "ns://row", + &format!("{}", 1_700_000_000_000_i64 + i as i64), + )) .unwrap(); let target = if i == needle_idx { stored_target.clone() } else { - format!("literal:string:{}", literal_percent_encode(&format!("row-{i}"))) + format!( + "literal:string:{}", + literal_percent_encode(&format!("row-{i}")) + ) }; store - .add_link(&make_link(&source, pred, &target, &format!("{}", 1_700_000_000_000_i64 + i as i64))) + .add_link(&make_link( + &source, + pred, + &target, + &format!("{}", 1_700_000_000_000_i64 + i as i64), + )) .unwrap(); } - let indexed = format!( - "SELECT ?source WHERE {{ ?source <{pred}> <{stored_target}> . }}" - ); + let indexed = format!("SELECT ?source WHERE {{ ?source <{pred}> <{stored_target}> . }}"); let filtered = format!( "SELECT ?source WHERE {{ \ ?source <{pred}> ?t . \ diff --git a/rust-executor/src/perspectives/model_query/projection.rs b/rust-executor/src/perspectives/model_query/projection.rs index 8e155ce9c..6bc8849ef 100644 --- a/rust-executor/src/perspectives/model_query/projection.rs +++ b/rust-executor/src/perspectives/model_query/projection.rs @@ -387,10 +387,7 @@ pub(super) fn build_projection_where_patterns( if looks_like_absolute_iri(val) { iris.push(format!("<{val}>")); } - patterns.push(format!( - " VALUES ?{var} {{ {} }}\n", - iris.join(" ") - )); + patterns.push(format!(" VALUES ?{var} {{ {} }}\n", iris.join(" "))); patterns.push(format!(" ?t <{pred}> ?{var} .\n")); } else { let escaped = escape_sparql_string(val); @@ -402,9 +399,7 @@ pub(super) fn build_projection_where_patterns( } WhereCondition::Bool(b) => { if is_literal_prop { - patterns.push(format!( - " ?t <{pred}> .\n" - )); + patterns.push(format!(" ?t <{pred}> .\n")); } else { let bval = if *b { "true" } else { "false" }; patterns.push(format!(" ?t <{pred}> ?{var} .\n")); @@ -416,9 +411,7 @@ pub(super) fn build_projection_where_patterns( WhereCondition::Number(n) => { if is_literal_prop { if let Some(num_str) = format_literal_number(*n) { - patterns.push(format!( - " ?t <{pred}> .\n" - )); + patterns.push(format!(" ?t <{pred}> .\n")); } else { patterns.push(" FILTER(false)\n".to_string()); } @@ -438,10 +431,7 @@ pub(super) fn build_projection_where_patterns( iris.push(format!("<{v}>")); } } - patterns.push(format!( - " VALUES ?{var} {{ {} }}\n", - iris.join(" ") - )); + patterns.push(format!(" VALUES ?{var} {{ {} }}\n", iris.join(" "))); patterns.push(format!(" ?t <{pred}> ?{var} .\n")); } else { let list = vals diff --git a/rust-executor/src/perspectives/model_query/sparql_builder.rs b/rust-executor/src/perspectives/model_query/sparql_builder.rs index d354ecb0f..f66e04f30 100644 --- a/rust-executor/src/perspectives/model_query/sparql_builder.rs +++ b/rust-executor/src/perspectives/model_query/sparql_builder.rs @@ -595,14 +595,10 @@ pub(super) fn build_query_patterns( } } let iv_var = format!("?_iv_{safe_name}"); - where_patterns.push(format!( - " VALUES {iv_var} {{ {} }}", - iris.join(" ") - )); - where_patterns.push(format!( - " ?source <{}> {iv_var} .", - prop.predicate - )); + where_patterns + .push(format!(" VALUES {iv_var} {{ {} }}", iris.join(" "))); + where_patterns + .push(format!(" ?source <{}> {iv_var} .", prop.predicate)); } else { let values_list = vals .iter() @@ -634,10 +630,8 @@ pub(super) fn build_query_patterns( } else { let iv_var = format!("?_iv_{safe_name}"); where_patterns.push(format!(" VALUES {iv_var} {{ {iris} }}")); - where_patterns.push(format!( - " ?source <{}> {iv_var} .", - prop.predicate - )); + where_patterns + .push(format!(" ?source <{}> {iv_var} .", prop.predicate)); } } else { let values_list = vals From 7ff5fbf42b8e2d171e51ab7321f1c6041e78e387 Mon Sep 17 00:00:00 2001 From: HexaField <10372036+HexaField@users.noreply.github.com> Date: Thu, 11 Jun 2026 10:41:15 +1000 Subject: [PATCH 15/19] review: address CodeRabbit comments on #837 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Five comments from CodeRabbit on the v3 literal-target refactor: * mod.rs (Major) — `migrate_signed_envelopes_to_plain_literals` now fail-stops perspective init on error. Previously the migration error was only warn-logged, so the perspective came up serving a mix of legacy `literal:json:` and migrated `literal:*` targets, and the new indexed WHERE filters silently missed the unmigrated rows. The error path now `log::error!`s and returns from the init spawn; the next executor restart can retry cleanly. * projection.rs:304 + sparql_builder.rs:512 (Major) — tighten the literal fast-path detection from `resolve_language.is_some()` to `resolve_language.as_deref() == Some("literal")`. Only the built-in `"literal"` resolver produces deterministic `literal:*` targets; any other resolver wraps values in author-signed expression IRIs and the former check would have emitted encoded-literal probes against them, silently dropping matches. * sparql_store.rs:1221 (Major) — the v3 migration's hand-rolled per- variant `format!("literal:{kind}:{...}", …)` block is gone. The migration now reuses the canonical `crate::languages::literal_encode` helper that fresh writes flow through, so migrated rows and new writes produce identical target IRIs. Without this, an integer-shaped float like `1.0` landed as `literal:number:1.0` from the migration while fresh writes / WHERE probes produced `literal:number:1` (via `format_literal_number`'s integer collapse), causing migrated rows to silently miss exact-match filters. * utils.rs:67 (Minor) — `validate_iri` now rejects any control or whitespace character (not just ASCII space), so `\n`/`\r`/`\t` and Unicode whitespace can no longer slip past the heuristic and emit malformed `<…>` IRIREFs in generated SPARQL. * integration_tests.rs:4480 (Minor) — the `bench_indexed_iri_vs_fn_parse_literal_filter` benchmark now parses each response and asserts exactly one row with `source == test://row/{needle_idx}`. The previous `contains` check would have passed silently if either plan started overmatching. Also drop the now-unused `utf8_percent_encode` / `NON_ALPHANUMERIC` imports in the migration path. JS test surface: * mcp-http.test.ts §5b — flip the "resolveLanguage for boolean/string properties" assertions to match the new contract: properties with `resolveLanguage: "literal"` now produce deterministic `literal:boolean:` / `literal:string:` targets, NOT the legacy `literal:json:` form. Provenance for the link as a whole lives on the RDF 1.2 reifier, so wrapping each property value in its own signed envelope was redundant and defeated indexed equality lookups. --- rust-executor/src/perspectives/mod.rs | 19 ++++++++- .../model_query/integration_tests.rs | 17 +++++++- .../perspectives/model_query/projection.rs | 12 ++++-- .../model_query/sparql_builder.rs | 6 ++- .../src/perspectives/model_query/utils.rs | 8 ++-- .../src/perspectives/sparql_store.rs | 28 ++++--------- tests/js/tests/mcp-http.test.ts | 41 +++++++++++-------- 7 files changed, 84 insertions(+), 47 deletions(-) diff --git a/rust-executor/src/perspectives/mod.rs b/rust-executor/src/perspectives/mod.rs index be3de4ff4..0e698c8ba 100644 --- a/rust-executor/src/perspectives/mod.rs +++ b/rust-executor/src/perspectives/mod.rs @@ -119,6 +119,13 @@ pub fn initialize_from_db() { Err(e) => log::warn!("Reifier migration for {}: {}", handle_clone.uuid, e), } + // Signed-envelope → plain-literal migration. The new indexed WHERE + // path probes `literal:*` targets directly, so if this migration + // fails midway through the perspective is left with a mix of legacy + // signed-envelope targets and new plain-literal targets — equality + // filters would then silently miss the unmigrated rows. Skip + // initialising the perspective on error so the next executor restart + // can retry the migration cleanly rather than serving stale results. match p.sparql_store.migrate_signed_envelopes_to_plain_literals() { Ok(count) if count > 0 => { log::info!( @@ -128,7 +135,17 @@ pub fn initialize_from_db() { ); } Ok(_) => {} // Already migrated or nothing to migrate - Err(e) => log::warn!("Signed-envelope migration for {}: {}", handle_clone.uuid, e), + Err(e) => { + log::error!( + "Signed-envelope migration failed for {}: {} — \ + skipping perspective init to avoid mixing legacy and \ + migrated link targets under the new indexed WHERE path. \ + Will retry on next executor restart.", + handle_clone.uuid, + e + ); + return; + } } // Rebuild SPARQL index from existing links diff --git a/rust-executor/src/perspectives/model_query/integration_tests.rs b/rust-executor/src/perspectives/model_query/integration_tests.rs index a30780661..6244ac268 100644 --- a/rust-executor/src/perspectives/model_query/integration_tests.rs +++ b/rust-executor/src/perspectives/model_query/integration_tests.rs @@ -4467,17 +4467,30 @@ fn bench_indexed_iri_vs_fn_parse_literal_filter() { let runs = 5; let mut indexed_total = std::time::Duration::ZERO; let mut filtered_total = std::time::Duration::ZERO; + let expected = format!("test://row/{needle_idx}"); for _ in 0..runs { let start = Instant::now(); let r = store.query(&indexed).unwrap(); indexed_total += start.elapsed(); - assert!(r.contains(&format!("test://row/{needle_idx}"))); + let rows: Vec = serde_json::from_str(&r).unwrap(); + assert_eq!(rows.len(), 1, "indexed query must return exactly 1 row"); + assert_eq!( + rows[0]["source"].as_str(), + Some(expected.as_str()), + "indexed query must return only the needle source", + ); let start = Instant::now(); let r = store.query(&filtered).unwrap(); filtered_total += start.elapsed(); - assert!(r.contains(&format!("test://row/{needle_idx}"))); + let rows: Vec = serde_json::from_str(&r).unwrap(); + assert_eq!(rows.len(), 1, "filtered query must return exactly 1 row"); + assert_eq!( + rows[0]["source"].as_str(), + Some(expected.as_str()), + "filtered query must return only the needle source", + ); } let indexed_us = (indexed_total.as_secs_f64() * 1_000_000.0) / runs as f64; diff --git a/rust-executor/src/perspectives/model_query/projection.rs b/rust-executor/src/perspectives/model_query/projection.rs index 6bc8849ef..5351bd1d5 100644 --- a/rust-executor/src/perspectives/model_query/projection.rs +++ b/rust-executor/src/perspectives/model_query/projection.rs @@ -290,9 +290,10 @@ pub(super) fn build_projection_where_patterns( // Resolve the target class's shape through the perspective cache so we // can translate property names in the projection's where-clause into the // predicate IRIs they map to in the store. The second tuple element - // records whether each property carries `resolve_language` so the - // condition branches below know to compare against the encoded - // `literal:*:` IRI form instead of going through `fn/parse_literal`. + // records whether each property carries `resolveLanguage: "literal"` — + // only that resolver stores deterministic `literal:*:` targets that we + // can probe directly. Other resolvers wrap values in author-signed + // expression IRIs, so we fall back to `fn/parse_literal` for those. let (pred_lookup, resolution_failed) = if let Some(ref target_name) = proj.target_class_name { match resolver.get_shape(target_name) { Ok(target_shape) => { @@ -301,7 +302,10 @@ pub(super) fn build_projection_where_patterns( if !p.predicate.is_empty() { map.insert( p.name.clone(), - (p.predicate.clone(), p.resolve_language.is_some()), + ( + p.predicate.clone(), + p.resolve_language.as_deref() == Some("literal"), + ), ); } } diff --git a/rust-executor/src/perspectives/model_query/sparql_builder.rs b/rust-executor/src/perspectives/model_query/sparql_builder.rs index f66e04f30..5f922de49 100644 --- a/rust-executor/src/perspectives/model_query/sparql_builder.rs +++ b/rust-executor/src/perspectives/model_query/sparql_builder.rs @@ -509,7 +509,11 @@ pub(super) fn build_query_patterns( continue; } let safe_name = prop_name.replace(|c: char| !c.is_alphanumeric(), "_"); - let is_literal_prop = prop.resolve_language.is_some(); + // Only `resolveLanguage: "literal"` stores deterministic + // `literal:*` targets we can probe directly. Other resolvers + // wrap values in author-signed expression IRIs, so they fall + // through to the FILTER-on-decoded path below. + let is_literal_prop = prop.resolve_language.as_deref() == Some("literal"); match condition { WhereCondition::String(val) => { if is_literal_prop { diff --git a/rust-executor/src/perspectives/model_query/utils.rs b/rust-executor/src/perspectives/model_query/utils.rs index 8592acd0b..549a73ecc 100644 --- a/rust-executor/src/perspectives/model_query/utils.rs +++ b/rust-executor/src/perspectives/model_query/utils.rs @@ -67,14 +67,16 @@ pub(super) fn looks_like_absolute_iri(s: &str) -> bool { } /// Validate a value for use inside an IRI `<…>`. Rejects characters that -/// would break or inject into a SPARQL IRI token. +/// would break or inject into a SPARQL IRI token, including all control and +/// whitespace characters (e.g. `\n`, `\r`, `\t`, U+00A0) which `validate_iri` +/// previously let through and which would emit malformed `<…>` IRIREFs. pub(super) fn validate_iri(s: &str) -> Result<&str, Error> { - if s.contains('>') + if s.chars().any(|c| c.is_control() || c.is_whitespace()) + || s.contains('>') || s.contains('<') || s.contains('{') || s.contains('}') || s.contains('"') - || s.contains(' ') { return Err(anyhow!("Invalid IRI component: '{}'", s)); } diff --git a/rust-executor/src/perspectives/sparql_store.rs b/rust-executor/src/perspectives/sparql_store.rs index 7483c870a..c4a0293f9 100644 --- a/rust-executor/src/perspectives/sparql_store.rs +++ b/rust-executor/src/perspectives/sparql_store.rs @@ -1137,7 +1137,7 @@ impl SparqlStore { log::info!("Migrating signed-envelope literal targets to plain literal form"); - use percent_encoding::{percent_decode_str, utf8_percent_encode, NON_ALPHANUMERIC}; + use percent_encoding::percent_decode_str; let rdf_reifies = NamedNodeRef::new_unchecked(RDF_REIFIES); let ont_author = NamedNodeRef::new_unchecked(ONT_AUTHOR); @@ -1218,24 +1218,14 @@ impl SparqlStore { _ => continue, // Not a signed envelope, leave as-is }; - // Encode the data field as a plain literal - let new_target = match &data { - serde_json::Value::String(s) => { - let encoded = utf8_percent_encode(s, NON_ALPHANUMERIC).to_string(); - format!("literal:string:{}", encoded) - } - serde_json::Value::Number(n) => { - format!("literal:number:{}", n) - } - serde_json::Value::Bool(b) => { - format!("literal:boolean:{}", b) - } - _ => { - let json_str = serde_json::to_string(&data).unwrap_or_default(); - let encoded = utf8_percent_encode(&json_str, NON_ALPHANUMERIC).to_string(); - format!("literal:json:{}", encoded) - } - }; + // Encode the inner `data` field as a plain literal IRI using the + // same canonical encoder that fresh writes flow through + // (`perspective_instance::link_target_for_value` → `literal_encode`). + // Hand-rolling the format here used to drift (e.g. integer-shaped + // floats landed as `literal:number:1.0` while the WHERE builders + // probed `literal:number:1`), causing migrated rows to silently + // miss equality filters. + let new_target = format!("literal:{}", crate::languages::literal_encode(&data)); if new_target == old_target { continue; // No change needed diff --git a/tests/js/tests/mcp-http.test.ts b/tests/js/tests/mcp-http.test.ts index f0b77875c..78901d621 100644 --- a/tests/js/tests/mcp-http.test.ts +++ b/tests/js/tests/mcp-http.test.ts @@ -998,8 +998,12 @@ describe("MCP HTTP Flux Chat Integration Test", function() { }); // ======================================================================== - // 5b. Resolve Language — verify properties with resolve_language produce - // proper literal:json: expressions instead of literal:string: + // 5b. Resolve Language — properties with resolveLanguage: "literal" must + // produce deterministic literal:boolean: / literal:number: / + // literal:string: targets (NOT the legacy literal:json: form). + // Per-link provenance lives on the RDF 1.2 reifier, so wrapping each + // property value in a signed expression envelope duplicates that and + // defeats indexed equality lookups in the WHERE builder. // ======================================================================== describe("5b. Resolve Language for Boolean/String Properties", function() { @@ -1019,7 +1023,7 @@ describe("MCP HTTP Flux Chat Integration Test", function() { console.log("channel_create with booleans:", resultStr); }); - it("should store boolean properties as literal:json: expressions, not literal:string:", async function() { + it("should store boolean properties as deterministic literal:boolean: targets", async function() { // Query the raw links to verify the encoding format var links = await callMcpTool(MCP_BASE_URL,'query_links', { perspective_id: perspectiveUuid, @@ -1028,17 +1032,18 @@ describe("MCP HTTP Flux Chat Integration Test", function() { }, mcpSessionId); console.log("isConversation links:", JSON.stringify(links)); - // The target should be a literal:json: expression (signed expression), - // NOT literal:string:false + // The target should be the deterministic literal:boolean: form. + // The legacy literal:json: form is no longer + // produced — provenance is carried by the reifier instead. var linksArr = Array.isArray(links) ? links : (links.links || []); expect(linksArr.length).to.be.greaterThan(0); var target = linksArr[0].data?.target || linksArr[0].target || ''; console.log("isConversation target:", target); - expect(target).to.not.include("literal:string:false"); - expect(target).to.include("literal:json:"); + expect(target).to.not.include("literal:json:"); + expect(target).to.equal("literal:boolean:false"); }); - it("should store string properties as literal:json: expressions when resolve_language is set", async function() { + it("should store string properties as deterministic literal:string: targets when resolve_language is set", async function() { var links = await callMcpTool(MCP_BASE_URL,'query_links', { perspective_id: perspectiveUuid, source: resolveTestChannelAddr, @@ -1050,8 +1055,9 @@ describe("MCP HTTP Flux Chat Integration Test", function() { expect(linksArr.length).to.be.greaterThan(0); var target = linksArr[0].data?.target || linksArr[0].target || ''; console.log("name target:", target); - // Should be a signed expression (literal:json:) not a raw string literal - expect(target).to.include("literal:json:"); + // Deterministic literal:string: form, not a signed envelope. + expect(target).to.not.include("literal:json:"); + expect(target).to.match(/^literal:string:/); }); it("should resolve boolean values via channel_set_isconversation", async function() { @@ -1062,7 +1068,7 @@ describe("MCP HTTP Flux Chat Integration Test", function() { }, mcpSessionId); expect(result.success).to.be.true; - // Verify the stored link target is a proper expression + // Verify the stored link target is the deterministic literal:boolean: form. var links = await callMcpTool(MCP_BASE_URL,'query_links', { perspective_id: perspectiveUuid, source: resolveTestChannelAddr, @@ -1072,8 +1078,8 @@ describe("MCP HTTP Flux Chat Integration Test", function() { expect(linksArr.length).to.be.greaterThan(0); var target = linksArr[0].data?.target || linksArr[0].target || ''; console.log("Updated isConversation target:", target); - expect(target).to.not.include("literal:string:true"); - expect(target).to.include("literal:json:"); + expect(target).to.not.include("literal:json:"); + expect(target).to.equal("literal:boolean:true"); }); it("should resolve string values via set_subject_property with resolve_language", async function() { @@ -1086,7 +1092,7 @@ describe("MCP HTTP Flux Chat Integration Test", function() { }, mcpSessionId); expect(result.success).to.be.true; - // Verify the stored link target uses literal:json: (signed expression) + // Verify the stored link target is the deterministic literal:string: form. var links = await callMcpTool(MCP_BASE_URL,'query_links', { perspective_id: perspectiveUuid, source: resolveTestChannelAddr, @@ -1096,7 +1102,8 @@ describe("MCP HTTP Flux Chat Integration Test", function() { expect(linksArr.length).to.be.greaterThan(0); var target = linksArr[0].data?.target || linksArr[0].target || ''; console.log("description target:", target); - expect(target).to.include("literal:json:"); + expect(target).to.not.include("literal:json:"); + expect(target).to.match(/^literal:string:/); }); it("should resolve boolean values via channel_update (dynamic update)", async function() { @@ -1116,8 +1123,8 @@ describe("MCP HTTP Flux Chat Integration Test", function() { expect(linksArr.length).to.be.greaterThan(0); var target = linksArr[0].data?.target || linksArr[0].target || ''; console.log("Updated isPinned target:", target); - expect(target).to.not.include("literal:string:false"); - expect(target).to.include("literal:json:"); + expect(target).to.not.include("literal:json:"); + expect(target).to.equal("literal:boolean:false"); }); }); From 561aa3807cafd4350bd2024b774578e4f09d0182 Mon Sep 17 00:00:00 2001 From: HexaField <10372036+HexaField@users.noreply.github.com> Date: Thu, 11 Jun 2026 10:47:17 +1000 Subject: [PATCH 16/19] fix(core): bypass createExpression for resolveLanguage: "literal" on writes MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `setProperty` was always routing values through `_perspective.createExpression` when `resolveLanguage` was set, which for the built-in `"literal"` resolver produced `literal:json:` targets — defeating the indexed WHERE path #837 puts in place and breaking round-trip assertions like `expect(await todo.title).to.equal("new title")` (the read decodes the envelope and surfaces the 4-key `{author, timestamp, data, proof}` object instead of `"new title"`). Mirror the Rust-side bypass already in `resolve_property_value`: when `resolveLanguage === "literal"`, encode the value with `Literal.from(value) .toUrl()` directly. Other resolvers still go through `createExpression` so non-literal resolveLanguage semantics are unchanged. Also fill in `Literal.get()`'s missing `boolean:` branch so callers that read a deterministic `literal:boolean:` target don't throw "Can't parse unknown literal" when consuming the new write format end-to-end. --- core/src/Literal.ts | 7 +++++++ core/src/model/Ad4mModel.ts | 13 ++++++++++++- 2 files changed, 19 insertions(+), 1 deletion(-) diff --git a/core/src/Literal.ts b/core/src/Literal.ts index d9a6107a6..1e1c54443 100644 --- a/core/src/Literal.ts +++ b/core/src/Literal.ts @@ -73,6 +73,13 @@ export class Literal { return parseFloat(numberString) } + if(body.startsWith("boolean:")) { + const boolString = decodeURIComponent(body.substring(8)) + if (boolString === "true") return true + if (boolString === "false") return false + throw new Error(`Can't parse boolean literal: ${boolString}`) + } + if(body.startsWith("json:")) { const json = body.substring(5) return JSON.parse(decodeURIComponent(json)) diff --git a/core/src/model/Ad4mModel.ts b/core/src/model/Ad4mModel.ts index 3fd122859..d36274245 100644 --- a/core/src/model/Ad4mModel.ts +++ b/core/src/model/Ad4mModel.ts @@ -1091,7 +1091,18 @@ export class Ad4mModel { } if (resolveLanguage) { - value = await this._perspective.createExpression(value, resolveLanguage); + // For the built-in "literal" resolver we produce the deterministic + // `literal:string:` / `:number:` / `:boolean:` / `:json:` URL directly + // instead of round-tripping through `createExpression`. The Rust + // mirror of this bypass lives in `resolve_property_value` — without + // both sides, the same value would land as `literal:json:` on writes through this path, defeating the indexed + // equality lookups the WHERE builders now use. + if (resolveLanguage === "literal") { + value = Literal.from(value).toUrl(); + } else { + value = await this._perspective.createExpression(value, resolveLanguage); + } } await this._perspective.executeAction(actions, this._baseExpression, [{ name: "value", value }], batchId); From f30b4d6a3fd105dffe7a241d60ed4f6fea8f9276 Mon Sep 17 00:00:00 2001 From: HexaField <10372036+HexaField@users.noreply.github.com> Date: Thu, 11 Jun 2026 11:04:52 +1000 Subject: [PATCH 17/19] =?UTF-8?q?fix(literal-bypass):=20leave=20URI-shaped?= =?UTF-8?q?=20strings=20raw=20=E2=80=94=20align=20write=20with=20query?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit CodeRabbit caught a divergence I introduced in the previous fixup: the TS-side `setProperty` bypass wrapped *every* string with `Literal.from(value).toUrl()`, so writing `"https://example.com"` to a `resolveLanguage: "literal"` property landed as `` while `queryToSPARQL()` still emitted `` for the same filter — exact-match WHERE clauses silently missed freshly written values. Align the three serialization paths so they all keep URI-shaped strings raw and only wrap primitives: * `core/src/model/query-sparql.ts` — export `valueToLiteralIri` so it's the single source of truth for write/read symmetry. * `core/src/model/Ad4mModel.ts` `setProperty` — route the literal bypass through `valueToLiteralIri` instead of calling `Literal.from(...).toUrl()` directly. * `rust-executor/src/perspectives/perspective_instance.rs` `resolve_property_value` — same logic for the `resolve_language == "literal"` short-circuit (strings with a URI scheme return as-is, everything else flows through `literal_encode`), matching the no-resolveLanguage branch immediately below. `looksLikeUri` / `URI_SCHEME_RE` use the identical `^[a-zA-Z][a-zA-Z0-9+\-._]*:` pattern on both sides so the write and query produce byte-identical IRIs for any given value. --- core/src/model/Ad4mModel.ts | 9 +++++++-- core/src/model/query-sparql.ts | 8 +++++++- .../src/perspectives/perspective_instance.rs | 15 +++++++++++++++ 3 files changed, 29 insertions(+), 3 deletions(-) diff --git a/core/src/model/Ad4mModel.ts b/core/src/model/Ad4mModel.ts index d36274245..ee7e7f4a6 100644 --- a/core/src/model/Ad4mModel.ts +++ b/core/src/model/Ad4mModel.ts @@ -11,7 +11,7 @@ import { isArrayType, determinePredicate, determineNamespace, buildModelFromJSON import type { SHACLShape } from "../shacl/SHACLShape"; import type { JSONSchemaProperty, JSONSchema, JSONSchemaToModelOptions } from "./json-schema"; -import { buildSPARQLQuery } from "./query-sparql"; +import { buildSPARQLQuery, valueToLiteralIri } from "./query-sparql"; import { ModelQueryBuilder } from "./ModelQueryBuilder"; import { normalizeValue, @@ -1098,8 +1098,13 @@ export class Ad4mModel { // both sides, the same value would land as `literal:json:` on writes through this path, defeating the indexed // equality lookups the WHERE builders now use. + // + // Route through `valueToLiteralIri` (also used by `queryToSPARQL`) + // so URI-shaped strings stay raw on both write AND read: writing + // `"https://example.com"` keeps it as ``, + // matching what WHERE filters generate for the same value. if (resolveLanguage === "literal") { - value = Literal.from(value).toUrl(); + value = valueToLiteralIri(value); } else { value = await this._perspective.createExpression(value, resolveLanguage); } diff --git a/core/src/model/query-sparql.ts b/core/src/model/query-sparql.ts index 87ea8ba7a..99ad591df 100644 --- a/core/src/model/query-sparql.ts +++ b/core/src/model/query-sparql.ts @@ -81,8 +81,14 @@ function looksLikeUri(value: string): boolean { * Convert a JS value to its literal: IRI form, matching how the Rust executor * stores property values that don't have a resolveLanguage set. * Strings that already look like URIs are returned as-is. + * + * Exported so write-side helpers (e.g. `Ad4mModel.setProperty`'s literal + * resolveLanguage bypass) can keep their on-disk form aligned with what + * `queryToSPARQL()` filters against — otherwise the same value can be + * stored as `` from one path while WHERE + * builders probe `` from the other. */ -function valueToLiteralIri(value: any): string { +export function valueToLiteralIri(value: any): string { if (typeof value === 'string') { if (looksLikeUri(value)) return value; return Literal.from(value).toUrl(); diff --git a/rust-executor/src/perspectives/perspective_instance.rs b/rust-executor/src/perspectives/perspective_instance.rs index 9f535da92..6963c6904 100644 --- a/rust-executor/src/perspectives/perspective_instance.rs +++ b/rust-executor/src/perspectives/perspective_instance.rs @@ -4028,6 +4028,21 @@ impl PerspectiveInstance { // SPARQL WHERE filters. Provenance for the link as a whole already // lives on the reifier; the literal payload doesn't need its own. if resolve_language == "literal" { + // Mirror the no-resolveLanguage branch below + the TS-side + // `valueToLiteralIri`: strings that already carry a URI + // scheme are stored as-is so they round-trip through the + // WHERE builders' `<…>` IRI probes without wrapping. Other + // values flow through the canonical `literal_encode` so + // migrated rows and fresh writes share one IRI shape. + if let serde_json::Value::String(s) = value { + static URI_SCHEME_RE: std::sync::OnceLock = + std::sync::OnceLock::new(); + let re = URI_SCHEME_RE + .get_or_init(|| regex::Regex::new(r"^[a-zA-Z][a-zA-Z0-9+\-._]*:").unwrap()); + if re.is_match(s) { + return Ok(s.clone()); + } + } let encoded = crate::languages::literal_encode(value); return Ok(format!("literal:{}", encoded)); } From 121e30800b7a9282cf149d95135a9ae23f5c1abc Mon Sep 17 00:00:00 2001 From: HexaField <10372036+HexaField@users.noreply.github.com> Date: Thu, 11 Jun 2026 12:05:44 +1000 Subject: [PATCH 18/19] =?UTF-8?q?test(prolog-and-literals):=20bump=20subsc?= =?UTF-8?q?ription-perf=20poll=20timeout=205s=20=E2=86=92=2030s?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The "should demonstrate subscription performance" test polled every 10ms with a hard 5s ceiling. That's plenty for steady-state subscription latency (sub-second locally) but CI workers regularly take longer to warm up the first subscription's GraphQL → executor → SPARQL → subscriber round-trip after a fresh `save()`, leading to spurious failures on otherwise-green runs. Match the 30s waitForCondition ceiling the surrounding tests already use (see `a5ad343ce test: bump waitForCondition timeouts to 60s for CI stability` for the precedent). Steady-state latency is still logged via `subscriptionLatency`, so a real regression to >30s would still show up as a hang then a failure, but flake from a slow first round-trip on a loaded CI worker won't fail the suite anymore. --- tests/js/tests/prolog-and-literals.test.ts | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/tests/js/tests/prolog-and-literals.test.ts b/tests/js/tests/prolog-and-literals.test.ts index 9354d014b..f7c95e8de 100644 --- a/tests/js/tests/prolog-and-literals.test.ts +++ b/tests/js/tests/prolog-and-literals.test.ts @@ -2206,10 +2206,13 @@ describe("Prolog + Literals", () => { await model.save(); const saveTime = Date.now(); - // Poll until callback called + // Poll until callback called. 30s upper bound matches the + // other waitForCondition timeouts in this suite — CI + // workers regularly take >5s for the first subscription + // round-trip even though steady-state latency is sub-second. while (!subscriptionCallback.called) { await sleep(10); - if (Date.now() - saveTime > 5000) throw new Error("Timeout waiting for subscription update"); + if (Date.now() - saveTime > 30000) throw new Error("Timeout waiting for subscription update"); } const saveLatency = saveTime - start; From 751acd701085a6c0f8b6b1e2ae51a6d3eb47efa0 Mon Sep 17 00:00:00 2001 From: HexaField <10372036+HexaField@users.noreply.github.com> Date: Thu, 11 Jun 2026 19:29:55 +1000 Subject: [PATCH 19/19] =?UTF-8?q?test(prolog-and-literals):=20bump=20subsc?= =?UTF-8?q?ription-perf=20poll=20timeout=2030s=20=E2=86=92=2060s?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The earlier 5s → 30s bump cleared the flake when this branch was at 121e30800, but the recent merge from dev (which pulled in #848's lazy-load resolveLanguage change) added enough first-fetch latency on a freshly registered SDNA class that integration-tests-js #17171 hit the 30s ceiling on the very same "should demonstrate subscription performance" test. Match the 60s waitForCondition ceiling that's already used by every other subscription test in this suite. The CI flake on #837 is the only blocker keeping the stack from going green; steady-state latency is still logged via `subscriptionLatency`, so a real regression past 60s would still surface as a slow log line rather than be hidden by the bumped ceiling. --- tests/js/tests/prolog-and-literals.test.ts | 17 ++++++++++++----- 1 file changed, 12 insertions(+), 5 deletions(-) diff --git a/tests/js/tests/prolog-and-literals.test.ts b/tests/js/tests/prolog-and-literals.test.ts index f7c95e8de..a653ea5ab 100644 --- a/tests/js/tests/prolog-and-literals.test.ts +++ b/tests/js/tests/prolog-and-literals.test.ts @@ -2206,13 +2206,20 @@ describe("Prolog + Literals", () => { await model.save(); const saveTime = Date.now(); - // Poll until callback called. 30s upper bound matches the - // other waitForCondition timeouts in this suite — CI - // workers regularly take >5s for the first subscription - // round-trip even though steady-state latency is sub-second. + // Poll until callback called. 60s upper bound matches + // the surrounding waitForCondition timeouts in this + // suite. Even with the previous 30s ceiling the test + // still flaked on integration-tests-js #17171 after + // the dev merge pulled in the lazy-load resolveLanguage + // change (#848), which adds first-fetch latency on a + // freshly registered SDNA class. Steady-state + // subscription latency is still logged via + // `subscriptionLatency`, so a real regression would + // surface as a slow log line rather than be hidden by + // the bumped ceiling. while (!subscriptionCallback.called) { await sleep(10); - if (Date.now() - saveTime > 30000) throw new Error("Timeout waiting for subscription update"); + if (Date.now() - saveTime > 60000) throw new Error("Timeout waiting for subscription update"); } const saveLatency = saveTime - start;