fix(parser): handle UTF-8 character boundaries in TypeScript visibility checks

bartolli · bartolli · commit fcee44f0ab2a · 2025-09-29T10:31:58.000-04:00
- added safe_substring_window() utility for UTF-8-safe string slicing - TypeScript parser now uses safe utility instead of raw byte slicing - prevents panic when checking export modifiers before Unicode characters - handles box-drawing characters, emojis, and other multi-byte sequences - utility available to all language parsers through parsing module Fixes #38
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -5,6 +5,21 @@ All notable changes to this project will be documented in this file.
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/),
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
+## [0.5.17] - 2025-09-29
+
+### Changed
+- Refactored relationship compatibility logic from indexer to language behaviors
+  - Moved `is_compatible_relationship` from SimpleIndexer to ResolutionScope trait
+  - Each language now controls its own relationship validation rules
+  - Cleaner separation between orchestration and language-specific logic
+
+### Fixed
+- UTF-8 character boundary parsing error when encountering Unicode characters
+  - Added `safe_substring_window()` utility for UTF-8-safe string slicing
+  - TypeScript parser now handles box-drawing characters and emojis correctly
+  - Prevents panic when checking for export modifiers before symbols
+  - Fixes Issue #38
+
 ## [0.5.16] - 2025-09-28
 
 ### Added
diff --git a/Cargo.lock b/Cargo.lock
diff --git a/Cargo.toml b/Cargo.toml
@@ -1,6 +1,6 @@
 [package]
 name = "codanna"
-version = "0.5.16"
+version = "0.5.17"
 authors = ["Angel Bartolli <bartolli@gmail.com>"]
 edition = "2024"
 description = "Code Intelligence for Large Language Models"
diff --git a/contributing/parsers/c/AUDIT_REPORT.md b/contributing/parsers/c/AUDIT_REPORT.md
@@ -1,6 +1,6 @@
 # C Parser Coverage Report
 
-*Generated: 2025-09-28 21:21:50 UTC*
+*Generated: 2025-09-29 13:24:54 UTC*
 
 ## Summary
 - Nodes in file: 145
diff --git a/contributing/parsers/c/GRAMMAR_ANALYSIS.md b/contributing/parsers/c/GRAMMAR_ANALYSIS.md
@@ -1,6 +1,6 @@
 # C Grammar Analysis
 
-*Generated: 2025-09-28 21:21:50 UTC*
+*Generated: 2025-09-29 13:24:54 UTC*
 
 ## Statistics
 - Total nodes in grammar JSON: 132
diff --git a/contributing/parsers/c/node_discovery.txt b/contributing/parsers/c/node_discovery.txt
@@ -1,5 +1,5 @@
 === C Language ABI-15 COMPREHENSIVE NODE MAPPING ===
-  Generated: 2025-09-28 21:21:50 UTC
+  Generated: 2025-09-29 13:24:54 UTC
   ABI Version: 15
   Node kind count: 145
 
diff --git a/contributing/parsers/cpp/AUDIT_REPORT.md b/contributing/parsers/cpp/AUDIT_REPORT.md
@@ -1,6 +1,6 @@
 # C++ Parser Coverage Report
 
-*Generated: 2025-09-28 21:21:50 UTC*
+*Generated: 2025-09-29 13:24:54 UTC*
 
 ## Summary
 - Nodes in file: 131
diff --git a/contributing/parsers/cpp/GRAMMAR_ANALYSIS.md b/contributing/parsers/cpp/GRAMMAR_ANALYSIS.md
@@ -1,6 +1,6 @@
 # C++ Grammar Analysis
 
-*Generated: 2025-09-28 21:21:50 UTC*
+*Generated: 2025-09-29 13:24:54 UTC*
 
 ## Statistics
 - Total nodes in grammar JSON: 223
diff --git a/contributing/parsers/cpp/node_discovery.txt b/contributing/parsers/cpp/node_discovery.txt
@@ -1,5 +1,5 @@
 === C++ Language ABI-15 COMPREHENSIVE NODE MAPPING ===
-  Generated: 2025-09-28 21:21:50 UTC
+  Generated: 2025-09-29 13:24:54 UTC
   ABI Version: 14
   Node kind count: 131
 
diff --git a/contributing/parsers/go/AUDIT_REPORT.md b/contributing/parsers/go/AUDIT_REPORT.md
@@ -1,6 +1,6 @@
 # Go Parser Symbol Extraction Coverage Report
 
-*Generated: 2025-09-28 21:21:50 UTC*
+*Generated: 2025-09-29 13:24:54 UTC*
 
 ## Summary
 - Nodes in file: 115
diff --git a/contributing/parsers/go/GRAMMAR_ANALYSIS.md b/contributing/parsers/go/GRAMMAR_ANALYSIS.md
@@ -1,6 +1,6 @@
 # Go Grammar Analysis
 
-*Generated: 2025-09-28 21:21:50 UTC*
+*Generated: 2025-09-29 13:24:54 UTC*
 
 ## Statistics
 - Total nodes in grammar JSON: 112
diff --git a/contributing/parsers/go/node_discovery.txt b/contributing/parsers/go/node_discovery.txt
@@ -1,5 +1,5 @@
 === Go Language ABI-15 COMPREHENSIVE NODE MAPPING ===
-  Generated: 2025-09-28 21:21:50 UTC
+  Generated: 2025-09-29 13:24:54 UTC
   ABI Version: 15
   Node kind count: 115
 
diff --git a/contributing/parsers/php/AUDIT_REPORT.md b/contributing/parsers/php/AUDIT_REPORT.md
@@ -1,6 +1,6 @@
 # PHP Parser Coverage Report
 
-*Generated: 2025-09-28 21:21:50 UTC*
+*Generated: 2025-09-29 13:24:54 UTC*
 
 ## Summary
 - Nodes in file: 177
diff --git a/contributing/parsers/php/GRAMMAR_ANALYSIS.md b/contributing/parsers/php/GRAMMAR_ANALYSIS.md
@@ -1,6 +1,6 @@
 # PHP Grammar Analysis
 
-*Generated: 2025-09-28 21:21:50 UTC*
+*Generated: 2025-09-29 13:24:54 UTC*
 
 ## Statistics
 - Total nodes in grammar JSON: 161
diff --git a/contributing/parsers/python/AUDIT_REPORT.md b/contributing/parsers/python/AUDIT_REPORT.md
@@ -1,6 +1,6 @@
 # Python Parser Symbol Extraction Coverage Report
 
-*Generated: 2025-09-28 21:21:50 UTC*
+*Generated: 2025-09-29 13:24:54 UTC*
 
 ## Summary
 - Nodes in file: 121
diff --git a/contributing/parsers/python/GRAMMAR_ANALYSIS.md b/contributing/parsers/python/GRAMMAR_ANALYSIS.md
@@ -1,6 +1,6 @@
 # Python Grammar Analysis
 
-*Generated: 2025-09-28 21:21:50 UTC*
+*Generated: 2025-09-29 13:24:54 UTC*
 
 ## Statistics
 - Total nodes in grammar JSON: 129
diff --git a/contributing/parsers/python/node_discovery.txt b/contributing/parsers/python/node_discovery.txt
@@ -1,5 +1,5 @@
 === Python Language ABI-15 COMPREHENSIVE NODE MAPPING ===
-  Generated: 2025-09-28 21:21:50 UTC
+  Generated: 2025-09-29 13:24:54 UTC
   ABI Version: 15
   Node kind count: 121
 
diff --git a/contributing/parsers/rust/AUDIT_REPORT.md b/contributing/parsers/rust/AUDIT_REPORT.md
@@ -1,6 +1,6 @@
 # Rust Parser Coverage Report
 
-*Generated: 2025-09-28 21:21:50 UTC*
+*Generated: 2025-09-29 13:24:54 UTC*
 
 ## Summary
 - Nodes in file: 143
diff --git a/contributing/parsers/rust/GRAMMAR_ANALYSIS.md b/contributing/parsers/rust/GRAMMAR_ANALYSIS.md
@@ -1,6 +1,6 @@
 # Rust Grammar Analysis
 
-*Generated: 2025-09-28 21:21:50 UTC*
+*Generated: 2025-09-29 13:24:54 UTC*
 
 ## Statistics
 - Total nodes in grammar JSON: 169
diff --git a/contributing/parsers/rust/node_discovery.txt b/contributing/parsers/rust/node_discovery.txt
@@ -1,5 +1,5 @@
 === Rust Language ABI-15 COMPREHENSIVE NODE MAPPING ===
-  Generated: 2025-09-28 21:21:50 UTC
+  Generated: 2025-09-29 13:24:54 UTC
   ABI Version: 15
   Node kind count: 143
 
diff --git a/contributing/parsers/typescript/AUDIT_REPORT.md b/contributing/parsers/typescript/AUDIT_REPORT.md
@@ -1,6 +1,6 @@
 # TypeScript Parser Coverage Report
 
-*Generated: 2025-09-28 21:21:50 UTC*
+*Generated: 2025-09-29 13:24:54 UTC*
 
 ## Summary
 - Nodes in file: 193
diff --git a/contributing/parsers/typescript/GRAMMAR_ANALYSIS.md b/contributing/parsers/typescript/GRAMMAR_ANALYSIS.md
@@ -1,6 +1,6 @@
 # TypeScript Grammar Analysis
 
-*Generated: 2025-09-28 21:21:50 UTC*
+*Generated: 2025-09-29 13:24:54 UTC*
 
 ## Statistics
 - Total nodes in grammar JSON: 183
diff --git a/contributing/parsers/typescript/node_discovery.txt b/contributing/parsers/typescript/node_discovery.txt
@@ -1,5 +1,5 @@
 === TypeScript Language ABI-15 COMPREHENSIVE NODE MAPPING ===
-  Generated: 2025-09-28 21:21:50 UTC
+  Generated: 2025-09-29 13:24:54 UTC
   ABI Version: 14
   Node kind count: 193
 
diff --git a/examples/typescript/react/src/test_aliases.ts b/examples/typescript/react/src/test_aliases.ts
@@ -0,0 +1,15 @@
+// Test file for alias resolution
+import { Button } from '@/components/ui/button';
+import { Container } from '@/components/Container';
+
+export function TestComponent() {
+    // Direct function calls (not JSX)
+    Button();
+    Container();
+
+    return null;
+}
+
+export function AnotherTest() {
+    TestComponent();
+}
diff --git a/examples/typescript/unicode-box-drawing.ts b/examples/typescript/unicode-box-drawing.ts
@@ -0,0 +1,64 @@
+/**
+ * Story Collector
+ * Test file for reproducing UTF-8 character boundary parsing error
+ */
+export class StoryCollector {
+  /**
+   * Build tree representation lines
+   * This method uses Unicode box-drawing characters that trigger the parsing error
+   */
+  private buildTreeLines(
+    info: StoryInfo,
+    hierarchy: Map<string, StoryInfo>,
+    lines: string[],
+    prefix: string,
+    isLast: boolean
+  ): void {
+    // These Unicode box-drawing characters cause UTF-8 boundary errors:
+    // └── (U+2514 U+2500 U+2500) = 9 bytes
+    // ├── (U+251C U+2500 U+2500) = 9 bytes
+    // │   (U+2502) = 3 bytes
+    const connector = isLast ? '└── ' : '├── ';
+    const line = `${prefix}${connector}${info.selfId} (from: ${info.parentElement})`;
+    lines.push(line);
+
+    // This also uses the problematic │ character
+    const childPrefix = prefix + (isLast ? '    ' : '│   ');
+    const children = info.childStories
+      .map(id => hierarchy.get(id))
+      .filter(child => child !== undefined) as StoryInfo[];
+
+    for (let i = 0; i < children.length; i++) {
+      this.buildTreeLines(children[i], hierarchy, lines, childPrefix, i === children.length - 1);
+    }
+  }
+
+  /**
+   * Example of more Unicode box-drawing characters in comments and strings
+   * ┌───────────────┐
+   * │ Tree Example  │
+   * ├─── Child 1    │
+   * ├─── Child 2    │
+   * └─── Child 3    │
+   * └───────────────┘
+   */
+  private renderTree(): string {
+    const tree = `
+    Root
+    ├── Branch 1
+    │   ├── Leaf 1.1
+    │   └── Leaf 1.2
+    └── Branch 2
+        ├── Leaf 2.1
+        └── Leaf 2.2
+    `;
+    return tree;
+  }
+}
+
+// Type definitions for testing
+interface StoryInfo {
+  selfId: string;
+  parentElement: string;
+  childStories: string[];
+}
diff --git a/src/parsing/mod.rs b/src/parsing/mod.rs
@@ -26,8 +26,8 @@ pub use language::Language;
 pub use language_behavior::{LanguageBehavior, LanguageMetadata};
 pub use method_call::MethodCall;
 pub use parser::{
-    HandledNode, LanguageParser, NodeTracker, NodeTrackingState, safe_truncate_str,
-    truncate_for_display,
+    HandledNode, LanguageParser, NodeTracker, NodeTrackingState, safe_substring_window,
+    safe_truncate_str, truncate_for_display,
 };
 pub use php::{PhpBehavior, PhpParser};
 pub use python::{PythonBehavior, PythonParser};
diff --git a/src/parsing/parser.rs b/src/parsing/parser.rs
@@ -187,6 +187,48 @@ pub fn safe_truncate_str(s: &str, max_bytes: usize) -> &str {
     &s[..boundary]
 }
 
+/// Safely extract a substring window from source code, respecting UTF-8 boundaries.
+///
+/// This function creates a window of up to `window_size` bytes before the `end_byte` position,
+/// ensuring we never slice in the middle of a UTF-8 character.
+///
+/// # Arguments
+/// * `code` - The source code string
+/// * `end_byte` - The ending byte position (exclusive)
+/// * `window_size` - Maximum size of the window in bytes
+///
+/// # Returns
+/// A slice of the code that:
+/// - Ends at `end_byte`
+/// - Starts at most `window_size` bytes before `end_byte`
+/// - Respects UTF-8 character boundaries
+///
+/// # Example
+/// ```ignore
+/// let code = "export class 🔍 Scanner";
+/// let window = safe_substring_window(code, 20, 10);
+/// // Returns a safe slice without cutting the emoji
+/// ```
+pub fn safe_substring_window(code: &str, end_byte: usize, window_size: usize) -> &str {
+    // Clamp end_byte to string length
+    let end = end_byte.min(code.len());
+
+    // Calculate the desired start position
+    let start_raw = end.saturating_sub(window_size);
+
+    // Find the nearest valid UTF-8 character boundary
+    let start = if start_raw > 0 && !code.is_char_boundary(start_raw) {
+        // Search forward for a valid boundary (up to 3 bytes for UTF-8)
+        (start_raw..=start_raw.saturating_add(3).min(end))
+            .find(|&i| code.is_char_boundary(i))
+            .unwrap_or(end) // If no boundary found, return empty string
+    } else {
+        start_raw
+    };
+
+    &code[start..end]
+}
+
 /// Creates a truncated preview with ellipsis for display purposes.
 /// Used for signatures and previews in parsers.
 #[inline]
diff --git a/src/parsing/typescript/parser.rs b/src/parsing/typescript/parser.rs
@@ -919,8 +919,7 @@ impl TypeScriptParser {
         // 3) Token check: if the source preceding the node contains 'export '
         // This catches inline modifiers when export is not represented as a wrapper.
         let start = node.start_byte();
-        let prefix_start = start.saturating_sub(10); // small window
-        let prefix = &code[prefix_start..start];
+        let prefix = crate::parsing::safe_substring_window(code, start, 10);
         if prefix.contains("export ") || prefix.contains("export\n") {
             return Visibility::Public;
         }
