|
| 1 | +//! Per-string quality scoring. |
| 2 | +//! |
| 3 | +//! Raw printable-byte scans always pick up some amount of noise: |
| 4 | +//! repeated-character runs (`AAAAAAAA`), single-symbol filler |
| 5 | +//! (`////////`, `++++++++`), low-entropy pattern bytes |
| 6 | +//! (`\x20\x20\x20\x20`), and similar. These pass `is_printable` |
| 7 | +//! but carry no useful information for an analyst. |
| 8 | +//! |
| 9 | +//! [`string_quality`] returns a score in `[0.0, 1.0]` that combines |
| 10 | +//! a handful of simple, predictable signals: |
| 11 | +//! |
| 12 | +//! * **Character diversity** — how many distinct characters the |
| 13 | +//! string contains, normalized against its length. |
| 14 | +//! * **Longest single-character run** — penalizes strings that are |
| 15 | +//! mostly one character repeated. |
| 16 | +//! * **Alphanumeric content** — the fraction of bytes that are |
| 17 | +//! `[A-Za-z0-9]`, the strongest single signal that a string |
| 18 | +//! carries human-readable content. |
| 19 | +//! * **Length bonus** — very short strings get a small score |
| 20 | +//! floor so legitimate 4-6 character literals aren't penalized |
| 21 | +//! into the dirt. |
| 22 | +//! |
| 23 | +//! The function is deliberately allocation-free: it does a single |
| 24 | +//! O(n) pass over the string's bytes. |
| 25 | +
|
| 26 | +/// Score `s` in `[0.0, 1.0]`. Higher = more likely meaningful text. |
| 27 | +/// |
| 28 | +/// The empty string scores `0.0`. A typical English-language string |
| 29 | +/// scores in the `0.7..1.0` range. Single-character or near-single- |
| 30 | +/// character runs (`AAAAAA`, `//////`) score below `0.3`. |
| 31 | +pub fn string_quality(s: &str) -> f64 { |
| 32 | + let bytes = s.as_bytes(); |
| 33 | + let n = bytes.len(); |
| 34 | + if n == 0 { |
| 35 | + return 0.0; |
| 36 | + } |
| 37 | + |
| 38 | + // O(n) sweep: count alphanumeric, track distinct bytes (256-bit |
| 39 | + // bitset on the stack), find the longest single-character run. |
| 40 | + let mut seen = [false; 256]; |
| 41 | + let mut distinct: u32 = 0; |
| 42 | + let mut alnum: u32 = 0; |
| 43 | + let mut prev: u8 = 0; |
| 44 | + let mut run: u32 = 1; |
| 45 | + let mut max_run: u32 = 1; |
| 46 | + for (i, &b) in bytes.iter().enumerate() { |
| 47 | + if !seen[b as usize] { |
| 48 | + seen[b as usize] = true; |
| 49 | + distinct += 1; |
| 50 | + } |
| 51 | + if b.is_ascii_alphanumeric() { |
| 52 | + alnum += 1; |
| 53 | + } |
| 54 | + if i > 0 { |
| 55 | + if b == prev { |
| 56 | + run += 1; |
| 57 | + if run > max_run { |
| 58 | + max_run = run; |
| 59 | + } |
| 60 | + } else { |
| 61 | + run = 1; |
| 62 | + } |
| 63 | + } |
| 64 | + prev = b; |
| 65 | + } |
| 66 | + |
| 67 | + let n_f = n as f64; |
| 68 | + // Component scores, each in [0, 1]. |
| 69 | + let diversity = (distinct as f64 / n_f).clamp(0.0, 1.0); |
| 70 | + let alnum_ratio = (alnum as f64 / n_f).clamp(0.0, 1.0); |
| 71 | + // Longer single-character runs are worse. 1 == no repeats at |
| 72 | + // all, n == the whole string is one character. |
| 73 | + let run_penalty = 1.0 - ((max_run as f64 - 1.0) / n_f).clamp(0.0, 1.0); |
| 74 | + // Combine alnum_ratio with diversity multiplicatively so that |
| 75 | + // a single repeated alphanumeric character (e.g. `AAAAAAAA`) |
| 76 | + // doesn't get a high score on the alnum signal alone — it |
| 77 | + // needs *both* good alphanumeric content and a non-trivial |
| 78 | + // distinct-character count to score well. |
| 79 | + let effective_alnum = alnum_ratio * diversity; |
| 80 | + // Very short strings: give a small floor so they aren't |
| 81 | + // punished out of existence by the diversity term (which can't |
| 82 | + // exceed `n / n = 1.0` but for `n=4` and 3 distinct chars is |
| 83 | + // already only 0.75). |
| 84 | + let length_floor = if n <= 6 { 0.2 } else { 0.0 }; |
| 85 | + |
| 86 | + // Weighted combination. effective_alnum carries the bulk of |
| 87 | + // the signal; the run penalty is a secondary check that hits |
| 88 | + // pure single-character runs hard. |
| 89 | + let composite = 0.6 * effective_alnum + 0.4 * run_penalty; |
| 90 | + (composite + length_floor).clamp(0.0, 1.0) |
| 91 | +} |
| 92 | + |
| 93 | +#[cfg(test)] |
| 94 | +mod tests { |
| 95 | + use super::*; |
| 96 | + |
| 97 | + #[test] |
| 98 | + fn empty_string_is_zero() { |
| 99 | + assert_eq!(string_quality(""), 0.0); |
| 100 | + } |
| 101 | + |
| 102 | + #[test] |
| 103 | + fn high_quality_strings_score_high() { |
| 104 | + assert!(string_quality("Hello, world!") > 0.65); |
| 105 | + assert!(string_quality("kernel32.dll") > 0.65); |
| 106 | + assert!(string_quality("InitializeCriticalSection") > 0.7); |
| 107 | + } |
| 108 | + |
| 109 | + #[test] |
| 110 | + fn single_character_runs_score_low() { |
| 111 | + assert!(string_quality("AAAAAAAA") < 0.35); |
| 112 | + assert!(string_quality("////////") < 0.35); |
| 113 | + assert!(string_quality("++++++++++") < 0.35); |
| 114 | + } |
| 115 | + |
| 116 | + #[test] |
| 117 | + fn assembly_byte_runs_are_not_classified_as_noise() { |
| 118 | + // `AWAVAUATSH` is the byte encoding of `push r15; push r14; |
| 119 | + // push r13; push r12; push rbx; push rax` — it looks like |
| 120 | + // text but is actually executable code. Content-based |
| 121 | + // quality scoring can't distinguish it from real text |
| 122 | + // (well-mixed uppercase letters), so it passes the noise |
| 123 | + // filter; the dedicated `skip_code_sections` option is what |
| 124 | + // suppresses these. Asserting the score is non-trivial |
| 125 | + // documents this limitation explicitly. |
| 126 | + let q = string_quality("AWAVAUATSH"); |
| 127 | + assert!(q > 0.5, "assembly run scores {q}, expected > 0.5"); |
| 128 | + } |
| 129 | + |
| 130 | + #[test] |
| 131 | + fn short_strings_get_a_floor() { |
| 132 | + // 4-character strings shouldn't be punished into the dirt. |
| 133 | + let q = string_quality("HTTP"); |
| 134 | + assert!(q > 0.45, "short alphanumeric should clear 0.45, got {q}"); |
| 135 | + } |
| 136 | +} |
0 commit comments