diff --git a/README.md b/README.md index b057e26..dd50769 100644 --- a/README.md +++ b/README.md @@ -5,7 +5,7 @@ [![CI](https://github.com/p10node/office-engine/actions/workflows/ci.yml/badge.svg)](https://github.com/p10node/office-engine/actions/workflows/ci.yml) [![License: MIT](https://img.shields.io/npm/l/@p10node/office-engine)](LICENSE) -View + edit **DOCX / XLSX / PPTX** directly in the browser, open and edit **DOC / XLS / PPT** (Office 97-2003), **RTF** and **OpenDocument** (ODT / ODS / ODP), view Apple **Pages**, **Keynote** and **Numbers** files and Microsoft **Visio** drawings, open **CSV**, text, Markdown, HTML and SVG as editable sheets / documents, and draw **Mermaid**, **PlantUML** and **Graphviz** diagrams (no server, no extra library). Loads progressively like Google Docs, handles heavy files, one-line React embed, saves back to the original format. +View + edit **DOCX / XLSX / PPTX** directly in the browser, open and edit **DOC / XLS / PPT** (Office 97-2003), **RTF** and **OpenDocument** (ODT / ODS / ODP), view Apple **Pages**, **Keynote** and **Numbers** files and Microsoft **Visio** drawings, open **CSV**, text, Markdown, HTML and SVG as editable sheets / documents and source code with syntax colors, and draw **Mermaid**, **PlantUML** and **Graphviz** diagrams (no server, no extra library). Loads progressively like Google Docs, handles heavy files, one-line React embed, saves back to the original format. **Zero runtime dependencies, zero servers.** Parsing, layout, formulas, diagrams and ZIP are all in the library, running in a Web Worker: nothing leaves the browser. @@ -74,7 +74,8 @@ Also online at [p10node.github.io/office-engine](https://p10node.github.io/offic - **CSV, text, Markdown, HTML, SVG**: `.csv` / `.tsv` open as a workbook (streamed: a million rows in about 2 s; past 12 million cells the rows are read from the file as they are shown, so there is no row limit), `.txt` / `.md` / `.html` as a document, `.svg` / `.svgz` as a picture (`.gz` files are unpacked first), from any `src` (file, Blob, URL); format from the extension, MIME type or content; UTF-8 / UTF-16 / Windows-1252; editable. Markdown, HTML and SVG are saved back in their own format (CSV and text as `.xlsx` / `.docx`) - **Diagrams as code**: `.mmd` (Mermaid), `.puml` (PlantUML), `.dot` / `.gv` (Graphviz) open as a document with the drawing; ` ```mermaid ` / ` ```plantuml ` / ` ```dot ` blocks in Markdown and `
` in HTML are drawn too. Drawn by a built-in layout engine in the worker: no server, no network, no dependency
 - **Code side by side**: Markdown, HTML, SVG and diagram files show their source next to the document (**Code** button); edit it and the document is rebuilt as you type; edit the document and the Markdown / HTML is built again from it
-- **File menu**: new, open (also DOC / XLS / PPT / RTF / OpenDocument / Pages / Keynote / Numbers / Visio / CSV / text / Markdown / HTML / SVG / diagrams), import CSV, rename, download as PDF / HTML / text / CSV, document details, print
+- **Syntax highlighting**: Markdown code blocks are colored by their fence's language (JavaScript / TypeScript / JSX, JSON, Python, CSS, HTML / XML, YAML, shell, SQL, Go, Rust, Java, C / C++ / C#, and more), stay editable, and are saved back with their language and exact code; the colors are only drawn, never stored. Source files (`main.ts`, `script.py`, `config.json`…, or any text with the `language` prop) open in a **code view** with line numbers, in light and dark, editable and saved as plain text. One built-in tokenizer, no dependency ([Features](docs/features.md#syntax-highlighting))
+- **File menu**: new, open (also DOC / XLS / PPT / RTF / OpenDocument / Pages / Keynote / Numbers / Visio / CSV / text / Markdown / HTML / SVG / diagrams / source code), import CSV, rename, download as PDF / HTML / text / CSV, document details, print
 - **Comments**: threads with replies, resolve, edit / delete, @mentions; on DOCX text, XLSX cells and PPTX slides, stored in the file the way Word, Excel and PowerPoint store them
 - **Real-time collaboration**: live edits, avatars, collaborators' cursors, offline edits sync on reconnect
 - **Dark mode and theming**: `theme="dark"` or `"auto"` (follows the OS), or a theme object with design tokens for colors, type and shape ([Theming](docs/theming.md)); documents keep their own colors
@@ -104,7 +105,7 @@ The renderers also ship on their own, to show diagrams and Markdown in any page
 | [`@p10node/mermaid`](packages/mermaid)   | Mermaid → SVG                                                       | 26 KB       |
 | [`@p10node/plantuml`](packages/plantuml) | PlantUML → SVG (no Java)                                            | 29 KB       |
 | [`@p10node/graphviz`](packages/graphviz) | Graphviz DOT → SVG (no WebAssembly)                                 | 16 KB       |
-| [`@p10node/markdown`](packages/markdown) | Markdown → safe HTML, diagram code blocks drawn                     | 39 KB       |
+| [`@p10node/markdown`](packages/markdown) | Markdown → safe HTML, diagram blocks drawn, code blocks colored     | 54 KB       |
 
 ```ts
 import { render } from '@p10node/mermaid'
diff --git a/docs/api.md b/docs/api.md
index 6f8f656..ac04601 100644
--- a/docs/api.md
+++ b/docs/api.md
@@ -50,8 +50,9 @@ CSS is auto-injected, no import needed (or import `@p10node/office-engine/styles
 
 | Prop               | Type                                           | Default   | Notes                                                                                                                    |
 |--------------------|------------------------------------------------|-----------|--------------------------------------------------------------------------------------------------------------------------|
-| `src`              | `File \| Blob \| ArrayBuffer \| string \| URL` | –         | Office (also 97-2003); OpenDocument; RTF; Pages / Keynote / Numbers; Visio; CSV / TSV, text, Markdown, HTML, diagrams    |
+| `src`              | `File \| Blob \| ArrayBuffer \| string \| URL` | –         | Office (also 97-2003); OpenDocument; RTF; Pages / Keynote / Numbers; Visio; CSV / TSV, text, Markdown, HTML, diagrams, source code |
 | `name`             | `string`                                       | auto      | Display name; its extension also tells the format of a `Blob` / `ArrayBuffer`                                            |
+| `language`         | `string`                                       | –         | The file is source code in this language (`'typescript'`, `'ts'`, `'python'`, `'c++'`…): a text file opens in the code view, colored in it, whatever its name / MIME type say (an unknown language: plain code). Without it, `main.ts`, `script.py`, `config.json`… are told by their extension or MIME type |
 | `toolbar`          | `boolean`                                      | `true`    | Top bar (name, status, progress)                                                                                         |
 | `toolbarExtra`     | `ReactNode`                                    | –         | Extra buttons on the right of the toolbar                                                                                |
 | `zoom`             | `number \| 'fit'`                              | `'fit'`   | Initial zoom (docx/pptx)                                                                                                 |
@@ -73,7 +74,7 @@ CSS is auto-injected, no import needed (or import `@p10node/office-engine/styles
 | `icons`            | `Icons`                                        | built-in  | Your own icons: a map by `IconName` or a resolver `(name, { size }) => node`; see [Icons](#icons)                        |
 | `onRename`         | `(name) => void`                               | –         | File → Rename result (extension included)                                                                                |
 | `onClose`          | `() => void`                                   | –         | Adds File → Close                                                                                                        |
-| `onSave`           | `(blob, name) => void`                         | download  | Receives saved file (downloads by default): the Office file, or a Markdown / HTML / SVG file as itself                   |
+| `onSave`           | `(blob, name) => void`                         | download  | Receives saved file (downloads by default): the Office file, or a Markdown / HTML / SVG / source file as itself          |
 | `onDirtyChange`    | `(dirty) => void`                              | –         | Whether there are unsaved changes                                                                                        |
 | `collab`           | `CollabOptions`                                | –         | Real-time collaboration, see [collaboration](collaboration.md)                                                           |
 | `comments`         | `CommentOptions \| false`                      | `{}`      | Comments (see below); `false` hides them                                                                                 |
@@ -205,7 +206,7 @@ import { IconsProvider, XlsxView } from '@p10node/office-engine'
 
 ```
 
-``: attributes `src`, `name`, `editable`, `toolbar="false"`, `thumbnails="false"`, `save="false"`, `download="false"`, `file-menu="false"`, `open="false"`, `capture-select-all="false"`, `theme="dark"` (or `"auto"`), `collab` / `room` / `user` / `base` (real-time collaboration; `user` also writes the comments), `comments="false"`; properties `.file = File` and `.icons` (map or resolver, as `icons` above); events `load`, `error`, `comment` (`detail`: a `CommentEvent`).
+``: attributes `src`, `name`, `language` (a source file's, as the prop), `editable`, `toolbar="false"`, `thumbnails="false"`, `save="false"`, `download="false"`, `file-menu="false"`, `open="false"`, `capture-select-all="false"`, `theme="dark"` (or `"auto"`), `collab` / `room` / `user` / `base` (real-time collaboration; `user` also writes the comments), `comments="false"`; properties `.file = File` and `.icons` (map or resolver, as `icons` above); events `load`, `error`, `comment` (`detail`: a `CommentEvent`).
 
 ## Core API (no React)
 
@@ -215,7 +216,8 @@ import { DocumentClient } from '@p10node/office-engine/core'
 const doc = await DocumentClient.open(fileOrUrl, { onProgress: console.log })
 doc.kind                                   // 'docx' | 'xlsx' | 'pptx'
 doc.info.from                              // 'pages' / 'keynote' / 'numbers' / 'visio': an Apple or Visio file, converted to docx / pptx / xlsx / pptx (view only)
-doc.info.imported                          // 'csv' / 'tsv' / 'text' / 'markdown' / 'html' / 'svg' / 'mermaid' / 'plantuml' / 'graphviz': a plain file, built as xlsx / docx (editable)
+doc.info.imported                          // 'csv' / 'tsv' / 'text' / 'markdown' / 'html' / 'svg' / 'mermaid' / 'plantuml' / 'graphviz' / 'code': a plain file, built as xlsx / docx (editable)
+doc.info.language                          // 'code' (a source file): its language, 'typescript' / 'python' / 'json'… (the hint's own word when it is not one the highlighter knows)
 doc.info.converted                         // 'doc' / 'xls' / 'ppt' / 'rtf' / 'odt' / 'ods' / 'odp': converted to docx / xlsx / pptx (editable, saved as such)
 doc.info.ext                               // 'xlsm' / 'xltx' / 'docm' / 'dotx' / 'pptm' / 'potx' / 'ppsx' …: the file is that variant, and is saved as it (undefined: plain docx / xlsx / pptx)
 doc.on('docx:blocks', (blocks) => {})      // docx: streamed blocks
@@ -233,13 +235,21 @@ await doc.call('xlsx.text', 0, ',') // the sheet as CSV (a Blob; '\t' for TSV),
 // open(src, { residentCells: 12_000_000 }): cells of a CSV / TSV kept in memory; the other rows are read from the file as they are shown
 // open(src, { encoding: 'windows-1258' }): the encoding of a text file (CSV, TSV, text, Markdown, HTML…), a TextDecoder label; without it the
 // byte order mark, the declared charset, UTF-8, then (Vietnamese text) Windows-1258, then Windows-1252 decide. An unknown label is an error
+// open(src, { language: 'python' }): a text file is source code in this language (as the `language` prop)
 // the worker also emits `warning` ({ text }) when a file opened but not whole: a CSV with more than 16,384 columns (the viewer shows it as a message)
 const comments = await doc.call('comments.list')  // DocComment[]: threads are roots + replies (threadsOf)
 doc.on('comment', ({ id, c }) => {})       // a comment set / deleted (c null), here or by a collaborator
 await doc.call('comments.set', c.id, { ...c, done: true }) // XLSX / PPTX threads carry their place (`at`)
 const blob = await doc.call('save')        // edited file
-await doc.call('plain.source')             // text of a Markdown / HTML / diagram file (null for other files)
+await doc.call('plain.source')             // text of a Markdown / HTML / diagram / source file (null for other files)
+await doc.call('plain.original')           // a source file's bytes as opened (null for other files)
 doc.destroy()
+
+// syntax colors, DOM-free (the tokenizer of the code view, the Markdown code blocks and @p10node/markdown)
+import { highlight, highlightLines, languageOf } from '@p10node/office-engine/core'
+highlight('ts', 'const a = 1') // [['k', 'const'], ['', ' a = '], ['n', '1']]: c comment, k keyword, s string, n number / constant,
+                               // f function, y type, a attribute / property, t tag, m meta, o operator ('' plain)
+languageOf('c++')              // 'cpp' (undefined: a language it does not color)
 ```
 
 ## Standalone packages
@@ -253,6 +263,7 @@ renderPage() // 
, … of the pag
 
 import { render as markdown } from '@p10node/markdown'
 el.innerHTML = markdown(text) // safe HTML; ```mermaid / ```plantuml / ```dot drawn
+el.innerHTML = markdown(text, { highlight: true }) // and code blocks colored: …
 ```
 
 ## CSP
diff --git a/docs/features.md b/docs/features.md
index c7ca46b..3f0df93 100644
--- a/docs/features.md
+++ b/docs/features.md
@@ -130,10 +130,11 @@ A `src` (file, Blob, buffer or URL) that is not an Office package opens too, con
 |-------------------------------|---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|
 | `.csv`, `.tsv` / `.tab`       | workbook, one sheet named after the file; the rows stream in as the file is read (a million rows open like a large workbook; past 12 million cells the rows are read from the file by pages as they are shown); numbers and booleans typed, codes like `007` kept as text |
 | `.txt`, `.text`, `.log`       | document, one paragraph per line                                                                                                                                                                                                                                          |
-| `.md`, `.markdown`            | document: headings, lists, bold / italic / strike, links, code, quotes, tables (as tab-separated lines)                                                                                                                                                                   |
+| `.md`, `.markdown`            | document: headings, lists, bold / italic / strike, links, code (code blocks colored by their language, see [Syntax highlighting](#syntax-highlighting)), quotes, tables (as tab-separated lines)                                                                          |
 | `.html`, `.htm`, `.xhtml`     | document: headings, paragraphs, lists, inline formatting and links; scripts and styles are dropped                                                                                                                                                                        |
 | `.svg`                        | the picture alone (no page), scaled down to fit the view, at its own size (`width` / `height` in any unit, else the `viewBox`) otherwise; the file is kept as it is, given the SVG namespace when it lacks one; *Download as* Word gives a document holding the picture   |
 | `.svgz`, `.csv.gz`, `.md.gz`… | the file inside: a gzip-compressed plain file is unpacked whole in the worker (native `DecompressionStream`), the name inside telling the format                                                                                                                          |
+| `.ts`, `.py`, `.json`, `.css` | source code: the [code view](#syntax-highlighting), the text colored by its language (also `.yaml`, `.sql`, `.sh`, `.go`, `.rs`, `.java`, `.c`, `.cpp`, `.cs`, `.xml`…)                                                                                                   |
 
 - **Format**: the extension (of `name`, the File or the URL), else the MIME type (Blob type, `Content-Type`), else the content (SVG, HTML, delimited rows, or text). A gzip file (`.svgz`, `.csv.gz`) is unpacked first and the name inside decides. A file named `.docx` / `.xlsx` / `.pptx` / … is read as a package, so an HTML error page served for a missing file still fails (with a message saying it is a web page); the old names `.doc` / `.xls` / `.xlt` are told by their bytes: a web page, MHTML ("Save as Web Page"), CSV, TSV or text saved under such a name opens as what it is. The bytes also decide when the name is wrong the other way: a real `.doc` / `.xls` named `.docx` / `.xlsx` is converted. A file that is none of these says what it is ("This is a PDF file…", an image, an empty file, an Excel 2003 XML spreadsheet), a password-protected one says to save it without a password, a damaged one (a cut-short download, a part that does not unpack or fails its checksum, a main part that is not XML) says it is damaged
 - **CSV**: comma, semicolon (then `12,5` is a number) or tab, detected from the first line; quoted fields, newlines inside quotes
@@ -143,8 +144,34 @@ A `src` (file, Blob, buffer or URL) that is not an Office package opens too, con
 - **Excel's limit**: a sheet of more than 1,048,576 rows cannot be saved as a workbook: the bar says so, saving fails with a message, and *Download as* CSV / TSV gives the rows (written in the worker, page by page)
 - **Encoding**: the `encoding` open parameter when given (a `TextDecoder` label, e.g. `windows-1258`), else the byte order mark (UTF-8, UTF-16), the declared `charset`, UTF-8, else Windows-1258 when the bytes read as Vietnamese in it (tone marks after vowels: Excel's CSV on Vietnamese Windows; the text is put in NFC), else Windows-1252 (Excel's CSV on other Windows)
 - **Columns**: a sheet holds 16,384 columns (up to XFD); the columns of a wider CSV after it are not loaded and the viewer says so (a message; `warning` event from the worker)
-- The bar shows a `CSV` / `TSV` / `TXT` / `MD` / `HTML` / `SVG` badge
-- **Saving**: a Markdown, HTML or SVG file is saved **as itself** (`.md` / `.html` / `.svg`, also what `onSave` receives). Markdown and HTML are built again from the document when it was edited there: headings, lists (nested), bold / italic / underline / strike, links, code, colors, tables, pictures (embedded as data URLs; a diagram block becomes its drawing); the text reads with its tracked changes accepted. An unedited file is saved as it is. CSV / TSV and text are saved as a workbook / document (`.xlsx` / `.docx`); *Download as* Word / CSV / text / Markdown gives the other formats
+- The bar shows a `CSV` / `TSV` / `TXT` / `MD` / `HTML` / `SVG` badge (a source file: its extension, `TS`, `PY`…)
+- **Saving**: a Markdown, HTML or SVG file is saved **as itself** (`.md` / `.html` / `.svg`, also what `onSave` receives). Markdown and HTML are built again from the document when it was edited there: headings, lists (nested), bold / italic / underline / strike, links, code (a code block with its fence, its language and its code as it is), colors, tables, pictures (embedded as data URLs; a diagram block becomes its drawing); the text reads with its tracked changes accepted. An unedited file is saved as it is. CSV / TSV and text are saved as a workbook / document (`.xlsx` / `.docx`); *Download as* Word / CSV / text / Markdown gives the other formats
+
+### Syntax highlighting
+
+Code is colored by its language, by one built-in tokenizer (in the core, no dependency, no server) shared by the code view, Markdown code blocks, the source pane and [`@p10node/markdown`](../packages/markdown).
+
+| Language                    | Names (code fences, the `language` prop)                  | Files                                            |
+|-----------------------------|-----------------------------------------------------------|--------------------------------------------------|
+| JavaScript (with JSX)       | `javascript`, `js`, `jsx`, `mjs`, `cjs`, `node`           | `.js`, `.mjs`, `.cjs`, `.jsx`                    |
+| TypeScript, TSX             | `typescript`, `ts`, `mts`, `cts`, `tsx`                   | `.ts`, `.mts`, `.cts`, `.tsx`                    |
+| JSON (comments allowed)     | `json`, `jsonc`, `json5`                                  | `.json`, `.jsonc`, `.json5`                      |
+| Python                      | `python`, `py`, `python3`                                 | `.py`, `.pyw`                                    |
+| CSS, SCSS / Less            | `css`, `scss`, `sass`, `less`                             | `.css`, `.scss`, `.less`                         |
+| HTML (scripts, styles), XML | `html`, `xhtml`, `vue`, `svelte`, `xml`, `svg`, `xsl`     | `.xml`, `.xsl`, `.xsd`, `.vue`, `.svelte`        |
+| YAML                        | `yaml`, `yml`                                             | `.yaml`, `.yml`                                  |
+| Shell                       | `shell`, `sh`, `bash`, `zsh`, `console`                   | `.sh`, `.bash`, `.zsh`                           |
+| SQL                         | `sql`, `mysql`, `postgresql`, `sqlite`, `tsql`            | `.sql`                                           |
+| Go, Rust, Java              | `go`, `golang`, `rust`, `rs`, `java`                      | `.go`, `.rs`, `.java`                            |
+| C, C++, C#                  | `c`, `h`, `cpp`, `c++`, `cc`, `hpp`, `csharp`, `cs`, `c#` | `.c`, `.h`, `.cpp`, `.cc`, `.cxx`, `.hpp`, `.cs` |
+| Markdown, diff, TOML, INI   | `markdown`, `md`, `diff`, `patch`, `toml`, `ini`          | `.toml`, `.ini`, `.cfg`, `.properties`, `.diff`  |
+| Mermaid, PlantUML, Graphviz | `mermaid`, `plantuml`, `puml`, `dot`, `graphviz`          | (open as drawings)                               |
+
+- **Markdown code blocks**: a fenced block (` ```js `, ` ~~~python `) is one code paragraph colored by the fence's language; a block in a language the tokenizer does not know is plain code; ` ```mermaid ` / ` ```plantuml ` / ` ```dot ` blocks are still drawn. Fences follow CommonMark: a longer fence holds shorter ones, a block ends at a fence of its character at least as long and with nothing after it, the fence's indentation is taken off its lines. The code is kept exactly (spaces, tabs, blank lines), and its language travels in the document as the paragraph style "Source Code (javascript)" (based on "Source Code", the style Pandoc gives code blocks): the Word document saved from it has the language as that style and the code in a code font, and **no colors** (the colors are drawn when the page is shown, never stored in the runs, so opening or showing a document never makes it unsaved or adds an undo step). Saved back as Markdown, an unedited block is the file's own text; an edited one is written with its fence, language and code
+- **Editing a code block**: the colors follow the text as it is typed; Enter is a new line in the block, indented as the line before it (Enter on an empty last line leaves the block for a new paragraph); pasted text keeps its lines in the block. Colors are not drawn over tracked changes (they keep their author's color)
+- **Code view**: a source file (`main.ts`, `script.py`, `config.json`…, told by its extension, else its MIME type, or the `language` prop / `` / the `language` open parameter, which wins) opens as its text with line numbers and colors, in the theme's colors (`srcKeyword`, `srcString`… in light and dark, see [Theming](theming.md)). **Edit** makes it editable as text; Save (and `onSave`) gives the plain text in its own name and extension, in its CRLF line breaks if it had them, and the file's own bytes (encoding, byte order mark) as long as it was not edited. *Download as*: the code in a Word document, PDF / print and web page (colored), plain text, Markdown (one code block). Comments are off for source files; with collaboration the code view is read only
+- **Source pane**: the **Code** pane of Markdown, HTML, SVG and diagram files colors the code blocks of a Markdown file in their own language, and the scripts and styles of a web page as JavaScript and CSS
+- **Bounded**: the tokenizer reads line by line (only what spans lines, such as a block comment or a template literal, is carried to the next), every scanner only moves forward and looks no further than its line, so the time is linear in the text (a 300 kB file takes a few tens of milliseconds); a text over 300,000 characters, a code block over 50,000 characters or a line over 10,000 characters (minified code) is shown plain. Lines already read are cached, so an edit reads again only the lines it changed, and only those are drawn again
 
 ### Diagrams: Mermaid, PlantUML, Graphviz
 
diff --git a/docs/theming.md b/docs/theming.md
index ef89f4b..b1b50bb 100644
--- a/docs/theming.md
+++ b/docs/theming.md
@@ -74,20 +74,24 @@ return 
B\n```\n\nEnd.\n' + +/** opens `text` as a file named `name` (a Blob, as a host passes one) */ +async function openText(v: Viewer, text: string, name: string, extra: Record = {}) { + await v.page.evaluate(([t, n, x]) => window.__setProps({ src: new Blob([t as string]), name: n as string, ...(x as object) }), [text, name, extra] as const) + await v.page.waitForFunction((n) => document.querySelector('.de-bar .de-title')?.textContent === n, name) +} + +/** WCAG contrast of two CSS rgb() colors */ +function contrast(a: string, b: string): number { + const lum = (c: string) => { + const [r, g, bl] = (c.match(/\d+(\.\d+)?/g) ?? []).slice(0, 3).map((x) => { + const v = Number(x) / 255 + return v <= 0.03928 ? v / 12.92 : ((v + 0.055) / 1.055) ** 2.4 + }) + return 0.2126 * r + 0.7152 * g + 0.0722 * bl + } + const [x, y] = [lum(a), lum(b)].sort((p, q) => q - p) + return (x + 0.05) / (y + 0.05) +} + +/** the color of each token class shown in `scope`, with the background behind it */ +async function tokenColors(v: Viewer, scope: string, bg: string) { + return v.page.evaluate( + ([s, b]) => { + const out: Record = {} + const back = getComputedStyle(document.querySelector(b)!).backgroundColor + for (const el of document.querySelectorAll(`${s} [class^="de-tk-"]`)) out[el.className] ??= [getComputedStyle(el).color, back] + return out + }, + [scope, bg] as const, + ) +} + +describe('syntax highlighting', () => { + test('a Markdown code block is colored by its language; opening it changes nothing', async () => { + // (the dark theme: the page stays paper, its code keeps colors readable on it) + const v = await viewer({ src: 'samples/sample.docx', onSave: true, theme: 'dark' }) + const { page } = v + await openText(v, MD, 'notes.md') + const block = page.locator('.de-page-inner .de-p.de-code[data-code="javascript"]') + await block.waitFor() + expect(await block.locator('.de-tk-k', { hasText: 'const' }).count()).toBe(1) + expect(await block.locator('.de-tk-s').first().innerText()).toBe('"Hello"') + expect(await block.locator('.de-tk-f', { hasText: 'log' }).count()).toBe(1) + // the text is the code, as it is + expect(await block.innerText()).toBe('const greeting = "Hello";\nconsole.log(greeting);') + // a language the viewer does not know: plain code; the diagram is drawn + const cobol = page.locator('.de-page-inner .de-p.de-code[data-code="cobol"]') + expect(await cobol.innerText()).toBe('MOVE A TO B.') + expect(await cobol.locator('[class^="de-tk-"]').count()).toBe(0) + expect(await page.locator('.de-page-inner img').count()).toBe(1) + expect(await page.locator('.de-pages').innerText()).not.toContain('flowchart') + // paper colors on the page: readable on white in the dark theme too + const colors = await tokenColors(v, '.de-p.de-code', '.de-page') + expect(Object.keys(colors).sort()).toEqual(['de-tk-f', 'de-tk-k', 'de-tk-s']) + for (const [fg, bg] of Object.values(colors)) expect(contrast(fg, bg)).toBeGreaterThan(4.5) + // not changed by being shown: no unsaved changes, nothing to undo + await v.edit() + await v.selectText('greeting', 'end') + await page.keyboard.press('Control+z') + await page.waitForTimeout(300) + expect(await block.innerText()).toBe('const greeting = "Hello";\nconsole.log(greeting);') + expect((await v.events('dirty')).filter((e) => e.dirty)).toEqual([]) + expect(await page.locator('.de-save-dirty').count()).toBe(0) + // File → Save: the file as it was + const n = (await v.events('save')).length + await v.menu('Save') + const saved = await v.waitEvent('save', n + 1) + expect(Buffer.from(saved.b64 as string, 'base64').toString('utf8')).toBe(MD) + // the source beside the document: the block colored in its language there too + await page.getByTestId('de-code-toggle').click() + const hl = page.locator('[data-testid=de-source] .de-src-hl') + await hl.waitFor() + expect(await hl.locator('.de-tk-k', { hasText: 'const' }).count()).toBe(1) + expect(await hl.locator('.de-tk-s', { hasText: '"Hello"' }).count()).toBe(1) + expect(v.errors).toEqual([]) + }) + + test('editing inside a code block: the colors follow, Enter is a new line of code, saved as the right Markdown', async () => { + const v = await viewer({ src: 'samples/sample.docx', onSave: true }) + const { page } = v + await openText(v, MD, 'notes.md') + await page.locator('.de-p.de-code[data-code="javascript"] .de-tk-s').waitFor() + await v.edit() + await v.selectText('"Hello"', 'end') + await page.keyboard.type(' + "!"') + await page.keyboard.press('End') + await page.keyboard.press('Enter') + await page.keyboard.type('done()') + const block = page.locator('.de-p.de-code[data-code="javascript"]') + await page.waitForFunction(() => document.querySelector('.de-p.de-code[data-code="javascript"]')?.textContent?.includes('done()')) + // still one block, colored as typed + expect(await page.locator('.de-p.de-code').count()).toBe(2) + expect(await block.locator('.de-tk-s', { hasText: '"!"' }).count()).toBe(1) + expect(await block.locator('.de-tk-f', { hasText: 'done' }).count()).toBe(1) + // Enter on the empty last line: out of the block, into a new paragraph + await v.selectText('console', 'start') + await page.keyboard.press('End') + await page.keyboard.press('Enter') + await page.keyboard.press('Enter') + await page.keyboard.type('Next') + await page.waitForFunction(() => [...document.querySelectorAll('.de-p:not(.de-code)')].some((p) => p.textContent === 'Next')) + const saved = await v.save() + expect(saved.name).toBe('notes.md') + expect(saved.bytes.toString('utf8')).toBe( + MD.replace('const greeting = "Hello";\n', 'const greeting = "Hello" + "!";\ndone()\n').replace('console.log(greeting);\n```\n\n', 'console.log(greeting);\n```\n\nNext\n\n'), + ) + // the Word document saved from it: the code, no colors + const word = await v.downloadAs('Word document') + const ps = await docxParas(word.bytes) + const js = ps.find((p) => p.code === 'javascript')! + expect(js.runs.map((r) => (r.t === 'text' ? r.text : '')).join('')).toBe('const greeting = "Hello" + "!";\ndone()\nconsole.log(greeting);') + expect(js.runs.every((r) => r.t !== 'text' || !r.d?.color)).toBe(true) + expect(v.errors).toEqual([]) + }) + + test('a .ts file opens in the code view: colored, edited as text, saved as it is', async () => { + const v = await viewer({ src: 'samples/sample.ts', onSave: true }) + const { page } = v + const code = page.getByTestId('de-code') + await code.waitFor() + expect(await v.title()).toBe('sample.ts') + expect(await page.locator('.de-badge').innerText()).toBe('TS') + expect(await code.getAttribute('data-language')).toBe('typescript') + expect(await page.locator('.de-page-inner').count()).toBe(0) + expect(await page.getByTestId('de-code-toggle').count()).toBe(0) + expect(await code.locator('.de-src-title').innerText()).toBe('TypeScript') + expect(await code.locator('.de-src-hl .de-tk-k', { hasText: 'interface' }).count()).toBe(1) + expect(await code.locator('.de-src-hl .de-tk-y', { hasText: 'Reading' }).count()).toBeGreaterThan(0) + expect(await code.locator('.de-src-hl .de-tk-c').count()).toBeGreaterThan(0) + const input = page.getByTestId('de-source-input') + const file = readFileSync(`${SAMPLES}/sample.ts`, 'utf8') + expect(await input.inputValue()).toBe(file) + // view: read only; not edited, Save gives the file as it is + expect(await input.getAttribute('readonly')).not.toBeNull() + const n = (await v.events('save')).length + await v.menu('Save') + const same = await v.waitEvent('save', n + 1) + expect(same.name).toBe('sample.ts') + expect(Buffer.from(same.b64 as string, 'base64').equals(readFileSync(`${SAMPLES}/sample.ts`))).toBe(true) + // edit: the text changes, the colors follow, the plain text is saved + await v.barButton('Edit').click() + expect(await input.getAttribute('readonly')).toBeNull() + const edited = file.replace('const SITES', 'const PLACES').replace('SITES.includes', 'PLACES.includes') + await input.fill(edited) + await page.waitForFunction(() => document.querySelector('[data-testid=de-code] .de-src-hl')?.textContent?.includes('PLACES')) + expect(await page.locator('.de-save-dirty').count()).toBe(1) + const saved = await v.save() + expect(saved.name).toBe('sample.ts') + // the plain text as edited, nothing of the colors + expect(saved.bytes.toString('utf8')).toBe(edited) + // File → Download as: the code in a Word document (one code paragraph, no colors), as a Markdown code block + const word = await v.downloadAs('Word document') + expect(word.name).toBe('sample.docx') + const ps = await docxParas(word.bytes) + expect(ps.map((p) => p.code)).toEqual(['typescript']) + expect(ps[0].runs.map((r) => (r.t === 'text' ? r.text : '')).join('')).toBe(edited) + expect(ps[0].runs.every((r) => r.t !== 'text' || !r.d)).toBe(true) + const md = await v.downloadAs('Markdown') + expect(md.text).toStartWith('```typescript\n/**') + expect(v.errors).toEqual([]) + }) + + test('a .py file, a language hint, an unknown language; readable colors in the dark theme', async () => { + const v = await viewer({ src: 'samples/sample.docx', theme: 'dark' }) + const { page } = v + await openText(v, 'import os\n\n@cache\ndef home(path: str) -> str:\n """Doc"""\n return f"{path}/x" # note\n', 'script.py') + const code = page.getByTestId('de-code') + await code.waitFor() + expect(await code.getAttribute('data-language')).toBe('python') + expect(await code.locator('.de-tk-m', { hasText: '@cache' }).count()).toBe(1) + expect(await code.locator('.de-tk-f', { hasText: 'home' }).count()).toBe(1) + const colors = await tokenColors(v, '[data-testid=de-code] .de-src-hl', '[data-testid=de-code] .de-src-scroll') + expect(Object.keys(colors).length).toBeGreaterThanOrEqual(5) + for (const [fg, bg] of Object.values(colors)) expect(contrast(fg, bg)).toBeGreaterThan(4.5) + // the dark theme's colors, not the light ones + expect(colors['de-tk-k'][0]).toBe('rgb(210, 168, 255)') + // the host says what the file is + await openText(v, 'SELECT id FROM users -- all\n', 'query.txt', { language: 'sql' }) + await page.waitForFunction(() => document.querySelector('[data-testid=de-code]')?.getAttribute('data-language') === 'sql') + expect(await code.locator('.de-tk-k', { hasText: 'SELECT' }).count()).toBe(1) + // a language with no colors: plain code + await openText(v, 'MOVE A TO B.\n', 'prog.txt', { language: 'cobol' }) + await page.waitForFunction(() => document.querySelector('.de-src-title')?.textContent === 'Source') + expect(await code.locator('[class^="de-tk-"]').count()).toBe(0) + expect(await code.locator('.de-src-hl').innerText()).toContain('MOVE A TO B.') + expect(v.errors).toEqual([]) + }) + + test('a source file keeps its bytes when not edited, its CRLF line breaks when edited', async () => { + const v = await viewer({ src: 'samples/sample.docx', onSave: true, editable: true }) + const { page } = v + const text = '\uFEFFx = 1\r\ny = 2\r\n' + await openText(v, text, 'crlf.py') + const input = page.getByTestId('de-source-input') + await input.waitFor() + expect(await input.inputValue()).toBe('x = 1\ny = 2\n') + const n = (await v.events('save')).length + await v.menu('Save') + const same = await v.waitEvent('save', n + 1) + expect(Buffer.from(same.b64 as string, 'base64').equals(Buffer.from(text, 'utf8'))).toBe(true) + await input.fill('x = 1\ny = 3\n') + const saved = await v.save() + expect(saved.name).toBe('crlf.py') + expect(saved.bytes.toString('utf8')).toBe('x = 1\r\ny = 3\r\n') + expect(v.errors).toEqual([]) + }) + + test(': the hint from the element', async () => { + const v = await viewer({}, { page: 'embed.html' }) + await v.page.evaluate(() => { + const wc = document.getElementById('wc')! + wc.setAttribute('language', 'python') + wc.setAttribute('src', 'samples/sample.ts') + }) + const code = v.page.locator('#wc [data-testid=de-code]') + await code.waitFor() + expect(await code.getAttribute('data-language')).toBe('python') + }) +}) diff --git a/e2e/harness.ts b/e2e/harness.ts index c2e8ce6..d09154a 100644 --- a/e2e/harness.ts +++ b/e2e/harness.ts @@ -101,8 +101,8 @@ export class Viewer { if (err && !(await this.events('load')).length) await this.fail(`document failed to load: ${err.message}`) const loaded = (await this.events('load')).at(-1) const k = kind ?? (loaded?.kind as keyof typeof READY) - // an SVG file shows its picture, not a document page - const sel = loaded?.imported === 'svg' ? '[data-testid=de-picture] img' : k && READY[k] + // an SVG file shows its picture, a source file its code, not a document page + const sel = loaded?.imported === 'svg' ? '[data-testid=de-picture] img' : loaded?.imported === 'code' ? '[data-testid=de-code]' : k && READY[k] if (sel) await this.page.locator(sel).first().waitFor({ timeout: 30_000 }).catch((e: Error) => this.fail(`${k} did not render`, e)) await this.page.waitForTimeout(400) } diff --git a/packages/markdown/README.md b/packages/markdown/README.md index fef20e6..1e44b28 100644 --- a/packages/markdown/README.md +++ b/packages/markdown/README.md @@ -1,6 +1,6 @@ # @p10node/markdown -Markdown → HTML that is safe to show, with ` ```mermaid `, ` ```plantuml ` and ` ```dot ` code blocks drawn as SVG. No DOM needed, no server, no dependency. +Markdown → HTML that is safe to show, with ` ```mermaid `, ` ```plantuml ` and ` ```dot ` code blocks drawn as SVG and, on request, other code blocks colored by their language. No DOM needed, no server, no dependency. Part of [office-engine](https://github.com/p10node/office-engine). Diagrams alone: [`@p10node/diagram`](../diagram). @@ -18,19 +18,38 @@ import { render, renderTo } from '@p10node/markdown' const html = render(markdownText) // string renderTo(document.getElementById('doc')!, markdownText) // into an element render(text, { diagrams: false, headingIds: false }) +render(text, { highlight: true }) // code blocks colored ``` -| Option | Default | Notes | -|--------------|---------|----------------------------------------------------------------------------------| -| `diagrams` | `true` | Draw ` ```mermaid `, ` ```plantuml ` / ` ```puml `, ` ```dot ` / ` ```graphviz ` | -| `headingIds` | `true` | `id` on headings from their text (`# Getting started` → `#getting-started`) | +| Option | Default | Notes | +|--------------|---------|--------------------------------------------------------------------------------------| +| `diagrams` | `true` | Draw ` ```mermaid `, ` ```plantuml ` / ` ```puml `, ` ```dot ` / ` ```graphviz ` | +| `headingIds` | `true` | `id` on headings from their text (`# Getting started` → `#getting-started`) | +| `highlight` | `false` | Color code blocks in a known language: their tokens in `` (below) | + +## Code colors + +With `highlight: true`, a code block in a known language has its tokens wrapped in spans, for a stylesheet of the page's own (the text stays escaped, the spans carry a class only). The languages: JavaScript / TypeScript / JSX / TSX, JSON, Python, CSS / SCSS / Less, HTML / XML, YAML, shell, SQL, Go, Rust, Java, C / C++ / C#, Markdown, diff, TOML / INI, Mermaid, PlantUML, Graphviz, with their usual names (` ```js `, ` ```ts `, ` ```py `, ` ```sh `, ` ```yml `, ` ```c++ `…). Another language is left plain. + +```css +.hl-comment { color: #6a737d; font-style: italic } +.hl-keyword { color: #8e3bc7 } +.hl-string { color: #1a7f37 } +.hl-number { color: #0b7a8a } +.hl-function { color: #2a51a8 } +.hl-type { color: #a3236b } +.hl-attr { color: #0b57d0 } +.hl-tag { color: #b42318 } +.hl-meta { color: #8a5a00 } +.hl-operator { color: #c2410c } +``` ## What is read - CommonMark blocks: ATX and setext headings, paragraphs, block quotes (nested, lazy lines), bullet and numbered lists (nested, tight / loose, `start`), fenced and indented code, thematic breaks - Inline: emphasis, strong, code spans, links and pictures (inline and reference), autolinks, backslash escapes, hard line breaks - GitHub: tables (column alignment), task lists, strikethrough, bare `https://` / `www.` links -- Code blocks get `class="language-…"`; diagrams `
`; a diagram that cannot be drawn shows why and its code (`
`) +- Code blocks get `class="language-…"` (and, with `highlight`, colored tokens); diagrams `
`; a diagram that cannot be drawn shows why and its code (`
`) ## Safety diff --git a/packages/markdown/package.json b/packages/markdown/package.json index 2f273d1..81b3397 100644 --- a/packages/markdown/package.json +++ b/packages/markdown/package.json @@ -1,7 +1,7 @@ { "name": "@p10node/markdown", "version": "0.1.0", - "description": "Markdown to safe HTML with Mermaid, PlantUML and Graphviz code blocks drawn as SVG; no dependency", + "description": "Markdown to safe HTML with Mermaid, PlantUML and Graphviz code blocks drawn as SVG and code blocks colored by their language; no dependency", "type": "module", "license": "MIT", "files": [ @@ -24,7 +24,8 @@ "html", "mermaid", "plantuml", - "graphviz" + "graphviz", + "syntax-highlighting" ], "homepage": "https://github.com/p10node/office-engine/tree/main/packages/markdown#readme", "repository": { diff --git a/packages/markdown/src/index.ts b/packages/markdown/src/index.ts index 305d009..1398a86 100644 --- a/packages/markdown/src/index.ts +++ b/packages/markdown/src/index.ts @@ -1,6 +1,7 @@ /** * @p10node/markdown: Markdown → safe HTML (CommonMark blocks, GitHub tables, task lists, strikethrough, bare links), - * with ```mermaid / ```plantuml / ```dot code blocks drawn as SVG. No DOM needed, no dependency. + * with ```mermaid / ```plantuml / ```dot code blocks drawn as SVG and, on request, other code blocks colored. No DOM + * needed, no dependency. */ import { markdownToHtml } from './markdown' @@ -9,6 +10,12 @@ export interface MarkdownOptions { diagrams?: boolean /** `id` attributes on headings (from their text) so `#links` reach them. Default true. */ headingIds?: boolean + /** + * Color code blocks in a known language (```js, ```python, ```yaml…): their tokens in `` + * (hl-keyword, hl-string, hl-comment, hl-number, hl-function, hl-type, hl-attr, hl-tag, hl-meta, hl-operator), for + * a stylesheet of the page's own. Default false. + */ + highlight?: boolean } /** diff --git a/packages/markdown/src/markdown.ts b/packages/markdown/src/markdown.ts index dc59c16..04de4da 100644 --- a/packages/markdown/src/markdown.ts +++ b/packages/markdown/src/markdown.ts @@ -5,6 +5,7 @@ * ```mermaid / ```plantuml / ```dot code blocks become SVG drawings. */ import { DIAGRAM_NAME, FENCE_LANG, renderDiagrams } from '../../../src/core/diagram' +import { highlight, languageOf } from '../../../src/core/syntax' import { safeHref } from '../../../src/core/url' export interface MarkdownOptions { @@ -12,6 +13,8 @@ export interface MarkdownOptions { diagrams?: boolean /** `id` attributes on headings (from their text) so `#links` reach them. Default true. */ headingIds?: boolean + /** Color code blocks in a known language: `` and the like around their tokens. Default false. */ + highlight?: boolean } interface Ctx { @@ -19,6 +22,21 @@ interface Ctx { diagrams: boolean ids: boolean slugs: Map + highlight: boolean +} + +/** the class of each token kind (see src/core/syntax) */ +const HL: Record = { + c: 'comment', k: 'keyword', s: 'string', n: 'number', o: 'operator', a: 'attr', t: 'tag', m: 'meta', f: 'function', y: 'type', h: 'heading', b: 'bold', i: 'italic', +} + +/** a code block's text as HTML: escaped, its tokens in spans when it is in a language the highlighter knows */ +function codeHtml(body: string, lang: string, ctx: Ctx): string { + const id = languageOf(lang) + if (!ctx.highlight || !id || id === 'text') return esc(body) + return highlight(id, body) + .map(([c, t]) => (c && HL[c] ? `${esc(t)}` : esc(t))) + .join('') } const esc = (s: string) => s.replace(/&/g, '&').replace(//g, '>').replace(/"/g, '"') @@ -37,7 +55,7 @@ const label = (s: string) => s.trim().replace(/\s+/g, ' ').toLowerCase() /** Markdown text → HTML. */ export function markdownToHtml(md: string, opts: MarkdownOptions = {}): string { const lines = md.replace(/^/, '').replace(/\r\n?/g, '\n').replace(/\t/g, ' ').split('\n') - const ctx: Ctx = { refs: new Map(), diagrams: opts.diagrams !== false, ids: opts.headingIds !== false, slugs: new Map() } + const ctx: Ctx = { refs: new Map(), diagrams: opts.diagrams !== false, ids: opts.headingIds !== false, slugs: new Map(), highlight: !!opts.highlight } // link reference definitions (outside code blocks) are read first and leave no text const kept: string[] = [] let fence: string | null = null @@ -270,7 +288,7 @@ function code(body: string, info: string, ctx: Ctx): string { ) } } - return `
${esc(body)}${body ? '\n' : ''}
` + return `
${codeHtml(body, lang, ctx)}${body ? '\n' : ''}
` } /** URLs a picture may load: web, relative, and data: images */ diff --git a/playground/main.tsx b/playground/main.tsx index 2d2df62..dcd4881 100644 --- a/playground/main.tsx +++ b/playground/main.tsx @@ -22,6 +22,7 @@ const SAMPLES = [ { label: 'sample.csv', url: 'samples/sample.csv' }, { label: 'sample.md', url: 'samples/sample.md' }, { label: 'sample.html', url: 'samples/sample.html' }, + { label: 'sample.ts', url: 'samples/sample.ts' }, { label: 'flowchart.mmd', url: 'samples/flowchart.mmd' }, { label: 'sequence.puml', url: 'samples/sequence.puml' }, { label: 'graph.dot', url: 'samples/graph.dot' }, diff --git a/playground/public/samples/sample.ts b/playground/public/samples/sample.ts new file mode 100644 index 0000000..c94dfa1 --- /dev/null +++ b/playground/public/samples/sample.ts @@ -0,0 +1,27 @@ +/** + * A source file opens as code: colored by its language (from its name, its MIME type or the `language` prop), + * with line numbers, edited as text and saved as it is. + */ +import { readFile } from 'node:fs/promises' + +interface Reading { + site: string + value: number + done?: boolean +} + +const SITES = ['North', 'South'] as const + +export async function load(path: string): Promise { + const text = await readFile(path, 'utf8') + return text + .split(/\r?\n/) + .filter((line) => line.trim() !== '') + .map((line) => { + const [site, value] = line.split(',') + return { site, value: Number(value), done: SITES.includes(site as (typeof SITES)[number]) } + }) +} + +// totals by site, as a template literal +export const summary = (rows: Reading[]) => `${rows.length} readings, ${rows.reduce((n, r) => n + r.value, 0)} in all` diff --git a/src/core/convert.ts b/src/core/convert.ts index 950dd7c..74c437f 100644 --- a/src/core/convert.ts +++ b/src/core/convert.ts @@ -4,7 +4,7 @@ */ import { officeHex } from './color' import { dict } from './dict' -import { HIGHLIGHT } from './docx/styles' +import { codeStyleName, HIGHLIGHT } from './docx/styles' import type { RunSet } from './docx/types' import { escAttr, escText } from './raw' import { safeHref } from './url' @@ -204,6 +204,11 @@ export interface ImportPara { list?: 'bullet' | 'number' align?: 'center' image?: ImportImage + /** + * A code block, in this language ('' for none): written in a "Source Code" paragraph style named after it, which + * the reader turns back into `Paragraph.code` (see codeStyleName) + */ + code?: string } export interface DocxOptions { @@ -495,6 +500,22 @@ function drawingXml(img: ImportImage, rid: string, id: number, maxW: number, max ) } +/** + * The paragraph styles of code blocks: "Source Code" (a code font, no proofing), and one based on it per language, + * named after it. The colors of the code are not in the document: they are drawn from the language when shown. + */ +function codeStylesXml(ids: Map): string { + if (!ids.size) return '' + const font = 'Courier New' + let x = + `` + + `` + for (const [lang, id] of ids) { + if (lang) x += `` + } + return x +} + /** Builds a document (Letter, 1" margins, Calibri 11) from paragraphs of formatted runs and pictures. */ export function parasToDocx(paras: ImportPara[], opts: DocxOptions = {}): Promise { const W = 'http://schemas.openxmlformats.org/wordprocessingml/2006/main' @@ -504,11 +525,26 @@ export function parasToDocx(paras: ImportPara[], opts: DocxOptions = {}): Promis const images: string[] = [] const links: string[] = [] const nums: ('bullet' | 'number')[] = [] + /** code paragraph styles by language: their ids */ + const codeIds = new Map() + const codeId = (lang: string) => { + let id = codeIds.get(lang) + if (id === undefined) { + // "SourceCode" is the style of code in no language, the others are based on it + const base = 'SourceCode' + (lang ? lang.replace(/[^A-Za-z0-9]/g, '').replace(/^[a-z]/, (c) => c.toUpperCase()) || 'Lang' : '') + const taken = new Set([...codeIds.values(), ...(lang ? ['SourceCode'] : [])]) + id = base + for (let k = 2; taken.has(id); k++) id = base + k + codeIds.set(lang, id) + } + return id + } let prevList: 'bullet' | 'number' | undefined let body = '' for (const p of paras) { let pPr = '' - if (p.heading && p.heading >= 1 && p.heading <= 6) pPr += `` + if (p.code !== undefined) pPr += `` + else if (p.heading && p.heading >= 1 && p.heading <= 6) pPr += `` else if (p.list) pPr += '' if (p.list) { if (p.list !== prevList) nums.push(p.list) @@ -546,6 +582,7 @@ export function parasToDocx(paras: ImportPara[], opts: DocxOptions = {}): Promis headingStyles() + '' + '' + + codeStylesXml(codeIds) + '' const rels = `${XML}` + diff --git a/src/core/docx/parser.ts b/src/core/docx/parser.ts index 1126e86..22d16bf 100644 --- a/src/core/docx/parser.ts +++ b/src/core/docx/parser.ts @@ -290,7 +290,8 @@ export class Converter { const base = merge(this.styles.docRs, layer?.rs, st.rs) const markRs = merge(base, readRPr(kid(pPr, 'rPr'), this.theme)) const sid = d.styleId ?? this.styles.defaultPara ?? '' - let cur: Paragraph = { t: 'p', ps, runs: [], markSz: markRs.sz, markFont: markRs.font, sid } + const code = this.styles.code(sid) + let cur: Paragraph = { t: 'p', ps, runs: [], markSz: markRs.sz, markFont: markRs.font, sid, ...(code !== undefined ? { code } : {}) } if (ps.pageBreakBefore) cur.pb = true let lst: 'bullet' | 'number' | undefined if (level && numPr) { @@ -351,7 +352,7 @@ export class Converter { else runs.splice(i, 1, { ...last, text: last.text.slice(0, -1) }, { ...last, text: last.text.slice(-1), cm }) }, pageBreak: () => { - cur = { t: 'p', ps, runs: [], markSz: markRs.sz, markFont: markRs.font, pb: true, sid } + cur = { t: 'p', ps, runs: [], markSz: markRs.sz, markFont: markRs.font, pb: true, sid, ...(code !== undefined ? { code } : {}) } if (pid !== undefined) cur.pid = pid parts.push(cur) }, diff --git a/src/core/docx/styles.ts b/src/core/docx/styles.ts index f8192bb..46ae104 100644 --- a/src/core/docx/styles.ts +++ b/src/core/docx/styles.ts @@ -201,6 +201,18 @@ const CANON_BY_NAME: Record = dict({ heading6: 'h6', }) +/** + * The paragraph style of code blocks in `lang` ('' : no language): "Source Code" (the name Pandoc gives its code + * blocks), "Source Code (js)". The language of a code block travels in the document as its style. + */ +export const codeStyleName = (lang: string) => (lang ? `Source Code (${lang})` : 'Source Code') + +/** the language of a style named by codeStyleName ('' : code in no language), undefined for any other style */ +export function codeStyleLang(name: string): string | undefined { + const m = /^source ?code(?: \((.*)\))?$/i.exec(name.trim()) + return m ? (m[1] ?? '') : undefined +} + function readDef(s: XEl): StyleDef | undefined { const id = s.a.styleId if (!id) return undefined @@ -222,6 +234,7 @@ export class StyleSheet { private paraCache = new Map() private charCache = new Map() private tblCache = new Map() + private codeCache = new Map() defaultPara?: string defaultTable?: string docRs: RunStyle = {} @@ -247,6 +260,20 @@ export class StyleSheet { if (!def) return this.defs.set(def.id, def) this.paraCache.clear() + this.codeCache.clear() + } + + /** the language of the code blocks of paragraph style `id` (a "Source Code" style, or one based on it), undefined for other styles */ + code(id: string | undefined): string | undefined { + if (!id) return undefined + let r = this.codeCache.get(id) + if (r === undefined) { + r = null + const chain = this.chain(id) + for (let k = chain.length - 1; k >= 0 && r === null; k--) if (chain[k].type === 'paragraph') r = codeStyleLang(chain[k].name) ?? null + this.codeCache.set(id, r) + } + return r ?? undefined } /** the element of paragraph style `id`: what the style says itself, nothing inherited */ diff --git a/src/core/docx/types.ts b/src/core/docx/types.ts index 4abf61f..be67eb9 100644 --- a/src/core/docx/types.ts +++ b/src/core/docx/types.ts @@ -147,6 +147,11 @@ export interface Paragraph { /** list kind + level when numbered */ lst?: 'bullet' | 'number' ilvl?: number + /** + * A code block (its paragraph style is "Source Code", or "Source Code ()"): the language its code is + * colored in when shown ('' : none) and written back with, as a Markdown fence. The colors are never in the runs. + */ + code?: string // ---- editor bookkeeping (main thread) ---- /** template paragraph (original pid) */ from?: number diff --git a/src/core/index.ts b/src/core/index.ts index b07624c..f90ac1d 100644 --- a/src/core/index.ts +++ b/src/core/index.ts @@ -14,3 +14,4 @@ export { Engine } from './worker/engine' export { DocumentClient, setDefaultWorker, type OpenOptions } from './worker/client' export type { Events, Methods, OpenResult } from './worker/protocol' export { threadsOf, newCommentId, type DocComment, type CommentUser, type CommentMention, type CommentAnchor, type CommentThread } from './comments' +export { highlight, highlightLines, languageOf, languageOfFile, LANGUAGE_NAME, type Token } from './syntax' diff --git a/src/core/plain.ts b/src/core/plain.ts index 6583d30..3ab704c 100644 --- a/src/core/plain.ts +++ b/src/core/plain.ts @@ -1,7 +1,8 @@ /** * Plain formats opened as Office files: CSV / TSV become a workbook, text / Markdown / HTML a document, an SVG file * and Mermaid / PlantUML / Graphviz diagrams a document with the picture (Markdown code blocks in those languages - * are drawn too). A gzip-compressed plain file (.svgz, .csv.gz) is unpacked first. + * are drawn too), a source file (code) a document of its code, shown as code. A gzip-compressed plain file (.svgz, + * .csv.gz) is unpacked first. * Used by the engine (a source that is not an Office package) and by File → Open. */ import { detectSeparator, markdownToHtml, parasToDocx, parseCsv, rowsToXlsx, textToParas, type ImportPara } from './convert' @@ -12,10 +13,12 @@ import { isSvg, svgImage } from './svgfile' import { dict } from './dict' import { composes, isWindows1258, knownEncoding } from './encoding' import { describeBinary, unreadText } from './filetype' +import { CODE_EXT, CODE_MIME } from './syntax' -export type PlainFormat = 'csv' | 'tsv' | 'text' | 'markdown' | 'html' | 'svg' | DiagramLang +export type PlainFormat = 'csv' | 'tsv' | 'text' | 'markdown' | 'html' | 'svg' | DiagramLang | 'code' export const PLAIN_EXT: Record = dict({ + ...Object.fromEntries(Object.keys(CODE_EXT).map((e) => [e, 'code' as const])), csv: 'csv', tsv: 'tsv', tab: 'tsv', @@ -49,6 +52,7 @@ const PACKAGE_EXT = new Set([ ]) const MIME: Record = dict({ + ...Object.fromEntries(Object.keys(CODE_MIME).map((m) => [m, 'code' as const])), 'text/csv': 'csv', 'application/csv': 'csv', 'text/comma-separated-values': 'csv', @@ -65,8 +69,8 @@ const MIME: Record = dict({ 'text/x-plantuml': 'plantuml', }) -/** formats shown with their source next to the document (and edited there) */ -export const HAS_SOURCE = new Set(['markdown', 'html', 'svg', 'mermaid', 'plantuml', 'graphviz']) +/** formats whose text is kept to be shown: next to the document (and edited there), or (code) as the view itself */ +export const HAS_SOURCE = new Set(['markdown', 'html', 'svg', 'mermaid', 'plantuml', 'graphviz', 'code']) /** a file name extension and MIME type for each plain format (sources saved from the editor) */ export const PLAIN_FILE: Record = { @@ -79,6 +83,8 @@ export const PLAIN_FILE: Record = { mermaid: { ext: 'mmd', mime: 'text/plain' }, plantuml: { ext: 'puml', mime: 'text/plain' }, graphviz: { ext: 'dot', mime: 'text/vnd.graphviz' }, + // (a source file keeps its own name) + code: { ext: 'txt', mime: 'text/plain' }, } /** what a plain format opens as */ @@ -92,6 +98,7 @@ export const PLAIN_KIND: Record = { mermaid: 'docx', plantuml: 'docx', graphviz: 'docx', + code: 'docx', } export const extOf = (name?: string) => /\.([^./\\]+)$/.exec(name ?? '')?.[1].toLowerCase() ?? '' @@ -221,9 +228,9 @@ function sniff(text: string, whole: boolean): PlainFormat { /** * The plain format of a file from its name, MIME type and first bytes (`whole`: they are the whole file), or null - * for an Office / iWork package or a binary file. + * for an Office / iWork package or a binary file. `language` (a host's hint): a text file is source code in it. */ -export function plainFormat(name: string | undefined, type: string | undefined, head: Uint8Array, whole: boolean): PlainFormat | null { +export function plainFormat(name: string | undefined, type: string | undefined, head: Uint8Array, whole: boolean, language?: string): PlainFormat | null { // ZIP (a package, whatever its name says); OLE: legacy binary Office or encrypted, the ZIP reader says so if ((head[0] === 0x50 && head[1] === 0x4b) || (head[0] === 0xd0 && head[1] === 0xcf)) return null const ext = extOf(name) @@ -237,7 +244,7 @@ export function plainFormat(name: string | undefined, type: string | undefined, } // a gzip-compressed plain file (.svgz, .csv.gz): the name inside says the format, also once the server has unpacked it const inner = unpackedName(name) - const byExt = PLAIN_EXT[ext] ?? (inner ? PLAIN_EXT[extOf(inner)] : undefined) + const byExt = language ? 'code' : PLAIN_EXT[ext] ?? (inner ? PLAIN_EXT[extOf(inner)] : undefined) if (isGzip(head)) { if (byExt) return byExt throw new Error(`${name ? `“${name}”` : 'The file'} is a gzip file: only gzip-compressed CSV / text / Markdown / HTML / SVG and diagram files open (.svgz, .csv.gz…)`) @@ -280,7 +287,40 @@ function diagramParas(lang: DiagramLang, src: string): ImportPara[] { } } -/** Markdown: the text, and code blocks in a diagram language (```mermaid, ```plantuml, ```dot) as drawings */ +/** a Markdown code fence: its indentation, its mark (three ` or ~ or more) and the first word of its info string */ +export interface Fence { + indent: number + mark: string + lang: string +} + +/** the code fence a Markdown line opens, or null */ +export function fenceOf(line: string): Fence | null { + const m = /^([ \t]*)(`{3,}|~{3,})[ \t]*([^\s`]*)/.exec(line) + return m ? { indent: m[1].length, mark: m[2], lang: m[3] } : null +} + +/** whether `line` closes the block `f` opened: its fence character, as many times or more, and nothing else */ +export function closesFence(line: string, f: Fence): boolean { + const t = line.trim() + if (t.length < f.mark.length) return false + for (let k = 0; k < t.length; k++) if (t[k] !== f.mark[0]) return false + return true +} + +/** a code block: one paragraph of its exact lines (less the fence's indentation), its language kept for colors and saving */ +function codePara(f: Fence, body: string[]): ImportPara { + const text = body + .map((l) => { + let k = 0 + while (k < f.indent && l[k] === ' ') k++ + return l.slice(k) + }) + .join('\n') + return { runs: text ? [{ text }] : [], code: f.lang } +} + +/** Markdown: the text; code blocks as code paragraphs, those in a diagram language (```mermaid, ```plantuml, ```dot) as drawings */ export function markdownParas(md: string, dom = true): ImportPara[] { const lines = md.replace(/^/, '').split(/\r\n|\r|\n/) const out: ImportPara[] = [] @@ -290,19 +330,18 @@ export function markdownParas(md: string, dom = true): ImportPara[] { text = [] } for (let i = 0; i < lines.length; i++) { - const fence = /^\s*(```|~~~)[`~]*\s*([\w-]*)/.exec(lines[i]) + const fence = fenceOf(lines[i]) if (!fence) { text.push(lines[i]) continue } - // the block ends like markdownToHtml ends it: a line starting with the same fence + // the block ends at its closing fence, or with the file let j = i + 1 - while (j < lines.length && !lines[j].trimStart().startsWith(fence[1])) j++ - const lang = FENCE_LANG[fence[2].toLowerCase()] - if (lang) { - flush() - out.push(...diagramParas(lang, lines.slice(i + 1, j).join('\n'))) - } else text.push(...lines.slice(i, j + 1)) + while (j < lines.length && !closesFence(lines[j], fence)) j++ + const lang = FENCE_LANG[fence.lang.toLowerCase()] + flush() + if (lang) out.push(...diagramParas(lang, lines.slice(i + 1, j).join('\n'))) + else out.push(codePara(fence, lines.slice(i + 1, j))) i = j } flush() @@ -345,8 +384,11 @@ function pictureDocx(paras: ImportPara[]): Promise { return parasToDocx(paras, { landscape: wide }) } -/** Builds the workbook (CSV, TSV) or document (text, Markdown, HTML, SVG, diagrams) a plain file opens as; `name` names the sheet. */ -export function plainToOffice(text: string, format: PlainFormat, name = ''): Promise { +/** + * Builds the workbook (CSV, TSV) or document (text, Markdown, HTML, SVG, diagrams, code) a plain file opens as; `name` + * names the sheet, `language` is a source file's. + */ +export function plainToOffice(text: string, format: PlainFormat, name = '', language = ''): Promise { switch (format) { case 'csv': case 'tsv': { @@ -365,5 +407,10 @@ export function plainToOffice(text: string, format: PlainFormat, name = ''): Pro case 'plantuml': case 'graphviz': return pictureDocx(diagramParas(format, text)) + case 'code': { + // one code paragraph (Download as Word); the code view shows the file's own text + const code = text.replace(/^\uFEFF/, '').replace(/\r\n?/g, '\n') + return parasToDocx([{ runs: code ? [{ text: code }] : [], code: language }]) + } } } diff --git a/src/core/plainmerge.ts b/src/core/plainmerge.ts index ffe012b..276028f 100644 --- a/src/core/plainmerge.ts +++ b/src/core/plainmerge.ts @@ -7,7 +7,10 @@ */ import type { Block, Inline, Paragraph } from './docx/types' import type { ImportPara } from './convert' -import { htmlParas, markdownParas } from './plain' +import { closesFence, fenceOf, htmlParas, markdownParas } from './plain' + +/** the text of a code block as it reads (tracked deletions and hidden text left out) */ +export const codeText = (runs: Inline[]) => runs.map((r) => (r.t === 'text' && !r.s.vanish && r.tc?.k !== 'del' ? r.text : '')).join('') export type MergeFormat = 'markdown' | 'html' @@ -37,9 +40,13 @@ function runsKey(parts: { key: string; text: string }[], heading: boolean): stri const norm = (c: string | undefined) => (c ?? '').replace('#', '').toLowerCase() +/** a code block: its language and its exact text (an edit of its spaces is an edit), as the document holds it */ +const codeSig = (lang: string, text: string) => `CODE|${lang}|${text.replace(/[\u0000-\u0008\u000b\u000c\u000e-\u001f\uFFFE\uFFFF]/g, '')}` + /** what the paragraph reads and looks like, for telling an edited one from one that came back as imported */ function importSig(p: ImportPara): string { if (p.image) return 'IMG' + if (p.code !== undefined) return codeSig(p.code, p.runs.map((r) => r.text).join('')) const link = (r: { href?: string }) => (r.href ? 'a' + r.href : '') const parts = p.runs.map((r) => { const s = r.set ?? {} @@ -54,6 +61,7 @@ function importSig(p: ImportPara): string { function blockSig(b: Block): string { if (b.t !== 'p') return 'TABLE' + Math.random() const p: Paragraph = b + if (p.code !== undefined) return codeSig(p.code, codeText(p.runs)) const lvl = p.ps.outline const heading = lvl !== undefined && lvl < 6 const list = p.marker ? (ORDERED.test(p.marker.text.trim()) ? 'number' : 'bullet') : '' @@ -79,7 +87,6 @@ function blockSig(b: Block): string { const lines = (s: string) => (s.match(/[^\r\n]*(?:\r\n|\r|\n)|[^\r\n]+$/g) ?? []) as string[] const LI = /^\s*(?:[-*+]|\d+[.)])\s/ -const FENCE = /^\s*(```|~~~)/ /** Markdown cut at its block boundaries (every character belongs to a chunk); `fine`: also at headings, rules and code blocks */ function markdownChunks(src: string, fine: boolean): string[] { @@ -102,12 +109,12 @@ function markdownChunks(src: string, fine: boolean): string[] { if (!cont) push() } blankBefore = blank && !!cur - const fence = FENCE.exec(line) + const fence = fenceOf(line) if (fence) { - // a code block is one unit, blank lines in it or not + // a code block is one unit, blank lines in it or not (it ends as markdownParas ends it) if (fine) push() let j = k + 1 - while (j < ls.length && !ls[j].trimStart().startsWith(fence[1])) j++ + while (j < ls.length && !closesFence(ls[j], fence)) j++ if (!cur) curStart = line cur += ls.slice(k, j + 1).join('') k = j diff --git a/src/core/syntax.ts b/src/core/syntax.ts new file mode 100644 index 0000000..3c25fe7 --- /dev/null +++ b/src/core/syntax.ts @@ -0,0 +1,1677 @@ +/** + * Syntax colors for source code: one tokenizer for the source pane, the whole-file code view, the code blocks of + * Markdown documents and @p10node/markdown. A text is read line by line, carrying to the next line only what spans + * lines (a block comment, a multi-line string, a template literal, a JSX element, a Markdown code block, a script in a + * web page), as a short state string. DOM-free. + * + * Bounded: every scanner moves forward on each step and looks no further than the end of its line, so the work is + * linear in the text; a text over `HIGHLIGHT_MAX` characters, or a line over `LINE_MAX`, is plain. + */ +import { dict } from './dict' + +/** + * [class, text]; class '' is plain text. Classes: c comment, k keyword, s string, n number / constant, o operator, + * a attribute / property / variable, t tag, m meta (decorator, directive, annotation), f function, y type, + * h heading, b bold, i italic. The texts of a line's tokens put together are the line. + */ +export type Token = [string, string] + +/** a longer text is plain */ +export const HIGHLIGHT_MAX = 300_000 +/** a longer line (minified code) is plain; what spans lines is carried over it as it was */ +const LINE_MAX = 10_000 +/** template literals / JSX elements / comments inside one another, at most */ +const DEPTH_MAX = 32 + +/** the tokens of one line, from the state at its start ('' : nothing open); returns the state at its end */ +type Lexer = (line: string, st: string, out: Token[]) => string + +/** frame separators of the states: code frames, a web page's script / style, a Markdown code block */ +const S1 = '\u0001' +const S2 = '\u0002' +const S3 = '\u0003' + +function put(out: Token[], c: string, t: string) { + if (!t) return + const last = out[out.length - 1] + if (last && last[0] === c) last[1] += t + else out.push([c, t]) +} + +const cc = (s: string, i: number) => s.charCodeAt(i) +const isDigit = (c: number) => c >= 48 && c <= 57 +const isHex = (c: number) => isDigit(c) || (c >= 97 && c <= 102) || (c >= 65 && c <= 70) +const isAlpha = (c: number) => (c >= 97 && c <= 122) || (c >= 65 && c <= 90) || c === 95 || (c > 127 && c !== 0xa0) +const isWord = (c: number) => isAlpha(c) || isDigit(c) +const isSpace = (c: number) => c === 32 || c === 9 || c === 11 || c === 12 || c === 0xa0 +const isUpper = (c: number) => c >= 65 && c <= 90 +const words = (s: string) => new Set(s.split(' ')) + +function wordEnd(s: string, i: number, dollar = false): number { + let j = i + while (j < s.length && (isWord(cc(s, j)) || (dollar && cc(s, j) === 36))) j++ + return j +} + +function spaceEnd(s: string, i: number): number { + let j = i + while (j < s.length && isSpace(cc(s, j))) j++ + return j +} + +/** the end of a number literal starting at `i` (a digit, or "." and a digit): 0x / 0b / 0o, _ separators, exponent, suffix */ +function numberEnd(s: string, i: number): number { + const n = s.length + let j = i + if (cc(s, j) === 48 && 'xXbBoO'.includes(s[j + 1] || ' ') && isHex(cc(s, j + 2))) { + j += 2 + while (j < n && (isHex(cc(s, j)) || s[j] === '_')) j++ + } else { + while (j < n && (isDigit(cc(s, j)) || s[j] === '_')) j++ + if (s[j] === '.' && isDigit(cc(s, j + 1))) { + j++ + while (j < n && (isDigit(cc(s, j)) || s[j] === '_')) j++ + } + if ((s[j] === 'e' || s[j] === 'E') && (isDigit(cc(s, j + 1)) || ((s[j + 1] === '+' || s[j + 1] === '-') && isDigit(cc(s, j + 2))))) { + j += 2 + while (j < n && isDigit(cc(s, j))) j++ + } + } + // suffixes: 10n, 1L, 2u, 1.5f, 8i32, 3j + while (j < n && isWord(cc(s, j))) j++ + return j +} + +/** past the closing quote of a string whose text starts at `i`; -1 when the line ends first */ +function quoteEnd(s: string, i: number, q: string, esc = true, doubled = false): number { + for (let j = i; j < s.length; j++) { + const c = s[j] + if (esc && c === '\\') j++ + else if (c === q) { + if (doubled && s[j + 1] === q) j++ + else return j + 1 + } + } + return -1 +} + +/** whether a "(" follows (spaces between): the name before it is called or defined */ +function callAt(s: string, j: number): boolean { + return s[spaceEnd(s, j)] === '(' +} + +/** a name in capitals (two characters or more): a constant */ +function allCaps(w: string): boolean { + if (w.length < 2) return false + let letter = false + for (let k = 0; k < w.length; k++) { + const c = cc(w, k) + if (c >= 97 && c <= 122) return false + if (isUpper(c)) letter = true + } + return letter +} + +// ---------------------------------------------------------------- C-like languages (JS / TS, Python, Go, Rust, Java, C, C++, C#, SQL) + +interface CLike { + kw: Set + consts?: Set + types?: Set + /** line comment openers */ + line?: string[] + /** slash-star block comments; nested (Rust) */ + block?: boolean + nested?: boolean + /** quotes of strings */ + quotes: string + /** quotes whose strings go on past the end of the line */ + multi?: string + /** quotes that open triple-quoted strings */ + triple?: string + /** a quote doubled escapes it, a backslash does not (SQL) */ + doubled?: boolean + /** quotes of names, not strings (SQL "col", `col`) */ + names?: string + /** template literals, regular expression literals, JSX, $ in names (JavaScript) */ + tmpl?: boolean + regex?: boolean + jsx?: boolean + dollar?: boolean + /** raw / verbatim strings: Go `…`, Rust r#"…"#, C++ R"x(…)x", C# @"…" */ + raw?: 'go' | 'rust' | 'cpp' | 'cs' + /** names written right before a quote that are part of the string (Python r"", f"") */ + prefixes?: RegExp + /** @decorators / @annotations: at the start of a line only, or anywhere */ + deco?: 'line' | 'any' + /** Rust #[attributes] */ + attrs?: boolean + /** # directives (C, C++, C#) */ + pre?: boolean + /** Rust 'lifetimes */ + lifetimes?: boolean + /** Rust name! macros */ + macros?: boolean + /** Capitalized names are types */ + caps?: boolean + /** NAMES_IN_CAPITALS are constants (default true) */ + constCaps?: boolean + /** keywords in any case (SQL) */ + ci?: boolean + /** characters that open variables (SQL @name, $1) */ + vars?: string + /** the name after these is a function / a type */ + fnKw?: Set + typeKw?: Set +} + +function clike(L: CLike): Lexer { + return (line, st, out) => { + const stack = st ? st.split(S1) : [] + const n = line.length + const push = (f: string) => stack.length < DEPTH_MAX && stack.push(f) > 0 + const setTop = (f: string) => (stack[stack.length - 1] = f) + let i = 0 + /** the last token was a value: "/" divides and "<" compares (else they open a regular expression / a JSX element) */ + let value = false + /** a regular expression not closed on this line: no other is looked for on it */ + let regexOk = !!L.regex + /** the class of the next name ("function f", "class C") */ + let pending = '' + /** the last token was ".": a member name follows (not a keyword) */ + let dot = false + /** nothing but spaces so far */ + let lead = true + // a directive: #include , #define, #region + if (L.pre && !stack.length) { + const m = /^([ \t]*)(#[ \t]*[A-Za-z]+)/.exec(line) + if (m) { + put(out, '', m[1]) + put(out, 'm', m[2]) + i = m[0].length + const inc = /^([ \t]*)(<[^<>]*>)/.exec(line.slice(i, i + 300)) + if (inc && /include|import/.test(m[2])) { + put(out, '', inc[1]) + put(out, 's', inc[2]) + i += inc[0].length + } + } + } + while (i < n) { + const top = stack.length ? stack[stack.length - 1] : '' + const k = top[0] + const ch = line[i] + const c = cc(line, i) + // ---------------- what spans lines + if (k === 'C') { + if (!L.nested) { + const e = line.indexOf('*/', i) + if (e < 0) { + put(out, 'c', line.slice(i)) + break + } + put(out, 'c', line.slice(i, e + 2)) + stack.pop() + i = e + 2 + continue + } + let d = Number(top.slice(1)) || 1 + let j = i + for (; j < n; j++) { + if (line[j] === '/' && line[j + 1] === '*') d = Math.min(d + 1, 999), j++ + else if (line[j] === '*' && line[j + 1] === '/') { + d-- + j++ + if (!d) break + } + } + if (!d) { + put(out, 'c', line.slice(i, j + 1)) + stack.pop() + i = j + 1 + continue + } + put(out, 'c', line.slice(i)) + setTop('C' + d) + break + } + if (k === 'S') { + const q = top[1] + const e = quoteEnd(line, i, q, !L.doubled, !!L.doubled) + const cls = L.names?.includes(q) ? 'a' : 's' + if (e < 0) { + put(out, cls, line.slice(i)) + break + } + put(out, cls, line.slice(i, e)) + stack.pop() + i = e + value = true + continue + } + if (k === 'T') { + const q3 = top[1].repeat(3) + let j = i + while (j < n && !line.startsWith(q3, j)) j += line[j] === '\\' ? 2 : 1 + if (j >= n) { + put(out, 's', line.slice(i)) + break + } + put(out, 's', line.slice(i, j + 3)) + stack.pop() + i = j + 3 + value = true + continue + } + if (k === 'B' || k === 'R' || k === 'Q') { + // Go `raw`, Rust r##"raw"##, C++ R"x(raw)x" + const close = k === 'B' ? '`' : k === 'R' ? '"' + '#'.repeat(Number(top.slice(1)) || 0) : ')' + top.slice(1) + '"' + const e = line.indexOf(close, i) + if (e < 0) { + put(out, 's', line.slice(i)) + break + } + put(out, 's', line.slice(i, e + close.length)) + stack.pop() + i = e + close.length + value = true + continue + } + if (k === 'V') { + // C# @"verbatim": "" is a quote + let j = i + while (j < n && !(line[j] === '"' && line[j + 1] !== '"')) j += line[j] === '"' ? 2 : 1 + if (j >= n) { + put(out, 's', line.slice(i)) + break + } + put(out, 's', line.slice(i, j + 1)) + stack.pop() + i = j + 1 + value = true + continue + } + if (k === 't') { + // template literal text, up to its end or a ${ expression + let j = i + while (j < n && line[j] !== '`' && !(line[j] === '$' && line[j + 1] === '{')) j += line[j] === '\\' ? 2 : 1 + if (j >= n) { + put(out, 's', line.slice(i)) + break + } + if (line[j] === '`') { + put(out, 's', line.slice(i, j + 1)) + stack.pop() + i = j + 1 + value = true + continue + } + put(out, 's', line.slice(i, j)) + put(out, push('e0') ? 'o' : 's', '${') + i = j + 2 + value = false + continue + } + if (k === 'x') { + // JSX children: text, {expressions}, elements + let j = i + while (j < n && line[j] !== '<' && line[j] !== '{') j++ + put(out, '', line.slice(i, j)) + i = j + if (i >= n) break + if (line[i] === '{') { + put(out, '', '{') + push('j0') + i++ + value = false + } else if (line[i + 1] === '/') { + put(out, 't', ' 2 + if (isSpace(c)) { + const j = spaceEnd(line, i) + put(out, '', line.slice(i, j)) + i = j + continue + } + if (ch === '/' && line[i + 1] === '>') { + put(out, 't', '/>') + stack.pop() + i += 2 + value = true + continue + } + if (ch === '>') { + put(out, 't', '>') + stack.pop() + i++ + if (closing) { + if (stack[stack.length - 1] === 'x') stack.pop() + } else if (line[spaceEnd(line, i)] !== '(') push('x') + // (a generic's "(": not an element) + value = true + continue + } + if (ch === '{') { + put(out, '', '{') + push('j0') + i++ + continue + } + if (ch === '"' || ch === "'") { + const e = quoteEnd(line, i + 1, ch, false) + put(out, 's', line.slice(i, e < 0 ? n : e)) + i = e < 0 ? n : e + continue + } + if (isAlpha(c) || c === 36) { + let j = i + while (j < n && (isWord(cc(line, j)) || '$-:.'.includes(line[j]))) j++ + put(out, named ? 'a' : 't', line.slice(i, j)) + if (!named) setTop(top + 'n') + i = j + continue + } + if (ch === '=') { + put(out, '', ch) + i++ + continue + } + // "= 0) { + put(out, cls, line.slice(i, e)) + i = e + continue + } + put(out, cls, line.slice(i)) + if (L.multi?.includes(ch)) push('S' + ch) + break + } + if (L.regex && ch === '/' && !value && regexOk) { + let j = i + 1 + let set = false + for (; j < n; j++) { + const d = line[j] + if (d === '\\') j++ + else if (d === '[') set = true + else if (d === ']') set = false + else if (d === '/' && !set) break + } + if (j < n) { + j++ + while (j < n && isAlpha(cc(line, j))) j++ + put(out, 's', line.slice(i, j)) + i = j + value = true + continue + } + regexOk = false + } + if (L.jsx && ch === '<' && !value && (isAlpha(cc(line, i + 1)) || line[i + 1] === '>') && push('g0')) { + put(out, 't', '<') + i++ + continue + } + if (L.deco && ch === '@' && isAlpha(cc(line, i + 1)) && (L.deco === 'any' || wasLead)) { + let j = wordEnd(line, i + 1) + while (line[j] === '.' && isAlpha(cc(line, j + 1))) j = wordEnd(line, j + 1) + put(out, 'm', line.slice(i, j)) + i = j + continue + } + if (L.attrs && ch === '#' && (line[i + 1] === '[' || (line[i + 1] === '!' && line[i + 2] === '['))) { + let j = line.indexOf('[', i) + let d = 0 + for (; j < n; j++) { + if (line[j] === '[') d++ + else if (line[j] === ']' && !--d) break + } + put(out, 'm', line.slice(i, Math.min(n, j + 1))) + i = Math.min(n, j + 1) + continue + } + if (L.vars?.includes(ch) && isWord(cc(line, i + 1))) { + const j = wordEnd(line, i + 1) + put(out, 'a', line.slice(i, j)) + i = j + value = true + continue + } + if (isDigit(c) || (ch === '.' && isDigit(cc(line, i + 1)))) { + const j = numberEnd(line, i) + put(out, 'n', line.slice(i, j)) + i = j + value = true + dot = false + pending = '' + continue + } + if (isAlpha(c) || (L.dollar && c === 36)) { + const j = wordEnd(line, i, L.dollar) + const w = line.slice(i, j) + const q = line[j] + // a string with a prefix: Python r"…" f'…', Rust b"…" r#"…"#, C++ R"x(…)x" u8"…" + if (q === '"' || q === "'" || (L.raw === 'rust' && q === '#')) { + const end = prefixed(L, line, i, j, w, stack, out) + if (end >= 0) { + i = end + value = true + continue + } + } + const key = L.ci ? w.toLowerCase() : w + let cls = '' + if (!dot && L.kw.has(key)) cls = 'k' + else if (!dot && L.consts?.has(key)) cls = 'n' + else if (pending) cls = pending + else if (!dot && L.types?.has(key)) cls = 'y' + else if (L.macros && q === '!' && line[j + 1] !== '=') cls = 'f' + else if (callAt(line, j)) cls = 'f' + else if (L.constCaps !== false && allCaps(w)) cls = 'n' + else if (L.caps && isUpper(c)) cls = 'y' + put(out, cls, w) + pending = cls === 'k' ? (L.fnKw?.has(key) ? 'f' : L.typeKw?.has(key) ? 'y' : '') : '' + value = cls !== 'k' || w === 'this' || w === 'super' || w === 'self' + dot = false + i = j + continue + } + // punctuation; the braces of ${…} and JSX {…} + if ((k === 'e' || k === 'j') && (ch === '{' || ch === '}')) { + const d = Number(top.slice(1)) || 0 + if (ch === '{') setTop(k + (d + 1)) + else if (d > 0) setTop(k + (d - 1)) + else { + stack.pop() + put(out, k === 'e' ? 'o' : '', '}') + i++ + value = true + continue + } + } + put(out, '', ch) + i++ + value = ch === ')' || ch === ']' + dot = ch === '.' + pending = '' + } + return stack.join(S1) + } +} + +function lineComment(openers: string[], line: string, i: number): boolean { + for (const m of openers) if (line.charCodeAt(i) === m.charCodeAt(0) && line.startsWith(m, i)) return true + return false +} + +/** a string written with a prefix name (`w`, from `i` to `j`); returns where it ends, or -1 when it is not one */ +function prefixed(L: CLike, line: string, i: number, j: number, w: string, stack: string[], out: Token[]): number { + const n = line.length + const q = line[j] + if (L.raw === 'rust' && (w === 'r' || w === 'br')) { + let h = j + while (line[h] === '#') h++ + if (line[h] !== '"') return -1 + const close = '"' + '#'.repeat(h - j) + const e = line.indexOf(close, h + 1) + if (e >= 0) { + put(out, 's', line.slice(i, e + close.length)) + return e + close.length + } + put(out, 's', line.slice(i)) + if (stack.length < DEPTH_MAX) stack.push('R' + (h - j)) + return n + } + if (L.raw === 'cpp' && q === '"' && /^(?:u8|[uUL])?R$/.test(w)) { + // R"delim( : a delimiter of 16 characters at most + let open = j + 1 + while (open < n && open <= j + 17 && line[open] !== '(') open++ + if (line[open] !== '(') return -1 + const delim = line.slice(j + 1, open) + if (/[\s\\)]/.test(delim)) return -1 + const close = ')' + delim + '"' + const e = line.indexOf(close, open + 1) + if (e >= 0) { + put(out, 's', line.slice(i, e + close.length)) + return e + close.length + } + put(out, 's', line.slice(i)) + if (stack.length < DEPTH_MAX) stack.push('Q' + delim) + return n + } + const ok = L.prefixes ? L.prefixes.test(w) : (L.raw === 'rust' && w === 'b') || (L.raw === 'cpp' && /^(?:u8|[uUL])$/.test(w)) + if (!ok || q === '#') return -1 + if (L.triple?.includes(q) && line.startsWith(q + q + q, j)) { + put(out, 's', line.slice(i, j + 3)) + if (stack.length < DEPTH_MAX) stack.push('T' + q) + return j + 3 + } + // raw strings keep their backslashes, but a backslash still escapes the quote (Python) + const e = quoteEnd(line, j + 1, q) + if (e >= 0) { + put(out, 's', line.slice(i, e)) + return e + } + put(out, 's', line.slice(i)) + if (L.multi?.includes(q) && stack.length < DEPTH_MAX) stack.push('S' + q) + return n +} + +const JS_KW = 'break case catch class const continue debugger default delete do else export extends finally for function if import in instanceof let new return super switch this throw try typeof var void while with yield async await of static get set from as' +const JS_CONST = words('true false null undefined NaN Infinity') +const FN_JS = words('function') +const TYPE_JS = words('class interface type enum extends implements namespace') + +const javascript = clike({ + kw: words(JS_KW), consts: JS_CONST, line: ['//'], block: true, quotes: `"'`, tmpl: true, regex: true, jsx: true, dollar: true, deco: 'any', caps: true, fnKw: FN_JS, typeKw: TYPE_JS, +}) +const TS = { + kw: words(JS_KW + ' type interface enum namespace module declare abstract implements readonly private protected public override keyof infer is asserts satisfies unique global'), + consts: JS_CONST, types: words('string number boolean any unknown never object symbol bigint'), line: ['//'], block: true, quotes: `"'`, tmpl: true, regex: true, dollar: true, deco: 'any' as const, caps: true, fnKw: FN_JS, typeKw: TYPE_JS, +} +const typescript = clike(TS) +const tsx = clike({ ...TS, jsx: true }) + +const python = clike({ + kw: words('and as assert async await break class continue def del elif else except finally for from global if import in is lambda nonlocal not or pass raise return try while with yield'), + consts: words('True False None NotImplemented Ellipsis __debug__'), + types: words('int float complex str bytes bytearray bool list dict set frozenset tuple object type range memoryview'), + line: ['#'], quotes: `"'`, triple: `"'`, prefixes: /^(?:[rbuf]|rb|br|fr|rf)$/i, deco: 'line', caps: true, fnKw: words('def'), typeKw: words('class'), +}) + +const go = clike({ + kw: words('break case chan const continue default defer else fallthrough for func go goto if import interface map package range return select struct switch type var'), + consts: words('true false nil iota'), + types: words('bool byte complex64 complex128 error float32 float64 int int8 int16 int32 int64 rune string uint uint8 uint16 uint32 uint64 uintptr any comparable'), + line: ['//'], block: true, quotes: `"'`, raw: 'go', fnKw: words('func'), typeKw: words('type'), +}) + +const rust = clike({ + kw: words('as async await break const continue crate dyn else enum extern fn for if impl in let loop match mod move mut pub ref return self static struct super trait type union unsafe use where while yield'), + consts: words('true false'), + types: words('bool char str i8 i16 i32 i64 i128 isize u8 u16 u32 u64 u128 usize f32 f64 Self'), + line: ['//'], block: true, nested: true, quotes: `"'`, multi: '"', raw: 'rust', attrs: true, lifetimes: true, macros: true, caps: true, + fnKw: words('fn'), typeKw: words('struct enum trait type union'), +}) + +const java = clike({ + kw: words('abstract assert break case catch class continue default do else enum extends final finally for if implements import instanceof interface native new package permits private protected public record return sealed static strictfp super switch synchronized this throw throws transient try var volatile while yield'), + consts: words('true false null'), + types: words('boolean byte char double float int long short void'), + line: ['//'], block: true, quotes: `"'`, triple: '"', deco: 'any', caps: true, typeKw: words('class interface enum record'), +}) + +const C_KW = 'auto break case const continue default do else enum extern for goto if inline register restrict return sizeof static struct switch typedef union volatile while _Alignas _Alignof _Atomic _Generic _Noreturn _Static_assert _Thread_local' +const C_TYPES = 'char double float int long short signed unsigned void bool _Bool size_t ssize_t ptrdiff_t intptr_t uintptr_t int8_t int16_t int32_t int64_t uint8_t uint16_t uint32_t uint64_t wchar_t FILE' +const c = clike({ + kw: words(C_KW), consts: words('NULL true false'), types: words(C_TYPES), line: ['//'], block: true, quotes: `"'`, raw: 'cpp', pre: true, typeKw: words('struct union enum'), +}) +const cpp = clike({ + kw: words( + C_KW + + ' alignas alignof and and_eq asm bitand bitor catch class compl concept consteval constexpr constinit const_cast co_await co_return co_yield decltype delete dynamic_cast explicit export friend mutable namespace new noexcept not not_eq operator or or_eq private protected public reinterpret_cast requires static_assert static_cast template this thread_local throw try typeid typename using virtual xor xor_eq override final', + ), + consts: words('NULL true false nullptr'), types: words(C_TYPES + ' char8_t char16_t char32_t'), line: ['//'], block: true, quotes: `"'`, raw: 'cpp', pre: true, caps: true, + typeKw: words('class struct union enum namespace'), +}) +const csharp = clike({ + kw: words( + 'abstract as base break case catch checked class const continue default delegate do else enum event explicit extern finally fixed for foreach goto if implicit in interface internal is lock namespace new operator out override params private protected public readonly ref return sealed sizeof stackalloc static struct switch this throw try typeof unchecked unsafe using virtual volatile while async await var dynamic get set init record required yield when where partial global nameof with', + ), + consts: words('true false null'), + types: words('bool byte char decimal double float int long object sbyte short string uint ulong ushort void nint nuint'), + line: ['//'], block: true, quotes: `"'`, triple: '"', raw: 'cs', pre: true, caps: true, typeKw: words('class struct interface enum record namespace'), +}) + +const sql = clike({ + kw: words( + 'select from where and or not insert into values update set delete create alter drop table view index trigger procedure function database schema if exists primary key foreign references unique check default constraint join inner left right full outer cross on using group by order asc desc having limit offset union intersect except all distinct as case when then else end is like ilike in between returning with recursive begin commit rollback transaction grant revoke truncate replace declare cursor fetch over partition window rows range current row lateral natural top merge matched cascade restrict add column rename to temporary temp materialized do return returns language', + ), + consts: words('true false null'), + types: words('int integer smallint bigint tinyint serial bigserial decimal numeric real float double precision char varchar nvarchar nchar text clob blob bytea boolean bool date time timestamp timestamptz interval json jsonb uuid xml money bit varbinary binary'), + line: ['--'], block: true, quotes: `'"\``, multi: `'`, names: '"`', doubled: true, ci: true, vars: '@$', constCaps: false, +}) + +// ---------------------------------------------------------------- JSON + +const json: Lexer = (line, st, out) => { + const n = line.length + let i = 0 + if (st === 'C') { + const e = line.indexOf('*/') + if (e < 0) { + put(out, 'c', line) + return 'C' + } + put(out, 'c', line.slice(0, e + 2)) + i = e + 2 + } + while (i < n) { + const ch = line[i] + const c = cc(line, i) + if (ch === '"' || ch === "'") { + let e = quoteEnd(line, i + 1, ch) + if (e < 0) e = n + // a key, when a colon follows + put(out, line[spaceEnd(line, e)] === ':' ? 'a' : 's', line.slice(i, e)) + i = e + continue + } + if (ch === '/' && line[i + 1] === '/') { + put(out, 'c', line.slice(i)) + break + } + if (ch === '/' && line[i + 1] === '*') { + const e = line.indexOf('*/', i + 2) + if (e < 0) { + put(out, 'c', line.slice(i)) + return 'C' + } + put(out, 'c', line.slice(i, e + 2)) + i = e + 2 + continue + } + const sign = (ch === '-' || ch === '+') && (isDigit(cc(line, i + 1)) || line[i + 1] === '.') + if (isDigit(c) || sign || (ch === '.' && isDigit(cc(line, i + 1)))) { + const j = numberEnd(line, sign ? i + 1 : i) + put(out, 'n', line.slice(i, j)) + i = j + continue + } + if (isAlpha(c) || c === 36) { + const j = wordEnd(line, i, true) + const w = line.slice(i, j) + put(out, /^(?:true|false|null|NaN|Infinity)$/.test(w) ? 'n' : line[spaceEnd(line, j)] === ':' ? 'a' : '', w) + i = j + continue + } + put(out, '', ch) + i++ + } + return '' +} + +// ---------------------------------------------------------------- CSS, SCSS / Less + +function css(scss: boolean): Lexer { + return (line, st, out) => { + const n = line.length + let depth = parseInt(st, 10) || 0 + let inC = st.includes('C') + /** in a value (after "prop:"), or the prelude of an @rule */ + let val = st.includes('V') + let at = false + /** the end of a nested rule's selector found ahead (a "{" before any ";") */ + let selEnd = -1 + let i = 0 + while (i < n) { + const ch = line[i] + const c = cc(line, i) + if (inC) { + const e = line.indexOf('*/', i) + if (e < 0) { + put(out, 'c', line.slice(i)) + break + } + put(out, 'c', line.slice(i, e + 2)) + i = e + 2 + inC = false + continue + } + if (ch === '/' && line[i + 1] === '*') { + put(out, 'c', '/*') + inC = true + i += 2 + continue + } + if (scss && ch === '/' && line[i + 1] === '/' && line[i - 1] !== ':') { + put(out, 'c', line.slice(i)) + break + } + if (isSpace(c)) { + const j = spaceEnd(line, i) + put(out, '', line.slice(i, j)) + i = j + continue + } + if (ch === '"' || ch === "'") { + const e = quoteEnd(line, i + 1, ch) + put(out, 's', line.slice(i, e < 0 ? n : e)) + i = e < 0 ? n : e + continue + } + if (ch === '{' || ch === '}' || ch === ';') { + if (ch === '{') depth = Math.min(depth + 1, 99) + else if (ch === '}') depth = Math.max(0, depth - 1) + val = at = false + selEnd = -1 + put(out, '', ch) + i++ + continue + } + if (ch === '@' && isAlpha(cc(line, i + 1))) { + let j = i + 1 + while (j < n && (isWord(cc(line, j)) || line[j] === '-')) j++ + put(out, 'k', line.slice(i, j)) + // Less variables (@x: 1) take a value + if (line[spaceEnd(line, j)] === ':') val = true + else at = true + i = j + continue + } + if (ch === '$' && scss && isAlpha(cc(line, i + 1))) { + let j = i + 1 + while (j < n && (isWord(cc(line, j)) || line[j] === '-')) j++ + put(out, 'a', line.slice(i, j)) + if (line[spaceEnd(line, j)] === ':') val = true + i = j + continue + } + const ident = isAlpha(c) || (ch === '-' && (isAlpha(cc(line, i + 1)) || line[i + 1] === '-')) + if (val || at) { + const sign = (ch === '-' || ch === '+') && (isDigit(cc(line, i + 1)) || (line[i + 1] === '.' && isDigit(cc(line, i + 2)))) + if ((isDigit(c) || sign || (ch === '.' && isDigit(cc(line, i + 1)))) && !isWord(cc(line, i - 1))) { + let j = i + 1 + while (j < n && (isDigit(cc(line, j)) || line[j] === '.')) j++ + while (j < n && (isAlpha(cc(line, j)) || line[j] === '%')) j++ + put(out, 'n', line.slice(i, j)) + i = j + continue + } + if (ch === '#' && isHex(cc(line, i + 1))) { + const j = wordEnd(line, i + 1) + put(out, 'n', line.slice(i, j)) + i = j + continue + } + if (ch === '!' && isAlpha(cc(line, i + 1))) { + const j = wordEnd(line, i + 1) + put(out, 'k', line.slice(i, j)) + i = j + continue + } + if (ident) { + let j = i + while (j < n && (isWord(cc(line, j)) || line[j] === '-')) j++ + const w = line.slice(i, j) + put(out, line[j] === '(' ? 'f' : w.startsWith('--') || (at && line[spaceEnd(line, j)] === ':') ? 'a' : '', w) + i = j + continue + } + put(out, '', ch) + i++ + continue + } + // in a rule: a property, unless a "{" comes before the next ";" or "}" (a nested rule: a:hover { … }) + if (depth > 0 && i > selEnd && ident) { + let j = i + while (j < n && (isWord(cc(line, j)) || line[j] === '-')) j++ + const colon = spaceEnd(line, j) + if (line[colon] === ':') { + let e = colon + while (e < n && line[e] !== ';' && line[e] !== '{' && line[e] !== '}') e++ + if (line[e] !== '{') { + put(out, 'a', line.slice(i, j)) + put(out, '', line.slice(j, colon + 1)) + i = colon + 1 + val = true + continue + } + selEnd = e + } + } + // selectors + if ((ch === '.' || ch === '#') && (isAlpha(cc(line, i + 1)) || line[i + 1] === '-')) { + let j = i + 1 + while (j < n && (isWord(cc(line, j)) || line[j] === '-')) j++ + put(out, 'a', line.slice(i, j)) + i = j + continue + } + if (ch === ':' && (isAlpha(cc(line, i + 1)) || line[i + 1] === ':' || line[i + 1] === '-')) { + let j = i + 1 + if (line[j] === ':') j++ + while (j < n && (isWord(cc(line, j)) || line[j] === '-')) j++ + put(out, 'k', line.slice(i, j)) + i = j + continue + } + if (ch === '[') { + const e = line.indexOf(']', i) + put(out, 'a', line.slice(i, e < 0 ? n : e + 1)) + i = e < 0 ? n : e + 1 + continue + } + if (ident || ch === '&' || ch === '*') { + let j = i + 1 + while (j < n && (isWord(cc(line, j)) || line[j] === '-')) j++ + put(out, ch === '&' ? 'k' : 't', line.slice(i, j)) + i = j + continue + } + put(out, '', ch) + i++ + } + return `${depth || ''}${inC ? 'C' : ''}${val ? 'V' : ''}` + } +} + +// ---------------------------------------------------------------- HTML, XML (and SVG) + +const END_SCRIPT = /<\/script\b/gi +const END_STYLE = /<\/style\b/gi +const ENTITY = /&(?:#\d{1,7}|#x[0-9a-fA-F]{1,6}|\w{1,32});/y + +/** the tokens of web page / XML text; `\n' + expect(cls('html', t, '')).toBe('k') + expect(cls('html', t, '')).toBe('c') + expect(cls('html', t, 'const')).toBe('k') + expect(cls('html', t, '// c')).toBe('c') + expect(cls('html', t, '')).toBe('m') + expect(cls('xml', x, '')).toBe('c') + }) + + test('YAML: keys, values, comments, items, anchors, block scalars', () => { + const t = '---\nname: demo # the name\nport: 8080\non: [push, pull_request]\nsteps:\n - run: |\n echo hi\n echo "x: y"\n - "quoted key": &a yes\nafter: *a' + expect(cls('yaml', t, '---')).toBe('k') + expect(cls('yaml', t, 'name')).toBe('a') + expect(cls('yaml', t, 'demo')).toBe('s') + expect(cls('yaml', t, '# the name')).toBe('c') + expect(cls('yaml', t, '8080')).toBe('n') + expect(cls('yaml', t, 'push')).toBe('s') + expect(cls('yaml', t, '-', 3)).toBe('o') + expect(cls('yaml', t, 'run')).toBe('a') + expect(cls('yaml', t, '|')).toBe('o') + expect(cls('yaml', t, ' echo "x: y"')).toBe('s') + expect(cls('yaml', t, '"quoted key"')).toBe('a') + expect(cls('yaml', t, '&a')).toBe('m') + expect(cls('yaml', t, 'yes')).toBe('n') + expect(cls('yaml', t, 'after')).toBe('a') + expect(cls('yaml', t, '*a')).toBe('m') + }) + + test('shell: commands, variables, strings, comments, here-documents', () => { + const t = '#!/bin/bash\n# build\nNAME="w $USER"\nif [ -n "$1" ]; then\n echo ${NAME} | tr a b > out.txt\nfi\ncat < { + const t = "SELECT id, \"Name\" FROM users u -- all\nWHERE u.note = 'it''s' AND count(*) > 10;\n/* multi\nline */ select 1" + expect(cls('sql', t, 'SELECT')).toBe('k') + expect(cls('sql', t, 'select')).toBe('k') + expect(cls('sql', t, '"Name"')).toBe('a') + expect(cls('sql', t, '-- all')).toBe('c') + expect(cls('sql', t, "'it''s'")).toBe('s') + expect(cls('sql', t, 'count')).toBe('f') + expect(cls('sql', t, '10')).toBe('n') + expect(cls('sql', t, 'line */')).toBe('c') + }) + + test('Go, Rust, Java, C, C++, C#', () => { + const go = 'package main\nfunc main() {\n\ts := `raw\nstring`\n\tvar n int = 0\n}' + expect(cls('go', go, 'func')).toBe('k') + expect(cls('go', go, 'main', 1)).toBe('f') + expect(cls('go', go, 'string`')).toBe('s') + expect(cls('go', go, 'int')).toBe('y') + const rs = "#[derive(Debug)]\nfn f<'a>(x: &'a str) -> Option { /* a /* b */ c */ println!(\"{}\", r#\"q\"#); 'c' }" + expect(cls('rust', rs, '#[derive(Debug)]')).toBe('m') + expect(cls('rust', rs, "'a", 0)).toBe('k') + expect(cls('rust', rs, 'str')).toBe('y') + expect(cls('rust', rs, 'Option')).toBe('y') + expect(cls('rust', rs, ' c */')).toBe('c') + expect(cls('rust', rs, 'println')).toBe('f') + expect(cls('rust', rs, 'r#"q"#')).toBe('s') + expect(cls('rust', rs, "'c'")).toBe('s') + const java = '@Override\npublic String text() {\n return """\n block\n """;\n}' + expect(cls('java', java, '@Override')).toBe('m') + expect(cls('java', java, 'String')).toBe('y') + expect(cls('java', java, 'block')).toBe('s') + const cc = '#include \n#define MAX 10\nint main(void) { char *s = "x"; return MAX; }' + expect(cls('c', cc, '#include')).toBe('m') + expect(cls('c', cc, '')).toBe('s') + expect(cls('c', cc, 'int')).toBe('y') + expect(cls('c', cc, 'main')).toBe('f') + expect(cls('c', cc, 'MAX', 1)).toBe('n') + const cpp = 'auto s = R"x(a "quoted" )\nstring)x";\nclass Widget : public Base {};\nstd::vector v{nullptr};' + expect(cls('cpp', cpp, 'string)x"')).toBe('s') + expect(cls('cpp', cpp, 'Widget')).toBe('y') + expect(cls('cpp', cpp, 'nullptr')).toBe('n') + const cs = 'var p = @"C:\\dir\n""quoted""";\nvar q = $"{a} b";\n#region X\npublic class C { }' + expect(cls('csharp', cs, '""quoted"""')).toBe('s') + expect(cls('csharp', cs, '$"{a} b"')).toBe('s') + expect(cls('csharp', cs, '#region')).toBe('m') + expect(cls('csharp', cs, 'class')).toBe('k') + }) + + test('Markdown: code blocks in their own language; diff; TOML', () => { + const md = '# Title\n\n```js\nconst a = "b" // c\n```\n\n````md\n```js\nnot closed by this\n```\n````\n\ntext **bold**' + expect(cls('markdown', md, '# Title')).toBe('h') + expect(cls('markdown', md, 'const')).toBe('k') + expect(cls('markdown', md, '"b"')).toBe('s') + expect(cls('markdown', md, '// c')).toBe('c') + expect(cls('markdown', md, '**bold**')).toBe('b') + // a fence with a language does not close a block + expect(cls('markdown', md, '```js', 1)).toBe('k') + // still JavaScript inside the outer block + expect(cls('markdown', md, 'this')).toBe('k') + expect(cls('diff', '--- a/x\n+++ b/x\n@@ -1 +1 @@\n-old\n+new\n same', '-old')).toBe('t') + expect(cls('diff', '--- a/x\n+++ b/x\n@@ -1 +1 @@\n-old\n+new\n same', '+new')).toBe('s') + const toml = '[server]\nhost = "a" # c\nport = 80\ndebug = true\nnote = """\nmulti\n"""' + expect(cls('toml', toml, '[server]')).toBe('y') + expect(cls('toml', toml, 'host ')).toBe('a') + expect(cls('toml', toml, '"a"')).toBe('s') + expect(cls('toml', toml, '80')).toBe('n') + expect(cls('toml', toml, 'multi')).toBe('s') + }) +}) + +describe('syntax: bounded', () => { + const LANGS = ['javascript', 'typescript', 'tsx', 'json', 'python', 'css', 'scss', 'html', 'xml', 'yaml', 'shell', 'sql', 'go', 'rust', 'java', 'c', 'cpp', 'csharp', 'markdown', 'mermaid', 'plantuml', 'graphviz', 'diff', 'toml', 'ini'] + + test('the tokens always put the text back together, whatever the text', () => { + // pieces that open and close what the scanners follow, in random order + const pieces = ['"', "'", '`', '${', '}', '{', '/*', '*/', '//', '#', '<', '>', '', '', '\n', ' ', 'a', '1', '.', '\\', 'r#"', '"""', "'''", '[', ']', '(', ')', ':', '- ', '|', '@x', '$x', '```', '~~~', '', 'R"x(', ')x"', '@"', '\t', '\r\n', 'é', '😀'] + let seed = 7 + const rnd = () => ((seed = (seed * 1103515245 + 12345) & 0x7fffffff) / 0x7fffffff) + for (let round = 0; round < 40; round++) { + let text = '' + for (let k = 0; k < 400; k++) text += pieces[Math.floor(rnd() * pieces.length)] + for (const lang of LANGS) { + const toks = highlight(lang, text) + expect(joined(toks)).toBe(text) + expect(toks.every((t) => t[1].length > 0)).toBe(true) + const lines = highlightLines(lang, text) + expect(lines.length).toBe(text.split('\n').length) + } + } + }) + + test('a text over the limit is plain; a line over the limit is plain and keeps the state around it', () => { + const big = 'const x = 1\n'.repeat(1000) + expect(highlight('javascript', big, 100)).toEqual(highlight(undefined, big)) + const long = '/* open\n' + 'x'.repeat(20_000) + '\nstill */ const' + const lines = highlightLines('javascript', long) + expect(lines[1]).toEqual([['', 'x'.repeat(20_000)]]) + expect(lines[2][0]).toEqual(['c', 'still */']) + expect(lines[2].at(-1)).toEqual(['k', 'const']) + }) + + test('nesting is capped', () => { + const deep = 'const s = ' + '`${'.repeat(5_000) + '\nnext()' + const toks = highlight('javascript', deep) + expect(joined(toks)).toBe(deep) + }) + + test('pathological inputs stay fast', () => { + // long lines of what opens a string, a comment, a tag, a link, an emphasis… and never closes it: a scanner that + // looks ahead to the end of the line from each of them is quadratic (seconds here), a linear one takes milliseconds + const text = (s: string) => Array.from({ length: 2 }, () => s.repeat(Math.floor(9_000 / s.length))).join('\n') + const inputs = ['"', '"\\', '`', '/', '/*', ' worst) [worst, what] = [ms, `${lang} ${JSON.stringify(s)}`] + } + } + expect(`${what}: ${worst < 1000 ? 'fast' : `${Math.round(worst)} ms`}`).toBe(`${what}: fast`) + }, 120_000) + + test('an edit reads again only the lines it changed (the others come from the cache)', () => { + const text = Array.from({ length: 200 }, (_, k) => `const v${k} = ${k} // line ${k}`).join('\n') + const a = highlightLines('javascript', text) + const b = highlightLines('javascript', text.replace('v100 =', 'v100 +=')) + expect(b[99]).toBe(a[99]) + expect(b[101]).toBe(a[101]) + expect(b[100]).not.toBe(a[100]) + }) +})