From ba4f7622c2209b4d6f693b1241f344e4b52c8942 Mon Sep 17 00:00:00 2001 From: "replicas-connector[bot]" Date: Sun, 4 Oct 2026 07:01:45 +0000 Subject: [PATCH 01/18] Add recoverable syncs to raw registry integrations MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Implement provider-specific tokens, incremental windows, durable scan progress, and safe restart recovery across the eight raw ingestion integrations. Update registry metadata, immutable releases, documentation, and the ingestion authoring skill with the stable sync quality bar. Co-Authored-By: Marc Höffl Co-Authored-By: replicas-connector[bot] --- .changeset/checkpointed-registry-ingestion.md | 7 + apps/docs/src/components/RegistryApps.astro | 4 +- .../src/components/RegistryReference.astro | 4 +- .../src/content/docs/api-sync/agent-skill.md | 3 +- .../src/content/docs/api-sync/operations.md | 2 +- .../docs/api-sync/registry-authoring.md | 4 +- apps/docs/src/content/docs/cli/ingest.md | 2 +- apps/docs/src/content/docs/cli/registry.md | 2 +- .../src/content/docs/integrations/attio.mdx | 17 +- .../content/docs/integrations/circleback.mdx | 19 +- .../src/content/docs/integrations/github.mdx | 18 +- .../docs/integrations/google-calendar.mdx | 32 +- .../content/docs/integrations/google-meet.mdx | 44 ++- .../src/content/docs/integrations/lemlist.mdx | 20 +- .../src/content/docs/integrations/linear.mdx | 20 +- .../src/content/docs/integrations/slack.mdx | 33 +- apps/docs/src/content/docs/plugins/ingest.md | 2 +- apps/docs/src/data/registry.ts | 6 +- packages/cli/src/commands/registry.ts | 9 +- packages/cli/src/registry/model.ts | 2 +- .../cli/src/test/registry/artifact.test.ts | 13 + packages/cli/src/test/registry/cli.test.ts | 12 +- registry/attio/README.md | 10 +- registry/attio/checkpoints.ts | 60 +++ registry/attio/client.ts | 59 +++ registry/attio/manifest.json | 19 +- registry/attio/pipeline.ts | 19 +- registry/attio/releases/0.1.3.json | 369 ++++++++++++++++++ registry/attio/sources/entries.ts | 22 +- registry/attio/sources/list-attributes.ts | 22 +- registry/attio/sources/lists.ts | 8 +- registry/attio/sources/members.ts | 13 +- registry/attio/sources/notes.ts | 13 +- registry/attio/sources/object-attributes.ts | 20 +- registry/attio/sources/objects.ts | 8 +- registry/attio/sources/records.ts | 24 +- registry/attio/sources/tasks.ts | 13 +- registry/attio/tests/attio.test.ts | 191 ++++++++- registry/attio/tests/install.e2e.test.ts | 14 +- registry/circleback/README.md | 25 +- registry/circleback/index.ts | 119 ++++-- registry/circleback/manifest.json | 21 +- registry/circleback/releases/0.2.0.json | 101 +++++ registry/circleback/tests/basic.test.ts | 141 ++++++- registry/github/README.md | 24 +- registry/github/index.ts | 134 +++++-- registry/github/manifest.json | 36 +- registry/github/releases/0.2.0.json | 115 ++++++ registry/github/tests/basic.test.ts | 107 +++++ registry/google-calendar/README.md | 40 +- registry/google-calendar/index.ts | 137 +++++-- registry/google-calendar/manifest.json | 30 +- registry/google-calendar/releases/0.2.0.json | 103 +++++ registry/google-calendar/tests/basic.test.ts | 185 ++++++++- registry/google-meet/README.md | 55 ++- registry/google-meet/index.ts | 364 ++++++++++++++--- registry/google-meet/manifest.json | 58 ++- registry/google-meet/releases/0.2.0.json | 144 +++++++ registry/google-meet/tests/basic.test.ts | 251 ++++++++++++ registry/lemlist/README.md | 26 +- registry/lemlist/index.ts | 94 ++++- registry/lemlist/manifest.json | 33 +- registry/lemlist/releases/0.2.0.json | 112 ++++++ registry/lemlist/tests/basic.test.ts | 128 ++++++ registry/linear/README.md | 26 +- registry/linear/index.ts | 105 +++-- registry/linear/manifest.json | 31 +- registry/linear/releases/0.2.0.json | 96 +++++ registry/linear/tests/basic.test.ts | 137 +++++++ registry/slack/README.md | 22 +- registry/slack/client.ts | 29 +- registry/slack/config.ts | 7 + registry/slack/manifest.json | 154 ++++++-- registry/slack/pipeline.ts | 3 +- registry/slack/releases/0.2.0.json | 208 ++++++++++ registry/slack/sources/messages.ts | 137 +++++-- registry/slack/state.ts | 104 +++++ registry/slack/tests/checkpoints.test.ts | 198 ++++++++++ registry/slack/tests/install.e2e.test.ts | 18 +- registry/slack/tests/slack.test.ts | 44 ++- skills/chkit-ingestion/SKILL.md | 25 +- .../chkit-ingestion/references/stable-sync.md | 58 +++ 82 files changed, 4623 insertions(+), 521 deletions(-) create mode 100644 .changeset/checkpointed-registry-ingestion.md create mode 100644 registry/attio/checkpoints.ts create mode 100644 registry/attio/releases/0.1.3.json create mode 100644 registry/circleback/releases/0.2.0.json create mode 100644 registry/github/releases/0.2.0.json create mode 100644 registry/github/tests/basic.test.ts create mode 100644 registry/google-calendar/releases/0.2.0.json create mode 100644 registry/google-meet/releases/0.2.0.json create mode 100644 registry/google-meet/tests/basic.test.ts create mode 100644 registry/lemlist/releases/0.2.0.json create mode 100644 registry/lemlist/tests/basic.test.ts create mode 100644 registry/linear/releases/0.2.0.json create mode 100644 registry/linear/tests/basic.test.ts create mode 100644 registry/slack/releases/0.2.0.json create mode 100644 registry/slack/state.ts create mode 100644 registry/slack/tests/checkpoints.test.ts create mode 100644 skills/chkit-ingestion/references/stable-sync.md diff --git a/.changeset/checkpointed-registry-ingestion.md b/.changeset/checkpointed-registry-ingestion.md new file mode 100644 index 00000000..041b1da8 --- /dev/null +++ b/.changeset/checkpointed-registry-ingestion.md @@ -0,0 +1,7 @@ +--- +"chkit": patch +--- + +Allow registry resources to declare cursor checkpoints and timestamp windows alongside full scans. Upgrade the eight raw API integrations with provider-specific sync tokens, durable scan progress, overlapping incremental ranges, and restart recovery, while preserving historical registry releases. + +Update the ingestion authoring skill with the registry sync quality bar, provider capability selection, durable recovery rules, and required failure/restart verification. diff --git a/apps/docs/src/components/RegistryApps.astro b/apps/docs/src/components/RegistryApps.astro index 0a73066a..7f50caf6 100644 --- a/apps/docs/src/components/RegistryApps.astro +++ b/apps/docs/src/components/RegistryApps.astro @@ -1,5 +1,5 @@ --- -import { registryApps } from '../data/registry'; +import { registryApps, registryStrategyLabel } from '../data/registry'; ---
    @@ -10,7 +10,7 @@ import { registryApps } from '../data/registry';

    {app.title}

    -

    {app.resources.length} resources · {app.strategies.map((strategy) => strategy === 'full' ? 'Full reads' : strategy).join(', ')} · {app.language === 'typescript' ? 'TypeScript' : app.language}

    +

    {app.resources.length} resources · {app.strategies.map(registryStrategyLabel).join(', ')} · {app.language === 'typescript' ? 'TypeScript' : app.language}

    v{app.version} diff --git a/apps/docs/src/components/RegistryReference.astro b/apps/docs/src/components/RegistryReference.astro index 95f53a76..611c8e0d 100644 --- a/apps/docs/src/components/RegistryReference.astro +++ b/apps/docs/src/components/RegistryReference.astro @@ -1,5 +1,5 @@ --- -import { registryApps, type RegistryReferenceSection } from '../data/registry'; +import { registryApps, registryStrategyLabel, type RegistryReferenceSection } from '../data/registry'; interface Props { name: string; @@ -22,7 +22,7 @@ if (!app) throw new Error(`Unknown registry app: ${name}`); Environment variables{app.authentication?.env.map((env, index) => <>{index > 0 && ', '}{env}) ?? 'See provider setup'} ClickHouse{app.clickhouse} Coverage{app.resources.length} synced resources, {app.views?.length ?? 0} derived views - Sync strategy{app.strategies.map((strategy) => strategy === 'full' ? 'Full read on every run' : strategy).join(', ')} + Sync strategy{app.strategies.map(registryStrategyLabel).join(', ')} )} diff --git a/apps/docs/src/content/docs/api-sync/agent-skill.md b/apps/docs/src/content/docs/api-sync/agent-skill.md index 5c111198..0971a2ea 100644 --- a/apps/docs/src/content/docs/api-sync/agent-skill.md +++ b/apps/docs/src/content/docs/api-sync/agent-skill.md @@ -42,10 +42,11 @@ Include provider documentation, example responses, freshness needs, and any rest ## Skill scope -The skill covers the authoring steps and requirements for bounded readers, checkpoint ordering, write identity, and scheduling. Use the linked docs for complete examples and tradeoffs. +The skill covers bounded readers, provider-specific sync strategies, durable checkpoint ordering, write identity, and scheduling. For official registry integrations, it requires a source contract and recovery design, raw payload preservation, late-change reconciliation, and portable interruption/replay tests. Sync tokens, timestamp windows, and checkpointed full scans follow the provider's actual capabilities. Use the linked docs for complete examples and tradeoffs. ## Related pages - [Destinations and transformations](/api-sync/destinations/): choose storage and mapping for the source. - [Quickstart](/api-sync/quickstart/): a working first source. +- [Publish a registry](/api-sync/registry-authoring/): metadata, portable tests, and immutable releases. - [For AI agents](/ai-agents/): general chkit setup and schema workflows. diff --git a/apps/docs/src/content/docs/api-sync/operations.md b/apps/docs/src/content/docs/api-sync/operations.md index 295c16d0..237efd16 100644 --- a/apps/docs/src/content/docs/api-sync/operations.md +++ b/apps/docs/src/content/docs/api-sync/operations.md @@ -100,7 +100,7 @@ chkit ingest status --tag pipeline:helpdesk --json | Symptom | Check | |---|---| | No pipeline / no matching streams | Entry exports and exact tag spelling | -| No checkpoint after a successful run | Full syncs have no incremental bookmark | +| No checkpoint after a successful run | The bundled `fullSync()` strategy has no checkpoint; custom full-scan strategies may record completion | | Window restarts from the beginning | Timestamp progress commits after the whole window; reduce work or use safe intermediate provider state | | Duplicated logical records | Destination keys/versioning, query reconciliation, deduplication settings, and volatile mapped fields | | Incompatible checkpoint | Strategy ID/version or state format changed | diff --git a/apps/docs/src/content/docs/api-sync/registry-authoring.md b/apps/docs/src/content/docs/api-sync/registry-authoring.md index fd2d05e9..a8451b3b 100644 --- a/apps/docs/src/content/docs/api-sync/registry-authoring.md +++ b/apps/docs/src/content/docs/api-sync/registry-authoring.md @@ -103,13 +103,15 @@ The item name `registry` is reserved because `registry.json` contains the catalo | `logo` | Optional HTTP(S) URL of the app's logo; included in discovery metadata | | `chkit`, `ingest`, `clickhouse` | Supported CLI, ingestion package, and ClickHouse version ranges | | `root`, `entry`, `exports` | Installation directory, provider entry, and explicit public exports | -| `resources` | Resource names and descriptions, read scopes, sync strategy (`full`), optional `title`, default `table`, and `endpoints` with `method`, `path`, and provider `documentation` URL | +| `resources` | Resource names and descriptions, read scopes, sync strategy (`full`, `timestamp`, or `cursor`), optional `title`, default `table`, and `endpoints` with `method`, `path`, and provider `documentation` URL | | `authentication` | Authentication `method`, required `env` names, ordered `setup` steps, and the provider's credential setup `documentation` URL | | `views` | Derived views with a `name`, source resource `source`, and `description`; these reuse synced records | | `sync` | Sync `description`, external `schedule`, and `deletions` behavior | | `env` | Environment variable names mapped to example values, such as `{"ATTIO_API_TOKEN": ""}` | | `fileHashes` | SHA-256 values keyed by target path; generated by the builder | +Resource strategies describe selection: `full` performs complete scan cycles, including scans with completion checkpoints; `timestamp` selects time windows; `cursor` follows checkpointed provider state, such as a change token or a resumable snapshot traversal. Describe restart behavior, cursor lifetime, and deletion handling in `sync`. Metadata does not configure the runtime strategy. Older strict registry clients accept only `full`; templates advertising new labels must require a CLI version that accepts them. + Declare npm dependencies as `package@semver-range`. Every item must include `@chkit/core` and `@chkit/plugin-ingest`. Git URLs, local dependencies, and package-manager aliases are outside this format. Use blank values for secrets in `env`. Never include credentials in metadata or source artifacts. State resource limitations in the item's description and copied README, including deletion behavior and inaccessible API families. diff --git a/apps/docs/src/content/docs/cli/ingest.md b/apps/docs/src/content/docs/cli/ingest.md index afcd178f..f16ab2bd 100644 --- a/apps/docs/src/content/docs/cli/ingest.md +++ b/apps/docs/src/content/docs/cli/ingest.md @@ -62,7 +62,7 @@ Run at most one ingestion process per project and target at a time. See [Schedul `--backfill ` runs the selected streams under the checkpoint namespace `#backfill:`, so it never moves the scheduled checkpoint. Reusing an ID resumes that backfill's state. The ID must start with a letter or digit and contain only letters, digits, `_`, `.`, and `-`. -`--from` and `--to` set explicit bounds for `timestampWindow` streams and take precedence over the backfill's watermark. Full-sync and cursor strategies ignore them. Passing `--from` or `--to` without `--backfill` fails with exit code 2. +`--from` and `--to` set explicit bounds for `timestampWindow` streams and take precedence over the backfill's watermark. The bundled `fullSync()` and `cursorState()` strategies do not interpret them. Custom provider strategies may honor or reject date bounds; see the source's documentation. Passing `--from` or `--to` without `--backfill` fails with exit code 2. ### `list` diff --git a/apps/docs/src/content/docs/cli/registry.md b/apps/docs/src/content/docs/cli/registry.md index 1c3805a9..48dbe5ba 100644 --- a/apps/docs/src/content/docs/cli/registry.md +++ b/apps/docs/src/content/docs/cli/registry.md @@ -167,7 +167,7 @@ chkit registry inspect attio --json | jq '{command, schemaVersion, ok, action, o ## Related commands -- [Apps & integrations](/integrations/): browse logos, integration guides, and full sync coverage. +- [Apps & integrations](/integrations/): browse logos, integration guides, and sync coverage. - [`chkit add`](/cli/add/): install a built template into a project. - [Provider templates](/api-sync/templates/): source ownership and customization. - [Publish a registry](/api-sync/registry-authoring/): format and release guidance. diff --git a/apps/docs/src/content/docs/integrations/attio.mdx b/apps/docs/src/content/docs/integrations/attio.mdx index 87f6f730..bcf7bd76 100644 --- a/apps/docs/src/content/docs/integrations/attio.mdx +++ b/apps/docs/src/content/docs/integrations/attio.mdx @@ -12,7 +12,7 @@ import RegistryReference from '../../../components/RegistryReference.astro'; Attio logo -The [app registry](/integrations/) installs the Attio integration as editable TypeScript source. It includes nine raw tables, nine full-read streams, and three SQL views for people, companies, and deals. Each run reads the selected resources through the Attio REST API and updates their latest observed versions in ClickHouse. +The [app registry](/integrations/) installs the Attio integration as editable TypeScript source. It includes nine raw tables, nine checkpointed full-scan streams, and three SQL views for people, companies, and deals. Each complete cycle reads the selected resources through the Attio REST API and updates their latest observed versions in ClickHouse. @@ -180,9 +180,13 @@ Paginated reads start at offset zero, advance by the number of returned entities 5. The runtime batches rows and writes them to ClickHouse. The journal records execution and load progress. 6. SQL views project the raw records and reconcile repeated row IDs at query time with `FINAL`. -### Full reads and freshness +### Checkpointed full scans and freshness -All nine streams use full sync. Every run reads from the beginning, including a rerun after failure. The readers do not save pagination offsets or use an incremental modification cursor. A successful run can therefore show no checkpoint in `ingest status`; use the run outcome and destination queries to verify success. +All nine streams save validated scan-cycle checkpoints. A fresh cycle discovers and freezes the selected object/list IDs and slugs. After every parent is fully fetched and loaded, an empty completion marker flushes preceding rows and commits that parent to the frontier. After interruption, the reader skips completed parents and restarts the unfinished parent at offset zero. It resumes the frozen set before performing new discovery. New parents are included in the next full cycle. + +Notes, tasks, and members use workspace-wide scans. Their unfinished collections restart at offset zero; terminal completion is persisted even when a collection is empty. Objects and lists likewise record completion after their returned rows load. A completed cycle starts a fresh full read on the next run, so edits remain observable. These checkpoints are scan boundaries, not a modification watermark. + +`ingest status` exposes the cycle, phase, frozen parent set, and completed parents. A selection change rejects the saved scope rather than reusing a frontier for different objects/lists. Restore the configuration or use a new source identity for a deliberate fresh scan. A removed or inaccessible frozen parent fails visibly. Child response workspace and parent IDs are checked against frozen discovery identity before any rows are published. Existing installations that have no checkpoint begin the first checkpointed cycle without a state migration. The destination stores the latest **observed** entity. New observations replace earlier observations for the same row identity when ClickHouse reconciles versions. Freshness depends on the external schedule, scan duration, and successful API responses. There are no webhooks or continuous change capture in this integration. @@ -376,7 +380,7 @@ bunx chkit ingest status --tag provider:attio --json Repeated tags use AND matching. With the default source ID, `--tag stream:attio.primary.records` selects the same records stream; a custom `sourceId` changes that derived tag. -Inspect the run's per-stream outcomes and error messages. `status` reports committed checkpoints rather than complete run history; full reads have no incremental bookmark. Keep the ingestion journal as durable operational state. After fixing a token, permission, or destination issue, rerun `ingest run`; the Attio reader starts its collection scan again. +Inspect the run's per-stream outcomes and error messages. `status` reports committed scan-cycle checkpoints rather than complete run history. Keep the ingestion journal as durable operational state. After fixing a token, permission, or destination issue, rerun `ingest run`; the Attio reader resumes its frozen parent set and restarts any unfinished offset collection. Verify a known Attio record against its retained `raw.data`, update that record in a test workspace, rerun the sync, and confirm its view reflects the new observation. A row count alone does not establish full API coverage. Date-range backfill flags do not turn these full-read readers into historical or incremental readers; implementing those behaviors requires a source-specific strategy. @@ -387,7 +391,7 @@ Verify a known Attio record against its retained `raw.data`, update that record | Source record deleted or merged | Previously observed row remains; no deletion or merge reconciliation | | Token loses access or a parent is deselected | Existing rows remain in ClickHouse | | Source changes during offset pagination | A scan may miss or repeat entities; it is not a provider snapshot | -| Run stops after some batches | Loaded batches remain visible; the next run scans again | +| Run stops after some batches | Loaded batches remain visible; completed parents are skipped and unfinished offset collections restart | | Older source data loaded later | Later ingestion time wins; no source-version ordering | | Complete change history needed | Raw tables reconcile observations and do not form a permanent change log | | Meetings, call recordings, transcripts, emails, files, comments, or sequences needed | No readers are included for these resources | @@ -401,7 +405,8 @@ Deleting or deselecting a source entity does not remove it from a view. Applicat | File | Responsibility | |---|---| | `config.ts` | Source identity, destination names, parent selection, page sizes | -| `client.ts` | Authentication, response validation, pagination, row identity, request pacing | +| `client.ts` | Authentication, response validation, pagination, parent-completion boundaries, row identity, request pacing | +| `checkpoints.ts` | Validated scan cycles, frozen parent sets, and selection scope | | `sources/objects.ts`, `sources/object-attributes.ts` | Object and attribute readers with their raw table schemas | | `sources/records.ts` | Record reader, raw records table, people/company/deal views | | `sources/lists.ts`, `sources/list-attributes.ts`, `sources/entries.ts` | List resource readers with their raw table schemas | diff --git a/apps/docs/src/content/docs/integrations/circleback.mdx b/apps/docs/src/content/docs/integrations/circleback.mdx index 238c5f44..0655591d 100644 --- a/apps/docs/src/content/docs/integrations/circleback.mdx +++ b/apps/docs/src/content/docs/integrations/circleback.mdx @@ -23,7 +23,24 @@ Set `CIRCLEBACK_API_KEY` in the runtime environment before the ingestion run. Ed ## Sync behavior -The meetings endpoint is scanned on every run. Missing transcripts remain null; removed meetings remain stored. The raw tables preserve provider fields and require ClickHouse 25.3 or later. See the installed README for configuration and [registry installation](/cli/add/). +Each new scan cycle reads all meetings with `ownership=All`, follows [provider Link continuations](https://circleback.ai/docs/api), and rechecks every observed transcript. The [meeting listing](https://circleback.ai/docs/api/meetings/list-meetings) has no documented modification-time filter or durable cursor lifetime; interrupted cycles restart pagination from the first page and skip only parent enrichment already acknowledged by the destination in that cycle. Unfinished parents are replayed. Completing the cycle clears the acknowledged ID set, so the next run rechecks every accessible meeting. A meeting timestamp is not assumed to track delayed transcript availability. + +Sink-acknowledged checkpoints include explicit source/filter scope, bounded active-cycle parent IDs, unavailable transcript diagnostics with distinct `forbidden` and `not_found` statuses, and the last fully completed cycle's `completedAt`. Neither transcript `403` nor `404` is assumed to mean temporary processing. A new cycle retries listed meetings even when their metadata is unchanged. IDs absent from the listing remain unavailable diagnostics, not executable queued work; they are retried only when listed again or resolved by an explicit reconciliation policy. Recovery can defer updates to acknowledged parents until the next cycle. Listing permission failures fail the stream. + +The editable `maxRetainedMeetings` setting bounds both recovery IDs and diagnostics at `10_000`; exceeding the bound fails visibly while preserving acknowledged progress. Source/filter scope changes require a new stream identity. + +```sh +bunx chkit ingest status --tag provider:circleback --json +``` + +Removed meetings remain stored; a full scan is not an atomic source snapshot. The raw tables preserve provider fields and require ClickHouse 25.3 or later. See the installed README and [registry installation](/cli/add/). + +## Test the reader + +```sh +bunx chkit add circleback --with-tests +bun test src/integrations/circleback/tests/basic.test.ts +``` ## Related pages diff --git a/apps/docs/src/content/docs/integrations/github.mdx b/apps/docs/src/content/docs/integrations/github.mdx index 7ff4f74a..062e4983 100644 --- a/apps/docs/src/content/docs/integrations/github.mdx +++ b/apps/docs/src/content/docs/integrations/github.mdx @@ -23,7 +23,23 @@ Set `GITHUB_TOKEN` in the runtime environment before the ingestion run. Edit `re ## Sync behavior -Each run scans configured repositories. Removed issues or stars remain stored. The raw tables preserve provider fields and require ClickHouse 25.3 or later. See the installed README for configuration and [registry installation](/cli/add/). +Issues use `since` with update-time ordering, a fixed local upper cutoff, and five minutes of overlap. GitHub has no server-side upper timestamp filter. The checkpoint advances only after the complete issue window and destination writes succeed; failed windows replay from their lower bound. The reader follows provider `Link` URLs and includes paginated issue comments plus pull request detail, reviews, review comments, commits and files. Provider caps are reflected in raw `_chkit_context` completeness fields. + +Stargazers remain full reads. Their checkpoint records the last completed scan, including an empty scan; page numbers and star creation timestamps are not removal cursors. Removed stars and deleted issues remain stored. Live pagination and child-only updates require periodic reconciliation or a durable webhook companion. [GitHub issue filters](https://docs.github.com/en/rest/issues/issues#list-repository-issues) and [pagination](https://docs.github.com/en/rest/using-the-rest-api/using-pagination-in-the-rest-api) describe the API behavior. + +```sh +bunx chkit ingest run --tag provider:github --tag resource:issues --backfill reconcile-2026-10-04 --from 2008-01-01 +bunx chkit ingest status --tag provider:github --json +``` + +Historical replay reads current observations, not historical versions. Use a new backfill ID for each full reconciliation. The raw tables preserve provider fields and require ClickHouse 25.3 or later. See the installed README for token permissions and [registry installation](/cli/add/). + +## Test the reader + +```sh +bunx chkit add github --with-tests +bun test src/integrations/github/tests/basic.test.ts +``` ## Related pages diff --git a/apps/docs/src/content/docs/integrations/google-calendar.mdx b/apps/docs/src/content/docs/integrations/google-calendar.mdx index bb3e2068..bcbe1dd3 100644 --- a/apps/docs/src/content/docs/integrations/google-calendar.mdx +++ b/apps/docs/src/content/docs/integrations/google-calendar.mdx @@ -1,31 +1,53 @@ --- title: Integrating ClickHouse with Google Calendar -description: Install the editable Google Calendar raw ingestion example. +description: Install canonical Google Calendar ingestion with resumable pages and sync-token checkpoints. --- -The Google Calendar registry item copies a small API reader and raw ClickHouse tables into a chkit project. +The Google Calendar registry item copies a canonical event reader with journaled page and sync-token progress into a chkit project. import RegistryReference from '../../../components/RegistryReference.astro'; ## Install ```sh -bunx chkit add google-calendar +bunx chkit add google-calendar --with-tests bunx chkit check bunx chkit generate --name add_google_calendar bunx chkit migrate --apply bunx chkit ingest run --tag provider:google-calendar +bunx chkit ingest status --tag provider:google-calendar ``` -Set `GOOGLE_CALENDAR_ACCESS_TOKEN` in the runtime environment before the ingestion run. Edit `calendarId`, `pastDays`, and `futureDays` in `src/integrations/google-calendar/index.ts` to select the calendar and time range. +Set `GOOGLE_CALENDAR_ACCESS_TOKEN` with `calendar.readonly` access before ingestion. Review `sourceId`, `calendarId`, and `database` in `src/integrations/google-calendar/index.ts`. `primary` is resolved through calendar metadata to detect a different account reusing the checkpoint. Schema imports do not request credentials or call Google. ## Sync behavior -Each run reads a configured time range. Events outside the range and deleted records are not reconciled. The raw tables preserve provider fields and require ClickHouse 25.3 or later. See the installed README for configuration and [registry installation](/cli/add/). +The first run reads canonical events with recurring masters and exceptions, using `singleEvents=false` and `showDeleted=true`. Later runs request changes through Google's `syncToken`. Cancellation payloads remain unmodified. Recurring occurrences require a separate bounded instances reader or application projection. + +Each acknowledged page stores its next-page position and retains the same input sync token. Only a loaded terminal page promotes `nextSyncToken`, including an empty page. Rerun after interruption to resume; failed loads leave the previous checkpoint. Requests use fixed parameters, and validation rejects changed source scope or malformed pagination. Native JSON requires ClickHouse 25.3 or later. + +Google forbids `timeMin`, `timeMax`, `updatedMin`, and `orderBy` on sync-token requests. This canonical reader rejects explicit `--from`/`--to` bounds. A named backfill without bounds runs an independent canonical sync. See [Google synchronization](https://developers.google.com/workspace/calendar/api/guides/sync) and [events.list parameters](https://developers.google.com/workspace/calendar/api/v3/reference/events/list). + +## Expiry and stored data + +`410 Gone` resets provider progress and starts a new baseline without deleting raw rows. A rejected page token replays its collection once. The destination keeps the latest observed payload per `[sourceId, resolvedCalendarId, event.id]`; absent records are not reconciled after a reset. A current-calendar view requires completed-baseline generations or explicit reconciliation before treating absence as removal. + +The checkpoint retains the recovery count until the terminal sync page is acknowledged. Pauses and successful nonterminal replay pages keep that count; a second token rejection for the unfinished sync fails across executions. Adjust the editable stream `budget.maxChunks`, execution duration, or polling frequency so the sync finishes while tokens remain valid. After reviewing coverage and correcting those conditions, migrate saved state explicitly or use a new stream identity for a fresh sync. + +Version 0.2.0 changes the former rolling occurrence-window semantics and row identities. Keep old tables as archives and start a new canonical dataset, or migrate identities explicitly; old expanded occurrences are not removed automatically. The installed README describes upgrade and recovery details. + +## Fixture verification + +```sh +bun test src/integrations/google-calendar/tests/basic.test.ts +``` + +Fixtures cover page resumption, empty deltas, sink failures before token promotion, cancellation payloads, expired tokens, and calendar-account changes. Schedule repeat runs externally with one ingestion process per destination at a time. ## Related pages - [App registry](/integrations/) - [Ingestion plugin](/plugins/ingest/) +- [Ingestion CLI](/cli/ingest/) diff --git a/apps/docs/src/content/docs/integrations/google-meet.mdx b/apps/docs/src/content/docs/integrations/google-meet.mdx index f6c66bbc..4993d5f1 100644 --- a/apps/docs/src/content/docs/integrations/google-meet.mdx +++ b/apps/docs/src/content/docs/integrations/google-meet.mdx @@ -1,31 +1,65 @@ --- title: Integrating ClickHouse with Google Meet -description: Install the editable Google Meet raw ingestion example. +description: Install resumable Google Meet ingestion with retained parent work and paged transcript entries. --- -The Google Meet registry item copies a small API reader and raw ClickHouse tables into a chkit project. +The Google Meet registry item copies raw readers with fixed discovery windows and journaled conference, transcript, and entry progress into a chkit project. import RegistryReference from '../../../components/RegistryReference.astro'; ## Install ```sh -bunx chkit add google-meet +bunx chkit add google-meet --with-tests bunx chkit check bunx chkit generate --name add_google_meet bunx chkit migrate --apply bunx chkit ingest run --tag provider:google-meet +bunx chkit ingest status --tag provider:google-meet ``` -Set `GOOGLE_MEET_ACCESS_TOKEN` in the runtime environment before the ingestion run. Edit `lookbackDays` in `src/integrations/google-meet/index.ts` to select the conference window. +Set `GOOGLE_MEET_ACCESS_TOKEN` with `meetings.space.readonly` access before ingestion. Review `sourceId`, `lookbackDays`, `overlapDays`, `windowDays`, and `database` in `src/integrations/google-meet/index.ts`. Keep the source identity tied to one Google account; these list responses do not identify the authenticated account. Schema imports do not call Google. ## Sync behavior -Each run reads a configured time range. Transcripts generated later require another scan of that range. The raw tables preserve provider fields and require ClickHouse 25.3 or later. See the installed README for configuration and [registry installation](/cli/add/). +Each stream independently discovers completed conferences through fixed `end_time` windows and ongoing conferences through `end_time IS NULL`. End-time discovery includes long-running calls that began before the lookback. The initial range defaults to 30 days, later discovery overlaps seven days, and each cycle advances by at most `windowDays`. + +Transcript and entry streams retain parent conferences until provider expiry and revisit them on completed cycles. An empty transcript list or a still-processing artifact stays eligible after discovery advances, so late transcripts are collected outside the discovery overlap. This is polling; Meet provides no modification/change token and no transcript or entry date filter. See [conference filters](https://developers.google.com/workspace/meet/api/reference/rest/v2/conferenceRecords/list) and [artifact behavior](https://developers.google.com/workspace/meet/api/guides/artifacts). + +## Checkpoints and recovery + +The journal stores fixed cycle bounds and nested parent, transcript-page, and entry-page positions. Candidate progress commits after its covering rows load; failed loads retain the preceding position. Rerun after interruption or budget exhaustion to continue. Rejected page tokens replay their scoped collection once, and repeated or malformed tokens fail visibly. + +Saved recovery counts cover each unfinished discovery phase, parent transcript collection, and transcript entry collection. They clear at the acknowledged terminal boundary, so pauses and nonterminal replay pages do not renew the allowance. A second rejection fails across executions. Adjust the editable `budget.maxChunks`, execution duration, or polling frequency, then review coverage before explicitly migrating saved state or using a new stream identity for a fresh scan. + +Streams permit 200 chunks per run and retain at most 1,000 conferences. These limits are editable; an oversized pending queue fails instead of losing parents. Source/window changes require explicit checkpoint migration or a new stream identity. Native JSON requires ClickHouse 25.3 or later. Schedule repeat runs externally with one ingestion process per destination at a time. + +Google removes conference records and API transcript entries 30 days after a call ends. Expiry during unfinished work fails with a coverage-gap message. Previously checked parents leave the queue on expiry, while stored raw rows remain. Expired or inaccessible resources do not become inferred deletions. See [conference retention](https://developers.google.com/workspace/meet/api/reference/rest/v2/conferenceRecords) and [artifact retention](https://developers.google.com/workspace/meet/api/guides/artifacts). + +## Data and historical ranges + +Raw identities are `[sourceId, provider.name]`. Transcript metadata includes `conference_name`; individual entry rows include `conference_name` and `transcript_name`. Entries stream page by page into `google_meet_transcript_entries_raw`. The API entries do not capture later edits to the separate Google Docs transcript file. + +Version 0.2.0 adds the entry destination and stores new transcript metadata without embedding every entry in one row. Existing tables remain available. Row identities now include the source label; keep older datasets as archives or migrate their identities deliberately. See the installed README for upgrade details. + +Explicit backfill bounds constrain conference **end times** and skip ongoing-call discovery. Reuse the same backfill ID and bounds to resume; expired API history remains unavailable. Artifact lists are parent-scoped and cannot themselves be date-filtered. + +```sh +bunx chkit ingest run --tag provider:google-meet --backfill october --from 2026-10-01T00:00:00Z --to 2026-10-05T00:00:00Z +``` + +## Fixture verification + +```sh +bun test src/integrations/google-meet/tests/basic.test.ts +``` + +Fixtures cover nested entry resumption, late transcripts from retained parents, fixed end-time and ongoing discovery, rejected tokens, sink failures, isolated backfill bounds, and expiry gaps. ## Related pages - [App registry](/integrations/) - [Ingestion plugin](/plugins/ingest/) +- [Ingestion CLI](/cli/ingest/) diff --git a/apps/docs/src/content/docs/integrations/lemlist.mdx b/apps/docs/src/content/docs/integrations/lemlist.mdx index 086b6516..b6501c0d 100644 --- a/apps/docs/src/content/docs/integrations/lemlist.mdx +++ b/apps/docs/src/content/docs/integrations/lemlist.mdx @@ -19,11 +19,29 @@ bunx chkit ingest run --tag provider:lemlist Set `LEMLIST_API_KEY` in the runtime environment before the ingestion run. Edit `src/integrations/lemlist/index.ts` to select activities, campaigns, or both. +Keep each stream ID and destination tied to one account. The reader does not resolve the key's account identity; switching accounts requires new stream IDs and destinations so previous watermarks cannot skip history. + ## Sync behavior -Both streams scan all accessible pages each run. Removed campaigns or activity records remain stored. The raw tables preserve provider fields and require ClickHouse 25.3 or later. See the installed README for configuration and [registry installation](/cli/add/). +Activities use inclusive `minDate`/`maxDate` filters on `createdAt`, thirty-day intervals, and one day of overlap. Each completed interval checkpoints only after its pages have destination acknowledgement. A failed interval starts at offset zero; an explicit backfill resumes its completed interval frontier in an isolated namespace. Empty intervals also commit progress. Offset pages remain mutable, so their offsets are never persisted as change cursors. + +The [activities API](https://developer.lemlist.com/api-reference/endpoints/activities/get-many-activities) can backfill metadata on older activities. Periodic historical replay refreshes such changes and late arrivals beyond the overlap. [Campaigns](https://developer.lemlist.com/api-reference/endpoints/campaigns/get-many-campaigns) remain full scans with creation-time ordering and a successful-scan `completedAt` checkpoint; creation-only incremental reads would miss campaign updates. + +```sh +bunx chkit ingest run --tag provider:lemlist --tag resource:activities --backfill reconcile-2026-10-04 --from 2000-01-01 +bunx chkit ingest status --tag provider:lemlist --json +``` + +Use a new backfill ID for each complete reconciliation; reusing an ID resumes that backfill. Narrowing a reused backfill's upper bound below its committed frontier is rejected. Removed campaigns and activities remain stored. The raw tables preserve provider fields and require ClickHouse 25.3 or later. See the installed README and [registry installation](/cli/add/). + +## Test the reader + +```sh +bunx chkit add lemlist --with-tests +bun test src/integrations/lemlist/tests/basic.test.ts +``` ## Related pages diff --git a/apps/docs/src/content/docs/integrations/linear.mdx b/apps/docs/src/content/docs/integrations/linear.mdx index d1c10ea5..e22ff9d2 100644 --- a/apps/docs/src/content/docs/integrations/linear.mdx +++ b/apps/docs/src/content/docs/integrations/linear.mdx @@ -19,11 +19,29 @@ bunx chkit ingest run --tag provider:linear Set `LINEAR_API_KEY` in the runtime environment before the ingestion run. Edit the GraphQL query in `src/integrations/linear/index.ts` to select the fields needed by the project. +Keep each stream ID and destination tied to one workspace. The reader does not resolve the token's workspace identity; switching workspaces requires a new stream ID and destination so the previous watermark cannot skip history. Reconcile older observations after adding requested fields. + ## Sync behavior -Every run reads all accessible issues. The example requests the first 100 comments per issue; extend the query for complete discussions. The raw tables preserve provider fields and require ClickHouse 25.3 or later. See the installed README for configuration and [registry installation](/cli/add/). +Issues use server-side bounded `updatedAt` windows, Relay cursors, five minutes of overlap, and explicit archive inclusion. All accessible comment pages are fetched before their issue is published. The timestamp checkpoint advances only after the whole window and destination writes succeed, including an empty window; failed windows replay from their lower bound. [Linear pagination](https://linear.app/developers/pagination) and [date filtering](https://linear.app/developers/filtering) document the selection controls. + +Deleted issues remain stored. Comment-only changes are not assumed to update the parent issue timestamp; use periodic reconciliation or a durable webhook companion to refresh old discussions. The API does not provide an atomic snapshot guarantee. + +```sh +bunx chkit ingest run --tag provider:linear --backfill reconcile-2026-10-04 --from 1970-01-01 +bunx chkit ingest status --tag provider:linear --json +``` + +Historical replay reads current issue observations, not historical versions. Use a new backfill ID for each full reconciliation. The raw tables preserve provider fields and require ClickHouse 25.3 or later. See the installed README for configuration and [registry installation](/cli/add/). + +## Test the reader + +```sh +bunx chkit add linear --with-tests +bun test src/integrations/linear/tests/basic.test.ts +``` ## Related pages diff --git a/apps/docs/src/content/docs/integrations/slack.mdx b/apps/docs/src/content/docs/integrations/slack.mdx index 2d93a762..8ba97edb 100644 --- a/apps/docs/src/content/docs/integrations/slack.mdx +++ b/apps/docs/src/content/docs/integrations/slack.mdx @@ -10,7 +10,7 @@ Sync Slack into ClickHouse to query conversation metadata, users, messages, and import RegistryReference from '../../../components/RegistryReference.astro'; -The [app registry](/integrations/) installs three raw tables and three full-read streams as editable TypeScript source. The integration preserves the complete objects returned by Slack; add SQL views for the fields and relationships needed by the project. +The [app registry](/integrations/) installs three raw tables, full scans for users and conversations, and checkpointed message history and threads as editable TypeScript source. The integration preserves the complete objects returned by Slack; add SQL views for the fields and relationships needed by the project. @@ -104,6 +104,9 @@ export const slackConfig: SlackConfig = { messagePageSize: 15, requestIntervalMs: 3000, messageIntervalMs: 60000, + historyFrom: '0.000000', + overlapMs: 86400000, + reconcileIntervalMs: 604800000, } ``` @@ -119,6 +122,9 @@ export const slackConfig: SlackConfig = { | `messagePageSize` | History and reply page size, integer `1`–`999` subject to Slack's app-specific limit; conservative default `15` | | `requestIntervalMs` | Delay before each authentication, conversation, and user request; default `3000` ms | | `messageIntervalMs` | Delay before each history and reply request; default `60000` ms | +| `historyFrom` | First message timestamp for bootstrap/reconciliation; exact string, default `0.000000` | +| `overlapMs` | Recent root overlap after the completed watermark; default one day | +| `reconcileIntervalMs` | Interval between full historical reads; default seven days; zero means every completed run | Set `channels: ['C012EXAMPLE', 'C034EXAMPLE']` to restrict conversation metadata and messages. Selection uses Slack IDs, not channel names. Unknown IDs, inaccessible conversations, or IDs excluded by `conversationTypes` fail visibly. User collection remains workspace-wide. @@ -163,25 +169,29 @@ Every API URL is relative to `https://slack.com/api/`. Each reader calls [`POST | `users` | `users.list` across the accessible workspace | | `messages` | Independent conversation discovery, then `conversations.history` for each selected conversation; `conversations.replies` for roots with `reply_count > 0` when replies are enabled | -The client follows `response_metadata.next_cursor` until it is absent, null, or empty. A short or empty page with a next cursor does not end the scan. Repeated cursors and `has_more: true` without a cursor fail explicitly rather than silently ending an incomplete scan. Each stream discovers its own workspace and parents, so the messages stream runs without first running the channels stream. +Metadata follows `response_metadata.next_cursor` until absent, null, or empty. Messages use exact `oldest` and `latest` time boundaries, preserving progress independently of temporary cursors. Empty message pages may use a live cursor; expiration restarts the unfinished interval. Repeated cursors and responses without a safe continuation fail visibly. Each stream discovers its workspace and parents independently. -For each channel, the reader finishes history pagination before fetching its discovered threads, avoiding an idle history cursor during long thread scans. History has no date bounds or subtype filtering. Each thread read includes every reply page and the parent returned by the replies endpoint. The parent keeps its history row identity; replies use their own timestamps and retain `thread_ts` as the root reference. +For each channel, history advances newest first. Each loaded page stores its discovered thread work, then completes that bounded queue before requesting older history. Replies advance oldest first with an independent timestamp boundary. Threads retain the parent returned by Slack under its history identity. No history cursor waits during long thread reads, and no temporary cursor enters durable state. ## How the sync works -Every run starts from the beginning of each selected resource. The readers do not save cursors, use incremental checkpoints, or bound message history by time. Re-reading old roots finds late replies to old threads when Slack still returns those roots and replies. It also refreshes returned message edits and reaction fields. Coverage remains limited by Slack retention, API visibility, scopes, and conversation access. +Channels and users remain full metadata reads. Messages bootstrap from `historyFrom` to a fixed cutoff, then select overlapping ranges from their completed watermark. A full historical reconciliation every seven days revisits old edits, newly threaded roots, and late replies outside recent overlap. New channels bootstrap retained history. Coverage depends on retention, API visibility, scopes, and access. + +Message checkpoints retain workspace/query scope, fixed bounds, channel position, exact history/reply frontiers, and a page-sized pending thread queue. State commits after its covered rows load. Failed or budget-limited runs resume those bounds and finish retained threads before advancing. The completed watermark moves only after all selected channels and child work finish. + +Keep source identity, selection, reply policy, and time configuration stable. Changing them fails against saved state instead of reinterpreting progress. Earlier templates had no message checkpoint: upgrading to version 0.2.0 bootstraps once while preserving existing raw tables and identities. ### Rate limits and scan duration -The default pipeline runs one stream, one fetch, and one load at a time, with a destination batch threshold of 500 rows. Requests wait before each attempt: metadata uses `requestIntervalMs: 3000`; history and replies use `messageIntervalMs: 60000` with pages of `15` messages. +The default pipeline runs one stream, one fetch, and one load at a time. Metadata batches use a 500-row threshold; messages flush each source page so progress commits promptly. Requests wait before each attempt: metadata uses `requestIntervalMs: 3000`; history and replies use `messageIntervalMs: 60000` with pages of `15` messages. Slack documents a one-request-per-minute limit and a 15-object page limit for new commercially distributed applications outside the Slack Marketplace. Marketplace and internal customer-built apps retain Tier 3 limits. See the current [`conversations.history` reference](https://docs.slack.dev/reference/methods/conversations.history/), [`conversations.replies` reference](https://docs.slack.dev/reference/methods/conversations.replies/), and [rate-limit clarification](https://docs.slack.dev/changelog/2025/06/03/rate-limits-clarity/). For an internal app entitled to Tier 3, edit the copied config to use `messagePageSize: 200` and `messageIntervalMs: 1200`. Confirm the app's actual limits before changing these settings. Slack may still return fewer objects than requested or apply `429` responses. -A full history read with many threads can take hours or days at the conservative defaults. Restrict conversation IDs and select only the necessary streams before scheduling runs. The CLI's default duration is `3600` seconds; set `--max-duration` for the expected scan. Increasing the duration does not add a resume cursor: a rerun after interruption reads from the beginning. +A full history read with many threads can take hours or days at the conservative defaults. Restrict conversation IDs and select only the necessary streams before scheduling runs. The CLI's default duration is `3600` seconds; set `--max-duration` for the expected scan. An interrupted message scan resumes its timestamp frontier and child work in the next run. Increasing duration allows more progress per execution. ### Retries and failures @@ -247,9 +257,11 @@ bunx chkit ingest run --tag provider:slack --tag resource:messages --max-duratio bunx chkit ingest status --tag provider:slack --json ``` -Repeated tags use AND matching. With the default source ID, `--tag stream:slack.primary.messages` selects the same stream. `status` reports committed checkpoints; a successful full read may have no incremental bookmark. Verify the execution's stream outcomes and query known retained messages rather than relying only on a row count. +Repeated tags use AND matching. With the default source ID, `--tag stream:slack.primary.messages` selects the same stream. `status` reports message checkpoints; full metadata reads have no incremental bookmark. Verify the execution's stream outcomes and query known retained messages rather than relying only on a row count. + +After the first run, compare a known message and a reply in an old thread with their retained `raw.data`. Add a reply to that old thread, then run a messages-only backfill with a new ID and `--from` before the root's timestamp to confirm it is collected. A normal incremental rerun revisits old roots during the next scheduled full reconciliation. -After the first run, compare a known message and a reply in an old thread with their retained `raw.data`. Add a reply to that test thread, rerun, and confirm it is collected. After fixing credentials, permissions, or destination failures, rerun ingestion; scans start again. Date-range backfill flags do not turn these full readers into bounded or incremental readers. +After fixing credentials, permissions, or destination failures, rerun ingestion; messages resume saved work and metadata scans restart. Messages support isolated bounded backfills through `--backfill --from --to `. Unfinished bounds must match the saved backfill; use another ID for a different range. Channels and users do not interpret date bounds. Historical observations loaded later can replace newer raw payloads because versioning uses ingestion time. ### Coverage and consistency limits @@ -263,12 +275,13 @@ Slack retention and access restrictions limit historical coverage. Changes durin |---|---| | `config.ts` | Source identity, destination names, conversation selection, replies, page sizes, pacing | | `client.ts` | Authentication, payload validation, pagination, request pacing, and row identity | +| `state.ts` | Validated message windows, exact timestamp arithmetic, and source binding | | `sources/channels.ts` | Conversation reader and raw table | | `sources/users.ts` | User reader and raw table | | `sources/messages.ts` | History and thread reader and raw messages table | | `pipeline.ts` | Enabled streams, tags, batching, retries, and concurrency | | `index.ts` | Discoverable pipeline and schema exports | -| `tests/slack.test.ts`, `tests/fixtures.ts` | Optional portable tests and sanitized mocked responses | +| `tests/slack.test.ts`, `tests/checkpoints.test.ts`, `tests/fixtures.ts` | Portable payload, restart, boundary, and reconciliation tests | Remove a stream from `pipeline.ts` to stop collecting a resource. Keep its schema export to preserve the existing table under migration management. Removing a schema export can generate a drop operation; review that migration. Revisit pacing when changing concurrency or page sizes and run fixture tests for customized readers. @@ -276,7 +289,7 @@ Remove a stream from `pipeline.ts` to stop collecting a resource. Keep its schem ```sh bunx chkit add slack --with-tests -bun test src/integrations/slack/tests/slack.test.ts +bun test src/integrations/slack/tests/slack.test.ts src/integrations/slack/tests/checkpoints.test.ts ``` The optional tests mock Slack responses and use an in-memory destination. They need Bun, with no Slack token or ClickHouse server. Adjust the test path for a custom `--path`. Installed files belong to the project; reinstalling does not overwrite local edits or merge newer integration versions automatically. diff --git a/apps/docs/src/content/docs/plugins/ingest.md b/apps/docs/src/content/docs/plugins/ingest.md index 82d28585..9c7ca830 100644 --- a/apps/docs/src/content/docs/plugins/ingest.md +++ b/apps/docs/src/content/docs/plugins/ingest.md @@ -168,7 +168,7 @@ chkit ingest run --backfill jan --from 2026-01-01 --to 2026-02-01 Every stream also carries the derived tags `pipeline:` and `stream:`. `schedule:` is a convention only: chkit never interprets it. A `--tag` filter that matches nothing fails before any work runs. -A backfill uses its own checkpoint namespace, so it never moves the scheduled bookmark. Reusing its ID reuses that state, but resumption depends on the strategy: explicit timestamp bounds take precedence over the watermark and reread that range. Full-sync and cursor strategies do not interpret date bounds. See [Backfill source data](/api-sync/operations/#backfill-source-data). +A backfill uses its own checkpoint namespace, so it never moves the scheduled bookmark. Reusing its ID reuses that state, but resumption depends on the strategy: explicit timestamp bounds take precedence over the watermark and reread that range. The bundled `fullSync()` and `cursorState()` strategies do not interpret date bounds; custom provider strategies may honor or reject them. See the source's documentation and [Backfill source data](/api-sync/operations/#backfill-source-data). `chkit check` verifies that every stream destination carries the ingestion metadata columns. See [`chkit ingest`](/cli/ingest/) for the full flag reference, exit codes, and JSON output. diff --git a/apps/docs/src/data/registry.ts b/apps/docs/src/data/registry.ts index 7a1dd529..7c25d35c 100644 --- a/apps/docs/src/data/registry.ts +++ b/apps/docs/src/data/registry.ts @@ -8,6 +8,10 @@ const registry: RegistryCatalog = JSON.parse(readFileSync(resolve('public/r/regi export type RegistryReferenceSection = 'overview' | 'authentication' | 'scopes' | 'resources' | 'views' | 'sync'; +export function registryStrategyLabel(strategy: 'full' | 'timestamp' | 'cursor'): string { + return { full: 'Full scans', timestamp: 'Timestamp windows', cursor: 'Cursor checkpoints' }[strategy]; +} + // The build validates and aggregates provider-local manifests before Astro loads this catalog. // HTML, raw Markdown, and structured data share the resulting metadata without CLI runtime dependencies. export const registryApps = registry.items.map((item) => ({ @@ -56,7 +60,7 @@ export function registryReferenceMarkdown(name: string, section: RegistryReferen ['Environment variables', app.authentication?.env.map((env) => `\`${env}\``).join(', ') ?? 'See provider setup'], ['ClickHouse', `\`${app.clickhouse}\``], ['Coverage', `${app.resources.length} synced resources, ${app.views?.length ?? 0} derived views`], - ['Sync strategy', app.strategies.map((strategy) => strategy === 'full' ? 'Full read on every run' : strategy).join(', ')], + ['Sync strategy', app.strategies.map(registryStrategyLabel).join(', ')], ]); case 'authentication': return app.authentication ? [ diff --git a/packages/cli/src/commands/registry.ts b/packages/cli/src/commands/registry.ts index 26e785df..5fc1e779 100644 --- a/packages/cli/src/commands/registry.ts +++ b/packages/cli/src/commands/registry.ts @@ -79,7 +79,7 @@ export async function cmdRegistry(command: 'add' | 'registry', argv: string[]): console.log(`Apps in ${catalog.name} (${items.length}):`) for (const item of items) { console.log(`\n${item.title} (${item.name}@${item.version})\n ${item.description}`) - console.log(` ${item.resourceCount} resource${item.resourceCount === 1 ? '' : 's'}; sync strategies: ${item.strategies.join(', ')}`) + console.log(` ${item.resourceCount} resource${item.resourceCount === 1 ? '' : 's'}; sync strategies: ${item.strategies.map(describeStrategy).join(', ')}`) if (item.documentation) console.log(` Guide: ${item.documentation}`) console.log(` Install: ${installCommand(`${item.name}@${item.version}`, values.registry)}`) } @@ -115,7 +115,7 @@ export async function cmdRegistry(command: 'add' | 'registry', argv: string[]): console.log(` ${resource.name}: ${resource.description}`) if (resource.title) console.log(` Title: ${resource.title}`) console.log(resource.table ? ` Default table: ${resource.table}` : ' Table: see source schema') - console.log(` Strategy: ${resource.strategy}; scopes: ${resource.scopes.join(', ') || 'none declared'}`) + console.log(` Strategy: ${describeStrategy(resource.strategy)}; scopes: ${resource.scopes.join(', ') || 'none declared'}`) for (const endpoint of resource.endpoints ?? []) console.log(` ${endpoint.method} ${endpoint.path} (${endpoint.documentation})`) } if (item.meta.chkit.views?.length) { @@ -140,6 +140,11 @@ export async function cmdRegistry(command: 'add' | 'registry', argv: string[]): else console.log(`Built ${result.items.length} template(s) into ${resolve(values.output ?? 'public/r')}`) } +function describeStrategy(strategy: 'full' | 'timestamp' | 'cursor'): string { + const labels = { full: 'full scans', timestamp: 'timestamp windows', cursor: 'cursor checkpoints' } + return `${strategy} (${labels[strategy]})` +} + function onePositional(positionals: string[], usage: string): string { const reference = positionals[0] if (!reference || positionals.length !== 1) throw new Error(`Usage: ${usage}`) diff --git a/packages/cli/src/registry/model.ts b/packages/cli/src/registry/model.ts index 2698a51a..e6057a4b 100644 --- a/packages/cli/src/registry/model.ts +++ b/packages/cli/src/registry/model.ts @@ -40,7 +40,7 @@ const metadataSchema = z.object({ table: z.string().min(1).optional(), description: z.string().min(1), scopes: z.array(z.string()), - strategy: z.literal('full'), + strategy: z.enum(['full', 'timestamp', 'cursor']), endpoints: z.array(z.object({ method: z.enum(['GET', 'POST']), path: z.string().startsWith('/'), diff --git a/packages/cli/src/test/registry/artifact.test.ts b/packages/cli/src/test/registry/artifact.test.ts index 2860956b..44c153a4 100644 --- a/packages/cli/src/test/registry/artifact.test.ts +++ b/packages/cli/src/test/registry/artifact.test.ts @@ -114,6 +114,19 @@ test.serial('presentation metadata is optional and accepts only HTTP or HTTPS UR } }) +test.serial('resource strategies preserve full metadata and accept timestamp and cursor checkpoints', async () => { + const fixture = await fixtures.create() + for (const strategy of ['full', 'timestamp', 'cursor']) { + const resources = fixture.item.meta.chkit.resources.map((resource) => ({ ...resource, strategy })) + const item = { ...fixture.item, meta: { chkit: { ...fixture.item.meta.chkit, resources } } } + expect(parseRegistryItem(item).meta.chkit.resources[0]?.strategy).toBe(strategy) + } + for (const strategy of ['offset', '', undefined]) { + const resources = fixture.item.meta.chkit.resources.map((resource) => ({ ...resource, strategy })) + expect(() => parseRegistryItem({ ...fixture.item, meta: { chkit: { ...fixture.item.meta.chkit, resources } } })).toThrow() + } +}) + test.serial('optional test roles and dependencies preserve legacy items and reject invalid entries', async () => { const fixture = await fixtures.create(true) expect(parseRegistryItem(fixture.item)).toEqual(fixture.item) diff --git a/packages/cli/src/test/registry/cli.test.ts b/packages/cli/src/test/registry/cli.test.ts index 0ff4f2b9..d0361743 100644 --- a/packages/cli/src/test/registry/cli.test.ts +++ b/packages/cli/src/test/registry/cli.test.ts @@ -94,7 +94,7 @@ test.serial('build, list and inspect roundtrip through the CLI without importing expect(inspectText.stdout).toContain('records: Fixture records') expect(inspectText.stdout).toContain('Table: see source schema') expect(inspectText.stdout).not.toContain('Table: records') - expect(inspectText.stdout).toContain('Strategy: full; scopes: record:read') + expect(inspectText.stdout).toContain('Strategy: full (full scans); scopes: record:read') expect(inspectText.stdout).not.toContain('Guide:') expect(inspectText.stdout).not.toContain('Logo:') }) @@ -104,8 +104,8 @@ test.serial('list and inspect expose integration guides, logos, complete resourc const documentation = 'https://example.com/registry/fixture/' const logo = 'https://example.com/logos/fixture.svg' const resources = [ - ...fixture.sourceItem.meta.chkit.resources, - { name: 'members', description: 'Workspace members', scopes: ['members:read', 'workspace:read'], strategy: 'full' as const }, + ...fixture.sourceItem.meta.chkit.resources.map((resource) => ({ ...resource, strategy: 'timestamp' as const })), + { name: 'members', description: 'Workspace members', scopes: ['members:read', 'workspace:read'], strategy: 'cursor' as const }, ] await writeManifest(fixture.manifest, { ...fixture.sourceItem, @@ -128,7 +128,7 @@ test.serial('list and inspect expose integration guides, logos, complete resourc command: 'registry', schemaVersion: 1, ok: true, action: 'list', items: [{ name: 'fixture', title: 'Fixture', description: 'A provider fixture', version: '1.0.1', - resourceCount: 2, resources, strategies: ['full'], documentation, logo, + resourceCount: 2, resources, strategies: ['timestamp', 'cursor'], documentation, logo, }], }) expect(inspectJson.exitCode).toBe(0) @@ -139,7 +139,7 @@ test.serial('list and inspect expose integration guides, logos, complete resourc }) expect(listText.exitCode).toBe(0) expect(listText.stdout).toContain('Fixture (fixture@1.0.1)') - expect(listText.stdout).toContain('2 resources; sync strategies: full') + expect(listText.stdout).toContain('2 resources; sync strategies: timestamp (timestamp windows), cursor (cursor checkpoints)') expect(listText.stdout).toContain(`Guide: ${documentation}`) expect(listText.stdout).toContain(`Install: chkit add fixture@1.0.1 --registry ${fixture.output}`) expect(inspectText.exitCode).toBe(0) @@ -147,7 +147,7 @@ test.serial('list and inspect expose integration guides, logos, complete resourc expect(inspectText.stdout).toContain(`Logo: ${logo}`) expect(inspectText.stdout).toContain(`Install: chkit add fixture@1.0.1 --registry ${fixture.output}`) expect(inspectText.stdout).toContain('members: Workspace members') - expect(inspectText.stdout).toContain('Strategy: full; scopes: members:read, workspace:read') + expect(inspectText.stdout).toContain('Strategy: cursor (cursor checkpoints); scopes: members:read, workspace:read') expect(inspectText.stdout).toContain('FIXTURE_REGION: "eu"') expect(await readdir(fixture.project)).toEqual([]) diff --git a/registry/attio/README.md b/registry/attio/README.md index 155f47cf..ef952477 100644 --- a/registry/attio/README.md +++ b/registry/attio/README.md @@ -32,7 +32,7 @@ The schema uses native JSON (ClickHouse 25.3 or newer). A direct ClickHouse conn ## Included resources -All nine streams perform full reads on each run. Every stream can run alone and discovers its own parent objects/lists; it does not rely on another stream having run first. +All nine streams perform checkpointed full scan cycles. Every stream runs independently; a new cycle discovers its own parent objects/lists. After interruption, parent-scoped streams keep the frozen parent set and resume after the last completely loaded parent. The unfinished parent restarts at offset zero because offsets into mutable collections are not durable change cursors. | Stream tag | Raw table | Required Attio scopes | | --- | --- | --- | @@ -54,7 +54,7 @@ API references: [attributes](https://docs.attio.com/rest-api/endpoint-reference/ ## Customize the source -Each file in `sources/` contains one resource's raw table and reader, such as `sources/notes.ts`, `sources/tasks.ts`, or `sources/members.ts`. `sources/records.ts` also defines the people, company, and deal views. `client.ts` shares HTTP and pagination logic; `pipeline.ts` composes the nine streams. +Each file in `sources/` contains one resource's raw table and reader, such as `sources/notes.ts`, `sources/tasks.ts`, or `sources/members.ts`. `sources/records.ts` also defines the people, company, and deal views. `client.ts` shares HTTP and pagination logic; `checkpoints.ts` validates scan state and selection scope; `pipeline.ts` composes the nine streams. Remove an entry from `pipeline.ts` to stop collecting that resource. Keep its schema export in `index.ts` to preserve already collected data. Removing schema exports can generate destructive migration operations, which should be reviewed. After removing a resource's stream and schema export, its local files can also be deleted. @@ -89,7 +89,9 @@ FROM default.attio_records_raw FINAL; All include the stable row `id`, `source_id`, `workspace_id`, `object_id`, `record_id`, nullable `created_at`, `web_url`, and `_chkit_ingested_at`. Missing optional strings yield empty strings, absent arrays yield empty arrays, and absent/unparseable dates or deal amounts yield null. A workspace without deals simply has an empty deals view. Edit these SQL projections to expose your custom fields, then generate and review a migration. The original payload remains available for further projections. -These views show the latest **observed** version per ID, not a guaranteed snapshot of Attio now. Full reads update returned IDs without clearing tables. Deleted, merged, inaccessible, or deselected entities remain until you implement reconciliation. Data is published batch by batch, so a failed run may have loaded some updates. Offset pagination over changing source data can miss or duplicate entities within a scan. There is no universal modification cursor, atomic snapshot replacement, source-version ordering, or complete change history in this template. +The journal stores scan cycles, selected parent IDs/slugs, completed parent IDs, and terminal completion. Parent-completion markers flush all preceding rows before advancing state. Workspace-wide notes/tasks/members restart an interrupted collection at offset zero; a completed cycle starts a fresh full read on the next run. New parents discovered during an interrupted cycle wait until that next cycle. An empty scan still records completion. Changing the object/list selection rejects the saved scope; restore it or use a new source identity for a deliberate fresh scan. A frozen parent that disappears or loses access fails visibly instead of being marked complete. Returned child workspace and parent IDs must match the frozen discovery identity; switching account tokens cannot publish rows from a different workspace under that frontier. Existing installations without a checkpoint start their first checkpointed cycle normally. + +These views show the latest **observed** version per ID, not a guaranteed snapshot of Attio now. Full reads update returned IDs without clearing tables. Deleted, merged, inaccessible, or deselected entities remain until you implement reconciliation. Data is published batch by batch, so a failed run may have loaded some updates. Offset pagination over changing source data can miss or duplicate entities within a scan. There is no universal modification cursor, atomic snapshot replacement, source-version ordering, or complete change history in this template. Checkpoints reduce repeated parent work; they do not turn offset scans into provider snapshots. No data-bearing chunk uses a page offset as its identity. ## Fixture tests @@ -99,7 +101,7 @@ Install the optional test suite with `bunx chkit add attio --with-tests`, then r bun test src/integrations/attio/tests/attio.test.ts ``` -For a custom install path, point the command at that folder's `tests/attio.test.ts`. Tests mock Attio HTTP responses and use in-memory ingestion destinations and journals. They require Bun but no Attio credentials or ClickHouse server. The fixture suite checks resource coverage, pagination, preserved custom fields, repeat runs, cancellation, and error handling. Adjust the fixtures and expectations when customizing the source. +For a custom install path, point the command at that folder's `tests/attio.test.ts`. Tests mock Attio HTTP responses and use in-memory ingestion destinations and journals. They require Bun but no Attio credentials or ClickHouse server. The fixture suite checks resource coverage, pagination, preserved custom fields, repeat runs, cancellation, errors, serialized checkpoint resumption, frozen parents, safe restart of unfinished offsets, empty completion, and rows loaded before a journal failure. Adjust the fixtures and expectations when customizing the source. ## Smoke check diff --git a/registry/attio/checkpoints.ts b/registry/attio/checkpoints.ts new file mode 100644 index 00000000..28b06d79 --- /dev/null +++ b/registry/attio/checkpoints.ts @@ -0,0 +1,60 @@ +import { cursorState, IngestConfigError, type ReadContext } from '@chkit/plugin-ingest' + +import { attioConfig } from './config.js' + +export interface ScanParent { + workspaceId: string + id: string + slug: string +} + +export interface AttioScanState { + scope: string + cycle: number + phase: 'scanning' | 'complete' + parents: readonly ScanParent[] + completedParents: readonly string[] +} + +export type AttioReadContext = ReadContext + +/** Full scans have completion checkpoints, never an inferred modification watermark. */ +export function scanCheckpoint(resource: string) { + return cursorState({ id: `attio.scan.${resource}`, version: 1, parse: parseScanState }) +} + +export function beginScan(context: AttioReadContext, resource: string): AttioScanState { + const parents = ['objects', 'object_attributes', 'records'].includes(resource) ? attioConfig.objects + : ['lists', 'list_attributes', 'entries'].includes(resource) ? attioConfig.lists : undefined + const scope = JSON.stringify([attioConfig.sourceId, resource, parents === undefined ? null : [...new Set(parents)].sort()]) + if (context.state && context.state.scope !== scope) { + throw new IngestConfigError('Attio checkpoint scope changed. Restore the parent selection or use a new sourceId for a deliberate fresh scan.') + } + if (context.state?.phase === 'scanning') return context.state + const cycle = (context.state?.cycle ?? 0) + 1 + if (!Number.isSafeInteger(cycle)) throw new IngestConfigError('Attio scan cycle exceeded the safe integer range.') + return { scope, cycle, phase: 'scanning', parents: [], completedParents: [] } +} + +function parseScanState(raw: unknown): AttioScanState { + if (!isObject(raw) || typeof raw.scope !== 'string' || typeof raw.cycle !== 'number' + || !Number.isSafeInteger(raw.cycle) || raw.cycle < 1 || (raw.phase !== 'scanning' && raw.phase !== 'complete') + || !Array.isArray(raw.parents) || !raw.parents.every(isParent) + || !Array.isArray(raw.completedParents) || !raw.completedParents.every((id): id is string => typeof id === 'string')) { + throw new IngestConfigError('Invalid Attio scan checkpoint.') + } + const ids = raw.parents.map((parent) => parent.id) + if (new Set(ids).size !== ids.length || new Set(raw.completedParents).size !== raw.completedParents.length + || raw.completedParents.some((id) => !ids.includes(id))) { + throw new IngestConfigError('Invalid Attio scan checkpoint parent frontier.') + } + return { scope: raw.scope, cycle: raw.cycle, phase: raw.phase, parents: raw.parents, completedParents: raw.completedParents } +} + +function isParent(value: unknown): value is ScanParent { + return isObject(value) && [value.workspaceId, value.id, value.slug].every((part) => typeof part === 'string' && part.length > 0) +} + +function isObject(value: unknown): value is Record { + return typeof value === 'object' && value !== null && !Array.isArray(value) +} diff --git a/registry/attio/client.ts b/registry/attio/client.ts index f704936b..d6396425 100644 --- a/registry/attio/client.ts +++ b/registry/attio/client.ts @@ -3,6 +3,7 @@ import { setTimeout as sleep } from 'node:timers/promises' import { HttpError, IngestConfigError, rawRows, type ErrorClassifier, type FetchContext } from '@chkit/plugin-ingest' import { attioConfig } from './config.js' +import { beginScan, type AttioReadContext, type ScanParent } from './checkpoints.js' export type AttioEntity = Record & { id: Record } @@ -21,6 +22,7 @@ interface CollectionRequest { /** Omit for the unpaginated objects, lists, and workspace_members endpoints. */ pageSize?: number valuesField?: 'values' | 'entry_values' + parent?: { field: 'object_id' | 'list_id'; id: string; workspaceId: string } } const defaultDeps: AttioClientDeps = { @@ -29,6 +31,59 @@ const defaultDeps: AttioClientDeps = { wait: async (milliseconds, signal) => { await sleep(milliseconds, undefined, { signal }) }, } +/** Offset pages are mutable: resume only after a completely loaded parent. */ +export async function* readParentCollections( + context: AttioReadContext, + input: { + resource: string + parents: 'objects' | 'lists' + idFields: readonly string[] + request: (parent: ScanParent) => CollectionRequest + }, + deps?: AttioClientDeps, +) { + let state = beginScan(context, input.resource) + if (context.state?.phase !== 'scanning') { + const parents = input.parents === 'objects' ? await discoverObjects(context, deps) : await discoverLists(context, deps) + state = { ...state, parents: parents.map((parent) => ({ + workspaceId: entityId(parent, 'workspace_id'), + id: entityId(parent, input.parents === 'objects' ? 'object_id' : 'list_id'), + slug: entitySlug(parent), + })) } + yield { rows: [], state, id: `scan:${state.cycle}:start` } + } + for (const parent of state.parents) { + if (state.completedParents.includes(parent.id)) continue + const request = { ...input.request(parent), parent: { + field: input.parents === 'objects' || input.resource === 'list_attributes' ? 'object_id' as const : 'list_id' as const, + id: parent.id, workspaceId: parent.workspaceId, + } } + for await (const page of readCollection(context, request, deps)) { + yield { rows: toAttioRows(page, input.resource, input.idFields, + input.parents === 'objects' ? { object_slug: parent.slug } : { list_slug: parent.slug }) } + } + state = { ...state, completedParents: [...state.completedParents, parent.id] } + // This empty marker flushes preceding rows before publishing the parent frontier. + yield { rows: [], state, id: `scan:${state.cycle}:parent:${parent.id}` } + } + yield { rows: [], state: { ...state, phase: 'complete' as const }, id: `scan:${state.cycle}:complete` } +} + +/** Workspace collections restart offset zero after interruption and record completion. */ +export async function* readCollectionScan( + context: AttioReadContext, + resource: string, + request: CollectionRequest, + deps?: AttioClientDeps, +) { + const state = beginScan(context, resource) + if (context.state?.phase !== 'scanning') yield { rows: [], state, id: `scan:${state.cycle}:start` } + for await (const page of readCollection(context, request, deps)) { + yield { rows: toAttioRows(page, resource, request.idFields) } + } + yield { rows: [], state: { ...state, phase: 'complete' as const }, id: `scan:${state.cycle}:complete` } +} + /** Each page is validated inside attempt, so executor retry/cancellation owns every request. */ export async function* readCollection( context: FetchContext, @@ -155,6 +210,10 @@ function selectEntities(entities: AttioEntity[], selection: readonly string[] | function parseEntity(value: unknown, request: CollectionRequest): AttioEntity { if (!isEntity(value)) throw new IngestConfigError(`Attio ${request.path} returned an entity without a valid id object.`) for (const field of request.idFields) entityId(value, field) + if (request.parent && (entityId(value, 'workspace_id') !== request.parent.workspaceId + || entityId(value, request.parent.field) !== request.parent.id)) { + throw new IngestConfigError(`Attio ${request.path} returned an entity from a different workspace or parent. Restore the original account token before resuming this scan.`) + } if (request.valuesField !== undefined) { const values = value[request.valuesField] if (!isObject(values) || !Object.values(values).every(Array.isArray)) { diff --git a/registry/attio/manifest.json b/registry/attio/manifest.json index 6db63bfc..147fa290 100644 --- a/registry/attio/manifest.json +++ b/registry/attio/manifest.json @@ -3,10 +3,10 @@ "name": "attio", "type": "registry:item", "title": "Attio", - "description": "Editable Attio CRM schemas, raw data readers, typed views, and full-read ingestion pipelines.", + "description": "Editable Attio CRM schemas, raw data readers, typed views, and checkpointed full-scan ingestion pipelines.", "dependencies": [ - "@chkit/core@^0.2.0-beta.8", - "@chkit/plugin-ingest@^0.2.0-beta.8" + "@chkit/core@^0.2.0-beta.10", + "@chkit/plugin-ingest@^0.2.0-beta.10" ], "files": [ { @@ -19,6 +19,11 @@ "type": "registry:file", "target": "src/integrations/attio/client.ts" }, + { + "path": "checkpoints.ts", + "type": "registry:file", + "target": "src/integrations/attio/checkpoints.ts" + }, { "path": "config.ts", "type": "registry:file", @@ -95,13 +100,13 @@ "meta": { "chkit": { "formatVersion": 1, - "version": "0.1.2", + "version": "0.1.3", "documentation": "https://chkit.obsessiondb.com/integrations/attio/", "logo": "https://chkit.obsessiondb.com/logos/attio.svg", "language": "typescript", "license": "MIT", - "chkit": "^0.2.0-beta.8", - "ingest": "^0.2.0-beta.8", + "chkit": "^0.2.0-beta.10", + "ingest": "^0.2.0-beta.10", "clickhouse": ">=25.3.0", "root": "src/integrations/attio", "entry": "index.ts", @@ -316,7 +321,7 @@ } ], "sync": { - "description": "Every run discovers the selected objects and lists and reads every page of each enabled resource. Stable row IDs retain the latest observed version in ClickHouse.", + "description": "Full scan cycles retain the latest observed raw entities. Interrupted parent collections resume from the frozen object/list set after the last completely loaded parent; unfinished offset scans restart at zero. A completed cycle starts fresh discovery and full reads on the next run.", "schedule": "Run chkit ingest run through cron, CI, or an existing scheduler; the template does not start a background worker.", "deletions": "Records absent from a later API response remain in ClickHouse. The template does not propagate deletions or consume webhooks." } diff --git a/registry/attio/pipeline.ts b/registry/attio/pipeline.ts index 002a9c84..759805fa 100644 --- a/registry/attio/pipeline.ts +++ b/registry/attio/pipeline.ts @@ -2,6 +2,7 @@ import { definePipeline, defineStream } from '@chkit/plugin-ingest' import { classifyAttioError } from './client.js' import { attioConfig } from './config.js' +import { scanCheckpoint } from './checkpoints.js' import { attioObjectsRaw, readObjects } from './sources/objects.js' import { attioObjectAttributesRaw, readObjectAttributes } from './sources/object-attributes.js' import { attioRecordsRaw, readRecords } from './sources/records.js' @@ -24,14 +25,14 @@ export const attio = definePipeline({ maxLoads: 1, retry: { retries: 5, minTimeout: 1_000, maxTimeout: 30_000, randomize: true }, streams: [ - defineStream({ ...streamOptions, id: `${attioConfig.sourceId}.objects`, tags: ['resource:objects'], destination: attioObjectsRaw, read: readObjects }), - defineStream({ ...streamOptions, id: `${attioConfig.sourceId}.object_attributes`, tags: ['resource:object_attributes'], destination: attioObjectAttributesRaw, read: readObjectAttributes }), - defineStream({ ...streamOptions, id: `${attioConfig.sourceId}.records`, tags: ['resource:records'], destination: attioRecordsRaw, read: readRecords }), - defineStream({ ...streamOptions, id: `${attioConfig.sourceId}.lists`, tags: ['resource:lists'], destination: attioListsRaw, read: readLists }), - defineStream({ ...streamOptions, id: `${attioConfig.sourceId}.list_attributes`, tags: ['resource:list_attributes'], destination: attioListAttributesRaw, read: readListAttributes }), - defineStream({ ...streamOptions, id: `${attioConfig.sourceId}.entries`, tags: ['resource:entries'], destination: attioEntriesRaw, read: readEntries }), - defineStream({ ...streamOptions, id: `${attioConfig.sourceId}.notes`, tags: ['resource:notes'], destination: attioNotesRaw, read: readNotes }), - defineStream({ ...streamOptions, id: `${attioConfig.sourceId}.tasks`, tags: ['resource:tasks'], destination: attioTasksRaw, read: readTasks }), - defineStream({ ...streamOptions, id: `${attioConfig.sourceId}.members`, tags: ['resource:members'], destination: attioMembersRaw, read: readMembers }), + defineStream({ ...streamOptions, id: `${attioConfig.sourceId}.objects`, tags: ['resource:objects'], incremental: scanCheckpoint('objects'), destination: attioObjectsRaw, read: readObjects }), + defineStream({ ...streamOptions, id: `${attioConfig.sourceId}.object_attributes`, tags: ['resource:object_attributes'], incremental: scanCheckpoint('object_attributes'), destination: attioObjectAttributesRaw, read: readObjectAttributes }), + defineStream({ ...streamOptions, id: `${attioConfig.sourceId}.records`, tags: ['resource:records'], incremental: scanCheckpoint('records'), destination: attioRecordsRaw, read: readRecords }), + defineStream({ ...streamOptions, id: `${attioConfig.sourceId}.lists`, tags: ['resource:lists'], incremental: scanCheckpoint('lists'), destination: attioListsRaw, read: readLists }), + defineStream({ ...streamOptions, id: `${attioConfig.sourceId}.list_attributes`, tags: ['resource:list_attributes'], incremental: scanCheckpoint('list_attributes'), destination: attioListAttributesRaw, read: readListAttributes }), + defineStream({ ...streamOptions, id: `${attioConfig.sourceId}.entries`, tags: ['resource:entries'], incremental: scanCheckpoint('entries'), destination: attioEntriesRaw, read: readEntries }), + defineStream({ ...streamOptions, id: `${attioConfig.sourceId}.notes`, tags: ['resource:notes'], incremental: scanCheckpoint('notes'), destination: attioNotesRaw, read: readNotes }), + defineStream({ ...streamOptions, id: `${attioConfig.sourceId}.tasks`, tags: ['resource:tasks'], incremental: scanCheckpoint('tasks'), destination: attioTasksRaw, read: readTasks }), + defineStream({ ...streamOptions, id: `${attioConfig.sourceId}.members`, tags: ['resource:members'], incremental: scanCheckpoint('members'), destination: attioMembersRaw, read: readMembers }), ], }) diff --git a/registry/attio/releases/0.1.3.json b/registry/attio/releases/0.1.3.json new file mode 100644 index 00000000..83eaf82c --- /dev/null +++ b/registry/attio/releases/0.1.3.json @@ -0,0 +1,369 @@ +{ + "$schema": "https://ui.shadcn.com/schema/registry-item.json", + "name": "attio", + "type": "registry:item", + "title": "Attio", + "description": "Editable Attio CRM schemas, raw data readers, typed views, and checkpointed full-scan ingestion pipelines.", + "dependencies": [ + "@chkit/core@^0.2.0-beta.10", + "@chkit/plugin-ingest@^0.2.0-beta.10" + ], + "devDependencies": [ + "@types/bun@^1.2.0" + ], + "files": [ + { + "path": "attio/README.md", + "type": "registry:file", + "target": "src/integrations/attio/README.md", + "content": "# Attio CRM ingestion\n\nThis is your source code: edit the API readers, schemas, views, and `pipeline.ts` directly. It uses the [chkit CLI](https://www.npmjs.com/package/chkit), `@chkit/core`, and `@chkit/plugin-ingest`; there is no runtime connection to the template registry.\n\n## First run\n\nSet `ATTIO_API_TOKEN` in your execution environment and configure the project's direct ClickHouse connection. A single-workspace API token works; an OAuth access token also works, but this template does not implement OAuth issuance or refresh. See [Attio authentication](https://docs.attio.com/rest-api/guides/authentication).\n\n### Create an Attio API key\n\n1. As a workspace admin, open the workspace-name dropdown in Attio and select **Workspace settings**.\n2. Open **Developers**, select **+ New access token**, and name the token, for example **chkit ClickHouse**.\n3. Grant the read scopes in the resource table below for the streams to run.\n4. Click the token on the Developers page to copy it. Set `ATTIO_API_TOKEN` in the project's `.env` file or the scheduler's secret environment.\n\nSee Attio's [API key instructions](https://attio.com/help/reference/apps/generating-an-api-key). Use a single-workspace API token for this setup; OAuth token creation and refresh require a separate application flow.\n\nBun loads a project `.env` automatically. With Node or a scheduler, explicitly supply environment variables using that runtime's environment-loading mechanism. `.env.example` only lists required variable names; it does not load values. Schema imports, `check`, and `ingest list` do not read the token or make API requests.\n\nReview `config.ts` before the first migration. The default target database is `default`; raw table and view names begin with `attio_`. Use a stable, non-secret `sourceId` for this source instance. For another installation, give it a distinct source ID and table prefix or database before exporting its pipeline. Changing these after ingestion changes stream/storage identities and requires a deliberate migration.\n\n```sh\nbunx chkit check\nbunx chkit ingest list --tag provider:attio\nbunx chkit generate --name add_attio\n# Review the generated SQL.\nbunx chkit migrate --apply\nbunx chkit ingest run --tag provider:attio\n```\n\nThe schema uses native JSON (ClickHouse 25.3 or newer). A direct ClickHouse connection is required; workbench-only execution is not supported. Use an external scheduler to repeat the run command, with one ingestion process per project/target at a time.\n\n## Included resources\n\nAll nine streams perform checkpointed full scan cycles. Every stream runs independently; a new cycle discovers its own parent objects/lists. After interruption, parent-scoped streams keep the frozen parent set and resume after the last completely loaded parent. The unfinished parent restarts at offset zero because offsets into mutable collections are not durable change cursors.\n\n| Stream tag | Raw table | Required Attio scopes |\n| --- | --- | --- |\n| `resource:objects` | `attio_objects_raw` | `object_configuration:read` |\n| `resource:object_attributes` | `attio_object_attributes_raw` | `object_configuration:read` |\n| `resource:records` | `attio_records_raw` | `object_configuration:read`, `record_permission:read` |\n| `resource:lists` | `attio_lists_raw` | `list_configuration:read` |\n| `resource:list_attributes` | `attio_list_attributes_raw` | `list_configuration:read` |\n| `resource:entries` | `attio_entries_raw` | `list_configuration:read`, `list_entry:read` |\n| `resource:notes` | `attio_notes_raw` | `note:read`, `object_configuration:read`, `record_permission:read` |\n| `resource:tasks` | `attio_tasks_raw` | `task:read`, `object_configuration:read`, `record_permission:read`, `user_management:read` |\n| `resource:members` | `attio_members_raw` | `user_management:read` |\n\nRecords include all accessible standard and custom objects returned by the [objects API](https://docs.attio.com/rest-api/endpoint-reference/objects/list-objects). Standard `users` records are distinct from [workspace members](https://docs.attio.com/rest-api/endpoint-reference/workspace-members/list-workspace-members). Attribute streams include archived attribute definitions; separate select-option/status catalogs and historical attribute values are not fetched. Active record/entry payloads retain their returned select/status values.\n\nThe template does not collect meetings, call recordings, transcripts, emails, files, comments, or historical attribute values. It does not receive webhooks or detect source deletions/merges. Coverage is limited to data the token can access. A permission error fails that stream with a diagnostic; it is never treated as an empty dataset.\n\nAPI references: [attributes](https://docs.attio.com/rest-api/endpoint-reference/attributes/list-attributes), [records](https://docs.attio.com/rest-api/endpoint-reference/records/list-records), [lists](https://docs.attio.com/rest-api/endpoint-reference/lists/list-all-lists), [entries](https://docs.attio.com/rest-api/endpoint-reference/entries/list-entries), [notes](https://docs.attio.com/rest-api/endpoint-reference/notes/list-notes), [tasks](https://docs.attio.com/rest-api/endpoint-reference/tasks/list-tasks).\n\n## Customize the source\n\nEach file in `sources/` contains one resource's raw table and reader, such as `sources/notes.ts`, `sources/tasks.ts`, or `sources/members.ts`. `sources/records.ts` also defines the people, company, and deal views. `client.ts` shares HTTP and pagination logic; `checkpoints.ts` validates scan state and selection scope; `pipeline.ts` composes the nine streams.\n\nRemove an entry from `pipeline.ts` to stop collecting that resource. Keep its schema export in `index.ts` to preserve already collected data. Removing schema exports can generate destructive migration operations, which should be reviewed. After removing a resource's stream and schema export, its local files can also be deleted.\n\nSet `objects` and `lists` in `config.ts` to arrays of API slugs or IDs to restrict discovery. `undefined` means all accessible parents; `[]` means none. Unknown configured parents fail visibly. Object selection affects objects, object attributes, and records. List selection affects lists, list attributes, and entries. Notes, tasks, and members remain workspace-wide; customize their readers separately if narrower scope is needed.\n\nTo run only records:\n\n```sh\nbunx chkit ingest run --tag provider:attio --tag resource:records\n```\n\n`pageSize` defaults to 500. Notes have a separate `notesPageSize` of 50, the documented maximum. Requests go through the ingestion executor's retry/cancellation context, with a 30-second request timeout. The serial pipeline leaves a 25ms gap before each request and a 125ms gap for notes; raising concurrency requires revisiting rate control. `429` responses retain `Retry-After` for executor backoff. Attio also limits query complexity; a permanently expensive query may need simpler filters/sorts rather than repeated retries. See [rate limits](https://docs.attio.com/rest-api/guides/rate-limiting).\n\n## Stored data and editable views\n\nEach raw table contains a stable `id`, native JSON `raw`, and chkit batch/run/ingestion-time metadata. The key includes source identity, resource kind, workspace ID, and every relevant object/list/entity ID. Names, emails, and domains are never keys.\n\nThe complete returned entity is retained under `raw.data`. The envelope adds `source_id`; record and object-attribute rows also include `object_slug`, and list-entry/list-attribute rows include `list_slug`. No custom attributes or multivalued arrays are flattened away. For example:\n\n```sql\nSELECT id, raw.data, _chkit_ingested_at\nFROM default.attio_records_raw FINAL;\n```\n\n`sources/records.ts` defines three ordinary `FINAL` views:\n\n| View | Editable projection |\n| --- | --- |\n| `attio_people` | name, email address array, company record ID array |\n| `attio_companies` | name, domain array, description |\n| `attio_deals` | name, stage, nullable numeric value, currency code, company/people record ID arrays |\n\nAll include the stable row `id`, `source_id`, `workspace_id`, `object_id`, `record_id`, nullable `created_at`, `web_url`, and `_chkit_ingested_at`. Missing optional strings yield empty strings, absent arrays yield empty arrays, and absent/unparseable dates or deal amounts yield null. A workspace without deals simply has an empty deals view. Edit these SQL projections to expose your custom fields, then generate and review a migration. The original payload remains available for further projections.\n\nThe journal stores scan cycles, selected parent IDs/slugs, completed parent IDs, and terminal completion. Parent-completion markers flush all preceding rows before advancing state. Workspace-wide notes/tasks/members restart an interrupted collection at offset zero; a completed cycle starts a fresh full read on the next run. New parents discovered during an interrupted cycle wait until that next cycle. An empty scan still records completion. Changing the object/list selection rejects the saved scope; restore it or use a new source identity for a deliberate fresh scan. A frozen parent that disappears or loses access fails visibly instead of being marked complete. Returned child workspace and parent IDs must match the frozen discovery identity; switching account tokens cannot publish rows from a different workspace under that frontier. Existing installations without a checkpoint start their first checkpointed cycle normally.\n\nThese views show the latest **observed** version per ID, not a guaranteed snapshot of Attio now. Full reads update returned IDs without clearing tables. Deleted, merged, inaccessible, or deselected entities remain until you implement reconciliation. Data is published batch by batch, so a failed run may have loaded some updates. Offset pagination over changing source data can miss or duplicate entities within a scan. There is no universal modification cursor, atomic snapshot replacement, source-version ordering, or complete change history in this template. Checkpoints reduce repeated parent work; they do not turn offset scans into provider snapshots. No data-bearing chunk uses a page offset as its identity.\n\n## Fixture tests\n\nInstall the optional test suite with `bunx chkit add attio --with-tests`, then run:\n\n```sh\nbun test src/integrations/attio/tests/attio.test.ts\n```\n\nFor a custom install path, point the command at that folder's `tests/attio.test.ts`. Tests mock Attio HTTP responses and use in-memory ingestion destinations and journals. They require Bun but no Attio credentials or ClickHouse server. The fixture suite checks resource coverage, pagination, preserved custom fields, repeat runs, cancellation, errors, serialized checkpoint resumption, frozen parents, safe restart of unfinished offsets, empty completion, and rows loaded before a journal failure. Adjust the fixtures and expectations when customizing the source.\n\n## Smoke check\n\nAfter a migration and one successful run, inspect `chkit ingest status --tag provider:attio` and query `SELECT count() FROM default.attio_records_raw FINAL`. Compare a known record's retained `raw.data` with its Attio API response. Run again after changing that record and verify its view reflects the new observation. This smoke check needs a real test workspace and database; the registry's sanitized fixture tests do not assert complete live API coverage.\n" + }, + { + "path": "attio/client.ts", + "type": "registry:file", + "target": "src/integrations/attio/client.ts", + "content": "import { setTimeout as sleep } from 'node:timers/promises'\n\nimport { HttpError, IngestConfigError, rawRows, type ErrorClassifier, type FetchContext } from '@chkit/plugin-ingest'\n\nimport { attioConfig } from './config.js'\nimport { beginScan, type AttioReadContext, type ScanParent } from './checkpoints.js'\n\nexport type AttioEntity = Record & { id: Record }\n\nexport interface AttioClientDeps {\n fetch: (url: string, init: RequestInit) => Promise\n token: () => string | undefined\n wait: (milliseconds: number, signal: AbortSignal) => Promise\n}\n\ninterface CollectionRequest {\n path: string\n idFields: readonly string[]\n method?: 'GET' | 'POST'\n body?: Record\n query?: Record\n /** Omit for the unpaginated objects, lists, and workspace_members endpoints. */\n pageSize?: number\n valuesField?: 'values' | 'entry_values'\n parent?: { field: 'object_id' | 'list_id'; id: string; workspaceId: string }\n}\n\nconst defaultDeps: AttioClientDeps = {\n fetch: (url, init) => fetch(url, init),\n token: () => process.env.ATTIO_API_TOKEN,\n wait: async (milliseconds, signal) => { await sleep(milliseconds, undefined, { signal }) },\n}\n\n/** Offset pages are mutable: resume only after a completely loaded parent. */\nexport async function* readParentCollections(\n context: AttioReadContext,\n input: {\n resource: string\n parents: 'objects' | 'lists'\n idFields: readonly string[]\n request: (parent: ScanParent) => CollectionRequest\n },\n deps?: AttioClientDeps,\n) {\n let state = beginScan(context, input.resource)\n if (context.state?.phase !== 'scanning') {\n const parents = input.parents === 'objects' ? await discoverObjects(context, deps) : await discoverLists(context, deps)\n state = { ...state, parents: parents.map((parent) => ({\n workspaceId: entityId(parent, 'workspace_id'),\n id: entityId(parent, input.parents === 'objects' ? 'object_id' : 'list_id'),\n slug: entitySlug(parent),\n })) }\n yield { rows: [], state, id: `scan:${state.cycle}:start` }\n }\n for (const parent of state.parents) {\n if (state.completedParents.includes(parent.id)) continue\n const request = { ...input.request(parent), parent: {\n field: input.parents === 'objects' || input.resource === 'list_attributes' ? 'object_id' as const : 'list_id' as const,\n id: parent.id, workspaceId: parent.workspaceId,\n } }\n for await (const page of readCollection(context, request, deps)) {\n yield { rows: toAttioRows(page, input.resource, input.idFields,\n input.parents === 'objects' ? { object_slug: parent.slug } : { list_slug: parent.slug }) }\n }\n state = { ...state, completedParents: [...state.completedParents, parent.id] }\n // This empty marker flushes preceding rows before publishing the parent frontier.\n yield { rows: [], state, id: `scan:${state.cycle}:parent:${parent.id}` }\n }\n yield { rows: [], state: { ...state, phase: 'complete' as const }, id: `scan:${state.cycle}:complete` }\n}\n\n/** Workspace collections restart offset zero after interruption and record completion. */\nexport async function* readCollectionScan(\n context: AttioReadContext,\n resource: string,\n request: CollectionRequest,\n deps?: AttioClientDeps,\n) {\n const state = beginScan(context, resource)\n if (context.state?.phase !== 'scanning') yield { rows: [], state, id: `scan:${state.cycle}:start` }\n for await (const page of readCollection(context, request, deps)) {\n yield { rows: toAttioRows(page, resource, request.idFields) }\n }\n yield { rows: [], state: { ...state, phase: 'complete' as const }, id: `scan:${state.cycle}:complete` }\n}\n\n/** Each page is validated inside attempt, so executor retry/cancellation owns every request. */\nexport async function* readCollection(\n context: FetchContext,\n request: CollectionRequest,\n deps: AttioClientDeps = defaultDeps,\n): AsyncGenerator {\n if (request.pageSize !== undefined && (!Number.isSafeInteger(request.pageSize) || request.pageSize <= 0)) {\n throw new IngestConfigError('Attio pageSize must be a positive safe integer.')\n }\n let offset = 0\n while (true) {\n context.signal.throwIfAborted()\n const currentOffset = offset\n const page = await context.attempt(\n (signal) => requestPage(request, currentOffset, signal, deps),\n { label: `${request.method ?? 'GET'} /v2${request.path}` },\n )\n if (page.length > 0) yield page\n if (request.pageSize === undefined || page.length < request.pageSize) return\n offset += page.length\n if (!Number.isSafeInteger(offset)) throw new IngestConfigError('Attio pagination offset exceeded the safe integer range.')\n }\n}\n\nexport async function discoverObjects(context: FetchContext, deps?: AttioClientDeps): Promise {\n for await (const objects of readCollection(context, { path: '/objects', idFields: ['workspace_id', 'object_id'] }, deps)) {\n return selectEntities(objects, attioConfig.objects, 'object_id')\n }\n return selectEntities([], attioConfig.objects, 'object_id')\n}\n\nexport async function discoverLists(context: FetchContext, deps?: AttioClientDeps): Promise {\n for await (const lists of readCollection(context, { path: '/lists', idFields: ['workspace_id', 'list_id'] }, deps)) {\n return selectEntities(lists, attioConfig.lists, 'list_id')\n }\n return selectEntities([], attioConfig.lists, 'list_id')\n}\n\n/** Original provider fields stay under data; metadata does not overwrite custom attributes. */\nexport function toAttioRows(\n items: readonly AttioEntity[],\n resource: string,\n idFields: readonly string[],\n metadata: { object_slug?: string; list_slug?: string } = {},\n) {\n return rawRows(\n items.map((data) => ({ source_id: attioConfig.sourceId, ...metadata, data })),\n ({ data }) => JSON.stringify([attioConfig.sourceId, resource, ...idFields.map((field) => entityId(data, field))]),\n )\n}\n\nexport function entityId(entity: AttioEntity, field: string): string {\n const value = entity.id[field]\n if (!value) throw new IngestConfigError(`Attio entity is missing id.${field}.`)\n return value\n}\n\nexport function entitySlug(entity: AttioEntity): string {\n if (typeof entity.api_slug !== 'string' || entity.api_slug.length === 0) {\n throw new IngestConfigError('Attio object/list is missing api_slug.')\n }\n return entity.api_slug\n}\n\n/** Invalid successful responses/configurations cannot be fixed by retrying. */\nexport const classifyAttioError: ErrorClassifier = (cause) => {\n if (cause instanceof IngestConfigError || cause instanceof SyntaxError) return { kind: 'permanent' }\n // HttpError preserves 429 Retry-After dates, 5xx retries, and permanent 4xx semantics.\n return undefined\n}\n\nasync function requestPage(\n request: CollectionRequest,\n offset: number,\n signal: AbortSignal,\n deps: AttioClientDeps,\n): Promise {\n const token = deps.token()?.trim()\n if (!token) throw new IngestConfigError('Set ATTIO_API_TOKEN before running Attio ingestion. Schema imports do not need credentials.')\n const url = new URL(`https://api.attio.com/v2${request.path}`)\n for (const [key, value] of Object.entries(request.query ?? {})) url.searchParams.set(key, value)\n const method = request.method ?? 'GET'\n const pagination = request.pageSize === undefined ? {} : { limit: request.pageSize, offset }\n if (method === 'GET') {\n for (const [key, value] of Object.entries(pagination)) url.searchParams.set(key, String(value))\n }\n // The default pipeline runs one request at a time: <=40 reads/s and <=8 notes/s.\n // Attio may lower limits and applies query-complexity limits; 429s still use executor backoff.\n await deps.wait(request.path === '/notes' ? 125 : 25, signal)\n signal.throwIfAborted()\n const response = await deps.fetch(url.toString(), {\n method,\n headers: { Authorization: `Bearer ${token}`, Accept: 'application/json', 'Content-Type': 'application/json' },\n ...(method === 'POST' ? { body: JSON.stringify({ ...request.body, ...pagination }) } : {}),\n signal: AbortSignal.any([signal, AbortSignal.timeout(30_000)]),\n redirect: 'error',\n })\n if (!response.ok) {\n const error = await HttpError.fromResponse(response)\n if (response.status === 401 || response.status === 403) {\n error.message += ' Check ATTIO_API_TOKEN and the resource scopes in the Attio README, or disable this stream.'\n }\n throw error\n }\n const payload: unknown = await response.json()\n if (!isObject(payload) || !Array.isArray(payload.data)) {\n throw new IngestConfigError(`Attio ${request.path} returned no data array.`)\n }\n return payload.data.map((value: unknown) => parseEntity(value, request))\n}\n\nfunction selectEntities(entities: AttioEntity[], selection: readonly string[] | undefined, idField: string): AttioEntity[] {\n // Validate slugs even when all resources are selected: records/views rely on this metadata.\n for (const entity of entities) entitySlug(entity)\n if (selection === undefined) return entities\n for (const wanted of selection) {\n if (!entities.some((entity) => entityId(entity, idField) === wanted || entitySlug(entity) === wanted)) {\n throw new IngestConfigError(`Configured Attio ${idField} or slug \"${wanted}\" was not found or is not accessible.`)\n }\n }\n return entities.filter((entity) => selection.includes(entityId(entity, idField)) || selection.includes(entitySlug(entity)))\n}\n\nfunction parseEntity(value: unknown, request: CollectionRequest): AttioEntity {\n if (!isEntity(value)) throw new IngestConfigError(`Attio ${request.path} returned an entity without a valid id object.`)\n for (const field of request.idFields) entityId(value, field)\n if (request.parent && (entityId(value, 'workspace_id') !== request.parent.workspaceId\n || entityId(value, request.parent.field) !== request.parent.id)) {\n throw new IngestConfigError(`Attio ${request.path} returned an entity from a different workspace or parent. Restore the original account token before resuming this scan.`)\n }\n if (request.valuesField !== undefined) {\n const values = value[request.valuesField]\n if (!isObject(values) || !Object.values(values).every(Array.isArray)) {\n throw new IngestConfigError(`Attio ${request.path} returned invalid ${request.valuesField}; expected arrays of attribute values.`)\n }\n }\n return value\n}\n\nfunction isEntity(value: unknown): value is AttioEntity {\n return isObject(value) && isObject(value.id) && Object.values(value.id).every((part) => typeof part === 'string')\n}\n\nfunction isObject(value: unknown): value is Record {\n return typeof value === 'object' && value !== null && !Array.isArray(value)\n}\n" + }, + { + "path": "attio/checkpoints.ts", + "type": "registry:file", + "target": "src/integrations/attio/checkpoints.ts", + "content": "import { cursorState, IngestConfigError, type ReadContext } from '@chkit/plugin-ingest'\n\nimport { attioConfig } from './config.js'\n\nexport interface ScanParent {\n workspaceId: string\n id: string\n slug: string\n}\n\nexport interface AttioScanState {\n scope: string\n cycle: number\n phase: 'scanning' | 'complete'\n parents: readonly ScanParent[]\n completedParents: readonly string[]\n}\n\nexport type AttioReadContext = ReadContext\n\n/** Full scans have completion checkpoints, never an inferred modification watermark. */\nexport function scanCheckpoint(resource: string) {\n return cursorState({ id: `attio.scan.${resource}`, version: 1, parse: parseScanState })\n}\n\nexport function beginScan(context: AttioReadContext, resource: string): AttioScanState {\n const parents = ['objects', 'object_attributes', 'records'].includes(resource) ? attioConfig.objects\n : ['lists', 'list_attributes', 'entries'].includes(resource) ? attioConfig.lists : undefined\n const scope = JSON.stringify([attioConfig.sourceId, resource, parents === undefined ? null : [...new Set(parents)].sort()])\n if (context.state && context.state.scope !== scope) {\n throw new IngestConfigError('Attio checkpoint scope changed. Restore the parent selection or use a new sourceId for a deliberate fresh scan.')\n }\n if (context.state?.phase === 'scanning') return context.state\n const cycle = (context.state?.cycle ?? 0) + 1\n if (!Number.isSafeInteger(cycle)) throw new IngestConfigError('Attio scan cycle exceeded the safe integer range.')\n return { scope, cycle, phase: 'scanning', parents: [], completedParents: [] }\n}\n\nfunction parseScanState(raw: unknown): AttioScanState {\n if (!isObject(raw) || typeof raw.scope !== 'string' || typeof raw.cycle !== 'number'\n || !Number.isSafeInteger(raw.cycle) || raw.cycle < 1 || (raw.phase !== 'scanning' && raw.phase !== 'complete')\n || !Array.isArray(raw.parents) || !raw.parents.every(isParent)\n || !Array.isArray(raw.completedParents) || !raw.completedParents.every((id): id is string => typeof id === 'string')) {\n throw new IngestConfigError('Invalid Attio scan checkpoint.')\n }\n const ids = raw.parents.map((parent) => parent.id)\n if (new Set(ids).size !== ids.length || new Set(raw.completedParents).size !== raw.completedParents.length\n || raw.completedParents.some((id) => !ids.includes(id))) {\n throw new IngestConfigError('Invalid Attio scan checkpoint parent frontier.')\n }\n return { scope: raw.scope, cycle: raw.cycle, phase: raw.phase, parents: raw.parents, completedParents: raw.completedParents }\n}\n\nfunction isParent(value: unknown): value is ScanParent {\n return isObject(value) && [value.workspaceId, value.id, value.slug].every((part) => typeof part === 'string' && part.length > 0)\n}\n\nfunction isObject(value: unknown): value is Record {\n return typeof value === 'object' && value !== null && !Array.isArray(value)\n}\n" + }, + { + "path": "attio/config.ts", + "type": "registry:file", + "target": "src/integrations/attio/config.ts", + "content": "export interface AttioConfig {\n /** Stable, non-secret name for this installation. Changing it creates new stream identities. */\n sourceId: string\n database: string\n tablePrefix: string\n /** Undefined reads every accessible object/list; otherwise use UUIDs or API slugs. */\n objects: readonly string[] | undefined\n lists: readonly string[] | undefined\n pageSize: number\n notesPageSize: number\n}\n\n// Keep these values stable across runs. Credentials are read only in client.ts at request time.\nexport const attioConfig: AttioConfig = {\n sourceId: 'attio.primary',\n database: 'default',\n tablePrefix: 'attio',\n objects: undefined,\n lists: undefined,\n pageSize: 500,\n notesPageSize: 50,\n}\n" + }, + { + "path": "attio/index.ts", + "type": "registry:file", + "target": "src/integrations/attio/index.ts", + "content": "export { attio } from './pipeline.js'\nexport { attioObjectsRaw } from './sources/objects.js'\nexport { attioObjectAttributesRaw } from './sources/object-attributes.js'\nexport { attioRecordsRaw, attioPeople, attioCompanies, attioDeals } from './sources/records.js'\nexport { attioListsRaw } from './sources/lists.js'\nexport { attioListAttributesRaw } from './sources/list-attributes.js'\nexport { attioEntriesRaw } from './sources/entries.js'\nexport { attioNotesRaw } from './sources/notes.js'\nexport { attioTasksRaw } from './sources/tasks.js'\nexport { attioMembersRaw } from './sources/members.js'\n" + }, + { + "path": "attio/pipeline.ts", + "type": "registry:file", + "target": "src/integrations/attio/pipeline.ts", + "content": "import { definePipeline, defineStream } from '@chkit/plugin-ingest'\n\nimport { classifyAttioError } from './client.js'\nimport { attioConfig } from './config.js'\nimport { scanCheckpoint } from './checkpoints.js'\nimport { attioObjectsRaw, readObjects } from './sources/objects.js'\nimport { attioObjectAttributesRaw, readObjectAttributes } from './sources/object-attributes.js'\nimport { attioRecordsRaw, readRecords } from './sources/records.js'\nimport { attioListsRaw, readLists } from './sources/lists.js'\nimport { attioListAttributesRaw, readListAttributes } from './sources/list-attributes.js'\nimport { attioEntriesRaw, readEntries } from './sources/entries.js'\nimport { attioNotesRaw, readNotes } from './sources/notes.js'\nimport { attioTasksRaw, readTasks } from './sources/tasks.js'\nimport { attioMembersRaw, readMembers } from './sources/members.js'\n\nconst streamOptions = { classifyError: classifyAttioError, batchSize: 500 }\n\n// Delete a stream entry to stop collecting it. Keep its schema export to retain existing data.\n// Every reader discovers its own parents; this array does not establish execution dependencies.\nexport const attio = definePipeline({\n id: attioConfig.sourceId,\n tags: ['provider:attio'],\n maxStreams: 1,\n maxFetches: 1,\n maxLoads: 1,\n retry: { retries: 5, minTimeout: 1_000, maxTimeout: 30_000, randomize: true },\n streams: [\n defineStream({ ...streamOptions, id: `${attioConfig.sourceId}.objects`, tags: ['resource:objects'], incremental: scanCheckpoint('objects'), destination: attioObjectsRaw, read: readObjects }),\n defineStream({ ...streamOptions, id: `${attioConfig.sourceId}.object_attributes`, tags: ['resource:object_attributes'], incremental: scanCheckpoint('object_attributes'), destination: attioObjectAttributesRaw, read: readObjectAttributes }),\n defineStream({ ...streamOptions, id: `${attioConfig.sourceId}.records`, tags: ['resource:records'], incremental: scanCheckpoint('records'), destination: attioRecordsRaw, read: readRecords }),\n defineStream({ ...streamOptions, id: `${attioConfig.sourceId}.lists`, tags: ['resource:lists'], incremental: scanCheckpoint('lists'), destination: attioListsRaw, read: readLists }),\n defineStream({ ...streamOptions, id: `${attioConfig.sourceId}.list_attributes`, tags: ['resource:list_attributes'], incremental: scanCheckpoint('list_attributes'), destination: attioListAttributesRaw, read: readListAttributes }),\n defineStream({ ...streamOptions, id: `${attioConfig.sourceId}.entries`, tags: ['resource:entries'], incremental: scanCheckpoint('entries'), destination: attioEntriesRaw, read: readEntries }),\n defineStream({ ...streamOptions, id: `${attioConfig.sourceId}.notes`, tags: ['resource:notes'], incremental: scanCheckpoint('notes'), destination: attioNotesRaw, read: readNotes }),\n defineStream({ ...streamOptions, id: `${attioConfig.sourceId}.tasks`, tags: ['resource:tasks'], incremental: scanCheckpoint('tasks'), destination: attioTasksRaw, read: readTasks }),\n defineStream({ ...streamOptions, id: `${attioConfig.sourceId}.members`, tags: ['resource:members'], incremental: scanCheckpoint('members'), destination: attioMembersRaw, read: readMembers }),\n ],\n})\n" + }, + { + "path": "attio/sources/objects.ts", + "type": "registry:file", + "target": "src/integrations/attio/sources/objects.ts", + "content": "import { rawTable } from '@chkit/plugin-ingest'\n\nimport { discoverObjects, toAttioRows, type AttioClientDeps } from '../client.js'\nimport { attioConfig } from '../config.js'\nimport { beginScan, type AttioReadContext } from '../checkpoints.js'\n\nexport const attioObjectsRaw = rawTable({ database: attioConfig.database, name: `${attioConfig.tablePrefix}_objects_raw` })\n\nexport async function* readObjects(context: AttioReadContext, deps?: AttioClientDeps) {\n const state = beginScan(context, 'objects')\n yield { rows: toAttioRows(await discoverObjects(context, deps), 'objects', ['workspace_id', 'object_id']), state: { ...state, phase: 'complete' as const } }\n}\n" + }, + { + "path": "attio/sources/object-attributes.ts", + "type": "registry:file", + "target": "src/integrations/attio/sources/object-attributes.ts", + "content": "import { rawTable } from '@chkit/plugin-ingest'\n\nimport { readParentCollections, type AttioClientDeps } from '../client.js'\nimport { attioConfig } from '../config.js'\nimport type { AttioReadContext } from '../checkpoints.js'\n\nexport const attioObjectAttributesRaw = rawTable({ database: attioConfig.database, name: `${attioConfig.tablePrefix}_object_attributes_raw` })\n\nexport async function* readObjectAttributes(context: AttioReadContext, deps?: AttioClientDeps) {\n yield* readParentCollections(context, {\n resource: 'object_attributes', parents: 'objects', idFields: ['workspace_id', 'object_id', 'attribute_id'],\n request: (parent) => ({\n path: `/objects/${encodeURIComponent(parent.id)}/attributes`,\n idFields: ['workspace_id', 'object_id', 'attribute_id'],\n query: { show_archived: 'true' },\n pageSize: attioConfig.pageSize,\n }),\n }, deps)\n}\n" + }, + { + "path": "attio/sources/records.ts", + "type": "registry:file", + "target": "src/integrations/attio/sources/records.ts", + "content": "import { view } from '@chkit/core'\nimport { rawTable } from '@chkit/plugin-ingest'\n\nimport { readParentCollections, type AttioClientDeps } from '../client.js'\nimport { attioConfig } from '../config.js'\nimport type { AttioReadContext } from '../checkpoints.js'\n\nexport const attioRecordsRaw = rawTable({ database: attioConfig.database, name: `${attioConfig.tablePrefix}_records_raw` })\n\n// Each view depends only on records. JSON extraction tolerates absent optional attributes.\nexport const attioPeople = recordView('people', `\n JSONExtractString(values, 'name', 1, 'full_name') AS name,\n arrayMap(value -> JSONExtractString(value, 'email_address'), JSONExtractArrayRaw(values, 'email_addresses')) AS email_addresses,\n arrayMap(value -> JSONExtractString(value, 'target_record_id'), JSONExtractArrayRaw(values, 'company')) AS company_record_ids`)\n\nexport const attioCompanies = recordView('companies', `\n JSONExtractString(values, 'name', 1, 'value') AS name,\n arrayMap(value -> JSONExtractString(value, 'domain'), JSONExtractArrayRaw(values, 'domains')) AS domains,\n JSONExtractString(values, 'description', 1, 'value') AS description`)\n\nexport const attioDeals = recordView('deals', `\n JSONExtractString(values, 'name', 1, 'value') AS name,\n JSONExtractString(values, 'stage', 1, 'status', 'title') AS stage,\n JSONExtract(values, 'value', 1, 'currency_value', 'Nullable(Float64)') AS value,\n JSONExtractString(values, 'value', 1, 'currency_code') AS currency_code,\n arrayMap(value -> JSONExtractString(value, 'target_record_id'), JSONExtractArrayRaw(values, 'associated_company')) AS company_record_ids,\n arrayMap(value -> JSONExtractString(value, 'target_record_id'), JSONExtractArrayRaw(values, 'associated_people')) AS people_record_ids`)\n\nexport async function* readRecords(context: AttioReadContext, deps?: AttioClientDeps) {\n yield* readParentCollections(context, {\n resource: 'records', parents: 'objects', idFields: ['workspace_id', 'object_id', 'record_id'],\n request: (parent) => ({\n path: `/objects/${encodeURIComponent(parent.id)}/records/query`,\n idFields: ['workspace_id', 'object_id', 'record_id'],\n method: 'POST',\n valuesField: 'values',\n pageSize: attioConfig.pageSize,\n }),\n }, deps)\n}\n\nfunction recordView(objectSlug: string, columns: string) {\n return view({\n database: attioConfig.database,\n name: `${attioConfig.tablePrefix}_${objectSlug}`,\n as: `WITH toJSONString(raw) AS payload,\n JSONExtractRaw(payload, 'data') AS record,\n JSONExtractRaw(record, 'values') AS values\nSELECT id,\n JSONExtractString(payload, 'source_id') AS source_id,\n JSONExtractString(record, 'id', 'workspace_id') AS workspace_id,\n JSONExtractString(record, 'id', 'object_id') AS object_id,\n JSONExtractString(record, 'id', 'record_id') AS record_id,\n parseDateTime64BestEffortOrNull(JSONExtractString(record, 'created_at'), 6, 'UTC') AS created_at,\n JSONExtractString(record, 'web_url') AS web_url,${columns},\n _chkit_ingested_at\nFROM ${quoteIdentifier(attioRecordsRaw.database)}.${quoteIdentifier(attioRecordsRaw.name)} FINAL\nWHERE JSONExtractString(payload, 'object_slug') = '${objectSlug}'`,\n })\n}\n\nfunction quoteIdentifier(value: string): string {\n return `\\`${value.replaceAll('\\\\', '\\\\\\\\').replaceAll('`', '\\\\`')}\\``\n}\n" + }, + { + "path": "attio/sources/lists.ts", + "type": "registry:file", + "target": "src/integrations/attio/sources/lists.ts", + "content": "import { rawTable } from '@chkit/plugin-ingest'\n\nimport { discoverLists, toAttioRows, type AttioClientDeps } from '../client.js'\nimport { attioConfig } from '../config.js'\nimport { beginScan, type AttioReadContext } from '../checkpoints.js'\n\nexport const attioListsRaw = rawTable({ database: attioConfig.database, name: `${attioConfig.tablePrefix}_lists_raw` })\n\nexport async function* readLists(context: AttioReadContext, deps?: AttioClientDeps) {\n const state = beginScan(context, 'lists')\n yield { rows: toAttioRows(await discoverLists(context, deps), 'lists', ['workspace_id', 'list_id']), state: { ...state, phase: 'complete' as const } }\n}\n" + }, + { + "path": "attio/sources/list-attributes.ts", + "type": "registry:file", + "target": "src/integrations/attio/sources/list-attributes.ts", + "content": "import { rawTable } from '@chkit/plugin-ingest'\n\nimport { readParentCollections, type AttioClientDeps } from '../client.js'\nimport { attioConfig } from '../config.js'\nimport type { AttioReadContext } from '../checkpoints.js'\n\nexport const attioListAttributesRaw = rawTable({ database: attioConfig.database, name: `${attioConfig.tablePrefix}_list_attributes_raw` })\n\nexport async function* readListAttributes(context: AttioReadContext, deps?: AttioClientDeps) {\n yield* readParentCollections(context, {\n resource: 'list_attributes', parents: 'lists', idFields: ['workspace_id', 'object_id', 'attribute_id'],\n request: (parent) => ({\n path: `/lists/${encodeURIComponent(parent.id)}/attributes`,\n idFields: ['workspace_id', 'object_id', 'attribute_id'],\n // Attio's shared attribute schema calls the parent list ID object_id too.\n query: { show_archived: 'true' },\n pageSize: attioConfig.pageSize,\n }),\n }, deps)\n}\n" + }, + { + "path": "attio/sources/entries.ts", + "type": "registry:file", + "target": "src/integrations/attio/sources/entries.ts", + "content": "import { rawTable } from '@chkit/plugin-ingest'\n\nimport { readParentCollections, type AttioClientDeps } from '../client.js'\nimport { attioConfig } from '../config.js'\nimport type { AttioReadContext } from '../checkpoints.js'\n\nexport const attioEntriesRaw = rawTable({ database: attioConfig.database, name: `${attioConfig.tablePrefix}_entries_raw` })\n\nexport async function* readEntries(context: AttioReadContext, deps?: AttioClientDeps) {\n yield* readParentCollections(context, {\n resource: 'entries', parents: 'lists', idFields: ['workspace_id', 'list_id', 'entry_id'],\n request: (parent) => ({\n path: `/lists/${encodeURIComponent(parent.id)}/entries/query`,\n idFields: ['workspace_id', 'list_id', 'entry_id'],\n method: 'POST',\n valuesField: 'entry_values',\n pageSize: attioConfig.pageSize,\n }),\n }, deps)\n}\n" + }, + { + "path": "attio/sources/notes.ts", + "type": "registry:file", + "target": "src/integrations/attio/sources/notes.ts", + "content": "import { IngestConfigError, rawTable } from '@chkit/plugin-ingest'\n\nimport { readCollectionScan, type AttioClientDeps } from '../client.js'\nimport { attioConfig } from '../config.js'\nimport type { AttioReadContext } from '../checkpoints.js'\n\nexport const attioNotesRaw = rawTable({ database: attioConfig.database, name: `${attioConfig.tablePrefix}_notes_raw` })\n\nexport async function* readNotes(context: AttioReadContext, deps?: AttioClientDeps) {\n if (attioConfig.notesPageSize > 50) throw new IngestConfigError('Attio notesPageSize cannot exceed 50.')\n yield* readCollectionScan(context, 'notes', {\n path: '/notes', idFields: ['workspace_id', 'note_id'], pageSize: attioConfig.notesPageSize,\n }, deps)\n}\n" + }, + { + "path": "attio/sources/tasks.ts", + "type": "registry:file", + "target": "src/integrations/attio/sources/tasks.ts", + "content": "import { rawTable } from '@chkit/plugin-ingest'\n\nimport { readCollectionScan, type AttioClientDeps } from '../client.js'\nimport { attioConfig } from '../config.js'\nimport type { AttioReadContext } from '../checkpoints.js'\n\nexport const attioTasksRaw = rawTable({ database: attioConfig.database, name: `${attioConfig.tablePrefix}_tasks_raw` })\n\nexport async function* readTasks(context: AttioReadContext, deps?: AttioClientDeps) {\n yield* readCollectionScan(context, 'tasks', {\n path: '/tasks', idFields: ['workspace_id', 'task_id'], pageSize: attioConfig.pageSize,\n query: { sort: 'created_at:asc' },\n }, deps)\n}\n" + }, + { + "path": "attio/sources/members.ts", + "type": "registry:file", + "target": "src/integrations/attio/sources/members.ts", + "content": "import { rawTable } from '@chkit/plugin-ingest'\n\nimport { readCollectionScan, type AttioClientDeps } from '../client.js'\nimport { attioConfig } from '../config.js'\nimport type { AttioReadContext } from '../checkpoints.js'\n\nexport const attioMembersRaw = rawTable({ database: attioConfig.database, name: `${attioConfig.tablePrefix}_members_raw` })\n\nexport async function* readMembers(context: AttioReadContext, deps?: AttioClientDeps) {\n yield* readCollectionScan(context, 'members', {\n path: '/workspace_members', idFields: ['workspace_id', 'workspace_member_id'],\n }, deps)\n}\n" + }, + { + "path": "attio/tests/attio.test.ts", + "type": "registry:file", + "target": "src/integrations/attio/tests/attio.test.ts", + "role": "test", + "content": "import { describe, expect, test } from 'bun:test'\n\nimport { HttpError, definePipeline, runIngestion, selectStreams, type SourceChunk } from '@chkit/plugin-ingest'\nimport { createMemoryDestination, createMemoryJournal } from '@chkit/plugin-ingest/testing'\n\nimport { classifyAttioError, readCollection, type AttioClientDeps } from '../client.js'\nimport { scanCheckpoint, type AttioReadContext } from '../checkpoints.js'\nimport * as definitions from '../index.js'\nimport { attio } from '../pipeline.js'\nimport { readObjects } from '../sources/objects.js'\nimport { readObjectAttributes } from '../sources/object-attributes.js'\nimport { readRecords } from '../sources/records.js'\nimport { readLists } from '../sources/lists.js'\nimport { readListAttributes } from '../sources/list-attributes.js'\nimport { readEntries } from '../sources/entries.js'\nimport { readNotes } from '../sources/notes.js'\nimport { readTasks } from '../sources/tasks.js'\nimport { readMembers } from '../sources/members.js'\nimport { customObject, fixtureCollections, fixtureDeps, peopleObject, person } from './fixtures.js'\n\nconst readers: Record AsyncIterable> = {\n objects: readObjects, object_attributes: readObjectAttributes, records: readRecords,\n lists: readLists, list_attributes: readListAttributes, entries: readEntries,\n notes: readNotes, tasks: readTasks, members: readMembers,\n}\n\ndescribe('Attio template', () => {\n test('exports every stream destination and all three views without reading credentials', () => {\n const exported = Object.values(definitions)\n expect(exported).toContain(attio)\n expect(attio.streams).toHaveLength(9)\n for (const stream of attio.streams) {\n expect(exported).toContain(stream.destination)\n expect(stream.incremental.id).toBe(`attio.scan.${stream.id.split('.').at(-1)}`)\n }\n expect([definitions.attioPeople, definitions.attioCompanies, definitions.attioDeals].map((view) => view.name))\n .toEqual(['attio_people', 'attio_companies', 'attio_deals'])\n })\n\n test('all streams ingest sanitized fixtures and reruns publish changed observations', async () => {\n let changed = false\n const deps = fixtureDeps((url) => {\n const data = fixtureCollections[url.pathname]\n if (data === undefined) throw new Error(`Unexpected fixture URL: ${url.pathname}`)\n return Response.json({ data: changed && url.pathname === '/v2/notes'\n ? [{ id: { workspace_id: 'workspace-1', note_id: 'note-1' }, content_markdown: '# Revised call' }]\n : data })\n })\n const pipeline = fixturePipeline(deps)\n const journal = createMemoryJournal()\n const destination = createMemoryDestination()\n const request = { selected: selectStreams([pipeline], []), backfill: undefined }\n const first = await runIngestion(request, { journal, destination })\n expect(first.ok).toBe(true)\n expect(first.streams.map((stream) => stream.rows)).toEqual([2, 2, 2, 1, 1, 1, 1, 1, 1])\n expect(destination.tables.size).toBe(9)\n expect(destination.tables.get('default.attio_records_raw')?.map((row) => row.raw)).toContainEqual({\n source_id: 'attio.primary', object_slug: 'people', data: person,\n })\n changed = true\n const second = await runIngestion(request, { journal, destination })\n expect(second.ok).toBe(true)\n const notes = destination.tables.get('default.attio_notes_raw') ?? []\n expect(notes).toHaveLength(2)\n expect(notes[0]?.id).toBe(notes[1]?.id)\n expect(notes[1]?.raw).toMatchObject({ data: { content_markdown: '# Revised call' } })\n })\n\n test('records discover parents independently and read all offset pages including custom objects', async () => {\n const requests: Array<{ path: string; body: unknown }> = []\n const firstPage = Array.from({ length: 500 }, (_, index) => ({ ...person, id: { ...person.id, record_id: `record-${index}` } }))\n const deps = fixtureDeps((url, init) => {\n const body = typeof init.body === 'string' ? JSON.parse(init.body) : undefined\n requests.push({ path: url.pathname, body })\n if (url.pathname === '/v2/objects') return Response.json({ data: [peopleObject, customObject] })\n if (url.pathname === '/v2/objects/object-people/records/query') {\n return Response.json({ data: body.offset === 0 ? firstPage : [{ ...person, id: { ...person.id, record_id: 'last' } }] })\n }\n if (url.pathname === '/v2/objects/object-custom/records/query') {\n return Response.json({ data: [{ id: { ...person.id, object_id: 'object-custom' }, values: {} }] })\n }\n throw new Error(`Unexpected URL ${url.pathname}`)\n })\n const chunks = await collect(readRecords(context(), deps))\n const rowChunks = chunks.filter((chunk) => chunk.rows.length > 0)\n expect(rowChunks.map((chunk) => chunk.rows.length)).toEqual([500, 1, 1])\n expect(requests.map((request) => request.body)).toEqual([undefined, { limit: 500, offset: 0 }, { limit: 500, offset: 500 }, { limit: 500, offset: 0 }])\n expect(rowChunks.every((chunk) => !('state' in chunk) && !('id' in chunk))).toBe(true)\n expect(rowChunks[0]?.rows[1]?.id).not.toBe(rowChunks[2]?.rows[0]?.id)\n expect(chunks.at(-1)?.state).toMatchObject({ phase: 'complete', completedParents: ['object-people', 'object-custom'] })\n })\n\n test('notes use their 50-item page limit, honor cancellation, and never fetch a second page after abort', async () => {\n const controller = new AbortController()\n const waits: number[] = []\n const urls: string[] = []\n const page = Array.from({ length: 50 }, (_, index) => ({ id: { workspace_id: 'workspace-1', note_id: `note-${index}` } }))\n const deps = fixtureDeps((url) => { urls.push(url.toString()); return Response.json({ data: page }) })\n deps.wait = async (milliseconds, signal) => { waits.push(milliseconds); signal.throwIfAborted() }\n const iterator = readNotes(context(controller.signal), deps)\n expect((await iterator.next()).value?.rows).toEqual([])\n const first = await iterator.next()\n expect(first.value?.rows).toHaveLength(50)\n controller.abort(new Error('fixture cancelled'))\n await expect(iterator.next()).rejects.toThrow('fixture cancelled')\n expect(urls).toHaveLength(1)\n expect(new URL(urls[0] ?? '').searchParams.get('limit')).toBe('50')\n expect(waits).toEqual([125])\n })\n\n test('GET pagination includes the terminating empty page for an exact multiple', async () => {\n const offsets: string[] = []\n const deps = fixtureDeps((url) => {\n const offset = url.searchParams.get('offset') ?? ''\n offsets.push(offset)\n return Response.json({ data: offset === '0' ? [{ id: { task_id: 'a' } }, { id: { task_id: 'b' } }] : [] })\n })\n const pages = await collect(readCollection(context(), { path: '/tasks', idFields: ['task_id'], pageSize: 2 }, deps))\n expect(pages.map((page) => page.length)).toEqual([2])\n expect(offsets).toEqual(['0', '2'])\n })\n\n test('malformed successful responses and missing identity fail rather than become empty data', async () => {\n for (const payload of [{}, { data: null }, { data: [{ id: { task_id: 'task-1' } }] }]) {\n const deps = fixtureDeps(() => Response.json(payload))\n await expect(collect(readTasks(context(), deps))).rejects.toThrow('Attio')\n }\n const malformedRecord = fixtureDeps((url) => Response.json({ data: url.pathname === '/v2/objects'\n ? [peopleObject] : [{ ...person, values: { name: 'not-an-array' } }] }))\n await expect(collect(readRecords(context(), malformedRecord))).rejects.toThrow('invalid values')\n })\n\n test('permission failures are permanent, actionable, and produce no rows', async () => {\n let calls = 0\n const deps = fixtureDeps(() => { calls += 1; return Response.json({ code: 'insufficient_scope' }, { status: 403 }) })\n const pipeline = fixturePipeline(deps, ['notes'])\n const destination = createMemoryDestination()\n const result = await runIngestion({ selected: selectStreams([pipeline], []), backfill: undefined }, {\n journal: createMemoryJournal(), destination,\n })\n expect(result.ok).toBe(false)\n expect(result.streams[0]?.error).toContain('resource scopes')\n expect(result.streams[0]?.rows).toBe(0)\n expect(calls).toBe(1)\n expect(destination.tables.size).toBe(0)\n })\n\n test('429 requests retry under executor authority and preserve Retry-After', async () => {\n let calls = 0\n const deps = fixtureDeps(() => {\n calls += 1\n return calls === 1\n ? Response.json({ code: 'rate_limit_exceeded' }, { status: 429, headers: { 'Retry-After': '0' } })\n : Response.json({ data: fixtureCollections['/v2/notes'] })\n })\n const pipeline = fixturePipeline(deps, ['notes'])\n const journal = createMemoryJournal()\n const result = await runIngestion({ selected: selectStreams([pipeline], []), backfill: undefined }, {\n journal, destination: createMemoryDestination(),\n })\n expect(result.ok).toBe(true)\n expect(calls).toBe(2)\n expect(journal.events.some((event) => event.eventKind === 'retry_scheduled' && event.errorClass === 'rate_limited')).toBe(true)\n const http = await HttpError.fromResponse(new Response(null, { status: 429, headers: { 'Retry-After': '2' } }))\n expect(http.retryAfterMs).toBe(2_000)\n expect(classifyAttioError(http, { kind: 'rate_limited', retryAfterMs: 2_000 })).toBeUndefined()\n })\n\n test('a missing token fails only when the reader runs', async () => {\n const deps = fixtureDeps(() => { throw new Error('must not fetch') })\n deps.token = () => undefined\n expect(attio.streams).toHaveLength(9)\n await expect(collect(readMembers(context(), deps))).rejects.toThrow('Set ATTIO_API_TOKEN')\n })\n\n test('a fresh runtime resumes the frozen parent set after the completed parent', async () => {\n const requests: string[] = []\n const deps = fixtureDeps((url) => {\n requests.push(url.pathname)\n const data = fixtureCollections[url.pathname]\n if (data === undefined) throw new Error(`Unexpected fixture URL ${url.pathname}`)\n return Response.json({ data })\n })\n const pipeline = fixturePipeline(deps, ['records'])\n const firstPipeline = definePipeline({ ...pipeline, streams: pipeline.streams.map((stream) => ({ ...stream, budget: { maxChunks: 3 } })) })\n const journal = createMemoryJournal()\n const destination = createMemoryDestination()\n const first = await runIngestion({ selected: selectStreams([firstPipeline], []), backfill: undefined }, { journal, destination })\n expect(first.streams[0]?.outcome).toBe('budget_exhausted')\n expect((await journal.readCheckpoint('attio.primary.records')).envelope?.state).toMatchObject({\n phase: 'scanning', cycle: 1, completedParents: ['object-people'],\n parents: [{ id: 'object-people', slug: 'people' }, { id: 'object-custom', slug: 'subscriptions' }],\n })\n\n requests.length = 0\n const restored = await restoredJournal(journal)\n const resumed = await runIngestion({ selected: selectStreams([fixturePipeline(deps, ['records'])], []), backfill: undefined }, {\n journal: restored, destination,\n })\n expect(resumed.ok).toBe(true)\n expect(requests).toEqual(['/v2/objects/object-custom/records/query'])\n expect((await restored.readCheckpoint('attio.primary.records')).envelope?.state).toMatchObject({ phase: 'complete', cycle: 1 })\n expect(destination.tables.get('default.attio_records_raw')).toHaveLength(2)\n\n requests.length = 0\n const next = await runIngestion({ selected: selectStreams([fixturePipeline(deps, ['records'])], []), backfill: undefined }, {\n journal: restored, destination,\n })\n expect(next.ok).toBe(true)\n expect(requests[0]).toBe('/v2/objects')\n expect((await restored.readCheckpoint('attio.primary.records')).envelope?.state).toMatchObject({ phase: 'complete', cycle: 2 })\n })\n\n test('an interrupted mutable parent restarts offset zero instead of bookmarking an offset', async () => {\n let fail = true\n const offsets: number[] = []\n const page = Array.from({ length: 500 }, (_, index) => ({ ...person, id: { ...person.id, record_id: `record-${index}` } }))\n const deps = fixtureDeps((url, init) => {\n if (url.pathname === '/v2/objects') return Response.json({ data: [peopleObject] })\n const { offset } = JSON.parse(String(init.body))\n offsets.push(offset)\n if (offset === 0) return Response.json({ data: page })\n return fail ? Response.json({ code: 'insufficient_scope' }, { status: 403 }) : Response.json({ data: [] })\n })\n const journal = createMemoryJournal()\n const destination = createMemoryDestination()\n const first = await runIngestion({ selected: selectStreams([fixturePipeline(deps, ['records'])], []), backfill: undefined }, { journal, destination })\n expect(first.ok).toBe(false)\n expect((await journal.readCheckpoint('attio.primary.records')).envelope?.state).toMatchObject({ phase: 'scanning', completedParents: [] })\n fail = false\n const restored = await restoredJournal(journal)\n const next = await runIngestion({ selected: selectStreams([fixturePipeline(deps, ['records'])], []), backfill: undefined }, {\n journal: restored, destination,\n })\n expect(next.ok).toBe(true)\n expect(offsets).toEqual([0, 500, 0, 500])\n expect((await restored.readCheckpoint('attio.primary.records')).envelope?.state).toMatchObject({ phase: 'complete', cycle: 1 })\n expect(destination.tables.get('default.attio_records_raw')).toHaveLength(500)\n })\n\n test('empty collections still checkpoint completion and start a new cycle on the next run', async () => {\n const deps = fixtureDeps(() => Response.json({ data: [] }))\n const journal = createMemoryJournal()\n const destination = createMemoryDestination()\n for (const cycle of [1, 2]) {\n const result = await runIngestion({ selected: selectStreams([fixturePipeline(deps, ['records', 'notes'])], []), backfill: undefined }, { journal, destination })\n expect(result.ok).toBe(true)\n for (const resource of ['records', 'notes']) {\n expect((await journal.readCheckpoint(`attio.primary.${resource}`)).envelope?.state).toMatchObject({ phase: 'complete', cycle })\n }\n }\n expect(destination.tables.size).toBe(0)\n })\n\n test('rows accepted before a lost journal write cannot mark their parent complete', async () => {\n const deps = fixtureDeps()\n const journal = createMemoryJournal()\n const destination = createMemoryDestination()\n const failingJournal = {\n ...journal,\n async append(events: Parameters[0]) {\n if (events.some((event) => event.eventKind === 'batch_committed' && Number(event.detail.rows) > 0)) {\n throw new Error('fixture journal unavailable after rows loaded')\n }\n await journal.append(events)\n },\n }\n const failed = await runIngestion({ selected: selectStreams([fixturePipeline(deps, ['records'])], []), backfill: undefined }, {\n journal: failingJournal, destination,\n })\n expect(failed.ok).toBe(false)\n expect((await journal.readCheckpoint('attio.primary.records')).envelope?.state).toMatchObject({ phase: 'scanning', completedParents: [] })\n expect(destination.tables.get('default.attio_records_raw')).toHaveLength(1)\n const restored = await restoredJournal(journal)\n const resumed = await runIngestion({ selected: selectStreams([fixturePipeline(deps, ['records'])], []), backfill: undefined }, {\n journal: restored, destination,\n })\n expect(resumed.ok).toBe(true)\n expect(destination.tables.get('default.attio_records_raw')).toHaveLength(2)\n expect((await restored.readCheckpoint('attio.primary.records')).envelope?.state).toMatchObject({ phase: 'complete' })\n }, 10_000)\n\n test('workspace note scans restart offset zero after an execution budget stops a page', async () => {\n const offsets: string[] = []\n const notes = Array.from({ length: 50 }, (_, index) => ({ id: { workspace_id: 'workspace-1', note_id: `note-${index}` } }))\n const deps = fixtureDeps((url) => {\n const offset = url.searchParams.get('offset') ?? ''\n offsets.push(offset)\n return Response.json({ data: offset === '0' ? notes : [] })\n })\n const pipeline = fixturePipeline(deps, ['notes'])\n const bounded = definePipeline({ ...pipeline, streams: pipeline.streams.map((stream) => ({ ...stream, budget: { maxChunks: 2 } })) })\n const journal = createMemoryJournal()\n const destination = createMemoryDestination()\n const first = await runIngestion({ selected: selectStreams([bounded], []), backfill: undefined }, { journal, destination })\n expect(first.streams[0]?.outcome).toBe('budget_exhausted')\n expect((await journal.readCheckpoint('attio.primary.notes')).envelope?.state).toMatchObject({ phase: 'scanning' })\n const restored = await restoredJournal(journal)\n const resumed = await runIngestion({ selected: selectStreams([fixturePipeline(deps, ['notes'])], []), backfill: undefined }, {\n journal: restored, destination,\n })\n expect(resumed.ok).toBe(true)\n expect(offsets).toEqual(['0', '0', '50'])\n expect((await restored.readCheckpoint('attio.primary.notes')).envelope?.state).toMatchObject({ phase: 'complete', cycle: 1 })\n })\n\n test('changed scope and malformed saved frontiers fail before another source request', async () => {\n const strategy = scanCheckpoint('records')\n const initial = { scope: 'changed-selection', cycle: 1, phase: 'scanning', parents: [], completedParents: [] }\n for (const invalid of [{ ...initial, cycle: -1 }, { ...initial, completedParents: ['unknown'] }, { ...initial, parents: [{}] }]) {\n expect(() => strategy.parseState(invalid)).toThrow('Invalid Attio scan checkpoint')\n }\n let requests = 0\n const deps = fixtureDeps(() => { requests += 1; return Response.json({ data: [] }) })\n await expect(collect(readRecords({ ...context(), state: strategy.parseState(initial) }, deps))).rejects.toThrow('checkpoint scope changed')\n expect(requests).toBe(0)\n })\n\n test.each(['workspace_id', 'object_id'])('resumed parents reject a response with a different %s', async (field) => {\n const pipeline = fixturePipeline(fixtureDeps(), ['records'])\n const bounded = definePipeline({ ...pipeline, streams: pipeline.streams.map((stream) => ({ ...stream, budget: { maxChunks: 1 } })) })\n const journal = createMemoryJournal()\n const destination = createMemoryDestination()\n const first = await runIngestion({ selected: selectStreams([bounded], []), backfill: undefined }, { journal, destination })\n expect(first.streams[0]?.outcome).toBe('budget_exhausted')\n const foreign = fixtureDeps(() => Response.json({ data: [{ ...person, id: { ...person.id, [field]: 'another-account-or-parent' } }] }))\n const restored = await restoredJournal(journal)\n const result = await runIngestion({ selected: selectStreams([fixturePipeline(foreign, ['records'])], []), backfill: undefined }, {\n journal: restored, destination,\n })\n expect(result.ok).toBe(false)\n expect(result.streams[0]?.error).toContain('different workspace or parent')\n expect(destination.tables.size).toBe(0)\n expect((await restored.readCheckpoint('attio.primary.records')).envelope?.state).toMatchObject({ phase: 'scanning', completedParents: [] })\n })\n})\n\nfunction fixturePipeline(deps: AttioClientDeps, selectedResources?: readonly string[]) {\n return definePipeline({\n ...attio,\n retry: { retries: 1, minTimeout: 0, maxTimeout: 0, randomize: false },\n streams: attio.streams.filter((stream) => selectedResources === undefined || selectedResources.includes(stream.id.split('.').at(-1) ?? '')).map((stream) => {\n const read = readers[stream.id.split('.').at(-1) ?? '']\n if (!read) throw new Error(`No fixture reader for ${stream.id}`)\n return { ...stream, read: (context: AttioReadContext) => read(context, deps) }\n }),\n })\n}\n\nfunction context(signal = new AbortController().signal): AttioReadContext {\n return { signal, attempt: async (operation) => operation(signal), streamId: 'fixture', state: undefined, selection: undefined, cutoff: new Date('2026-01-01T00:00:00Z') }\n}\n\nasync function collect(items: AsyncIterable): Promise {\n const result: T[] = []\n for await (const item of items) result.push(item)\n return result\n}\n\nasync function restoredJournal(previous: ReturnType) {\n const journal = createMemoryJournal()\n await journal.append(previous.events.map((event) => ({\n ...event, checkpoint: event.checkpoint === undefined ? undefined : JSON.parse(JSON.stringify(event.checkpoint)),\n })))\n return journal\n}\n" + }, + { + "path": "attio/tests/fixtures.ts", + "type": "registry:file", + "target": "src/integrations/attio/tests/fixtures.ts", + "role": "test", + "content": "import type { AttioClientDeps } from '../client.js'\n\nexport const peopleObject = { id: { workspace_id: 'workspace-1', object_id: 'object-people' }, api_slug: 'people' }\nexport const customObject = { id: { workspace_id: 'workspace-1', object_id: 'object-custom' }, api_slug: 'subscriptions' }\nconst salesList = { id: { workspace_id: 'workspace-1', list_id: 'list-sales' }, api_slug: 'sales' }\n\nexport const person = {\n id: { workspace_id: 'workspace-1', object_id: 'object-people', record_id: 'record-1' },\n created_at: '2026-01-01T00:00:00Z',\n web_url: 'https://app.attio.com/example/person/record-1',\n values: {\n name: [{ full_name: 'Ada Example', first_name: 'Ada', last_name: 'Example', attribute_type: 'personal-name', active_until: null }],\n email_addresses: [{ email_address: 'ada@example.test', attribute_type: 'email-address' }],\n custom_multi: [{ value: 'one' }, { value: 'two' }],\n },\n}\n\nexport const fixtureCollections: Record = {\n '/v2/objects': [peopleObject, customObject],\n '/v2/objects/object-people/records/query': [person],\n '/v2/objects/object-custom/records/query': [{ id: { ...person.id, object_id: 'object-custom' }, values: { custom_number: [{ value: 7 }] } }],\n '/v2/objects/object-people/attributes': [{ id: { ...peopleObject.id, attribute_id: 'attribute-1' }, api_slug: 'name', type: 'personal-name' }],\n '/v2/objects/object-custom/attributes': [{ id: { ...customObject.id, attribute_id: 'attribute-1' }, api_slug: 'custom_number', type: 'number' }],\n '/v2/lists': [salesList],\n '/v2/lists/list-sales/attributes': [{ id: { workspace_id: 'workspace-1', object_id: 'list-sales', attribute_id: 'attribute-1' }, api_slug: 'stage', type: 'status' }],\n '/v2/lists/list-sales/entries/query': [{ id: { ...salesList.id, entry_id: 'entry-1' }, parent_object: 'people', parent_record_id: 'record-1', entry_values: { stage: [{ status: { title: 'Prospect' } }] } }],\n '/v2/notes': [{ id: { workspace_id: 'workspace-1', note_id: 'note-1' }, parent_object: 'people', parent_record_id: 'record-1', content_markdown: '# First call' }],\n '/v2/tasks': [{ id: { workspace_id: 'workspace-1', task_id: 'task-1' }, content_plaintext: 'Follow up', is_completed: false }],\n '/v2/workspace_members': [{ id: { workspace_id: 'workspace-1', workspace_member_id: 'member-1' }, first_name: 'Grace' }],\n}\n\nexport function fixtureDeps(\n respond: (url: URL, init: RequestInit) => Response | Promise = (url) => {\n const data = fixtureCollections[url.pathname]\n if (data === undefined) throw new Error(`Unexpected fixture URL: ${url.pathname}`)\n return Response.json({ data })\n },\n): AttioClientDeps {\n return {\n token: () => 'fixture-token',\n wait: async () => {},\n fetch: (url, init) => Promise.resolve(respond(new URL(url), init)),\n }\n}\n" + } + ], + "meta": { + "chkit": { + "formatVersion": 1, + "version": "0.1.3", + "language": "typescript", + "license": "MIT", + "documentation": "https://chkit.obsessiondb.com/integrations/attio/", + "logo": "https://chkit.obsessiondb.com/logos/attio.svg", + "authentication": { + "method": "Bearer token", + "env": [ + "ATTIO_API_TOKEN" + ], + "setup": [ + "As a workspace admin, open the workspace-name dropdown in Attio and select Workspace settings.", + "Open Developers, select + New access token, and name the token (for example, chkit ClickHouse).", + "Grant the read scopes listed for the resources to sync.", + "Click the token on the Developers page to copy it. Set ATTIO_API_TOKEN in the project .env file or the scheduler secret environment." + ], + "documentation": "https://attio.com/help/reference/apps/generating-an-api-key" + }, + "chkit": "^0.2.0-beta.10", + "ingest": "^0.2.0-beta.10", + "clickhouse": ">=25.3.0", + "root": "src/integrations/attio", + "entry": "index.ts", + "exports": [ + "attio", + "attioObjectsRaw", + "attioObjectAttributesRaw", + "attioRecordsRaw", + "attioPeople", + "attioCompanies", + "attioDeals", + "attioListsRaw", + "attioListAttributesRaw", + "attioEntriesRaw", + "attioNotesRaw", + "attioTasksRaw", + "attioMembersRaw" + ], + "resources": [ + { + "name": "objects", + "title": "Objects", + "table": "attio_objects_raw", + "description": "Accessible standard and custom object definitions, IDs, API slug, singular and plural names, creation time", + "scopes": [ + "object_configuration:read" + ], + "strategy": "full", + "endpoints": [ + { + "method": "GET", + "path": "/objects", + "documentation": "https://docs.attio.com/rest-api/endpoint-reference/objects/list-objects" + } + ] + }, + { + "name": "object_attributes", + "title": "Object attributes", + "table": "attio_object_attributes_raw", + "description": "Attribute IDs, slug, title, description, type, flags, defaults, relationship definitions, configuration, creation time; includes archived definitions", + "scopes": [ + "object_configuration:read" + ], + "strategy": "full", + "endpoints": [ + { + "method": "GET", + "path": "/objects/{object_id}/attributes", + "documentation": "https://docs.attio.com/rest-api/endpoint-reference/attributes/list-attributes" + } + ] + }, + { + "name": "records", + "title": "Records", + "table": "attio_records_raw", + "description": "Current records for every selected object, IDs, creation time, Attio URL, and the complete values object with all returned attribute arrays", + "scopes": [ + "object_configuration:read", + "record_permission:read" + ], + "strategy": "full", + "endpoints": [ + { + "method": "POST", + "path": "/objects/{object_id}/records/query", + "documentation": "https://docs.attio.com/rest-api/endpoint-reference/records/list-records" + } + ] + }, + { + "name": "lists", + "title": "Lists", + "table": "attio_lists_raw", + "description": "List IDs, slug, name, parent objects, workspace and member access configuration, creator, creation time", + "scopes": [ + "list_configuration:read" + ], + "strategy": "full", + "endpoints": [ + { + "method": "GET", + "path": "/lists", + "documentation": "https://docs.attio.com/rest-api/endpoint-reference/lists/list-all-lists" + } + ] + }, + { + "name": "list_attributes", + "title": "List attributes", + "table": "attio_list_attributes_raw", + "description": "Attribute definitions for selected lists, including types, configuration, flags, defaults, relationships, and archived definitions", + "scopes": [ + "list_configuration:read" + ], + "strategy": "full", + "endpoints": [ + { + "method": "GET", + "path": "/lists/{list_id}/attributes", + "documentation": "https://docs.attio.com/rest-api/endpoint-reference/attributes/list-attributes" + } + ] + }, + { + "name": "entries", + "title": "List entries", + "table": "attio_entries_raw", + "description": "Entry IDs, parent object and record ID, creation time, and the complete entry_values object", + "scopes": [ + "list_configuration:read", + "list_entry:read" + ], + "strategy": "full", + "endpoints": [ + { + "method": "POST", + "path": "/lists/{list_id}/entries/query", + "documentation": "https://docs.attio.com/rest-api/endpoint-reference/entries/list-entries" + } + ] + }, + { + "name": "notes", + "title": "Notes", + "table": "attio_notes_raw", + "description": "Note IDs, parent record links, title, plaintext and Markdown content, tags, creator, creation time, and meeting reference when returned", + "scopes": [ + "note:read", + "object_configuration:read", + "record_permission:read" + ], + "strategy": "full", + "endpoints": [ + { + "method": "GET", + "path": "/notes", + "documentation": "https://docs.attio.com/rest-api/endpoint-reference/notes/list-notes" + } + ] + }, + { + "name": "tasks", + "title": "Tasks", + "table": "attio_tasks_raw", + "description": "Task IDs, plaintext content, deadline, completion state and time, linked records, assignees, creator, creation time", + "scopes": [ + "task:read", + "object_configuration:read", + "record_permission:read", + "user_management:read" + ], + "strategy": "full", + "endpoints": [ + { + "method": "GET", + "path": "/tasks", + "documentation": "https://docs.attio.com/rest-api/endpoint-reference/tasks/list-tasks" + } + ] + }, + { + "name": "members", + "title": "Workspace members", + "table": "attio_members_raw", + "description": "Member IDs, first and last name, email address, avatar URL, access level, creation time", + "scopes": [ + "user_management:read" + ], + "strategy": "full", + "endpoints": [ + { + "method": "GET", + "path": "/workspace_members", + "documentation": "https://docs.attio.com/rest-api/endpoint-reference/workspace-members/list-workspace-members" + } + ] + } + ], + "views": [ + { + "name": "attio_people", + "source": "records", + "description": "Names, email addresses, and linked companies for people records." + }, + { + "name": "attio_companies", + "source": "records", + "description": "Names, domains, and descriptions for company records." + }, + { + "name": "attio_deals", + "source": "records", + "description": "Deal names, stages, values, currencies, and linked people and companies." + } + ], + "sync": { + "description": "Full scan cycles retain the latest observed raw entities. Interrupted parent collections resume from the frozen object/list set after the last completely loaded parent; unfinished offset scans restart at zero. A completed cycle starts fresh discovery and full reads on the next run.", + "schedule": "Run chkit ingest run through cron, CI, or an existing scheduler; the template does not start a background worker.", + "deletions": "Records absent from a later API response remain in ClickHouse. The template does not propagate deletions or consume webhooks." + }, + "env": { + "ATTIO_API_TOKEN": "" + }, + "fileHashes": { + "src/integrations/attio/README.md": "8bdfb675eee45a3efac2b51c07b761320411ab2a8c43d8aca69997d13573095f", + "src/integrations/attio/client.ts": "99765a125b81fe713b97997ab64bcddc11c8d6ea24891da27c97289236aa2cc3", + "src/integrations/attio/checkpoints.ts": "390f9530320c7f1dd6472bbe60f1492dd0e5f9b684f486f5a2361a40ffe889e6", + "src/integrations/attio/config.ts": "b0bd151a5d6dc5863500efdedc91481b8ebc3e090b03695f99c2e50177421aad", + "src/integrations/attio/index.ts": "74be5d3684988a95cef41a53ad44071b3d06000b8adb49bebbf30c3b2698a663", + "src/integrations/attio/pipeline.ts": "816ccf65620b3dde90e39ba48004871041252803e743f3b557948c4c64b6843a", + "src/integrations/attio/sources/objects.ts": "14b3824c7c7471799c841bc809ebe4aac12bd87cc72165862d48e0b50904c5f0", + "src/integrations/attio/sources/object-attributes.ts": "858119e59f3f7a8053ee686cdb4598ccc4a7ca4c7b438431b640104ffbcf6897", + "src/integrations/attio/sources/records.ts": "8a6942a6c9b8b67b1b0af77489263e3ca171ca28a46bb04ac5ec4746c3c40aba", + "src/integrations/attio/sources/lists.ts": "fcc8629dbe0fc2f75eedbc7ca639fda4bfeb3f4091a638c56ecb3124d5e3c007", + "src/integrations/attio/sources/list-attributes.ts": "c9a56ab01e5d43715313f3d75ff90a70b78aab73c49ac1a362d119e0fdb47d82", + "src/integrations/attio/sources/entries.ts": "5d3c89f88a6ae5861803b9eab9278cbaf31ec357ec81572d414230a2d8bf6b5d", + "src/integrations/attio/sources/notes.ts": "ac5e42c27b48965bc936d7933ef0483184df4db6867558fd4d772471b8d34c6c", + "src/integrations/attio/sources/tasks.ts": "d7dfad36ad2cbf15058b7f6ef543582fb96c167d2dd883ddeeb758caf914ff3a", + "src/integrations/attio/sources/members.ts": "502c1110a3f4c06b877066116b377bcc6567dee70522573c7c8c54af776b8c57", + "src/integrations/attio/tests/attio.test.ts": "874d5d3c28dab6ea8fb86023991b7eeb265ec23df02f5a0935cfd0aebcd05495", + "src/integrations/attio/tests/fixtures.ts": "c40e4d9bacbcd070e70bdb77351bbb350ac5ba249d2a06dcf9151fd278d07a15" + } + } + } +} diff --git a/registry/attio/sources/entries.ts b/registry/attio/sources/entries.ts index de7af6de..2352168e 100644 --- a/registry/attio/sources/entries.ts +++ b/registry/attio/sources/entries.ts @@ -1,20 +1,20 @@ -import { rawTable, type FetchContext } from '@chkit/plugin-ingest' +import { rawTable } from '@chkit/plugin-ingest' -import { discoverLists, entityId, entitySlug, readCollection, toAttioRows, type AttioClientDeps } from '../client.js' +import { readParentCollections, type AttioClientDeps } from '../client.js' import { attioConfig } from '../config.js' +import type { AttioReadContext } from '../checkpoints.js' export const attioEntriesRaw = rawTable({ database: attioConfig.database, name: `${attioConfig.tablePrefix}_entries_raw` }) -export async function* readEntries(context: FetchContext, deps?: AttioClientDeps) { - for (const list of await discoverLists(context, deps)) { - for await (const page of readCollection(context, { - path: `/lists/${encodeURIComponent(entityId(list, 'list_id'))}/entries/query`, - method: 'POST', +export async function* readEntries(context: AttioReadContext, deps?: AttioClientDeps) { + yield* readParentCollections(context, { + resource: 'entries', parents: 'lists', idFields: ['workspace_id', 'list_id', 'entry_id'], + request: (parent) => ({ + path: `/lists/${encodeURIComponent(parent.id)}/entries/query`, idFields: ['workspace_id', 'list_id', 'entry_id'], + method: 'POST', valuesField: 'entry_values', pageSize: attioConfig.pageSize, - }, deps)) { - yield { rows: toAttioRows(page, 'entries', ['workspace_id', 'list_id', 'entry_id'], { list_slug: entitySlug(list) }) } - } - } + }), + }, deps) } diff --git a/registry/attio/sources/list-attributes.ts b/registry/attio/sources/list-attributes.ts index c1c0ef21..44464ca3 100644 --- a/registry/attio/sources/list-attributes.ts +++ b/registry/attio/sources/list-attributes.ts @@ -1,20 +1,20 @@ -import { rawTable, type FetchContext } from '@chkit/plugin-ingest' +import { rawTable } from '@chkit/plugin-ingest' -import { discoverLists, entityId, entitySlug, readCollection, toAttioRows, type AttioClientDeps } from '../client.js' +import { readParentCollections, type AttioClientDeps } from '../client.js' import { attioConfig } from '../config.js' +import type { AttioReadContext } from '../checkpoints.js' export const attioListAttributesRaw = rawTable({ database: attioConfig.database, name: `${attioConfig.tablePrefix}_list_attributes_raw` }) -export async function* readListAttributes(context: FetchContext, deps?: AttioClientDeps) { - for (const list of await discoverLists(context, deps)) { - for await (const page of readCollection(context, { - path: `/lists/${encodeURIComponent(entityId(list, 'list_id'))}/attributes`, - // Attio's shared attribute schema calls the parent list ID "object_id" too. +export async function* readListAttributes(context: AttioReadContext, deps?: AttioClientDeps) { + yield* readParentCollections(context, { + resource: 'list_attributes', parents: 'lists', idFields: ['workspace_id', 'object_id', 'attribute_id'], + request: (parent) => ({ + path: `/lists/${encodeURIComponent(parent.id)}/attributes`, idFields: ['workspace_id', 'object_id', 'attribute_id'], + // Attio's shared attribute schema calls the parent list ID object_id too. query: { show_archived: 'true' }, pageSize: attioConfig.pageSize, - }, deps)) { - yield { rows: toAttioRows(page, 'list_attributes', ['workspace_id', 'object_id', 'attribute_id'], { list_slug: entitySlug(list) }) } - } - } + }), + }, deps) } diff --git a/registry/attio/sources/lists.ts b/registry/attio/sources/lists.ts index 6f8b5227..70880ff5 100644 --- a/registry/attio/sources/lists.ts +++ b/registry/attio/sources/lists.ts @@ -1,10 +1,12 @@ -import { rawTable, type FetchContext } from '@chkit/plugin-ingest' +import { rawTable } from '@chkit/plugin-ingest' import { discoverLists, toAttioRows, type AttioClientDeps } from '../client.js' import { attioConfig } from '../config.js' +import { beginScan, type AttioReadContext } from '../checkpoints.js' export const attioListsRaw = rawTable({ database: attioConfig.database, name: `${attioConfig.tablePrefix}_lists_raw` }) -export async function* readLists(context: FetchContext, deps?: AttioClientDeps) { - yield { rows: toAttioRows(await discoverLists(context, deps), 'lists', ['workspace_id', 'list_id']) } +export async function* readLists(context: AttioReadContext, deps?: AttioClientDeps) { + const state = beginScan(context, 'lists') + yield { rows: toAttioRows(await discoverLists(context, deps), 'lists', ['workspace_id', 'list_id']), state: { ...state, phase: 'complete' as const } } } diff --git a/registry/attio/sources/members.ts b/registry/attio/sources/members.ts index fa0eaf87..7f07e5b5 100644 --- a/registry/attio/sources/members.ts +++ b/registry/attio/sources/members.ts @@ -1,14 +1,13 @@ -import { rawTable, type FetchContext } from '@chkit/plugin-ingest' +import { rawTable } from '@chkit/plugin-ingest' -import { readCollection, toAttioRows, type AttioClientDeps } from '../client.js' +import { readCollectionScan, type AttioClientDeps } from '../client.js' import { attioConfig } from '../config.js' +import type { AttioReadContext } from '../checkpoints.js' export const attioMembersRaw = rawTable({ database: attioConfig.database, name: `${attioConfig.tablePrefix}_members_raw` }) -export async function* readMembers(context: FetchContext, deps?: AttioClientDeps) { - for await (const page of readCollection(context, { +export async function* readMembers(context: AttioReadContext, deps?: AttioClientDeps) { + yield* readCollectionScan(context, 'members', { path: '/workspace_members', idFields: ['workspace_id', 'workspace_member_id'], - }, deps)) { - yield { rows: toAttioRows(page, 'members', ['workspace_id', 'workspace_member_id']) } - } + }, deps) } diff --git a/registry/attio/sources/notes.ts b/registry/attio/sources/notes.ts index 69380748..dc0fba5a 100644 --- a/registry/attio/sources/notes.ts +++ b/registry/attio/sources/notes.ts @@ -1,15 +1,14 @@ -import { IngestConfigError, rawTable, type FetchContext } from '@chkit/plugin-ingest' +import { IngestConfigError, rawTable } from '@chkit/plugin-ingest' -import { readCollection, toAttioRows, type AttioClientDeps } from '../client.js' +import { readCollectionScan, type AttioClientDeps } from '../client.js' import { attioConfig } from '../config.js' +import type { AttioReadContext } from '../checkpoints.js' export const attioNotesRaw = rawTable({ database: attioConfig.database, name: `${attioConfig.tablePrefix}_notes_raw` }) -export async function* readNotes(context: FetchContext, deps?: AttioClientDeps) { +export async function* readNotes(context: AttioReadContext, deps?: AttioClientDeps) { if (attioConfig.notesPageSize > 50) throw new IngestConfigError('Attio notesPageSize cannot exceed 50.') - for await (const page of readCollection(context, { + yield* readCollectionScan(context, 'notes', { path: '/notes', idFields: ['workspace_id', 'note_id'], pageSize: attioConfig.notesPageSize, - }, deps)) { - yield { rows: toAttioRows(page, 'notes', ['workspace_id', 'note_id']) } - } + }, deps) } diff --git a/registry/attio/sources/object-attributes.ts b/registry/attio/sources/object-attributes.ts index 343e8bda..2793a2a6 100644 --- a/registry/attio/sources/object-attributes.ts +++ b/registry/attio/sources/object-attributes.ts @@ -1,19 +1,19 @@ -import { rawTable, type FetchContext } from '@chkit/plugin-ingest' +import { rawTable } from '@chkit/plugin-ingest' -import { discoverObjects, entityId, entitySlug, readCollection, toAttioRows, type AttioClientDeps } from '../client.js' +import { readParentCollections, type AttioClientDeps } from '../client.js' import { attioConfig } from '../config.js' +import type { AttioReadContext } from '../checkpoints.js' export const attioObjectAttributesRaw = rawTable({ database: attioConfig.database, name: `${attioConfig.tablePrefix}_object_attributes_raw` }) -export async function* readObjectAttributes(context: FetchContext, deps?: AttioClientDeps) { - for (const object of await discoverObjects(context, deps)) { - for await (const page of readCollection(context, { - path: `/objects/${encodeURIComponent(entityId(object, 'object_id'))}/attributes`, +export async function* readObjectAttributes(context: AttioReadContext, deps?: AttioClientDeps) { + yield* readParentCollections(context, { + resource: 'object_attributes', parents: 'objects', idFields: ['workspace_id', 'object_id', 'attribute_id'], + request: (parent) => ({ + path: `/objects/${encodeURIComponent(parent.id)}/attributes`, idFields: ['workspace_id', 'object_id', 'attribute_id'], query: { show_archived: 'true' }, pageSize: attioConfig.pageSize, - }, deps)) { - yield { rows: toAttioRows(page, 'object_attributes', ['workspace_id', 'object_id', 'attribute_id'], { object_slug: entitySlug(object) }) } - } - } + }), + }, deps) } diff --git a/registry/attio/sources/objects.ts b/registry/attio/sources/objects.ts index 6cbf8d7e..6a4fba5d 100644 --- a/registry/attio/sources/objects.ts +++ b/registry/attio/sources/objects.ts @@ -1,10 +1,12 @@ -import { rawTable, type FetchContext } from '@chkit/plugin-ingest' +import { rawTable } from '@chkit/plugin-ingest' import { discoverObjects, toAttioRows, type AttioClientDeps } from '../client.js' import { attioConfig } from '../config.js' +import { beginScan, type AttioReadContext } from '../checkpoints.js' export const attioObjectsRaw = rawTable({ database: attioConfig.database, name: `${attioConfig.tablePrefix}_objects_raw` }) -export async function* readObjects(context: FetchContext, deps?: AttioClientDeps) { - yield { rows: toAttioRows(await discoverObjects(context, deps), 'objects', ['workspace_id', 'object_id']) } +export async function* readObjects(context: AttioReadContext, deps?: AttioClientDeps) { + const state = beginScan(context, 'objects') + yield { rows: toAttioRows(await discoverObjects(context, deps), 'objects', ['workspace_id', 'object_id']), state: { ...state, phase: 'complete' as const } } } diff --git a/registry/attio/sources/records.ts b/registry/attio/sources/records.ts index a4d4e887..5133f0b9 100644 --- a/registry/attio/sources/records.ts +++ b/registry/attio/sources/records.ts @@ -1,8 +1,9 @@ import { view } from '@chkit/core' -import { rawTable, type FetchContext } from '@chkit/plugin-ingest' +import { rawTable } from '@chkit/plugin-ingest' -import { discoverObjects, entityId, entitySlug, readCollection, toAttioRows, type AttioClientDeps } from '../client.js' +import { readParentCollections, type AttioClientDeps } from '../client.js' import { attioConfig } from '../config.js' +import type { AttioReadContext } from '../checkpoints.js' export const attioRecordsRaw = rawTable({ database: attioConfig.database, name: `${attioConfig.tablePrefix}_records_raw` }) @@ -25,20 +26,17 @@ export const attioDeals = recordView('deals', ` arrayMap(value -> JSONExtractString(value, 'target_record_id'), JSONExtractArrayRaw(values, 'associated_company')) AS company_record_ids, arrayMap(value -> JSONExtractString(value, 'target_record_id'), JSONExtractArrayRaw(values, 'associated_people')) AS people_record_ids`) -export async function* readRecords(context: FetchContext, deps?: AttioClientDeps) { - // Discovery is local to this reader; the metadata stream need not have run. - for (const object of await discoverObjects(context, deps)) { - for await (const page of readCollection(context, { - path: `/objects/${encodeURIComponent(entityId(object, 'object_id'))}/records/query`, - method: 'POST', +export async function* readRecords(context: AttioReadContext, deps?: AttioClientDeps) { + yield* readParentCollections(context, { + resource: 'records', parents: 'objects', idFields: ['workspace_id', 'object_id', 'record_id'], + request: (parent) => ({ + path: `/objects/${encodeURIComponent(parent.id)}/records/query`, idFields: ['workspace_id', 'object_id', 'record_id'], + method: 'POST', valuesField: 'values', pageSize: attioConfig.pageSize, - }, deps)) { - // No checkpoint or page identity: offsets are not change cursors, and pages can mutate. - yield { rows: toAttioRows(page, 'records', ['workspace_id', 'object_id', 'record_id'], { object_slug: entitySlug(object) }) } - } - } + }), + }, deps) } function recordView(objectSlug: string, columns: string) { diff --git a/registry/attio/sources/tasks.ts b/registry/attio/sources/tasks.ts index 862dd561..8ed6c60d 100644 --- a/registry/attio/sources/tasks.ts +++ b/registry/attio/sources/tasks.ts @@ -1,15 +1,14 @@ -import { rawTable, type FetchContext } from '@chkit/plugin-ingest' +import { rawTable } from '@chkit/plugin-ingest' -import { readCollection, toAttioRows, type AttioClientDeps } from '../client.js' +import { readCollectionScan, type AttioClientDeps } from '../client.js' import { attioConfig } from '../config.js' +import type { AttioReadContext } from '../checkpoints.js' export const attioTasksRaw = rawTable({ database: attioConfig.database, name: `${attioConfig.tablePrefix}_tasks_raw` }) -export async function* readTasks(context: FetchContext, deps?: AttioClientDeps) { - for await (const page of readCollection(context, { +export async function* readTasks(context: AttioReadContext, deps?: AttioClientDeps) { + yield* readCollectionScan(context, 'tasks', { path: '/tasks', idFields: ['workspace_id', 'task_id'], pageSize: attioConfig.pageSize, query: { sort: 'created_at:asc' }, - }, deps)) { - yield { rows: toAttioRows(page, 'tasks', ['workspace_id', 'task_id']) } - } + }, deps) } diff --git a/registry/attio/tests/attio.test.ts b/registry/attio/tests/attio.test.ts index 609430cf..f43d7c9c 100644 --- a/registry/attio/tests/attio.test.ts +++ b/registry/attio/tests/attio.test.ts @@ -1,9 +1,10 @@ import { describe, expect, test } from 'bun:test' -import { HttpError, definePipeline, runIngestion, selectStreams, type FetchContext, type SourceChunk } from '@chkit/plugin-ingest' +import { HttpError, definePipeline, runIngestion, selectStreams, type SourceChunk } from '@chkit/plugin-ingest' import { createMemoryDestination, createMemoryJournal } from '@chkit/plugin-ingest/testing' import { classifyAttioError, readCollection, type AttioClientDeps } from '../client.js' +import { scanCheckpoint, type AttioReadContext } from '../checkpoints.js' import * as definitions from '../index.js' import { attio } from '../pipeline.js' import { readObjects } from '../sources/objects.js' @@ -17,7 +18,7 @@ import { readTasks } from '../sources/tasks.js' import { readMembers } from '../sources/members.js' import { customObject, fixtureCollections, fixtureDeps, peopleObject, person } from './fixtures.js' -const readers: Record AsyncIterable> = { +const readers: Record AsyncIterable> = { objects: readObjects, object_attributes: readObjectAttributes, records: readRecords, lists: readLists, list_attributes: readListAttributes, entries: readEntries, notes: readNotes, tasks: readTasks, members: readMembers, @@ -30,7 +31,7 @@ describe('Attio template', () => { expect(attio.streams).toHaveLength(9) for (const stream of attio.streams) { expect(exported).toContain(stream.destination) - expect(stream.incremental.id).toBe('chkit.full_sync') + expect(stream.incremental.id).toBe(`attio.scan.${stream.id.split('.').at(-1)}`) } expect([definitions.attioPeople, definitions.attioCompanies, definitions.attioDeals].map((view) => view.name)) .toEqual(['attio_people', 'attio_companies', 'attio_deals']) @@ -81,10 +82,12 @@ describe('Attio template', () => { throw new Error(`Unexpected URL ${url.pathname}`) }) const chunks = await collect(readRecords(context(), deps)) - expect(chunks.map((chunk) => chunk.rows.length)).toEqual([500, 1, 1]) + const rowChunks = chunks.filter((chunk) => chunk.rows.length > 0) + expect(rowChunks.map((chunk) => chunk.rows.length)).toEqual([500, 1, 1]) expect(requests.map((request) => request.body)).toEqual([undefined, { limit: 500, offset: 0 }, { limit: 500, offset: 500 }, { limit: 500, offset: 0 }]) - expect(chunks.every((chunk) => !('state' in chunk) && !('id' in chunk))).toBe(true) - expect(chunks[0]?.rows[1]?.id).not.toBe(chunks[2]?.rows[0]?.id) + expect(rowChunks.every((chunk) => !('state' in chunk) && !('id' in chunk))).toBe(true) + expect(rowChunks[0]?.rows[1]?.id).not.toBe(rowChunks[2]?.rows[0]?.id) + expect(chunks.at(-1)?.state).toMatchObject({ phase: 'complete', completedParents: ['object-people', 'object-custom'] }) }) test('notes use their 50-item page limit, honor cancellation, and never fetch a second page after abort', async () => { @@ -95,6 +98,7 @@ describe('Attio template', () => { const deps = fixtureDeps((url) => { urls.push(url.toString()); return Response.json({ data: page }) }) deps.wait = async (milliseconds, signal) => { waits.push(milliseconds); signal.throwIfAborted() } const iterator = readNotes(context(controller.signal), deps) + expect((await iterator.next()).value?.rows).toEqual([]) const first = await iterator.next() expect(first.value?.rows).toHaveLength(50) controller.abort(new Error('fixture cancelled')) @@ -168,6 +172,167 @@ describe('Attio template', () => { expect(attio.streams).toHaveLength(9) await expect(collect(readMembers(context(), deps))).rejects.toThrow('Set ATTIO_API_TOKEN') }) + + test('a fresh runtime resumes the frozen parent set after the completed parent', async () => { + const requests: string[] = [] + const deps = fixtureDeps((url) => { + requests.push(url.pathname) + const data = fixtureCollections[url.pathname] + if (data === undefined) throw new Error(`Unexpected fixture URL ${url.pathname}`) + return Response.json({ data }) + }) + const pipeline = fixturePipeline(deps, ['records']) + const firstPipeline = definePipeline({ ...pipeline, streams: pipeline.streams.map((stream) => ({ ...stream, budget: { maxChunks: 3 } })) }) + const journal = createMemoryJournal() + const destination = createMemoryDestination() + const first = await runIngestion({ selected: selectStreams([firstPipeline], []), backfill: undefined }, { journal, destination }) + expect(first.streams[0]?.outcome).toBe('budget_exhausted') + expect((await journal.readCheckpoint('attio.primary.records')).envelope?.state).toMatchObject({ + phase: 'scanning', cycle: 1, completedParents: ['object-people'], + parents: [{ id: 'object-people', slug: 'people' }, { id: 'object-custom', slug: 'subscriptions' }], + }) + + requests.length = 0 + const restored = await restoredJournal(journal) + const resumed = await runIngestion({ selected: selectStreams([fixturePipeline(deps, ['records'])], []), backfill: undefined }, { + journal: restored, destination, + }) + expect(resumed.ok).toBe(true) + expect(requests).toEqual(['/v2/objects/object-custom/records/query']) + expect((await restored.readCheckpoint('attio.primary.records')).envelope?.state).toMatchObject({ phase: 'complete', cycle: 1 }) + expect(destination.tables.get('default.attio_records_raw')).toHaveLength(2) + + requests.length = 0 + const next = await runIngestion({ selected: selectStreams([fixturePipeline(deps, ['records'])], []), backfill: undefined }, { + journal: restored, destination, + }) + expect(next.ok).toBe(true) + expect(requests[0]).toBe('/v2/objects') + expect((await restored.readCheckpoint('attio.primary.records')).envelope?.state).toMatchObject({ phase: 'complete', cycle: 2 }) + }) + + test('an interrupted mutable parent restarts offset zero instead of bookmarking an offset', async () => { + let fail = true + const offsets: number[] = [] + const page = Array.from({ length: 500 }, (_, index) => ({ ...person, id: { ...person.id, record_id: `record-${index}` } })) + const deps = fixtureDeps((url, init) => { + if (url.pathname === '/v2/objects') return Response.json({ data: [peopleObject] }) + const { offset } = JSON.parse(String(init.body)) + offsets.push(offset) + if (offset === 0) return Response.json({ data: page }) + return fail ? Response.json({ code: 'insufficient_scope' }, { status: 403 }) : Response.json({ data: [] }) + }) + const journal = createMemoryJournal() + const destination = createMemoryDestination() + const first = await runIngestion({ selected: selectStreams([fixturePipeline(deps, ['records'])], []), backfill: undefined }, { journal, destination }) + expect(first.ok).toBe(false) + expect((await journal.readCheckpoint('attio.primary.records')).envelope?.state).toMatchObject({ phase: 'scanning', completedParents: [] }) + fail = false + const restored = await restoredJournal(journal) + const next = await runIngestion({ selected: selectStreams([fixturePipeline(deps, ['records'])], []), backfill: undefined }, { + journal: restored, destination, + }) + expect(next.ok).toBe(true) + expect(offsets).toEqual([0, 500, 0, 500]) + expect((await restored.readCheckpoint('attio.primary.records')).envelope?.state).toMatchObject({ phase: 'complete', cycle: 1 }) + expect(destination.tables.get('default.attio_records_raw')).toHaveLength(500) + }) + + test('empty collections still checkpoint completion and start a new cycle on the next run', async () => { + const deps = fixtureDeps(() => Response.json({ data: [] })) + const journal = createMemoryJournal() + const destination = createMemoryDestination() + for (const cycle of [1, 2]) { + const result = await runIngestion({ selected: selectStreams([fixturePipeline(deps, ['records', 'notes'])], []), backfill: undefined }, { journal, destination }) + expect(result.ok).toBe(true) + for (const resource of ['records', 'notes']) { + expect((await journal.readCheckpoint(`attio.primary.${resource}`)).envelope?.state).toMatchObject({ phase: 'complete', cycle }) + } + } + expect(destination.tables.size).toBe(0) + }) + + test('rows accepted before a lost journal write cannot mark their parent complete', async () => { + const deps = fixtureDeps() + const journal = createMemoryJournal() + const destination = createMemoryDestination() + const failingJournal = { + ...journal, + async append(events: Parameters[0]) { + if (events.some((event) => event.eventKind === 'batch_committed' && Number(event.detail.rows) > 0)) { + throw new Error('fixture journal unavailable after rows loaded') + } + await journal.append(events) + }, + } + const failed = await runIngestion({ selected: selectStreams([fixturePipeline(deps, ['records'])], []), backfill: undefined }, { + journal: failingJournal, destination, + }) + expect(failed.ok).toBe(false) + expect((await journal.readCheckpoint('attio.primary.records')).envelope?.state).toMatchObject({ phase: 'scanning', completedParents: [] }) + expect(destination.tables.get('default.attio_records_raw')).toHaveLength(1) + const restored = await restoredJournal(journal) + const resumed = await runIngestion({ selected: selectStreams([fixturePipeline(deps, ['records'])], []), backfill: undefined }, { + journal: restored, destination, + }) + expect(resumed.ok).toBe(true) + expect(destination.tables.get('default.attio_records_raw')).toHaveLength(2) + expect((await restored.readCheckpoint('attio.primary.records')).envelope?.state).toMatchObject({ phase: 'complete' }) + }, 10_000) + + test('workspace note scans restart offset zero after an execution budget stops a page', async () => { + const offsets: string[] = [] + const notes = Array.from({ length: 50 }, (_, index) => ({ id: { workspace_id: 'workspace-1', note_id: `note-${index}` } })) + const deps = fixtureDeps((url) => { + const offset = url.searchParams.get('offset') ?? '' + offsets.push(offset) + return Response.json({ data: offset === '0' ? notes : [] }) + }) + const pipeline = fixturePipeline(deps, ['notes']) + const bounded = definePipeline({ ...pipeline, streams: pipeline.streams.map((stream) => ({ ...stream, budget: { maxChunks: 2 } })) }) + const journal = createMemoryJournal() + const destination = createMemoryDestination() + const first = await runIngestion({ selected: selectStreams([bounded], []), backfill: undefined }, { journal, destination }) + expect(first.streams[0]?.outcome).toBe('budget_exhausted') + expect((await journal.readCheckpoint('attio.primary.notes')).envelope?.state).toMatchObject({ phase: 'scanning' }) + const restored = await restoredJournal(journal) + const resumed = await runIngestion({ selected: selectStreams([fixturePipeline(deps, ['notes'])], []), backfill: undefined }, { + journal: restored, destination, + }) + expect(resumed.ok).toBe(true) + expect(offsets).toEqual(['0', '0', '50']) + expect((await restored.readCheckpoint('attio.primary.notes')).envelope?.state).toMatchObject({ phase: 'complete', cycle: 1 }) + }) + + test('changed scope and malformed saved frontiers fail before another source request', async () => { + const strategy = scanCheckpoint('records') + const initial = { scope: 'changed-selection', cycle: 1, phase: 'scanning', parents: [], completedParents: [] } + for (const invalid of [{ ...initial, cycle: -1 }, { ...initial, completedParents: ['unknown'] }, { ...initial, parents: [{}] }]) { + expect(() => strategy.parseState(invalid)).toThrow('Invalid Attio scan checkpoint') + } + let requests = 0 + const deps = fixtureDeps(() => { requests += 1; return Response.json({ data: [] }) }) + await expect(collect(readRecords({ ...context(), state: strategy.parseState(initial) }, deps))).rejects.toThrow('checkpoint scope changed') + expect(requests).toBe(0) + }) + + test.each(['workspace_id', 'object_id'])('resumed parents reject a response with a different %s', async (field) => { + const pipeline = fixturePipeline(fixtureDeps(), ['records']) + const bounded = definePipeline({ ...pipeline, streams: pipeline.streams.map((stream) => ({ ...stream, budget: { maxChunks: 1 } })) }) + const journal = createMemoryJournal() + const destination = createMemoryDestination() + const first = await runIngestion({ selected: selectStreams([bounded], []), backfill: undefined }, { journal, destination }) + expect(first.streams[0]?.outcome).toBe('budget_exhausted') + const foreign = fixtureDeps(() => Response.json({ data: [{ ...person, id: { ...person.id, [field]: 'another-account-or-parent' } }] })) + const restored = await restoredJournal(journal) + const result = await runIngestion({ selected: selectStreams([fixturePipeline(foreign, ['records'])], []), backfill: undefined }, { + journal: restored, destination, + }) + expect(result.ok).toBe(false) + expect(result.streams[0]?.error).toContain('different workspace or parent') + expect(destination.tables.size).toBe(0) + expect((await restored.readCheckpoint('attio.primary.records')).envelope?.state).toMatchObject({ phase: 'scanning', completedParents: [] }) + }) }) function fixturePipeline(deps: AttioClientDeps, selectedResources?: readonly string[]) { @@ -177,13 +342,13 @@ function fixturePipeline(deps: AttioClientDeps, selectedResources?: readonly str streams: attio.streams.filter((stream) => selectedResources === undefined || selectedResources.includes(stream.id.split('.').at(-1) ?? '')).map((stream) => { const read = readers[stream.id.split('.').at(-1) ?? ''] if (!read) throw new Error(`No fixture reader for ${stream.id}`) - return { ...stream, read: (context: FetchContext) => read(context, deps) } + return { ...stream, read: (context: AttioReadContext) => read(context, deps) } }), }) } -function context(signal = new AbortController().signal): FetchContext { - return { signal, attempt: async (operation) => operation(signal) } +function context(signal = new AbortController().signal): AttioReadContext { + return { signal, attempt: async (operation) => operation(signal), streamId: 'fixture', state: undefined, selection: undefined, cutoff: new Date('2026-01-01T00:00:00Z') } } async function collect(items: AsyncIterable): Promise { @@ -191,3 +356,11 @@ async function collect(items: AsyncIterable): Promise { for await (const item of items) result.push(item) return result } + +async function restoredJournal(previous: ReturnType) { + const journal = createMemoryJournal() + await journal.append(previous.events.map((event) => ({ + ...event, checkpoint: event.checkpoint === undefined ? undefined : JSON.parse(JSON.stringify(event.checkpoint)), + }))) + return journal +} diff --git a/registry/attio/tests/install.e2e.test.ts b/registry/attio/tests/install.e2e.test.ts index 313c442d..6749b9d5 100644 --- a/registry/attio/tests/install.e2e.test.ts +++ b/registry/attio/tests/install.e2e.test.ts @@ -9,6 +9,7 @@ import { createClickHouseDestination, definePipeline, runIngestion, selectStream import { createMemoryJournal } from '@chkit/plugin-ingest/testing' import { buildRegistryCatalog } from '../../../packages/cli/src/registry/build.js' +import { CLI_VERSION } from '../../../packages/cli/src/runtime/version.js' import { formatTestDiagnostic, runCli } from '../../../packages/cli/src/test/e2e-testkit.js' import { readRegistrySourceCatalog } from '../../../scripts/registry-catalog.js' import { fixtureDeps, person } from './fixtures.js' @@ -33,7 +34,14 @@ describe.serial('built Attio template installed into a consumer', () => { const directory = await mkdtemp(join(tmpdir(), 'chkit-attio-installed-')) temporary.push(directory) const outputDir = join(directory, 'registry') - await buildRegistryCatalog({ catalog: await readRegistrySourceCatalog(), sourceRoot: join(root, 'registry'), outputDir }) + const catalog = await readRegistrySourceCatalog() + // Exercise unreleased source with workspace packages; published artifacts keep their release minima. + const workspaceCatalog = { ...catalog, items: catalog.items.map((item) => item.name === 'attio' ? { + ...item, + dependencies: [`@chkit/core@${CLI_VERSION}`, `@chkit/plugin-ingest@${CLI_VERSION}`], + meta: { chkit: { ...item.meta.chkit, version: `${item.meta.chkit.version}-workspace`, chkit: CLI_VERSION, ingest: CLI_VERSION } }, + } : item) } + await buildRegistryCatalog({ catalog: workspaceCatalog, sourceRoot: join(root, 'registry'), outputDir }) const project = join(directory, 'project') await writeProject(project, root, env.clickhouseDatabase, prefix) const cliEnv = { @@ -86,8 +94,8 @@ describe.serial('built Attio template installed into a consumer', () => { { id: { workspace_id: 'workspace-1', object_id: 'custom' }, api_slug: 'subscriptions' }, ] }) if (url.pathname === '/v2/objects/people/records/query') return Response.json({ data: [ - { ...person, values: { ...person.values, name: [{ full_name: revision === 1 ? 'Ada Example' : 'Ada Updated' }], email_addresses: [{ email_address: 'one@example.test' }, { email_address: 'two@example.test' }] } }, - { id: { ...person.id, record_id: 'empty-attributes' }, values: {} }, + { ...person, id: { ...person.id, object_id: 'people' }, values: { ...person.values, name: [{ full_name: revision === 1 ? 'Ada Example' : 'Ada Updated' }], email_addresses: [{ email_address: 'one@example.test' }, { email_address: 'two@example.test' }] } }, + { id: { ...person.id, object_id: 'people', record_id: 'empty-attributes' }, values: {} }, ] }) if (url.pathname === '/v2/objects/custom/records/query') return Response.json({ data: [{ id: { workspace_id: 'workspace-1', object_id: 'custom', record_id: 'custom-1' }, values: { unmodeled: [{ value: 'retained' }] } }] }) throw new Error(`Unexpected provider fixture request: ${url.pathname}`) diff --git a/registry/circleback/README.md b/registry/circleback/README.md index 72a53a7a..70c36008 100644 --- a/registry/circleback/README.md +++ b/registry/circleback/README.md @@ -16,4 +16,27 @@ bunx chkit ingest run --tag provider:circleback - `meetings` → `circleback_meetings_raw` -The meetings endpoint is scanned on every run. Missing transcripts remain null; removed meetings remain stored. A failed API request fails the stream. Raw rows use provider IDs and keep the last observed value; there is no deletion reconciliation. Edit the readers, select fewer streams, or add SQL views for a specific use case. Run one ingestion process per ClickHouse target at a time. +## Sync and recovery + +Every new scan cycle reads all accessible meetings using `ownership=All` and the provider's `Link` continuations. The documented listing has no modification-time filter, and no durable lifetime is promised for its opaque page cursor. Failed scans restart pagination from the first page and skip only parents whose metadata and transcript observation were already acknowledged by the destination in that active cycle. Unfinished parent enrichment is replayed. Exhausting the listing clears the active cycle, so the following run observes every accessible meeting again. Changes to parents skipped during recovery are refreshed in that next cycle. The reader does not assume a meeting's `updatedAt` tracks transcript availability. + +The checkpoint includes the explicit source/filter `scope`, the active cycle's start time and acknowledged `completedMeetingIds`, `completedAt` for the last fully exhausted cycle, and `unavailableTranscripts` diagnostics with distinct statuses and times checked. Each parent observation and its recovery state commit together after destination acknowledgement. A final empty marker records scan completion, including empty scans. Loaded observations can remain visible after a later failure, while `completedAt` stays at the previous complete scan. + +Transcript `403` responses retain `forbidden`; `404` responses retain `not_found`. Both preserve a null transcript with `_chkit_transcript_status` in raw JSON. Neither status is interpreted as proof of temporary processing. A new cycle retries listed meetings even if their metadata did not change. Unavailable IDs absent from a later listing remain diagnostics because absence is not proof of deletion. These diagnostics are not an executable work queue: no independent request is made for an ID absent from the listing. They are retried when the meeting is listed again; an explicit reconciliation policy may resolve IDs that remain inaccessible. Listing permission failures and other failed requests fail the stream. + +The `maxRetainedMeetings` setting defaults to `10_000`, bounding active parent IDs and unavailable diagnostics. Exceeding a bound fails visibly and keeps committed recovery progress; resolve old diagnostics or raise the setting deliberately for a larger account. When changing authenticated accounts, change `sourceIdentity`, the stream ID and the destination identity. A changed source/filter scope is rejected rather than reusing another source's progress. + +```sh +bunx chkit ingest status --tag provider:circleback --json +``` + +Removed meetings remain stored. Full scans over a changing API are not atomic provider snapshots. Run one ingestion process per ClickHouse target at a time. + +## Fixture tests + +```sh +bunx chkit add circleback --with-tests +bun test src/integrations/circleback/tests/basic.test.ts +``` + +The portable suite covers continuation links, unavailable-to-available transcripts, sink failure, bounded cycle recovery, unavailable diagnostics and first-page replay without live credentials. diff --git a/registry/circleback/index.ts b/registry/circleback/index.ts index 4e60b1a4..15275882 100644 --- a/registry/circleback/index.ts +++ b/registry/circleback/index.ts @@ -1,49 +1,122 @@ -import { definePipeline, defineStream, HttpError, rawRows, rawTable } from '@chkit/plugin-ingest' +import { cursorState, definePipeline, defineStream, HttpError, IngestConfigError, rawRows, rawTable } from '@chkit/plugin-ingest' const database = 'default' const baseUrl = 'https://circleback.ai/api' +// Change the source identity and stream ID when switching authenticated accounts. +const sourceIdentity = 'circleback.primary' +const ownership = 'All' +const scope = JSON.stringify({ sourceIdentity, ownership }) +// Bounds durable parent progress and unavailable diagnostics. Raise deliberately for larger accounts. +const maxRetainedMeetings = 10_000 interface Meeting { id: string; [key: string]: unknown } +interface UnavailableTranscript { id: string; status: 'forbidden' | 'not_found'; checkedAt: string } +interface ScanCycle { startedAt: string; completedMeetingIds: string[] } +interface ScanState { scope: string; completedAt: string | null; scan: ScanCycle | null; unavailableTranscripts: UnavailableTranscript[] } export const circleback_meetingsRaw = rawTable({ database, name: 'circleback_meetings_raw' }) export const circlebackPipeline = definePipeline({ - id: 'circleback', tags: ['provider:circleback'], maxFetches: 1, + id: 'circleback', tags: ['provider:circleback'], maxStreams: 1, maxFetches: 1, streams: [defineStream({ id: 'circleback.meetings', tags: ['resource:meetings'], destination: circleback_meetingsRaw, + incremental: cursorState({ id: 'circleback.meetings.full_scan', version: 2, parse: parseState }), + // Candidate parent progress belongs to the corresponding loaded observation. + batchSize: 1, async *read(context) { - let next: string | undefined = `${baseUrl}/meetings?ownership=All` + const unavailable = new Map(context.state?.unavailableTranscripts.map((item) => [item.id, item]) ?? []) + const completed = new Set(context.state?.scan?.completedMeetingIds ?? []) + const startedAt = context.state?.scan?.startedAt ?? context.cutoff.toISOString() + let next: string | undefined = `${baseUrl}/meetings?ownership=${encodeURIComponent(ownership)}` const seen = new Set() while (next) { - if (seen.has(next)) throw new Error('Circleback repeated a page URL') + if (seen.has(next)) throw new IngestConfigError('Circleback repeated a page URL.') seen.add(next) const current: string = next - const response = await context.attempt((signal) => request(current, signal), { label: 'GET /meetings' }) - if (!response) throw new Error('Circleback returned no meetings response') - const meetings = await response.json() as Meeting[] - if (!Array.isArray(meetings)) throw new Error('Circleback meetings response is not an array') - const rows = [] - for (const meeting of meetings) { - if (!meeting.id) throw new Error('Circleback meeting has no ID') - const transcript = await context.attempt(async (signal) => { - const result = await request(`${baseUrl}/meeting/${encodeURIComponent(meeting.id)}/transcript`, signal, true) - return result ? await result.json() : null + const page = await context.attempt(async (signal) => { + const response = await request(current, signal) + const meetings: unknown = await response.json() + if (!Array.isArray(meetings) || !meetings.every(isMeeting)) throw new IngestConfigError('Circleback returned invalid meetings.') + return { meetings, next: nextLink(response.headers.get('link'), current) } + }, { label: 'GET /meetings' }) + for (const meeting of page.meetings) { + // Pages restart from the beginning. Only sink-acknowledged parents are skipped in this cycle. + if (completed.has(meeting.id)) continue + if (completed.size >= maxRetainedMeetings) throw new IngestConfigError('Circleback scan exceeds maxRetainedMeetings; raise that source setting before resuming.') + const outcome = await context.attempt(async (signal) => { + const url = `${baseUrl}/meeting/${encodeURIComponent(meeting.id)}/transcript` + const response = await request(url, signal, true) + if (response.status === 403) return { transcript: null, status: 'forbidden' as const } + if (response.status === 404) return { transcript: null, status: 'not_found' as const } + return { transcript: await response.json(), status: 'available' as const } }, { label: 'GET meeting transcript' }) - rows.push({ ...meeting, transcript }) + if (outcome.status === 'available') unavailable.delete(meeting.id) + else unavailable.set(meeting.id, { id: meeting.id, status: outcome.status, checkedAt: context.cutoff.toISOString() }) + if (unavailable.size > maxRetainedMeetings) throw new IngestConfigError('Circleback unavailable diagnostics exceed maxRetainedMeetings; resolve old entries or raise that source setting.') + completed.add(meeting.id) + // No assumption that a meeting timestamp tracks delayed transcript availability. + yield { + rows: rawRows([{ ...meeting, transcript: outcome.transcript, _chkit_transcript_status: outcome.status }], (item) => item.id), + state: { scope, completedAt: context.state?.completedAt ?? null, + scan: { startedAt, completedMeetingIds: [...completed] }, unavailableTranscripts: [...unavailable.values()] }, + } } - if (rows.length) yield { rows: rawRows(rows, (meeting) => meeting.id) } - next = response.headers.get('link')?.split(',').map((part) => /<([^>]+)>\s*;.*rel="?next"?/.exec(part)?.[1]).find(Boolean) - if (next) next = new URL(next, current).toString() + next = page.next } + // The next execution starts a fresh cycle and rechecks every accessible parent's latest data. + // IDs absent from this listing remain unavailable diagnostics, not executable pending work. + yield { rows: [], state: { scope, completedAt: context.cutoff.toISOString(), scan: null, + unavailableTranscripts: [...unavailable.values()] }, id: `completed-scan:${startedAt}` } }, })], }) -async function request(url: string, signal: AbortSignal, optional = false): Promise { - const token = process.env.CIRCLEBACK_API_KEY - if (!token) throw new Error('Set CIRCLEBACK_API_KEY') - const response = await fetch(url, { signal, headers: { Authorization: `Bearer ${token}`, Accept: 'application/json' } }) - if (optional && (response.status === 403 || response.status === 404)) return null +async function request(url: string, signal: AbortSignal, allowUnavailable = false): Promise { + const token = process.env.CIRCLEBACK_API_KEY?.trim() + if (!token) throw new IngestConfigError('Set CIRCLEBACK_API_KEY.') + const response = await fetch(url, { signal: AbortSignal.any([signal, AbortSignal.timeout(30_000)]), redirect: 'error', + headers: { Authorization: `Bearer ${token}`, Accept: 'application/json' } }) + if (allowUnavailable && (response.status === 403 || response.status === 404)) return response if (!response.ok) throw await HttpError.fromResponse(response) return response } + +function parseState(raw: unknown): ScanState { + if (!isObject(raw) || raw.scope !== scope || (raw.completedAt !== null && !isTimestamp(raw.completedAt)) || + (raw.scan !== null && !isScan(raw.scan)) || !Array.isArray(raw.unavailableTranscripts) || + raw.unavailableTranscripts.length > maxRetainedMeetings || !raw.unavailableTranscripts.every(isUnavailable) || + new Set(raw.unavailableTranscripts.map((item) => item.id)).size !== raw.unavailableTranscripts.length) { + throw new IngestConfigError('Circleback checkpoint is invalid or its source/filter scope changed; use a new stream identity for a different source.') + } + return { scope, completedAt: raw.completedAt, scan: raw.scan, unavailableTranscripts: raw.unavailableTranscripts } +} + +function isScan(value: unknown): value is ScanCycle { + return isObject(value) && isTimestamp(value.startedAt) && Array.isArray(value.completedMeetingIds) && + value.completedMeetingIds.length <= maxRetainedMeetings && value.completedMeetingIds.every((id: unknown) => typeof id === 'string' && id.length > 0) && + new Set(value.completedMeetingIds).size === value.completedMeetingIds.length +} + +function isMeeting(value: unknown): value is Meeting { + return isObject(value) && typeof value.id === 'string' && value.id.length > 0 +} + +function isUnavailable(value: unknown): value is UnavailableTranscript { + return isMeeting(value) && (value.status === 'forbidden' || value.status === 'not_found') && isTimestamp(value.checkedAt) +} + +function isTimestamp(value: unknown): value is string { + return typeof value === 'string' && Number.isFinite(Date.parse(value)) +} + +function nextLink(header: string | null, current: string): string | undefined { + const value = header?.split(',').map((part) => /<([^>]+)>\s*;.*rel="?next"?/.exec(part)?.[1]).find(Boolean) + if (!value) return undefined + const next = new URL(value, current) + if (next.origin !== 'https://circleback.ai') throw new IngestConfigError('Circleback next page has an unexpected origin.') + return next.toString() +} + +function isObject(value: unknown): value is Record { + return typeof value === 'object' && value !== null && !Array.isArray(value) +} diff --git a/registry/circleback/manifest.json b/registry/circleback/manifest.json index 110b1b1d..7ba19378 100644 --- a/registry/circleback/manifest.json +++ b/registry/circleback/manifest.json @@ -3,10 +3,10 @@ "name": "circleback", "type": "registry:item", "title": "Circleback", - "description": "Small, editable Circleback API ingestion example with raw ClickHouse tables.", + "description": "Editable Circleback full scan cycles with bounded parent recovery and durable transcript availability diagnostics.", "dependencies": [ - "@chkit/core@^0.2.0-beta.8", - "@chkit/plugin-ingest@^0.2.0-beta.8" + "@chkit/core@^0.2.0-beta.10", + "@chkit/plugin-ingest@^0.2.0-beta.10" ], "files": [ { @@ -29,13 +29,13 @@ "meta": { "chkit": { "formatVersion": 1, - "version": "0.1.0", + "version": "0.2.0", "documentation": "https://chkit.obsessiondb.com/integrations/circleback/", "logo": "https://chkit.obsessiondb.com/logos/circleback.svg", "language": "typescript", "license": "MIT", - "chkit": "^0.2.0-beta.8", - "ingest": "^0.2.0-beta.8", + "chkit": "^0.2.0-beta.10", + "ingest": "^0.2.0-beta.10", "clickhouse": ">=25.3.0", "root": "src/integrations/circleback", "entry": "index.ts", @@ -81,10 +81,13 @@ }, "views": [], "sync": { - "description": "A full read runs for each enabled stream. Stable IDs keep the latest observed record.", + "description": "Full scan cycles retain acknowledged parent enrichment for recovery; completing a cycle resets parent progress, while unavailable transcript diagnostics remain explicit.", "schedule": "Run bunx chkit ingest run --tag provider:circleback from a scheduler.", - "deletions": "The meetings endpoint is scanned on every run. Missing transcripts remain null; removed meetings remain stored." + "deletions": "Removed meetings remain stored. Transcript 403 and 404 responses retain distinct unavailable statuses; neither proves that a transcript is still processing." } } - } + }, + "devDependencies": [ + "@types/bun@^1.2.0" + ] } diff --git a/registry/circleback/releases/0.2.0.json b/registry/circleback/releases/0.2.0.json new file mode 100644 index 00000000..5ef3d017 --- /dev/null +++ b/registry/circleback/releases/0.2.0.json @@ -0,0 +1,101 @@ +{ + "$schema": "https://ui.shadcn.com/schema/registry-item.json", + "name": "circleback", + "type": "registry:item", + "title": "Circleback", + "description": "Editable Circleback full scan cycles with bounded parent recovery and durable transcript availability diagnostics.", + "dependencies": [ + "@chkit/core@^0.2.0-beta.10", + "@chkit/plugin-ingest@^0.2.0-beta.10" + ], + "devDependencies": [ + "@types/bun@^1.2.0" + ], + "files": [ + { + "path": "circleback/README.md", + "type": "registry:file", + "target": "src/integrations/circleback/README.md", + "content": "# Circleback raw ingestion example\n\nThis is a small editable starting point based on the Circleback source in [obsessiondb/brain](https://github.com/obsessiondb/brain/tree/main/scripts/brain-sync). It stores provider responses as native JSON without Brain's memory or sales projections.\n\nSet `CIRCLEBACK_API_KEY` in the runtime environment and configure a direct ClickHouse connection. Edit the reader in `index.ts` to select meetings and decide whether to fetch transcripts. Native JSON requires ClickHouse 25.3 or later.\n\n```sh\nbunx chkit check\nbunx chkit generate --name add_circleback\n# Review the migration before applying it.\nbunx chkit migrate --apply\nbunx chkit ingest run --tag provider:circleback\n```\n\n## Resources\n\n- `meetings` → `circleback_meetings_raw`\n\n## Sync and recovery\n\nEvery new scan cycle reads all accessible meetings using `ownership=All` and the provider's `Link` continuations. The documented listing has no modification-time filter, and no durable lifetime is promised for its opaque page cursor. Failed scans restart pagination from the first page and skip only parents whose metadata and transcript observation were already acknowledged by the destination in that active cycle. Unfinished parent enrichment is replayed. Exhausting the listing clears the active cycle, so the following run observes every accessible meeting again. Changes to parents skipped during recovery are refreshed in that next cycle. The reader does not assume a meeting's `updatedAt` tracks transcript availability.\n\nThe checkpoint includes the explicit source/filter `scope`, the active cycle's start time and acknowledged `completedMeetingIds`, `completedAt` for the last fully exhausted cycle, and `unavailableTranscripts` diagnostics with distinct statuses and times checked. Each parent observation and its recovery state commit together after destination acknowledgement. A final empty marker records scan completion, including empty scans. Loaded observations can remain visible after a later failure, while `completedAt` stays at the previous complete scan.\n\nTranscript `403` responses retain `forbidden`; `404` responses retain `not_found`. Both preserve a null transcript with `_chkit_transcript_status` in raw JSON. Neither status is interpreted as proof of temporary processing. A new cycle retries listed meetings even if their metadata did not change. Unavailable IDs absent from a later listing remain diagnostics because absence is not proof of deletion. These diagnostics are not an executable work queue: no independent request is made for an ID absent from the listing. They are retried when the meeting is listed again; an explicit reconciliation policy may resolve IDs that remain inaccessible. Listing permission failures and other failed requests fail the stream.\n\nThe `maxRetainedMeetings` setting defaults to `10_000`, bounding active parent IDs and unavailable diagnostics. Exceeding a bound fails visibly and keeps committed recovery progress; resolve old diagnostics or raise the setting deliberately for a larger account. When changing authenticated accounts, change `sourceIdentity`, the stream ID and the destination identity. A changed source/filter scope is rejected rather than reusing another source's progress.\n\n```sh\nbunx chkit ingest status --tag provider:circleback --json\n```\n\nRemoved meetings remain stored. Full scans over a changing API are not atomic provider snapshots. Run one ingestion process per ClickHouse target at a time.\n\n## Fixture tests\n\n```sh\nbunx chkit add circleback --with-tests\nbun test src/integrations/circleback/tests/basic.test.ts\n```\n\nThe portable suite covers continuation links, unavailable-to-available transcripts, sink failure, bounded cycle recovery, unavailable diagnostics and first-page replay without live credentials.\n" + }, + { + "path": "circleback/index.ts", + "type": "registry:file", + "target": "src/integrations/circleback/index.ts", + "content": "import { cursorState, definePipeline, defineStream, HttpError, IngestConfigError, rawRows, rawTable } from '@chkit/plugin-ingest'\n\nconst database = 'default'\nconst baseUrl = 'https://circleback.ai/api'\n// Change the source identity and stream ID when switching authenticated accounts.\nconst sourceIdentity = 'circleback.primary'\nconst ownership = 'All'\nconst scope = JSON.stringify({ sourceIdentity, ownership })\n// Bounds durable parent progress and unavailable diagnostics. Raise deliberately for larger accounts.\nconst maxRetainedMeetings = 10_000\n\ninterface Meeting { id: string; [key: string]: unknown }\ninterface UnavailableTranscript { id: string; status: 'forbidden' | 'not_found'; checkedAt: string }\ninterface ScanCycle { startedAt: string; completedMeetingIds: string[] }\ninterface ScanState { scope: string; completedAt: string | null; scan: ScanCycle | null; unavailableTranscripts: UnavailableTranscript[] }\n\nexport const circleback_meetingsRaw = rawTable({ database, name: 'circleback_meetings_raw' })\n\nexport const circlebackPipeline = definePipeline({\n id: 'circleback', tags: ['provider:circleback'], maxStreams: 1, maxFetches: 1,\n streams: [defineStream({\n id: 'circleback.meetings', tags: ['resource:meetings'], destination: circleback_meetingsRaw,\n incremental: cursorState({ id: 'circleback.meetings.full_scan', version: 2, parse: parseState }),\n // Candidate parent progress belongs to the corresponding loaded observation.\n batchSize: 1,\n async *read(context) {\n const unavailable = new Map(context.state?.unavailableTranscripts.map((item) => [item.id, item]) ?? [])\n const completed = new Set(context.state?.scan?.completedMeetingIds ?? [])\n const startedAt = context.state?.scan?.startedAt ?? context.cutoff.toISOString()\n let next: string | undefined = `${baseUrl}/meetings?ownership=${encodeURIComponent(ownership)}`\n const seen = new Set()\n while (next) {\n if (seen.has(next)) throw new IngestConfigError('Circleback repeated a page URL.')\n seen.add(next)\n const current: string = next\n const page = await context.attempt(async (signal) => {\n const response = await request(current, signal)\n const meetings: unknown = await response.json()\n if (!Array.isArray(meetings) || !meetings.every(isMeeting)) throw new IngestConfigError('Circleback returned invalid meetings.')\n return { meetings, next: nextLink(response.headers.get('link'), current) }\n }, { label: 'GET /meetings' })\n for (const meeting of page.meetings) {\n // Pages restart from the beginning. Only sink-acknowledged parents are skipped in this cycle.\n if (completed.has(meeting.id)) continue\n if (completed.size >= maxRetainedMeetings) throw new IngestConfigError('Circleback scan exceeds maxRetainedMeetings; raise that source setting before resuming.')\n const outcome = await context.attempt(async (signal) => {\n const url = `${baseUrl}/meeting/${encodeURIComponent(meeting.id)}/transcript`\n const response = await request(url, signal, true)\n if (response.status === 403) return { transcript: null, status: 'forbidden' as const }\n if (response.status === 404) return { transcript: null, status: 'not_found' as const }\n return { transcript: await response.json(), status: 'available' as const }\n }, { label: 'GET meeting transcript' })\n if (outcome.status === 'available') unavailable.delete(meeting.id)\n else unavailable.set(meeting.id, { id: meeting.id, status: outcome.status, checkedAt: context.cutoff.toISOString() })\n if (unavailable.size > maxRetainedMeetings) throw new IngestConfigError('Circleback unavailable diagnostics exceed maxRetainedMeetings; resolve old entries or raise that source setting.')\n completed.add(meeting.id)\n // No assumption that a meeting timestamp tracks delayed transcript availability.\n yield {\n rows: rawRows([{ ...meeting, transcript: outcome.transcript, _chkit_transcript_status: outcome.status }], (item) => item.id),\n state: { scope, completedAt: context.state?.completedAt ?? null,\n scan: { startedAt, completedMeetingIds: [...completed] }, unavailableTranscripts: [...unavailable.values()] },\n }\n }\n next = page.next\n }\n // The next execution starts a fresh cycle and rechecks every accessible parent's latest data.\n // IDs absent from this listing remain unavailable diagnostics, not executable pending work.\n yield { rows: [], state: { scope, completedAt: context.cutoff.toISOString(), scan: null,\n unavailableTranscripts: [...unavailable.values()] }, id: `completed-scan:${startedAt}` }\n },\n })],\n})\n\nasync function request(url: string, signal: AbortSignal, allowUnavailable = false): Promise {\n const token = process.env.CIRCLEBACK_API_KEY?.trim()\n if (!token) throw new IngestConfigError('Set CIRCLEBACK_API_KEY.')\n const response = await fetch(url, { signal: AbortSignal.any([signal, AbortSignal.timeout(30_000)]), redirect: 'error',\n headers: { Authorization: `Bearer ${token}`, Accept: 'application/json' } })\n if (allowUnavailable && (response.status === 403 || response.status === 404)) return response\n if (!response.ok) throw await HttpError.fromResponse(response)\n return response\n}\n\nfunction parseState(raw: unknown): ScanState {\n if (!isObject(raw) || raw.scope !== scope || (raw.completedAt !== null && !isTimestamp(raw.completedAt)) ||\n (raw.scan !== null && !isScan(raw.scan)) || !Array.isArray(raw.unavailableTranscripts) ||\n raw.unavailableTranscripts.length > maxRetainedMeetings || !raw.unavailableTranscripts.every(isUnavailable) ||\n new Set(raw.unavailableTranscripts.map((item) => item.id)).size !== raw.unavailableTranscripts.length) {\n throw new IngestConfigError('Circleback checkpoint is invalid or its source/filter scope changed; use a new stream identity for a different source.')\n }\n return { scope, completedAt: raw.completedAt, scan: raw.scan, unavailableTranscripts: raw.unavailableTranscripts }\n}\n\nfunction isScan(value: unknown): value is ScanCycle {\n return isObject(value) && isTimestamp(value.startedAt) && Array.isArray(value.completedMeetingIds) &&\n value.completedMeetingIds.length <= maxRetainedMeetings && value.completedMeetingIds.every((id: unknown) => typeof id === 'string' && id.length > 0) &&\n new Set(value.completedMeetingIds).size === value.completedMeetingIds.length\n}\n\nfunction isMeeting(value: unknown): value is Meeting {\n return isObject(value) && typeof value.id === 'string' && value.id.length > 0\n}\n\nfunction isUnavailable(value: unknown): value is UnavailableTranscript {\n return isMeeting(value) && (value.status === 'forbidden' || value.status === 'not_found') && isTimestamp(value.checkedAt)\n}\n\nfunction isTimestamp(value: unknown): value is string {\n return typeof value === 'string' && Number.isFinite(Date.parse(value))\n}\n\nfunction nextLink(header: string | null, current: string): string | undefined {\n const value = header?.split(',').map((part) => /<([^>]+)>\\s*;.*rel=\"?next\"?/.exec(part)?.[1]).find(Boolean)\n if (!value) return undefined\n const next = new URL(value, current)\n if (next.origin !== 'https://circleback.ai') throw new IngestConfigError('Circleback next page has an unexpected origin.')\n return next.toString()\n}\n\nfunction isObject(value: unknown): value is Record {\n return typeof value === 'object' && value !== null && !Array.isArray(value)\n}\n" + }, + { + "path": "circleback/tests/basic.test.ts", + "type": "registry:file", + "target": "src/integrations/circleback/tests/basic.test.ts", + "role": "test", + "content": "import { afterEach, expect, test } from 'bun:test'\nimport { IngestConfigError, runIngestion, selectStreams } from '@chkit/plugin-ingest'\nimport { createMemoryDestination, createMemoryJournal } from '@chkit/plugin-ingest/testing'\n\nimport { circlebackPipeline } from '../index.js'\n\nconst originalFetch = globalThis.fetch\nconst originalToken = process.env.CIRCLEBACK_API_KEY\n\nfunction setFetch(handler: (...args: Parameters) => Promise): void {\n globalThis.fetch = Object.assign(handler, { preconnect: originalFetch.preconnect })\n}\n\nafterEach(() => {\n globalThis.fetch = originalFetch\n if (originalToken === undefined) delete process.env.CIRCLEBACK_API_KEY\n else process.env.CIRCLEBACK_API_KEY = originalToken\n})\n\ntest.serial('Circleback follows the Link cursor and joins available transcripts', async () => {\n process.env.CIRCLEBACK_API_KEY = 'fixture'\n const paths: string[] = []\n setFetch(async (input) => {\n const url = new URL(String(input))\n paths.push(url.pathname + url.search)\n if (url.pathname.endsWith('/transcript')) return url.pathname.includes('/b/') ? new Response('', { status: 404 }) : Response.json([{ text: 'Hello' }])\n if (url.searchParams.has('cursor')) return Response.json([{ id: 'b', name: 'Second' }])\n return Response.json([{ id: 'a', name: 'First' }], { headers: { Link: '; rel=\"next\"' } })\n })\n const destination = createMemoryDestination()\n const result = await runIngestion({ selected: selectStreams([circlebackPipeline], []), backfill: undefined }, { journal: createMemoryJournal(), destination })\n expect(result.ok).toBe(true)\n expect(paths).toEqual(['/api/meetings?ownership=All', '/api/meeting/a/transcript', '/api/meetings?cursor=next', '/api/meeting/b/transcript'])\n const rows = destination.tables.get('default.circleback_meetings_raw') ?? []\n expect(rows.map((row) => row.id)).toEqual(['a', 'b'])\n expect(rows[0]?.raw).toMatchObject({ transcript: [{ text: 'Hello' }] })\n expect(rows[1]?.raw).toMatchObject({ transcript: null })\n})\n\ntest.serial('Circleback retains unavailable transcripts and retries them when metadata has not changed', async () => {\n process.env.CIRCLEBACK_API_KEY = 'fixture'\n let available = false\n const listingStarts: string[] = []\n setFetch(async (input) => {\n const url = new URL(String(input))\n if (url.pathname.endsWith('/transcript')) {\n if (available) return Response.json([{ text: 'ready' }])\n return new Response('unavailable', { status: url.pathname.includes('/a/') ? 403 : 404 })\n }\n listingStarts.push(url.search)\n return Response.json([{ id: 'a', updatedAt: '2025-01-01' }, { id: 'b', updatedAt: '2025-01-01' }])\n })\n const journal = createMemoryJournal()\n const destination = createMemoryDestination()\n const request = { selected: selectStreams([circlebackPipeline], []), backfill: undefined }\n expect((await runIngestion(request, { journal, destination, now: () => new Date('2026-01-01') })).ok).toBe(true)\n expect((await journal.readCheckpoint('circleback.meetings')).envelope?.state).toEqual({\n scope: JSON.stringify({ sourceIdentity: 'circleback.primary', ownership: 'All' }), scan: null,\n completedAt: '2026-01-01T00:00:00.000Z', unavailableTranscripts: [\n { id: 'a', status: 'forbidden', checkedAt: '2026-01-01T00:00:00.000Z' },\n { id: 'b', status: 'not_found', checkedAt: '2026-01-01T00:00:00.000Z' },\n ],\n })\n available = true\n expect((await runIngestion(request, { journal, destination, now: () => new Date('2026-01-02') })).ok).toBe(true)\n expect((await journal.readCheckpoint('circleback.meetings')).envelope?.state).toMatchObject({ completedAt: '2026-01-02T00:00:00.000Z', scan: null, unavailableTranscripts: [] })\n expect(listingStarts).toEqual(['?ownership=All', '?ownership=All'])\n expect(destination.tables.get('default.circleback_meetings_raw')?.at(-1)?.raw).toMatchObject({ transcript: [{ text: 'ready' }], _chkit_transcript_status: 'available' })\n})\n\ntest.serial('Circleback sink failures do not commit enrichment state', async () => {\n process.env.CIRCLEBACK_API_KEY = 'fixture'\n setFetch(async (input) => String(input).endsWith('/transcript') ? new Response('unavailable', { status: 404 }) : Response.json([{ id: 'a' }]))\n const journal = createMemoryJournal()\n const selected = selectStreams([circlebackPipeline], []).map((item) => ({ ...item, stream: { ...item.stream, retry: { retries: 0 } } }))\n const result = await runIngestion({ selected, backfill: undefined }, {\n journal, destination: { insert: async () => { throw new IngestConfigError('sink rejected') } },\n })\n expect(result.ok).toBe(false)\n expect((await journal.readCheckpoint('circleback.meetings')).envelope).toBeUndefined()\n})\n\ntest.serial('Circleback fails incomplete pagination without committing scan completion and restarts from the first page', async () => {\n process.env.CIRCLEBACK_API_KEY = 'fixture'\n let fail = true\n const starts: string[] = []\n setFetch(async (input) => {\n const url = new URL(String(input))\n if (url.pathname.endsWith('/transcript')) return new Response('unavailable', { status: 404 })\n if (url.searchParams.has('cursor')) return fail ? new Response('denied', { status: 403 }) : Response.json([])\n starts.push(url.search)\n return Response.json([{ id: 'a' }], { headers: { Link: '; rel=\"next\"' } })\n })\n const journal = createMemoryJournal()\n const destination = createMemoryDestination()\n const selected = selectStreams([circlebackPipeline], []).map((item) => ({ ...item, stream: { ...item.stream, retry: { retries: 0 } } }))\n const request = { selected, backfill: undefined }\n expect((await runIngestion(request, { journal, destination, now: () => new Date('2026-01-01') })).ok).toBe(false)\n expect((await journal.readCheckpoint('circleback.meetings')).envelope?.state).toMatchObject({ completedAt: null, unavailableTranscripts: [{ id: 'a' }] })\n fail = false\n expect((await runIngestion(request, { journal, destination, now: () => new Date('2026-01-02') })).ok).toBe(true)\n expect(starts).toEqual(['?ownership=All', '?ownership=All'])\n expect((await journal.readCheckpoint('circleback.meetings')).envelope?.state).toMatchObject({ completedAt: '2026-01-02T00:00:00.000Z' })\n})\n\ntest.serial('Circleback replays unfinished enrichment while skipping acknowledged parents until the next full cycle', async () => {\n process.env.CIRCLEBACK_API_KEY = 'fixture'\n let failSecond = true\n const transcripts: string[] = []\n const journal = createMemoryJournal()\n let acknowledgeFirst = () => {}\n const firstCommitted = new Promise((resolve) => { acknowledgeFirst = resolve })\n const append = journal.append\n journal.append = async (events) => {\n await append(events)\n if (events.some((event) => JSON.stringify(event.checkpoint?.state)?.includes('\"completedMeetingIds\":[\"a\"]'))) acknowledgeFirst()\n }\n setFetch(async (input) => {\n const url = new URL(String(input))\n if (!url.pathname.endsWith('/transcript')) return Response.json([{ id: 'a', custom: 'first' }, { id: 'b', custom: 'second' }])\n const id = url.pathname.includes('/a/') ? 'a' : 'b'\n transcripts.push(id)\n if (id === 'b' && failSecond) {\n await firstCommitted\n return new Response('temporarily unavailable', { status: 500 })\n }\n return Response.json([{ text: id }])\n })\n const destination = createMemoryDestination()\n const selected = selectStreams([circlebackPipeline], []).map((item) => ({ ...item, stream: { ...item.stream, retry: { retries: 0 } } }))\n const request = { selected, backfill: undefined }\n expect((await runIngestion(request, { journal, destination, now: () => new Date('2026-01-01') })).ok).toBe(false)\n expect((await journal.readCheckpoint('circleback.meetings')).envelope?.state).toMatchObject({\n completedAt: null, scan: { startedAt: '2026-01-01T00:00:00.000Z', completedMeetingIds: ['a'] },\n })\n failSecond = false\n expect((await runIngestion(request, { journal, destination, now: () => new Date('2026-01-02') })).ok).toBe(true)\n expect(transcripts).toEqual(['a', 'b', 'b'])\n expect((await journal.readCheckpoint('circleback.meetings')).envelope?.state).toMatchObject({ completedAt: '2026-01-02T00:00:00.000Z', scan: null })\n expect(destination.tables.get('default.circleback_meetings_raw')?.map((row) => row.id)).toEqual(['a', 'b'])\n expect((await runIngestion(request, { journal, destination, now: () => new Date('2026-01-03') })).ok).toBe(true)\n expect(transcripts).toEqual(['a', 'b', 'b', 'a', 'b'])\n})\n\ntest.serial('Circleback empty scans complete and retain absent unavailable IDs as diagnostics', async () => {\n process.env.CIRCLEBACK_API_KEY = 'fixture'\n let listed = true\n const transcripts: string[] = []\n setFetch(async (input) => {\n const url = new URL(String(input))\n if (url.pathname.endsWith('/transcript')) {\n transcripts.push(url.pathname)\n return new Response('not found', { status: 404 })\n }\n return Response.json(listed ? [{ id: 'a' }] : [])\n })\n const journal = createMemoryJournal()\n const destination = createMemoryDestination()\n const request = { selected: selectStreams([circlebackPipeline], []), backfill: undefined }\n expect((await runIngestion(request, { journal, destination, now: () => new Date('2026-01-01') })).ok).toBe(true)\n listed = false\n expect((await runIngestion(request, { journal, destination, now: () => new Date('2026-01-02') })).ok).toBe(true)\n expect(transcripts).toHaveLength(1)\n expect((await journal.readCheckpoint('circleback.meetings')).envelope?.state).toMatchObject({\n completedAt: '2026-01-02T00:00:00.000Z', scan: null,\n unavailableTranscripts: [{ id: 'a', status: 'not_found', checkedAt: '2026-01-01T00:00:00.000Z' }],\n })\n})\n\ntest('Circleback rejects changed source scopes and unbounded or duplicated recovery IDs', () => {\n const parse = circlebackPipeline.streams[0]?.incremental.parseState\n expect(parse).toBeDefined()\n const state = { scope: JSON.stringify({ sourceIdentity: 'circleback.primary', ownership: 'All' }), completedAt: null, scan: null, unavailableTranscripts: [] }\n expect(() => parse?.({ ...state, scope: 'different-source' })).toThrow('scope changed')\n expect(() => parse?.({ ...state, scan: { startedAt: '2026-01-01T00:00:00Z', completedMeetingIds: ['a', 'a'] } })).toThrow('checkpoint is invalid')\n expect(() => parse?.({ ...state, scan: { startedAt: '2026-01-01T00:00:00Z', completedMeetingIds: Array.from({ length: 10_001 }, (_, index) => String(index)) } })).toThrow('checkpoint is invalid')\n})\n" + } + ], + "meta": { + "chkit": { + "formatVersion": 1, + "version": "0.2.0", + "language": "typescript", + "license": "MIT", + "documentation": "https://chkit.obsessiondb.com/integrations/circleback/", + "logo": "https://chkit.obsessiondb.com/logos/circleback.svg", + "authentication": { + "method": "Bearer API key", + "env": [ + "CIRCLEBACK_API_KEY" + ], + "setup": [ + "Provide CIRCLEBACK_API_KEY with read access to the selected resources.", + "Keep the API key in the runtime environment and rotate it according to the provider policy." + ], + "documentation": "https://circleback.ai/docs/api" + }, + "chkit": "^0.2.0-beta.10", + "ingest": "^0.2.0-beta.10", + "clickhouse": ">=25.3.0", + "root": "src/integrations/circleback", + "entry": "index.ts", + "exports": [ + "circlebackPipeline", + "circleback_meetingsRaw" + ], + "resources": [ + { + "name": "meetings", + "title": "Meetings", + "table": "circleback_meetings_raw", + "description": "Meeting metadata and notes, with a transcript where access permits it.", + "scopes": [], + "strategy": "full", + "endpoints": [ + { + "method": "GET", + "path": "/meetings", + "documentation": "https://circleback.ai/docs/api/meetings/list-meetings" + }, + { + "method": "GET", + "path": "/meeting/{meetingId}/transcript", + "documentation": "https://circleback.ai/docs/api/meetings/get-meeting-transcript" + } + ] + } + ], + "views": [], + "sync": { + "description": "Full scan cycles retain acknowledged parent enrichment for recovery; completing a cycle resets parent progress, while unavailable transcript diagnostics remain explicit.", + "schedule": "Run bunx chkit ingest run --tag provider:circleback from a scheduler.", + "deletions": "Removed meetings remain stored. Transcript 403 and 404 responses retain distinct unavailable statuses; neither proves that a transcript is still processing." + }, + "env": { + "CIRCLEBACK_API_KEY": "" + }, + "fileHashes": { + "src/integrations/circleback/README.md": "37e37285a6b5678395c2b31db52dd1d5452d28d043361e84df0dddf9598bbf1f", + "src/integrations/circleback/index.ts": "4a55faa60b2e57a27c8a8f15c0cd6b6b4075d9208a7a55c553c3bb7dfcaaa023", + "src/integrations/circleback/tests/basic.test.ts": "371c302ed4c4c903a476884bd554833b60953938cbdc60fcd79e480457455acc" + } + } + } +} diff --git a/registry/circleback/tests/basic.test.ts b/registry/circleback/tests/basic.test.ts index bc7f2682..1cdcb14a 100644 --- a/registry/circleback/tests/basic.test.ts +++ b/registry/circleback/tests/basic.test.ts @@ -1,5 +1,5 @@ import { afterEach, expect, test } from 'bun:test' -import { runIngestion, selectStreams } from '@chkit/plugin-ingest' +import { IngestConfigError, runIngestion, selectStreams } from '@chkit/plugin-ingest' import { createMemoryDestination, createMemoryJournal } from '@chkit/plugin-ingest/testing' import { circlebackPipeline } from '../index.js' @@ -36,3 +36,142 @@ test.serial('Circleback follows the Link cursor and joins available transcripts' expect(rows[0]?.raw).toMatchObject({ transcript: [{ text: 'Hello' }] }) expect(rows[1]?.raw).toMatchObject({ transcript: null }) }) + +test.serial('Circleback retains unavailable transcripts and retries them when metadata has not changed', async () => { + process.env.CIRCLEBACK_API_KEY = 'fixture' + let available = false + const listingStarts: string[] = [] + setFetch(async (input) => { + const url = new URL(String(input)) + if (url.pathname.endsWith('/transcript')) { + if (available) return Response.json([{ text: 'ready' }]) + return new Response('unavailable', { status: url.pathname.includes('/a/') ? 403 : 404 }) + } + listingStarts.push(url.search) + return Response.json([{ id: 'a', updatedAt: '2025-01-01' }, { id: 'b', updatedAt: '2025-01-01' }]) + }) + const journal = createMemoryJournal() + const destination = createMemoryDestination() + const request = { selected: selectStreams([circlebackPipeline], []), backfill: undefined } + expect((await runIngestion(request, { journal, destination, now: () => new Date('2026-01-01') })).ok).toBe(true) + expect((await journal.readCheckpoint('circleback.meetings')).envelope?.state).toEqual({ + scope: JSON.stringify({ sourceIdentity: 'circleback.primary', ownership: 'All' }), scan: null, + completedAt: '2026-01-01T00:00:00.000Z', unavailableTranscripts: [ + { id: 'a', status: 'forbidden', checkedAt: '2026-01-01T00:00:00.000Z' }, + { id: 'b', status: 'not_found', checkedAt: '2026-01-01T00:00:00.000Z' }, + ], + }) + available = true + expect((await runIngestion(request, { journal, destination, now: () => new Date('2026-01-02') })).ok).toBe(true) + expect((await journal.readCheckpoint('circleback.meetings')).envelope?.state).toMatchObject({ completedAt: '2026-01-02T00:00:00.000Z', scan: null, unavailableTranscripts: [] }) + expect(listingStarts).toEqual(['?ownership=All', '?ownership=All']) + expect(destination.tables.get('default.circleback_meetings_raw')?.at(-1)?.raw).toMatchObject({ transcript: [{ text: 'ready' }], _chkit_transcript_status: 'available' }) +}) + +test.serial('Circleback sink failures do not commit enrichment state', async () => { + process.env.CIRCLEBACK_API_KEY = 'fixture' + setFetch(async (input) => String(input).endsWith('/transcript') ? new Response('unavailable', { status: 404 }) : Response.json([{ id: 'a' }])) + const journal = createMemoryJournal() + const selected = selectStreams([circlebackPipeline], []).map((item) => ({ ...item, stream: { ...item.stream, retry: { retries: 0 } } })) + const result = await runIngestion({ selected, backfill: undefined }, { + journal, destination: { insert: async () => { throw new IngestConfigError('sink rejected') } }, + }) + expect(result.ok).toBe(false) + expect((await journal.readCheckpoint('circleback.meetings')).envelope).toBeUndefined() +}) + +test.serial('Circleback fails incomplete pagination without committing scan completion and restarts from the first page', async () => { + process.env.CIRCLEBACK_API_KEY = 'fixture' + let fail = true + const starts: string[] = [] + setFetch(async (input) => { + const url = new URL(String(input)) + if (url.pathname.endsWith('/transcript')) return new Response('unavailable', { status: 404 }) + if (url.searchParams.has('cursor')) return fail ? new Response('denied', { status: 403 }) : Response.json([]) + starts.push(url.search) + return Response.json([{ id: 'a' }], { headers: { Link: '; rel="next"' } }) + }) + const journal = createMemoryJournal() + const destination = createMemoryDestination() + const selected = selectStreams([circlebackPipeline], []).map((item) => ({ ...item, stream: { ...item.stream, retry: { retries: 0 } } })) + const request = { selected, backfill: undefined } + expect((await runIngestion(request, { journal, destination, now: () => new Date('2026-01-01') })).ok).toBe(false) + expect((await journal.readCheckpoint('circleback.meetings')).envelope?.state).toMatchObject({ completedAt: null, unavailableTranscripts: [{ id: 'a' }] }) + fail = false + expect((await runIngestion(request, { journal, destination, now: () => new Date('2026-01-02') })).ok).toBe(true) + expect(starts).toEqual(['?ownership=All', '?ownership=All']) + expect((await journal.readCheckpoint('circleback.meetings')).envelope?.state).toMatchObject({ completedAt: '2026-01-02T00:00:00.000Z' }) +}) + +test.serial('Circleback replays unfinished enrichment while skipping acknowledged parents until the next full cycle', async () => { + process.env.CIRCLEBACK_API_KEY = 'fixture' + let failSecond = true + const transcripts: string[] = [] + const journal = createMemoryJournal() + let acknowledgeFirst = () => {} + const firstCommitted = new Promise((resolve) => { acknowledgeFirst = resolve }) + const append = journal.append + journal.append = async (events) => { + await append(events) + if (events.some((event) => JSON.stringify(event.checkpoint?.state)?.includes('"completedMeetingIds":["a"]'))) acknowledgeFirst() + } + setFetch(async (input) => { + const url = new URL(String(input)) + if (!url.pathname.endsWith('/transcript')) return Response.json([{ id: 'a', custom: 'first' }, { id: 'b', custom: 'second' }]) + const id = url.pathname.includes('/a/') ? 'a' : 'b' + transcripts.push(id) + if (id === 'b' && failSecond) { + await firstCommitted + return new Response('temporarily unavailable', { status: 500 }) + } + return Response.json([{ text: id }]) + }) + const destination = createMemoryDestination() + const selected = selectStreams([circlebackPipeline], []).map((item) => ({ ...item, stream: { ...item.stream, retry: { retries: 0 } } })) + const request = { selected, backfill: undefined } + expect((await runIngestion(request, { journal, destination, now: () => new Date('2026-01-01') })).ok).toBe(false) + expect((await journal.readCheckpoint('circleback.meetings')).envelope?.state).toMatchObject({ + completedAt: null, scan: { startedAt: '2026-01-01T00:00:00.000Z', completedMeetingIds: ['a'] }, + }) + failSecond = false + expect((await runIngestion(request, { journal, destination, now: () => new Date('2026-01-02') })).ok).toBe(true) + expect(transcripts).toEqual(['a', 'b', 'b']) + expect((await journal.readCheckpoint('circleback.meetings')).envelope?.state).toMatchObject({ completedAt: '2026-01-02T00:00:00.000Z', scan: null }) + expect(destination.tables.get('default.circleback_meetings_raw')?.map((row) => row.id)).toEqual(['a', 'b']) + expect((await runIngestion(request, { journal, destination, now: () => new Date('2026-01-03') })).ok).toBe(true) + expect(transcripts).toEqual(['a', 'b', 'b', 'a', 'b']) +}) + +test.serial('Circleback empty scans complete and retain absent unavailable IDs as diagnostics', async () => { + process.env.CIRCLEBACK_API_KEY = 'fixture' + let listed = true + const transcripts: string[] = [] + setFetch(async (input) => { + const url = new URL(String(input)) + if (url.pathname.endsWith('/transcript')) { + transcripts.push(url.pathname) + return new Response('not found', { status: 404 }) + } + return Response.json(listed ? [{ id: 'a' }] : []) + }) + const journal = createMemoryJournal() + const destination = createMemoryDestination() + const request = { selected: selectStreams([circlebackPipeline], []), backfill: undefined } + expect((await runIngestion(request, { journal, destination, now: () => new Date('2026-01-01') })).ok).toBe(true) + listed = false + expect((await runIngestion(request, { journal, destination, now: () => new Date('2026-01-02') })).ok).toBe(true) + expect(transcripts).toHaveLength(1) + expect((await journal.readCheckpoint('circleback.meetings')).envelope?.state).toMatchObject({ + completedAt: '2026-01-02T00:00:00.000Z', scan: null, + unavailableTranscripts: [{ id: 'a', status: 'not_found', checkedAt: '2026-01-01T00:00:00.000Z' }], + }) +}) + +test('Circleback rejects changed source scopes and unbounded or duplicated recovery IDs', () => { + const parse = circlebackPipeline.streams[0]?.incremental.parseState + expect(parse).toBeDefined() + const state = { scope: JSON.stringify({ sourceIdentity: 'circleback.primary', ownership: 'All' }), completedAt: null, scan: null, unavailableTranscripts: [] } + expect(() => parse?.({ ...state, scope: 'different-source' })).toThrow('scope changed') + expect(() => parse?.({ ...state, scan: { startedAt: '2026-01-01T00:00:00Z', completedMeetingIds: ['a', 'a'] } })).toThrow('checkpoint is invalid') + expect(() => parse?.({ ...state, scan: { startedAt: '2026-01-01T00:00:00Z', completedMeetingIds: Array.from({ length: 10_001 }, (_, index) => String(index)) } })).toThrow('checkpoint is invalid') +}) diff --git a/registry/github/README.md b/registry/github/README.md index 15699e22..38f0c629 100644 --- a/registry/github/README.md +++ b/registry/github/README.md @@ -17,4 +17,26 @@ bunx chkit ingest run --tag provider:github - `issues` → `github_issues_raw` - `stargazers` → `github_stargazers_raw` -Each run scans configured repositories. Removed issues or stars remain stored. A failed API request fails the stream. Raw rows use provider IDs and keep the last observed value; there is no deletion reconciliation. Edit the readers, select fewer streams, or add SQL views for a specific use case. Run one ingestion process per ClickHouse target at a time. +## Sync and recovery + +Issues use `since`, `sort=updated`, and `direction=asc`, then discard observations newer than the fixed execution cutoff. The initial scan starts at `2008-01-01`; later scans replay five minutes before the last committed watermark. GitHub has no `until` filter: following `Link` continuations over changing results is best-effort polling, not a source snapshot. The watermark advances only after the complete issue window and all its destination writes succeed. Failed windows start again from their lower bound; no page number is saved. + +Each issue includes all returned issue-comment pages. Pull requests additionally include detail, reviews, review comments, commits, and files. The provider can cap commits or files; `_chkit_context.commits_complete` and `files_complete` compare returned counts with the detail response. A child-request failure prevents publishing that parent and advancing the window. GitHub tokens need read access to issues and pull requests for this context. + +Stargazers always perform a full scan. Their checkpoint's `completedAt` records a successful scan, including an empty one; it is not an incremental star cursor. Removed stars and deleted issues remain stored. Child-only changes may not update the parent issue, and offset pagination can miss records under concurrent mutations. Periodically replay the issue history using a new backfill ID, or supplement polling with a durable webhook integration: + +```sh +bunx chkit ingest run --tag provider:github --tag resource:issues --backfill reconcile-2026-10-04 --from 2008-01-01 +bunx chkit ingest status --tag provider:github --json +``` + +This re-reads current observations by their last-update timestamp; it does not reconstruct historical versions. Reusing an ID addresses that backfill's checkpoint namespace. Raw payloads keep provider fields, and mutable observations intentionally have no declared chunk ID so a changed replay is not suppressed by a page identity. Run one ingestion process per ClickHouse target at a time. + +## Fixture tests + +```sh +bunx chkit add github --with-tests +bun test src/integrations/github/tests/basic.test.ts +``` + +The portable suite covers continuation links, cutoff filtering, PR context, replay after source/sink failures, overlap and empty full-scan completion without live credentials. diff --git a/registry/github/index.ts b/registry/github/index.ts index ddd8291c..728cdc51 100644 --- a/registry/github/index.ts +++ b/registry/github/index.ts @@ -1,30 +1,59 @@ -import { definePipeline, defineStream, HttpError, rawRows, rawTable, type FetchContext } from '@chkit/plugin-ingest' +import { definePipeline, defineStream, HttpError, IngestConfigError, rawRows, rawTable, timestampWindow, type ErrorClassifier, type FetchContext, type IncrementalStrategy } from '@chkit/plugin-ingest' const database = 'default' -// Supply owner/repo names; one stable stream per repository. +// One stable stream per repository. Reset its checkpoint when changing its scope. const repositories = ['obsessiondb/chkit'] +const overlapMs = 5 * 60 * 1000 -interface Issue { number: number; [key: string]: unknown } +interface Issue { number: number; updated_at: string; pull_request?: { url: string }; [key: string]: unknown } interface Stargazer { user: { id: number }; [key: string]: unknown } +interface ScanState { completedAt: string } export const github_issuesRaw = rawTable({ database, name: 'github_issues_raw' }) export const github_stargazersRaw = rawTable({ database, name: 'github_stargazers_raw' }) +const stargazerScan: IncrementalStrategy = { + id: 'github.stargazers.full_scan', version: 1, + parseState(raw) { + if (!isObject(raw) || typeof raw.completedAt !== 'string' || !Number.isFinite(Date.parse(raw.completedAt))) { + throw new IngestConfigError('GitHub stargazer checkpoint has no valid completedAt.') + } + return { completedAt: raw.completedAt } + }, + plan: ({ cutoff }) => cutoff.toISOString(), + complete: ({ selection }) => ({ completedAt: selection }), +} + export const githubPipeline = definePipeline({ - id: 'github', tags: ['provider:github'], maxFetches: 1, + id: 'github', tags: ['provider:github'], maxStreams: 1, maxFetches: 1, streams: repositories.flatMap((repo) => [ defineStream({ id: `github.issues.${repo.replace('/', '.')}`, tags: ['resource:issues'], destination: github_issuesRaw, + // GitHub's since filter is strict; overlap includes equal timestamps and recent changes. + incremental: timestampWindow({ start: new Date('2008-01-01T00:00:00Z'), overlapMs }), + classifyError: classifyGitHubError, async *read(context) { - for await (const page of pages(context, repo, 'issues', 'application/vnd.github+json')) { - yield { rows: rawRows(page, (issue) => JSON.stringify([repo, issue.number])) } + for await (const page of pages(context, `/repos/${repo}/issues`, 'application/vnd.github+json', { + state: 'all', sort: 'updated', direction: 'asc', since: context.selection.from.toISOString(), + })) { + for (const issue of page) { + if (!Number.isSafeInteger(issue.number) || !Number.isFinite(Date.parse(issue.updated_at))) { + throw new IngestConfigError('GitHub issue has no valid number or updated_at.') + } + // REST has no until parameter. Enforce the cutoff locally; live pages are not snapshots. + if (Date.parse(issue.updated_at) > context.selection.to.getTime()) continue + const complete = await withIssueContext(context, repo, issue) + yield { rows: rawRows([complete], (item) => JSON.stringify([repo, item.number])) } + } } }, }), defineStream({ id: `github.stargazers.${repo.replace('/', '.')}`, tags: ['resource:stargazers'], destination: github_stargazersRaw, + incremental: stargazerScan, classifyError: classifyGitHubError, async *read(context) { - for await (const page of pages(context, repo, 'stargazers', 'application/vnd.github.star+json')) { + for await (const page of pages(context, `/repos/${repo}/stargazers`, 'application/vnd.github.star+json')) { + if (page.some((star) => !star.user || !Number.isSafeInteger(star.user.id))) throw new IngestConfigError('GitHub stargazer has no user ID.') yield { rows: rawRows(page, (star) => JSON.stringify([repo, star.user.id])) } } }, @@ -32,23 +61,78 @@ export const githubPipeline = definePipeline({ ]), }) -async function* pages(context: FetchContext, repo: string, resource: string, accept: string): AsyncGenerator { - let page = 1 - while (true) { - const url = new URL(`https://api.github.com/repos/${repo}/${resource}`) - url.searchParams.set('per_page', '100') - url.searchParams.set('page', String(page)) - if (resource === 'issues') url.searchParams.set('state', 'all') - const items = await context.attempt(async (signal) => { - const token = process.env.GITHUB_TOKEN - if (!token) throw new Error('Set GITHUB_TOKEN') - const response = await fetch(url, { signal, headers: { Authorization: `Bearer ${token}`, Accept: accept, 'X-GitHub-Api-Version': '2022-11-28' } }) - if (!response.ok) throw await HttpError.fromResponse(response) - return await response.json() as T[] - }, { label: `GET /repos/{owner}/{repo}/${resource}` }) - if (!Array.isArray(items)) throw new Error(`GitHub ${resource} response is not an array`) - if (items.length) yield items - if (items.length < 100) return - page++ +/** A 403 can mean rate exhaustion; ordinary permission failures remain permanent. */ +function classifyGitHubError(cause: unknown): ReturnType { + if (!(cause instanceof HttpError) || (cause.status !== 403 && cause.status !== 429)) return undefined + const headers = cause.response.headers + if (headers.get('x-ratelimit-remaining') !== '0' && !headers.has('retry-after')) return undefined + const resetSeconds = Number(headers.get('x-ratelimit-reset')) + const untilReset = Number.isFinite(resetSeconds) && resetSeconds > 0 ? Math.max(0, resetSeconds * 1000 - Date.now()) : undefined + return { kind: 'rate_limited', retryAfterMs: cause.retryAfterMs ?? untilReset } +} + +async function* pages(context: FetchContext, path: string, accept: string, params: Record = {}): AsyncGenerator { + const first = new URL(`https://api.github.com${path}`) + for (const [key, value] of Object.entries({ ...params, per_page: '100' })) first.searchParams.set(key, value) + let next: string | undefined = first.toString() + const seen = new Set() + while (next) { + if (seen.has(next)) throw new IngestConfigError('GitHub repeated a page URL.') + seen.add(next) + const current: string = next + const result = await context.attempt(async (signal) => { + const response = await request(current, accept, signal) + const items: unknown = await response.json() + if (!Array.isArray(items)) throw new IngestConfigError(`GitHub ${path} response is not an array.`) + return { items: items as T[], next: nextLink(response.headers.get('link'), current) } + }, { label: `GET ${path}` }) + if (result.items.length) yield result.items + next = result.next } } + +/** Load complete child conversations before publishing the parent observation. */ +async function withIssueContext(context: FetchContext, repo: string, issue: Issue) { + const comment_items = await collect(context, `/repos/${repo}/issues/${issue.number}/comments`) + if (!issue.pull_request) return { ...issue, comment_items } + const path = `/repos/${repo}/pulls/${issue.number}` + const pull_detail = await context.attempt(async (signal) => { + const response = await request(`https://api.github.com${path}`, 'application/vnd.github+json', signal) + const detail: unknown = await response.json() + if (!isObject(detail)) throw new IngestConfigError('GitHub pull request detail is not an object.') + return detail + }, { label: 'GET pull request detail' }) + const reviews = await collect(context, `${path}/reviews`) + const review_comments = await collect(context, `${path}/comments`) + const commits = await collect(context, `${path}/commits`) + const files = await collect(context, `${path}/files`) + return { ...issue, pull_detail, comment_items, reviews, review_comments, commits, files, + _chkit_context: { files_complete: files.length === pull_detail.changed_files, commits_complete: commits.length === pull_detail.commits } } +} + +async function collect(context: FetchContext, path: string): Promise[]> { + const items: Record[] = [] + for await (const page of pages>(context, path, 'application/vnd.github+json')) items.push(...page) + return items +} + +async function request(url: string, accept: string, signal: AbortSignal): Promise { + const token = process.env.GITHUB_TOKEN?.trim() + if (!token) throw new IngestConfigError('Set GITHUB_TOKEN.') + const response = await fetch(url, { signal: AbortSignal.any([signal, AbortSignal.timeout(30_000)]), redirect: 'error', + headers: { Authorization: `Bearer ${token}`, Accept: accept, 'X-GitHub-Api-Version': '2022-11-28' } }) + if (!response.ok) throw await HttpError.fromResponse(response) + return response +} + +function nextLink(header: string | null, current: string): string | undefined { + const value = header?.split(',').map((part) => /<([^>]+)>\s*;.*rel="?next"?/.exec(part)?.[1]).find(Boolean) + if (!value) return undefined + const next = new URL(value, current) + if (next.origin !== 'https://api.github.com') throw new IngestConfigError('GitHub next page has an unexpected origin.') + return next.toString() +} + +function isObject(value: unknown): value is Record { + return typeof value === 'object' && value !== null && !Array.isArray(value) +} diff --git a/registry/github/manifest.json b/registry/github/manifest.json index 1d27fe1a..4f531378 100644 --- a/registry/github/manifest.json +++ b/registry/github/manifest.json @@ -3,10 +3,10 @@ "name": "github", "type": "registry:item", "title": "GitHub", - "description": "Small, editable GitHub API ingestion example with raw ClickHouse tables.", + "description": "Editable GitHub timestamp ingestion with complete issue and pull request context and full stargazer scans.", "dependencies": [ - "@chkit/core@^0.2.0-beta.8", - "@chkit/plugin-ingest@^0.2.0-beta.8" + "@chkit/core@^0.2.0-beta.10", + "@chkit/plugin-ingest@^0.2.0-beta.10" ], "files": [ { @@ -18,18 +18,24 @@ "path": "index.ts", "type": "registry:file", "target": "src/integrations/github/index.ts" + }, + { + "path": "tests/basic.test.ts", + "type": "registry:file", + "target": "src/integrations/github/tests/basic.test.ts", + "role": "test" } ], "meta": { "chkit": { "formatVersion": 1, - "version": "0.1.0", + "version": "0.2.0", "documentation": "https://chkit.obsessiondb.com/integrations/github/", "logo": "https://chkit.obsessiondb.com/logos/github.svg", "language": "typescript", "license": "MIT", - "chkit": "^0.2.0-beta.8", - "ingest": "^0.2.0-beta.8", + "chkit": "^0.2.0-beta.10", + "ingest": "^0.2.0-beta.10", "clickhouse": ">=25.3.0", "root": "src/integrations/github", "entry": "index.ts", @@ -43,9 +49,12 @@ "name": "issues", "title": "Issues and pull requests", "table": "github_issues_raw", - "description": "Issue and pull request summaries for the configured repositories", - "scopes": [], - "strategy": "full", + "description": "Issues and pull requests updated inside a timestamp window, including paginated comments and pull request context.", + "scopes": [ + "Issues:read", + "Pull requests:read" + ], + "strategy": "timestamp", "endpoints": [ { "method": "GET", @@ -86,10 +95,13 @@ }, "views": [], "sync": { - "description": "A full read runs for each enabled stream. Stable IDs keep the latest observed record.", + "description": "Issues use completed timestamp windows with five-minute overlap; stargazers record successful full-scan completion.", "schedule": "Run bunx chkit ingest run --tag provider:github from a scheduler.", - "deletions": "Each run scans configured repositories. Removed issues or stars remain stored." + "deletions": "Deleted issues and removed stars remain stored. Periodic historical reconciliation refreshes child context; absence does not generate tombstones." } } - } + }, + "devDependencies": [ + "@types/bun@^1.2.0" + ] } diff --git a/registry/github/releases/0.2.0.json b/registry/github/releases/0.2.0.json new file mode 100644 index 00000000..1d560e6c --- /dev/null +++ b/registry/github/releases/0.2.0.json @@ -0,0 +1,115 @@ +{ + "$schema": "https://ui.shadcn.com/schema/registry-item.json", + "name": "github", + "type": "registry:item", + "title": "GitHub", + "description": "Editable GitHub timestamp ingestion with complete issue and pull request context and full stargazer scans.", + "dependencies": [ + "@chkit/core@^0.2.0-beta.10", + "@chkit/plugin-ingest@^0.2.0-beta.10" + ], + "devDependencies": [ + "@types/bun@^1.2.0" + ], + "files": [ + { + "path": "github/README.md", + "type": "registry:file", + "target": "src/integrations/github/README.md", + "content": "# GitHub raw ingestion example\n\nThis is a small editable starting point based on the GitHub source in [obsessiondb/brain](https://github.com/obsessiondb/brain/tree/main/scripts/brain-sync). It stores provider responses as native JSON without Brain's memory or sales projections.\n\nSet `GITHUB_TOKEN` in the runtime environment and configure a direct ClickHouse connection. Edit `repositories` in `index.ts` before the first migration. Native JSON requires ClickHouse 25.3 or later.\n\n```sh\nbunx chkit check\nbunx chkit generate --name add_github\n# Review the migration before applying it.\nbunx chkit migrate --apply\nbunx chkit ingest run --tag provider:github\n```\n\n## Resources\n\n- `issues` → `github_issues_raw`\n- `stargazers` → `github_stargazers_raw`\n\n## Sync and recovery\n\nIssues use `since`, `sort=updated`, and `direction=asc`, then discard observations newer than the fixed execution cutoff. The initial scan starts at `2008-01-01`; later scans replay five minutes before the last committed watermark. GitHub has no `until` filter: following `Link` continuations over changing results is best-effort polling, not a source snapshot. The watermark advances only after the complete issue window and all its destination writes succeed. Failed windows start again from their lower bound; no page number is saved.\n\nEach issue includes all returned issue-comment pages. Pull requests additionally include detail, reviews, review comments, commits, and files. The provider can cap commits or files; `_chkit_context.commits_complete` and `files_complete` compare returned counts with the detail response. A child-request failure prevents publishing that parent and advancing the window. GitHub tokens need read access to issues and pull requests for this context.\n\nStargazers always perform a full scan. Their checkpoint's `completedAt` records a successful scan, including an empty one; it is not an incremental star cursor. Removed stars and deleted issues remain stored. Child-only changes may not update the parent issue, and offset pagination can miss records under concurrent mutations. Periodically replay the issue history using a new backfill ID, or supplement polling with a durable webhook integration:\n\n```sh\nbunx chkit ingest run --tag provider:github --tag resource:issues --backfill reconcile-2026-10-04 --from 2008-01-01\nbunx chkit ingest status --tag provider:github --json\n```\n\nThis re-reads current observations by their last-update timestamp; it does not reconstruct historical versions. Reusing an ID addresses that backfill's checkpoint namespace. Raw payloads keep provider fields, and mutable observations intentionally have no declared chunk ID so a changed replay is not suppressed by a page identity. Run one ingestion process per ClickHouse target at a time.\n\n## Fixture tests\n\n```sh\nbunx chkit add github --with-tests\nbun test src/integrations/github/tests/basic.test.ts\n```\n\nThe portable suite covers continuation links, cutoff filtering, PR context, replay after source/sink failures, overlap and empty full-scan completion without live credentials.\n" + }, + { + "path": "github/index.ts", + "type": "registry:file", + "target": "src/integrations/github/index.ts", + "content": "import { definePipeline, defineStream, HttpError, IngestConfigError, rawRows, rawTable, timestampWindow, type ErrorClassifier, type FetchContext, type IncrementalStrategy } from '@chkit/plugin-ingest'\n\nconst database = 'default'\n// One stable stream per repository. Reset its checkpoint when changing its scope.\nconst repositories = ['obsessiondb/chkit']\nconst overlapMs = 5 * 60 * 1000\n\ninterface Issue { number: number; updated_at: string; pull_request?: { url: string }; [key: string]: unknown }\ninterface Stargazer { user: { id: number }; [key: string]: unknown }\ninterface ScanState { completedAt: string }\n\nexport const github_issuesRaw = rawTable({ database, name: 'github_issues_raw' })\nexport const github_stargazersRaw = rawTable({ database, name: 'github_stargazers_raw' })\n\nconst stargazerScan: IncrementalStrategy = {\n id: 'github.stargazers.full_scan', version: 1,\n parseState(raw) {\n if (!isObject(raw) || typeof raw.completedAt !== 'string' || !Number.isFinite(Date.parse(raw.completedAt))) {\n throw new IngestConfigError('GitHub stargazer checkpoint has no valid completedAt.')\n }\n return { completedAt: raw.completedAt }\n },\n plan: ({ cutoff }) => cutoff.toISOString(),\n complete: ({ selection }) => ({ completedAt: selection }),\n}\n\nexport const githubPipeline = definePipeline({\n id: 'github', tags: ['provider:github'], maxStreams: 1, maxFetches: 1,\n streams: repositories.flatMap((repo) => [\n defineStream({\n id: `github.issues.${repo.replace('/', '.')}`, tags: ['resource:issues'], destination: github_issuesRaw,\n // GitHub's since filter is strict; overlap includes equal timestamps and recent changes.\n incremental: timestampWindow({ start: new Date('2008-01-01T00:00:00Z'), overlapMs }),\n classifyError: classifyGitHubError,\n async *read(context) {\n for await (const page of pages(context, `/repos/${repo}/issues`, 'application/vnd.github+json', {\n state: 'all', sort: 'updated', direction: 'asc', since: context.selection.from.toISOString(),\n })) {\n for (const issue of page) {\n if (!Number.isSafeInteger(issue.number) || !Number.isFinite(Date.parse(issue.updated_at))) {\n throw new IngestConfigError('GitHub issue has no valid number or updated_at.')\n }\n // REST has no until parameter. Enforce the cutoff locally; live pages are not snapshots.\n if (Date.parse(issue.updated_at) > context.selection.to.getTime()) continue\n const complete = await withIssueContext(context, repo, issue)\n yield { rows: rawRows([complete], (item) => JSON.stringify([repo, item.number])) }\n }\n }\n },\n }),\n defineStream({\n id: `github.stargazers.${repo.replace('/', '.')}`, tags: ['resource:stargazers'], destination: github_stargazersRaw,\n incremental: stargazerScan, classifyError: classifyGitHubError,\n async *read(context) {\n for await (const page of pages(context, `/repos/${repo}/stargazers`, 'application/vnd.github.star+json')) {\n if (page.some((star) => !star.user || !Number.isSafeInteger(star.user.id))) throw new IngestConfigError('GitHub stargazer has no user ID.')\n yield { rows: rawRows(page, (star) => JSON.stringify([repo, star.user.id])) }\n }\n },\n }),\n ]),\n})\n\n/** A 403 can mean rate exhaustion; ordinary permission failures remain permanent. */\nfunction classifyGitHubError(cause: unknown): ReturnType {\n if (!(cause instanceof HttpError) || (cause.status !== 403 && cause.status !== 429)) return undefined\n const headers = cause.response.headers\n if (headers.get('x-ratelimit-remaining') !== '0' && !headers.has('retry-after')) return undefined\n const resetSeconds = Number(headers.get('x-ratelimit-reset'))\n const untilReset = Number.isFinite(resetSeconds) && resetSeconds > 0 ? Math.max(0, resetSeconds * 1000 - Date.now()) : undefined\n return { kind: 'rate_limited', retryAfterMs: cause.retryAfterMs ?? untilReset }\n}\n\nasync function* pages(context: FetchContext, path: string, accept: string, params: Record = {}): AsyncGenerator {\n const first = new URL(`https://api.github.com${path}`)\n for (const [key, value] of Object.entries({ ...params, per_page: '100' })) first.searchParams.set(key, value)\n let next: string | undefined = first.toString()\n const seen = new Set()\n while (next) {\n if (seen.has(next)) throw new IngestConfigError('GitHub repeated a page URL.')\n seen.add(next)\n const current: string = next\n const result = await context.attempt(async (signal) => {\n const response = await request(current, accept, signal)\n const items: unknown = await response.json()\n if (!Array.isArray(items)) throw new IngestConfigError(`GitHub ${path} response is not an array.`)\n return { items: items as T[], next: nextLink(response.headers.get('link'), current) }\n }, { label: `GET ${path}` })\n if (result.items.length) yield result.items\n next = result.next\n }\n}\n\n/** Load complete child conversations before publishing the parent observation. */\nasync function withIssueContext(context: FetchContext, repo: string, issue: Issue) {\n const comment_items = await collect(context, `/repos/${repo}/issues/${issue.number}/comments`)\n if (!issue.pull_request) return { ...issue, comment_items }\n const path = `/repos/${repo}/pulls/${issue.number}`\n const pull_detail = await context.attempt(async (signal) => {\n const response = await request(`https://api.github.com${path}`, 'application/vnd.github+json', signal)\n const detail: unknown = await response.json()\n if (!isObject(detail)) throw new IngestConfigError('GitHub pull request detail is not an object.')\n return detail\n }, { label: 'GET pull request detail' })\n const reviews = await collect(context, `${path}/reviews`)\n const review_comments = await collect(context, `${path}/comments`)\n const commits = await collect(context, `${path}/commits`)\n const files = await collect(context, `${path}/files`)\n return { ...issue, pull_detail, comment_items, reviews, review_comments, commits, files,\n _chkit_context: { files_complete: files.length === pull_detail.changed_files, commits_complete: commits.length === pull_detail.commits } }\n}\n\nasync function collect(context: FetchContext, path: string): Promise[]> {\n const items: Record[] = []\n for await (const page of pages>(context, path, 'application/vnd.github+json')) items.push(...page)\n return items\n}\n\nasync function request(url: string, accept: string, signal: AbortSignal): Promise {\n const token = process.env.GITHUB_TOKEN?.trim()\n if (!token) throw new IngestConfigError('Set GITHUB_TOKEN.')\n const response = await fetch(url, { signal: AbortSignal.any([signal, AbortSignal.timeout(30_000)]), redirect: 'error',\n headers: { Authorization: `Bearer ${token}`, Accept: accept, 'X-GitHub-Api-Version': '2022-11-28' } })\n if (!response.ok) throw await HttpError.fromResponse(response)\n return response\n}\n\nfunction nextLink(header: string | null, current: string): string | undefined {\n const value = header?.split(',').map((part) => /<([^>]+)>\\s*;.*rel=\"?next\"?/.exec(part)?.[1]).find(Boolean)\n if (!value) return undefined\n const next = new URL(value, current)\n if (next.origin !== 'https://api.github.com') throw new IngestConfigError('GitHub next page has an unexpected origin.')\n return next.toString()\n}\n\nfunction isObject(value: unknown): value is Record {\n return typeof value === 'object' && value !== null && !Array.isArray(value)\n}\n" + }, + { + "path": "github/tests/basic.test.ts", + "type": "registry:file", + "target": "src/integrations/github/tests/basic.test.ts", + "role": "test", + "content": "import { afterEach, expect, test } from 'bun:test'\nimport { IngestConfigError, runIngestion, selectStreams } from '@chkit/plugin-ingest'\nimport { createMemoryDestination, createMemoryJournal } from '@chkit/plugin-ingest/testing'\n\nimport { githubPipeline } from '../index.js'\n\nconst originalFetch = globalThis.fetch\nconst originalToken = process.env.GITHUB_TOKEN\nconst cutoff = new Date('2026-01-02T00:00:00Z')\n\nfunction setFetch(handler: (...args: Parameters) => Promise): void {\n globalThis.fetch = Object.assign(handler, { preconnect: originalFetch.preconnect })\n process.env.GITHUB_TOKEN = 'fixture'\n}\n\nfunction selected(resource = 'issues') {\n return selectStreams([githubPipeline], [`resource:${resource}`]).map((item) => ({ ...item, stream: { ...item.stream, batchSize: 1, retry: { retries: 0 } } }))\n}\n\nafterEach(() => {\n globalThis.fetch = originalFetch\n if (originalToken === undefined) delete process.env.GITHUB_TOKEN\n else process.env.GITHUB_TOKEN = originalToken\n})\n\ntest.serial('GitHub follows short-page Link continuations, bounds updates and preserves complete PR context', async () => {\n const paths: string[] = []\n setFetch(async (input) => {\n const url = new URL(String(input))\n paths.push(url.pathname + url.search)\n if (url.pathname.endsWith('/issues')) {\n expect(url.searchParams.get('since')).toBe('2008-01-01T00:00:00.000Z')\n expect(url.searchParams.get('sort')).toBe('updated')\n expect(url.searchParams.get('direction')).toBe('asc')\n if (url.searchParams.has('page')) return Response.json([\n { number: 2, updated_at: '2026-01-01T12:00:00Z', pull_request: { url: 'detail' } },\n { number: 3, updated_at: '2026-01-03T00:00:00Z' },\n ])\n return Response.json([{ number: 1, updated_at: '2026-01-01T12:00:00Z' }], {\n headers: { Link: `; rel=\"next\"` },\n })\n }\n if (url.pathname.endsWith('/pulls/2')) return Response.json({ changed_files: 2, commits: 1, custom: 'preserved' })\n if (url.pathname.endsWith('/files')) return Response.json([{ filename: 'file.ts' }])\n if (url.pathname.endsWith('/commits')) return Response.json([{ sha: 'a' }])\n return Response.json([{ id: 11, body: 'context' }])\n })\n const journal = createMemoryJournal()\n const destination = createMemoryDestination()\n const result = await runIngestion({ selected: selected(), backfill: undefined }, { journal, destination, now: () => cutoff })\n expect(result.ok).toBe(true)\n const rows = destination.tables.get('default.github_issues_raw') ?? []\n expect(rows.map((row) => row.id)).toEqual(['[\"obsessiondb/chkit\",1]', '[\"obsessiondb/chkit\",2]'])\n expect(rows[1]?.raw).toMatchObject({ pull_detail: { custom: 'preserved' }, comment_items: [{ id: 11 }], _chkit_context: { files_complete: false, commits_complete: true } })\n expect(paths.some((path) => path.includes('/issues/3/comments'))).toBe(false)\n expect((await journal.readCheckpoint('github.issues.obsessiondb.chkit')).envelope?.state).toEqual({ watermark: cutoff.toISOString() })\n})\n\ntest.serial('GitHub child failures replay the issue window and destination failures cannot advance it', async () => {\n let failComments = true\n const starts: string[] = []\n setFetch(async (input) => {\n const url = new URL(String(input))\n if (url.pathname.endsWith('/issues')) {\n starts.push(url.searchParams.get('since') ?? '')\n return Response.json([{ number: 1, updated_at: '2026-01-01T12:00:00Z' }])\n }\n return failComments ? new Response('denied', { status: 403 }) : Response.json([])\n })\n const journal = createMemoryJournal()\n const destination = createMemoryDestination()\n const request = { selected: selected(), backfill: undefined }\n expect((await runIngestion(request, { journal, destination, now: () => cutoff })).ok).toBe(false)\n expect((await journal.readCheckpoint('github.issues.obsessiondb.chkit')).envelope).toBeUndefined()\n failComments = false\n expect((await runIngestion(request, { journal, destination: { insert: async () => { throw new IngestConfigError('sink rejected') } }, now: () => cutoff })).ok).toBe(false)\n expect((await journal.readCheckpoint('github.issues.obsessiondb.chkit')).envelope).toBeUndefined()\n expect((await runIngestion(request, { journal, destination, now: () => cutoff })).ok).toBe(true)\n expect(starts).toEqual(Array(3).fill('2008-01-01T00:00:00.000Z'))\n})\n\ntest.serial('GitHub subsequent issue scans overlap the committed watermark', async () => {\n const starts: string[] = []\n setFetch(async (input) => {\n starts.push(new URL(String(input)).searchParams.get('since') ?? '')\n return Response.json([])\n })\n const journal = createMemoryJournal()\n const destination = createMemoryDestination()\n const request = { selected: selected(), backfill: undefined }\n expect((await runIngestion(request, { journal, destination, now: () => cutoff })).ok).toBe(true)\n expect((await runIngestion(request, { journal, destination, now: () => new Date('2026-01-03T00:00:00Z') })).ok).toBe(true)\n expect(starts).toEqual(['2008-01-01T00:00:00.000Z', '2026-01-01T23:55:00.000Z'])\n})\n\ntest.serial('GitHub empty stargazer scans persist completion and always read again', async () => {\n const paths: string[] = []\n setFetch(async (input) => { paths.push(String(input)); return Response.json([]) })\n const journal = createMemoryJournal()\n const destination = createMemoryDestination()\n const request = { selected: selected('stargazers'), backfill: undefined }\n expect((await runIngestion(request, { journal, destination, now: () => cutoff })).ok).toBe(true)\n expect((await runIngestion(request, { journal, destination, now: () => new Date('2026-01-03T00:00:00Z') })).ok).toBe(true)\n expect(paths).toHaveLength(2)\n expect(paths.every((path) => !new URL(path).searchParams.has('since'))).toBe(true)\n expect((await journal.readCheckpoint('github.stargazers.obsessiondb.chkit')).envelope?.state).toEqual({ completedAt: '2026-01-03T00:00:00.000Z' })\n})\n" + } + ], + "meta": { + "chkit": { + "formatVersion": 1, + "version": "0.2.0", + "language": "typescript", + "license": "MIT", + "documentation": "https://chkit.obsessiondb.com/integrations/github/", + "logo": "https://chkit.obsessiondb.com/logos/github.svg", + "authentication": { + "method": "Bearer token", + "env": [ + "GITHUB_TOKEN" + ], + "setup": [ + "Provide GITHUB_TOKEN with read access to the selected resources.", + "Supply a new access token when the existing token expires." + ], + "documentation": "https://docs.github.com/en/authentication/keeping-your-account-and-data-secure/managing-your-personal-access-tokens" + }, + "chkit": "^0.2.0-beta.10", + "ingest": "^0.2.0-beta.10", + "clickhouse": ">=25.3.0", + "root": "src/integrations/github", + "entry": "index.ts", + "exports": [ + "githubPipeline", + "github_issuesRaw", + "github_stargazersRaw" + ], + "resources": [ + { + "name": "issues", + "title": "Issues and pull requests", + "table": "github_issues_raw", + "description": "Issues and pull requests updated inside a timestamp window, including paginated comments and pull request context.", + "scopes": [ + "Issues:read", + "Pull requests:read" + ], + "strategy": "timestamp", + "endpoints": [ + { + "method": "GET", + "path": "/repos/{owner}/{repo}/issues", + "documentation": "https://docs.github.com/en/rest/issues/issues#list-repository-issues" + } + ] + }, + { + "name": "stargazers", + "title": "Stargazers", + "table": "github_stargazers_raw", + "description": "Repository stargazers with starred_at when the token permits it", + "scopes": [], + "strategy": "full", + "endpoints": [ + { + "method": "GET", + "path": "/repos/{owner}/{repo}/stargazers", + "documentation": "https://docs.github.com/en/rest/activity/starring#list-stargazers" + } + ] + } + ], + "views": [], + "sync": { + "description": "Issues use completed timestamp windows with five-minute overlap; stargazers record successful full-scan completion.", + "schedule": "Run bunx chkit ingest run --tag provider:github from a scheduler.", + "deletions": "Deleted issues and removed stars remain stored. Periodic historical reconciliation refreshes child context; absence does not generate tombstones." + }, + "env": { + "GITHUB_TOKEN": "" + }, + "fileHashes": { + "src/integrations/github/README.md": "dfa40213490ff65c8d946b53863ee4c506c4237cdd6a4883e37e81e7727b900e", + "src/integrations/github/index.ts": "d46a583543188f2a435998487fbcc90ee63c0a9988c7d78b0fc220825d0bc3ab", + "src/integrations/github/tests/basic.test.ts": "e3bed6edb47951e99715463d69effbe2f5172c6d27db0613f3e7ea327e845f39" + } + } + } +} diff --git a/registry/github/tests/basic.test.ts b/registry/github/tests/basic.test.ts new file mode 100644 index 00000000..4ed7ae05 --- /dev/null +++ b/registry/github/tests/basic.test.ts @@ -0,0 +1,107 @@ +import { afterEach, expect, test } from 'bun:test' +import { IngestConfigError, runIngestion, selectStreams } from '@chkit/plugin-ingest' +import { createMemoryDestination, createMemoryJournal } from '@chkit/plugin-ingest/testing' + +import { githubPipeline } from '../index.js' + +const originalFetch = globalThis.fetch +const originalToken = process.env.GITHUB_TOKEN +const cutoff = new Date('2026-01-02T00:00:00Z') + +function setFetch(handler: (...args: Parameters) => Promise): void { + globalThis.fetch = Object.assign(handler, { preconnect: originalFetch.preconnect }) + process.env.GITHUB_TOKEN = 'fixture' +} + +function selected(resource = 'issues') { + return selectStreams([githubPipeline], [`resource:${resource}`]).map((item) => ({ ...item, stream: { ...item.stream, batchSize: 1, retry: { retries: 0 } } })) +} + +afterEach(() => { + globalThis.fetch = originalFetch + if (originalToken === undefined) delete process.env.GITHUB_TOKEN + else process.env.GITHUB_TOKEN = originalToken +}) + +test.serial('GitHub follows short-page Link continuations, bounds updates and preserves complete PR context', async () => { + const paths: string[] = [] + setFetch(async (input) => { + const url = new URL(String(input)) + paths.push(url.pathname + url.search) + if (url.pathname.endsWith('/issues')) { + expect(url.searchParams.get('since')).toBe('2008-01-01T00:00:00.000Z') + expect(url.searchParams.get('sort')).toBe('updated') + expect(url.searchParams.get('direction')).toBe('asc') + if (url.searchParams.has('page')) return Response.json([ + { number: 2, updated_at: '2026-01-01T12:00:00Z', pull_request: { url: 'detail' } }, + { number: 3, updated_at: '2026-01-03T00:00:00Z' }, + ]) + return Response.json([{ number: 1, updated_at: '2026-01-01T12:00:00Z' }], { + headers: { Link: `; rel="next"` }, + }) + } + if (url.pathname.endsWith('/pulls/2')) return Response.json({ changed_files: 2, commits: 1, custom: 'preserved' }) + if (url.pathname.endsWith('/files')) return Response.json([{ filename: 'file.ts' }]) + if (url.pathname.endsWith('/commits')) return Response.json([{ sha: 'a' }]) + return Response.json([{ id: 11, body: 'context' }]) + }) + const journal = createMemoryJournal() + const destination = createMemoryDestination() + const result = await runIngestion({ selected: selected(), backfill: undefined }, { journal, destination, now: () => cutoff }) + expect(result.ok).toBe(true) + const rows = destination.tables.get('default.github_issues_raw') ?? [] + expect(rows.map((row) => row.id)).toEqual(['["obsessiondb/chkit",1]', '["obsessiondb/chkit",2]']) + expect(rows[1]?.raw).toMatchObject({ pull_detail: { custom: 'preserved' }, comment_items: [{ id: 11 }], _chkit_context: { files_complete: false, commits_complete: true } }) + expect(paths.some((path) => path.includes('/issues/3/comments'))).toBe(false) + expect((await journal.readCheckpoint('github.issues.obsessiondb.chkit')).envelope?.state).toEqual({ watermark: cutoff.toISOString() }) +}) + +test.serial('GitHub child failures replay the issue window and destination failures cannot advance it', async () => { + let failComments = true + const starts: string[] = [] + setFetch(async (input) => { + const url = new URL(String(input)) + if (url.pathname.endsWith('/issues')) { + starts.push(url.searchParams.get('since') ?? '') + return Response.json([{ number: 1, updated_at: '2026-01-01T12:00:00Z' }]) + } + return failComments ? new Response('denied', { status: 403 }) : Response.json([]) + }) + const journal = createMemoryJournal() + const destination = createMemoryDestination() + const request = { selected: selected(), backfill: undefined } + expect((await runIngestion(request, { journal, destination, now: () => cutoff })).ok).toBe(false) + expect((await journal.readCheckpoint('github.issues.obsessiondb.chkit')).envelope).toBeUndefined() + failComments = false + expect((await runIngestion(request, { journal, destination: { insert: async () => { throw new IngestConfigError('sink rejected') } }, now: () => cutoff })).ok).toBe(false) + expect((await journal.readCheckpoint('github.issues.obsessiondb.chkit')).envelope).toBeUndefined() + expect((await runIngestion(request, { journal, destination, now: () => cutoff })).ok).toBe(true) + expect(starts).toEqual(Array(3).fill('2008-01-01T00:00:00.000Z')) +}) + +test.serial('GitHub subsequent issue scans overlap the committed watermark', async () => { + const starts: string[] = [] + setFetch(async (input) => { + starts.push(new URL(String(input)).searchParams.get('since') ?? '') + return Response.json([]) + }) + const journal = createMemoryJournal() + const destination = createMemoryDestination() + const request = { selected: selected(), backfill: undefined } + expect((await runIngestion(request, { journal, destination, now: () => cutoff })).ok).toBe(true) + expect((await runIngestion(request, { journal, destination, now: () => new Date('2026-01-03T00:00:00Z') })).ok).toBe(true) + expect(starts).toEqual(['2008-01-01T00:00:00.000Z', '2026-01-01T23:55:00.000Z']) +}) + +test.serial('GitHub empty stargazer scans persist completion and always read again', async () => { + const paths: string[] = [] + setFetch(async (input) => { paths.push(String(input)); return Response.json([]) }) + const journal = createMemoryJournal() + const destination = createMemoryDestination() + const request = { selected: selected('stargazers'), backfill: undefined } + expect((await runIngestion(request, { journal, destination, now: () => cutoff })).ok).toBe(true) + expect((await runIngestion(request, { journal, destination, now: () => new Date('2026-01-03T00:00:00Z') })).ok).toBe(true) + expect(paths).toHaveLength(2) + expect(paths.every((path) => !new URL(path).searchParams.has('since'))).toBe(true) + expect((await journal.readCheckpoint('github.stargazers.obsessiondb.chkit')).envelope?.state).toEqual({ completedAt: '2026-01-03T00:00:00.000Z' }) +}) diff --git a/registry/google-calendar/README.md b/registry/google-calendar/README.md index 8f569696..d7bfceee 100644 --- a/registry/google-calendar/README.md +++ b/registry/google-calendar/README.md @@ -1,8 +1,8 @@ -# Google Calendar raw ingestion example +# Google Calendar raw ingestion -This is a small editable starting point based on the Google-Calendar source in [obsessiondb/brain](https://github.com/obsessiondb/brain/tree/main/scripts/brain-sync). It stores provider responses as native JSON without Brain's memory or sales projections. +This editable reader stores canonical Google Calendar responses as native JSON and checkpoints provider sync tokens in the chkit ingestion journal. It follows the recoverable ingestion patterns in [obsessiondb/brain](https://github.com/obsessiondb/brain/tree/main/scripts/brain-sync), using Google's change-token protocol for canonical events. -Set `GOOGLE_CALENDAR_ACCESS_TOKEN` in the runtime environment and configure a direct ClickHouse connection. Edit `calendarId`, `pastDays`, and `futureDays` in `index.ts` before the first migration. Native JSON requires ClickHouse 25.3 or later. +Set `GOOGLE_CALENDAR_ACCESS_TOKEN` in the runtime environment and configure a direct ClickHouse connection. Review `sourceId`, `calendarId`, and `database` in `index.ts`. Native JSON requires ClickHouse 25.3 or later. Keep `sourceId` tied to one OAuth installation. Schema imports do not need credentials; requests read the token at runtime and do not refresh it. ```sh bunx chkit check @@ -10,10 +10,40 @@ bunx chkit generate --name add_google_calendar # Review the migration before applying it. bunx chkit migrate --apply bunx chkit ingest run --tag provider:google-calendar +bunx chkit ingest status --tag provider:google-calendar ``` -## Resources +## Resources and identities - `events` → `google_calendar_events_raw` -Each run reads a configured time range. Events outside the range and deleted records are not reconciled. A failed API request fails the stream. Raw rows use provider IDs and keep the last observed value; there is no deletion reconciliation. Edit the readers, select fewer streams, or add SQL views for a specific use case. Run one ingestion process per ClickHouse target at a time. +The first run reads canonical events with `singleEvents=false` and `showDeleted=true`: one-off events, recurring series masters, exceptions, and cancellation payloads. It does not expand every occurrence of a recurring series. Raw row IDs are `[sourceId, resolvedCalendarId, event.id]`; calendar metadata resolves `primary` to a stable calendar ID and prevents a different account from silently reusing its checkpoint. + +The raw destination keeps the latest observed payload per identity. A recurring master contains recurrence rules; applications needing occurrences should add a separate bounded instances reader or derive an occurrence projection. A cancelled payload can be sparse and remains unmodified. + +## Checkpoints and recovery + +Each acknowledged event page checkpoints its next page token while retaining the same input sync token. Only an acknowledged terminal page promotes `nextSyncToken`. Empty terminal pages also commit progress. A failed load leaves the preceding checkpoint, so rerunning resumes safely and may replay observations. Requests have a 30-second timeout. Each run permits 200 page chunks; budget exhaustion preserves progress for the next run. The executor owns retries, cancellation, and sink acknowledgements; run one ingestion process per ClickHouse target at a time. + +Subsequent runs send the committed `syncToken` and ingest Google's changes, including cancelled events. All pages use fixed request parameters. `syncToken` cannot be combined with `timeMin`, `timeMax`, `updatedMin`, or `orderBy`, so this reader rejects explicit `--from`/`--to` bounds. A separately named backfill without date bounds has an isolated checkpoint and performs its own canonical sync. See [Google's synchronization guide](https://developers.google.com/workspace/calendar/api/guides/sync) and [events.list](https://developers.google.com/workspace/calendar/api/v3/reference/events/list). + +`410 Gone` on an incremental request clears provider progress and starts a full baseline. A rejected page token replays its current sync once; repeated rejection fails visibly. Neither recovery path drops stored raw rows. A record absent from the rebuilt baseline can remain in ClickHouse, so this table is a latest-observed archive, not a reconciled current calendar. A current-state projection needs completed-baseline generations or explicit reconciliation before treating absence as removal. + +The recovery count stays in the checkpoint until the terminal sync page has destination acknowledgement. Pausing immediately after recovery or successfully replaying a nonterminal page keeps that count. A second token rejection for the same unfinished sync fails across scheduled executions. Edit the stream's `budget.maxChunks` to change its chunk cap, choose sufficient `--max-duration`, and run frequently enough to finish while provider tokens remain valid. After correcting those conditions and reviewing coverage, migrate the saved state explicitly or use a new stream identity for a deliberate fresh sync. + +Checkpoint state validates source/query scope and resolved calendar identity. Changing source settings requires an explicit state migration or another stream ID; the reader never silently reinterprets old progress. + +## Upgrading from 0.1.x + +Version 0.2.0 replaces the rolling occurrence window with canonical change-token sync and changes row identities to include the resolved calendar and installation. Keep the existing raw table as an archive and use a new table/stream identity for a fresh canonical dataset, or deliberately migrate existing row identities. Existing expanded occurrences are not removed automatically. + +## Fixture tests + +```sh +bunx chkit add google-calendar --with-tests +bun test src/integrations/google-calendar/tests/basic.test.ts +``` + +Fixtures verify page resumption, empty deltas, terminal token promotion after loads, cancellation payloads, token expiry, scope changes, and malformed pagination without live credentials. + +The [Google Calendar integration guide](https://chkit.obsessiondb.com/integrations/google-calendar/) covers installation and sync behavior. diff --git a/registry/google-calendar/index.ts b/registry/google-calendar/index.ts index ffe7d2dd..9f662359 100644 --- a/registry/google-calendar/index.ts +++ b/registry/google-calendar/index.ts @@ -1,45 +1,132 @@ -import { definePipeline, defineStream, HttpError, rawRows, rawTable, type FetchContext } from '@chkit/plugin-ingest' +import { cursorState, definePipeline, defineStream, HttpError, IngestConfigError, rawRows, rawTable, type FetchContext } from '@chkit/plugin-ingest' const database = 'default' +// Keep this identity tied to one OAuth installation. Use another stream ID for another source. +const sourceId = 'google-calendar.primary' const calendarId = 'primary' -// Change these bounds to the events relevant to the project. -const pastDays = 365 -const futureDays = 180 +const scope = JSON.stringify([sourceId, calendarId, { singleEvents: false, showDeleted: true, maxResults: 250 }]) interface Event { id: string; [key: string]: unknown } -interface Page { items?: Event[]; nextPageToken?: string } +interface Page { items: Event[]; nextPageToken?: string; nextSyncToken?: string } +interface CalendarState { + scope: string + calendarId: string + syncToken?: string + pageToken?: string + /** Recovery attempts for this unfinished sync, including previous executions. */ + recoveryCount?: number +} export const google_calendar_eventsRaw = rawTable({ database, name: 'google_calendar_events_raw' }) export const google_calendarPipeline = definePipeline({ id: 'google-calendar', tags: ['provider:google-calendar'], maxFetches: 1, - streams: [defineStream({ id: 'google-calendar.events', tags: ['resource:events'], destination: google_calendar_eventsRaw, + streams: [defineStream({ + id: 'google-calendar.events', tags: ['resource:events'], destination: google_calendar_eventsRaw, + incremental: { + ...cursorState({ id: 'google-calendar.sync-token', version: 1, parse: parseState }), + plan({ state, range }) { + if (range?.from || range?.to) throw new IngestConfigError('Calendar canonical sync does not accept date bounds. Use a separate bounded instances reader for an occurrence backfill.') + return state + }, + }, + batchSize: 1, + budget: { maxChunks: 200, maxChunkRows: 250 }, + classifyError: (cause) => cause instanceof IngestConfigError ? { kind: 'permanent' } : undefined, async *read(context) { - let pageToken: string | undefined + const resolvedCalendarId = await resolveCalendar(context) + if (context.state && context.state.calendarId !== resolvedCalendarId) { + throw new IngestConfigError('Calendar checkpoint belongs to a different authenticated calendar. Use a separate stream ID for the new source.') + } + let state: CalendarState = context.state ?? { scope, calendarId: resolvedCalendarId } const seen = new Set() - do { - const page = await readPage(context, pageToken) - if (!Array.isArray(page.items ?? [])) throw new Error('Calendar items are not an array') - if (page.items?.length) yield { rows: rawRows(page.items, (event) => JSON.stringify([calendarId, event.id])) } - pageToken = page.nextPageToken - if (pageToken && seen.has(pageToken)) throw new Error('Calendar repeated a page token') - if (pageToken) seen.add(pageToken) - } while (pageToken) + if (state.pageToken) seen.add(state.pageToken) + while (true) { + const page = await readPage(context, state) + if ('reset' in page) { + if ((state.recoveryCount ?? 0) >= 1) throw new IngestConfigError('Calendar unfinished sync repeatedly rejected a token across resumed runs. Adjust the editable stream chunk budget, run duration, or polling frequency, then explicitly migrate the saved state or use a new stream identity after reviewing coverage.') + seen.clear() + // An expired sync token starts a new baseline; a rejected page token + // replays the current sync from its original, committed input token. + state = { scope, calendarId: resolvedCalendarId, recoveryCount: (state.recoveryCount ?? 0) + 1, ...(page.reset === 'page' && state.syncToken ? { syncToken: state.syncToken } : {}) } + yield { rows: [], state, id: 'calendar-reset' } + continue + } + if (page.nextPageToken && seen.has(page.nextPageToken)) throw new IngestConfigError('Calendar repeated a page token.') + if (page.nextPageToken) seen.add(page.nextPageToken) + state = page.nextPageToken + ? { ...state, pageToken: page.nextPageToken } + // Only acknowledgement of this terminal page clears recovery debt. + : { scope, calendarId: resolvedCalendarId, syncToken: requiredString(page.nextSyncToken, 'nextSyncToken') } + const rows = rawRows(page.items, (event) => JSON.stringify([sourceId, resolvedCalendarId, event.id])) + // Empty terminal pages must commit the new sync token too. + yield { rows, state, ...(rows.length === 0 ? { id: 'calendar-empty-page' } : {}) } + if (!page.nextPageToken) return + } }, })], }) -async function readPage(context: FetchContext, pageToken?: string): Promise { +async function resolveCalendar(context: FetchContext): Promise { + return context.attempt(async (signal) => { + const payload = await request(`calendars/${encodeURIComponent(calendarId)}`, signal) + return requiredString(payload.id, 'calendar id') + }, { label: 'GET calendar metadata' }) +} + +async function readPage(context: FetchContext, state: CalendarState): Promise { return context.attempt(async (signal) => { - const token = process.env.GOOGLE_CALENDAR_ACCESS_TOKEN - if (!token) throw new Error('Set GOOGLE_CALENDAR_ACCESS_TOKEN') - const url = new URL(`https://www.googleapis.com/calendar/v3/calendars/${encodeURIComponent(calendarId)}/events`) - const timeMin = new Date(Date.now() - pastDays * 86_400_000).toISOString() - const timeMax = new Date(Date.now() + futureDays * 86_400_000).toISOString() - for (const [key, value] of Object.entries({ singleEvents: 'true', showDeleted: 'true', maxResults: '250', timeMin, timeMax })) url.searchParams.set(key, value) - if (pageToken) url.searchParams.set('pageToken', pageToken) - const response = await fetch(url, { signal, headers: { Authorization: `Bearer ${token}` } }) + const url = new URL(`https://www.googleapis.com/calendar/v3/calendars/${encodeURIComponent(state.calendarId)}/events`) + for (const [key, value] of Object.entries({ singleEvents: 'false', showDeleted: 'true', maxResults: '250' })) url.searchParams.set(key, value) + if (state.syncToken) url.searchParams.set('syncToken', state.syncToken) + if (state.pageToken) url.searchParams.set('pageToken', state.pageToken) + const response = await fetch(url, { signal: AbortSignal.any([signal, AbortSignal.timeout(30_000)]), headers: authorization(), redirect: 'error' }) + if (response.status === 410 && state.syncToken) return { reset: 'sync' } + if ((response.status === 400 || response.status === 410) && state.pageToken) return { reset: 'page' } if (!response.ok) throw await HttpError.fromResponse(response) - return await response.json() as Page + const payload: unknown = await response.json() + if (!isObject(payload) || (payload.items !== undefined && !Array.isArray(payload.items))) throw new IngestConfigError('Calendar items are not an array.') + const items = (payload.items ?? []).map((item: unknown) => { + if (!isObject(item)) throw new IngestConfigError('Calendar returned an invalid event.') + return { ...item, id: requiredString(item.id, 'event id') } + }) + const nextPageToken = optionalString(payload.nextPageToken, 'nextPageToken') + const nextSyncToken = optionalString(payload.nextSyncToken, 'nextSyncToken') + if (nextPageToken && nextSyncToken) throw new IngestConfigError('Calendar returned both a page and sync token.') + if (!nextPageToken && !nextSyncToken) throw new IngestConfigError('Calendar terminal page has no nextSyncToken.') + return { items, nextPageToken, nextSyncToken } }, { label: 'GET calendar events' }) } + +async function request(path: string, signal: AbortSignal): Promise> { + const response = await fetch(`https://www.googleapis.com/calendar/v3/${path}`, { signal: AbortSignal.any([signal, AbortSignal.timeout(30_000)]), headers: authorization(), redirect: 'error' }) + if (!response.ok) throw await HttpError.fromResponse(response) + const payload: unknown = await response.json() + if (!isObject(payload)) throw new IngestConfigError('Calendar returned an invalid response.') + return payload +} + +function parseState(raw: unknown): CalendarState { + if (!isObject(raw) || raw.scope !== scope) throw new IngestConfigError('Calendar checkpoint scope changed. Migrate the state explicitly or use a new stream ID.') + if (raw.recoveryCount !== undefined && raw.recoveryCount !== 0 && raw.recoveryCount !== 1) throw new IngestConfigError('Calendar checkpoint recoveryCount must be zero or one.') + return { scope, calendarId: requiredString(raw.calendarId, 'checkpoint calendar id'), syncToken: optionalString(raw.syncToken, 'checkpoint syncToken'), pageToken: optionalString(raw.pageToken, 'checkpoint pageToken'), recoveryCount: raw.recoveryCount } +} + +function authorization(): Record { + const token = process.env.GOOGLE_CALENDAR_ACCESS_TOKEN?.trim() + if (!token) throw new IngestConfigError('Set GOOGLE_CALENDAR_ACCESS_TOKEN.') + return { Authorization: `Bearer ${token}` } +} + +function optionalString(value: unknown, field: string): string | undefined { + return value === undefined ? undefined : requiredString(value, field) +} + +function requiredString(value: unknown, field: string): string { + if (typeof value !== 'string' || value.trim().length === 0) throw new IngestConfigError(`Calendar ${field} must be a non-empty string.`) + return value +} + +function isObject(value: unknown): value is Record { + return typeof value === 'object' && value !== null && !Array.isArray(value) +} diff --git a/registry/google-calendar/manifest.json b/registry/google-calendar/manifest.json index 0e05de09..b58b4ee4 100644 --- a/registry/google-calendar/manifest.json +++ b/registry/google-calendar/manifest.json @@ -3,10 +3,10 @@ "name": "google-calendar", "type": "registry:item", "title": "Google Calendar", - "description": "Small, editable Google Calendar API ingestion example with raw ClickHouse tables.", + "description": "Editable Google Calendar API ingestion with journaled checkpoints and raw ClickHouse tables.", "dependencies": [ - "@chkit/core@^0.2.0-beta.8", - "@chkit/plugin-ingest@^0.2.0-beta.8" + "@chkit/core@^0.2.0-beta.10", + "@chkit/plugin-ingest@^0.2.0-beta.10" ], "files": [ { @@ -29,13 +29,13 @@ "meta": { "chkit": { "formatVersion": 1, - "version": "0.1.0", + "version": "0.2.0", "documentation": "https://chkit.obsessiondb.com/integrations/google-calendar/", "logo": "https://chkit.obsessiondb.com/logos/google-calendar.svg", "language": "typescript", "license": "MIT", - "chkit": "^0.2.0-beta.8", - "ingest": "^0.2.0-beta.8", + "chkit": "^0.2.0-beta.10", + "ingest": "^0.2.0-beta.10", "clickhouse": ">=25.3.0", "root": "src/integrations/google-calendar", "entry": "index.ts", @@ -48,12 +48,17 @@ "name": "events", "title": "Events", "table": "google_calendar_events_raw", - "description": "Events from the configured calendar, including cancelled events", + "description": "Canonical events, recurring masters, exceptions, and cancellation payloads from the configured calendar", "scopes": [ "https://www.googleapis.com/auth/calendar.readonly" ], - "strategy": "full", + "strategy": "cursor", "endpoints": [ + { + "method": "GET", + "path": "/calendars/{calendarId}", + "documentation": "https://developers.google.com/workspace/calendar/api/v3/reference/calendars/get" + }, { "method": "GET", "path": "/calendars/{calendarId}/events", @@ -78,10 +83,13 @@ }, "views": [], "sync": { - "description": "A full read runs for each enabled stream. Stable IDs keep the latest observed record.", + "description": "Initial canonical event read followed by Google sync-token deltas. Acknowledged pages resume from journaled state.", "schedule": "Run bunx chkit ingest run --tag provider:google-calendar from a scheduler.", - "deletions": "Each run reads a configured time range. Events outside the range and deleted records are not reconciled." + "deletions": "Cancellation payloads are retained. An expired token reboots provider progress without clearing raw observations; absent records are not reconciled." } } - } + }, + "devDependencies": [ + "@types/bun@^1.2.0" + ] } diff --git a/registry/google-calendar/releases/0.2.0.json b/registry/google-calendar/releases/0.2.0.json new file mode 100644 index 00000000..5cc1eee4 --- /dev/null +++ b/registry/google-calendar/releases/0.2.0.json @@ -0,0 +1,103 @@ +{ + "$schema": "https://ui.shadcn.com/schema/registry-item.json", + "name": "google-calendar", + "type": "registry:item", + "title": "Google Calendar", + "description": "Editable Google Calendar API ingestion with journaled checkpoints and raw ClickHouse tables.", + "dependencies": [ + "@chkit/core@^0.2.0-beta.10", + "@chkit/plugin-ingest@^0.2.0-beta.10" + ], + "devDependencies": [ + "@types/bun@^1.2.0" + ], + "files": [ + { + "path": "google-calendar/README.md", + "type": "registry:file", + "target": "src/integrations/google-calendar/README.md", + "content": "# Google Calendar raw ingestion\n\nThis editable reader stores canonical Google Calendar responses as native JSON and checkpoints provider sync tokens in the chkit ingestion journal. It follows the recoverable ingestion patterns in [obsessiondb/brain](https://github.com/obsessiondb/brain/tree/main/scripts/brain-sync), using Google's change-token protocol for canonical events.\n\nSet `GOOGLE_CALENDAR_ACCESS_TOKEN` in the runtime environment and configure a direct ClickHouse connection. Review `sourceId`, `calendarId`, and `database` in `index.ts`. Native JSON requires ClickHouse 25.3 or later. Keep `sourceId` tied to one OAuth installation. Schema imports do not need credentials; requests read the token at runtime and do not refresh it.\n\n```sh\nbunx chkit check\nbunx chkit generate --name add_google_calendar\n# Review the migration before applying it.\nbunx chkit migrate --apply\nbunx chkit ingest run --tag provider:google-calendar\nbunx chkit ingest status --tag provider:google-calendar\n```\n\n## Resources and identities\n\n- `events` → `google_calendar_events_raw`\n\nThe first run reads canonical events with `singleEvents=false` and `showDeleted=true`: one-off events, recurring series masters, exceptions, and cancellation payloads. It does not expand every occurrence of a recurring series. Raw row IDs are `[sourceId, resolvedCalendarId, event.id]`; calendar metadata resolves `primary` to a stable calendar ID and prevents a different account from silently reusing its checkpoint.\n\nThe raw destination keeps the latest observed payload per identity. A recurring master contains recurrence rules; applications needing occurrences should add a separate bounded instances reader or derive an occurrence projection. A cancelled payload can be sparse and remains unmodified.\n\n## Checkpoints and recovery\n\nEach acknowledged event page checkpoints its next page token while retaining the same input sync token. Only an acknowledged terminal page promotes `nextSyncToken`. Empty terminal pages also commit progress. A failed load leaves the preceding checkpoint, so rerunning resumes safely and may replay observations. Requests have a 30-second timeout. Each run permits 200 page chunks; budget exhaustion preserves progress for the next run. The executor owns retries, cancellation, and sink acknowledgements; run one ingestion process per ClickHouse target at a time.\n\nSubsequent runs send the committed `syncToken` and ingest Google's changes, including cancelled events. All pages use fixed request parameters. `syncToken` cannot be combined with `timeMin`, `timeMax`, `updatedMin`, or `orderBy`, so this reader rejects explicit `--from`/`--to` bounds. A separately named backfill without date bounds has an isolated checkpoint and performs its own canonical sync. See [Google's synchronization guide](https://developers.google.com/workspace/calendar/api/guides/sync) and [events.list](https://developers.google.com/workspace/calendar/api/v3/reference/events/list).\n\n`410 Gone` on an incremental request clears provider progress and starts a full baseline. A rejected page token replays its current sync once; repeated rejection fails visibly. Neither recovery path drops stored raw rows. A record absent from the rebuilt baseline can remain in ClickHouse, so this table is a latest-observed archive, not a reconciled current calendar. A current-state projection needs completed-baseline generations or explicit reconciliation before treating absence as removal.\n\nThe recovery count stays in the checkpoint until the terminal sync page has destination acknowledgement. Pausing immediately after recovery or successfully replaying a nonterminal page keeps that count. A second token rejection for the same unfinished sync fails across scheduled executions. Edit the stream's `budget.maxChunks` to change its chunk cap, choose sufficient `--max-duration`, and run frequently enough to finish while provider tokens remain valid. After correcting those conditions and reviewing coverage, migrate the saved state explicitly or use a new stream identity for a deliberate fresh sync.\n\nCheckpoint state validates source/query scope and resolved calendar identity. Changing source settings requires an explicit state migration or another stream ID; the reader never silently reinterprets old progress.\n\n## Upgrading from 0.1.x\n\nVersion 0.2.0 replaces the rolling occurrence window with canonical change-token sync and changes row identities to include the resolved calendar and installation. Keep the existing raw table as an archive and use a new table/stream identity for a fresh canonical dataset, or deliberately migrate existing row identities. Existing expanded occurrences are not removed automatically.\n\n## Fixture tests\n\n```sh\nbunx chkit add google-calendar --with-tests\nbun test src/integrations/google-calendar/tests/basic.test.ts\n```\n\nFixtures verify page resumption, empty deltas, terminal token promotion after loads, cancellation payloads, token expiry, scope changes, and malformed pagination without live credentials.\n\nThe [Google Calendar integration guide](https://chkit.obsessiondb.com/integrations/google-calendar/) covers installation and sync behavior.\n" + }, + { + "path": "google-calendar/index.ts", + "type": "registry:file", + "target": "src/integrations/google-calendar/index.ts", + "content": "import { cursorState, definePipeline, defineStream, HttpError, IngestConfigError, rawRows, rawTable, type FetchContext } from '@chkit/plugin-ingest'\n\nconst database = 'default'\n// Keep this identity tied to one OAuth installation. Use another stream ID for another source.\nconst sourceId = 'google-calendar.primary'\nconst calendarId = 'primary'\nconst scope = JSON.stringify([sourceId, calendarId, { singleEvents: false, showDeleted: true, maxResults: 250 }])\n\ninterface Event { id: string; [key: string]: unknown }\ninterface Page { items: Event[]; nextPageToken?: string; nextSyncToken?: string }\ninterface CalendarState {\n scope: string\n calendarId: string\n syncToken?: string\n pageToken?: string\n /** Recovery attempts for this unfinished sync, including previous executions. */\n recoveryCount?: number\n}\n\nexport const google_calendar_eventsRaw = rawTable({ database, name: 'google_calendar_events_raw' })\n\nexport const google_calendarPipeline = definePipeline({\n id: 'google-calendar', tags: ['provider:google-calendar'], maxFetches: 1,\n streams: [defineStream({\n id: 'google-calendar.events', tags: ['resource:events'], destination: google_calendar_eventsRaw,\n incremental: {\n ...cursorState({ id: 'google-calendar.sync-token', version: 1, parse: parseState }),\n plan({ state, range }) {\n if (range?.from || range?.to) throw new IngestConfigError('Calendar canonical sync does not accept date bounds. Use a separate bounded instances reader for an occurrence backfill.')\n return state\n },\n },\n batchSize: 1,\n budget: { maxChunks: 200, maxChunkRows: 250 },\n classifyError: (cause) => cause instanceof IngestConfigError ? { kind: 'permanent' } : undefined,\n async *read(context) {\n const resolvedCalendarId = await resolveCalendar(context)\n if (context.state && context.state.calendarId !== resolvedCalendarId) {\n throw new IngestConfigError('Calendar checkpoint belongs to a different authenticated calendar. Use a separate stream ID for the new source.')\n }\n let state: CalendarState = context.state ?? { scope, calendarId: resolvedCalendarId }\n const seen = new Set()\n if (state.pageToken) seen.add(state.pageToken)\n while (true) {\n const page = await readPage(context, state)\n if ('reset' in page) {\n if ((state.recoveryCount ?? 0) >= 1) throw new IngestConfigError('Calendar unfinished sync repeatedly rejected a token across resumed runs. Adjust the editable stream chunk budget, run duration, or polling frequency, then explicitly migrate the saved state or use a new stream identity after reviewing coverage.')\n seen.clear()\n // An expired sync token starts a new baseline; a rejected page token\n // replays the current sync from its original, committed input token.\n state = { scope, calendarId: resolvedCalendarId, recoveryCount: (state.recoveryCount ?? 0) + 1, ...(page.reset === 'page' && state.syncToken ? { syncToken: state.syncToken } : {}) }\n yield { rows: [], state, id: 'calendar-reset' }\n continue\n }\n if (page.nextPageToken && seen.has(page.nextPageToken)) throw new IngestConfigError('Calendar repeated a page token.')\n if (page.nextPageToken) seen.add(page.nextPageToken)\n state = page.nextPageToken\n ? { ...state, pageToken: page.nextPageToken }\n // Only acknowledgement of this terminal page clears recovery debt.\n : { scope, calendarId: resolvedCalendarId, syncToken: requiredString(page.nextSyncToken, 'nextSyncToken') }\n const rows = rawRows(page.items, (event) => JSON.stringify([sourceId, resolvedCalendarId, event.id]))\n // Empty terminal pages must commit the new sync token too.\n yield { rows, state, ...(rows.length === 0 ? { id: 'calendar-empty-page' } : {}) }\n if (!page.nextPageToken) return\n }\n },\n })],\n})\n\nasync function resolveCalendar(context: FetchContext): Promise {\n return context.attempt(async (signal) => {\n const payload = await request(`calendars/${encodeURIComponent(calendarId)}`, signal)\n return requiredString(payload.id, 'calendar id')\n }, { label: 'GET calendar metadata' })\n}\n\nasync function readPage(context: FetchContext, state: CalendarState): Promise {\n return context.attempt(async (signal) => {\n const url = new URL(`https://www.googleapis.com/calendar/v3/calendars/${encodeURIComponent(state.calendarId)}/events`)\n for (const [key, value] of Object.entries({ singleEvents: 'false', showDeleted: 'true', maxResults: '250' })) url.searchParams.set(key, value)\n if (state.syncToken) url.searchParams.set('syncToken', state.syncToken)\n if (state.pageToken) url.searchParams.set('pageToken', state.pageToken)\n const response = await fetch(url, { signal: AbortSignal.any([signal, AbortSignal.timeout(30_000)]), headers: authorization(), redirect: 'error' })\n if (response.status === 410 && state.syncToken) return { reset: 'sync' }\n if ((response.status === 400 || response.status === 410) && state.pageToken) return { reset: 'page' }\n if (!response.ok) throw await HttpError.fromResponse(response)\n const payload: unknown = await response.json()\n if (!isObject(payload) || (payload.items !== undefined && !Array.isArray(payload.items))) throw new IngestConfigError('Calendar items are not an array.')\n const items = (payload.items ?? []).map((item: unknown) => {\n if (!isObject(item)) throw new IngestConfigError('Calendar returned an invalid event.')\n return { ...item, id: requiredString(item.id, 'event id') }\n })\n const nextPageToken = optionalString(payload.nextPageToken, 'nextPageToken')\n const nextSyncToken = optionalString(payload.nextSyncToken, 'nextSyncToken')\n if (nextPageToken && nextSyncToken) throw new IngestConfigError('Calendar returned both a page and sync token.')\n if (!nextPageToken && !nextSyncToken) throw new IngestConfigError('Calendar terminal page has no nextSyncToken.')\n return { items, nextPageToken, nextSyncToken }\n }, { label: 'GET calendar events' })\n}\n\nasync function request(path: string, signal: AbortSignal): Promise> {\n const response = await fetch(`https://www.googleapis.com/calendar/v3/${path}`, { signal: AbortSignal.any([signal, AbortSignal.timeout(30_000)]), headers: authorization(), redirect: 'error' })\n if (!response.ok) throw await HttpError.fromResponse(response)\n const payload: unknown = await response.json()\n if (!isObject(payload)) throw new IngestConfigError('Calendar returned an invalid response.')\n return payload\n}\n\nfunction parseState(raw: unknown): CalendarState {\n if (!isObject(raw) || raw.scope !== scope) throw new IngestConfigError('Calendar checkpoint scope changed. Migrate the state explicitly or use a new stream ID.')\n if (raw.recoveryCount !== undefined && raw.recoveryCount !== 0 && raw.recoveryCount !== 1) throw new IngestConfigError('Calendar checkpoint recoveryCount must be zero or one.')\n return { scope, calendarId: requiredString(raw.calendarId, 'checkpoint calendar id'), syncToken: optionalString(raw.syncToken, 'checkpoint syncToken'), pageToken: optionalString(raw.pageToken, 'checkpoint pageToken'), recoveryCount: raw.recoveryCount }\n}\n\nfunction authorization(): Record {\n const token = process.env.GOOGLE_CALENDAR_ACCESS_TOKEN?.trim()\n if (!token) throw new IngestConfigError('Set GOOGLE_CALENDAR_ACCESS_TOKEN.')\n return { Authorization: `Bearer ${token}` }\n}\n\nfunction optionalString(value: unknown, field: string): string | undefined {\n return value === undefined ? undefined : requiredString(value, field)\n}\n\nfunction requiredString(value: unknown, field: string): string {\n if (typeof value !== 'string' || value.trim().length === 0) throw new IngestConfigError(`Calendar ${field} must be a non-empty string.`)\n return value\n}\n\nfunction isObject(value: unknown): value is Record {\n return typeof value === 'object' && value !== null && !Array.isArray(value)\n}\n" + }, + { + "path": "google-calendar/tests/basic.test.ts", + "type": "registry:file", + "target": "src/integrations/google-calendar/tests/basic.test.ts", + "role": "test", + "content": "import { afterEach, expect, test } from 'bun:test'\nimport { runIngestion, selectStreams } from '@chkit/plugin-ingest'\nimport { createMemoryDestination, createMemoryJournal } from '@chkit/plugin-ingest/testing'\n\nimport { google_calendarPipeline } from '../index.js'\n\nconst originalFetch = globalThis.fetch\nconst originalToken = process.env.GOOGLE_CALENDAR_ACCESS_TOKEN\n\nfunction setFetch(handler: (url: URL) => Response | Promise, calendar = () => 'calendar@example.com'): void {\n process.env.GOOGLE_CALENDAR_ACCESS_TOKEN = 'fixture'\n globalThis.fetch = Object.assign(async (input: Parameters[0]) => {\n const url = new URL(String(input))\n return url.pathname.endsWith('/events') ? handler(url) : Response.json({ id: calendar() })\n }, { preconnect: originalFetch.preconnect })\n}\n\nfunction request(maxChunks?: number) {\n const pipeline = { ...google_calendarPipeline, retry: { retries: 0 }, streams: google_calendarPipeline.streams.map((stream) => ({ ...stream, ...(maxChunks === undefined ? {} : { budget: { maxChunks } }) })) }\n return { selected: selectStreams([pipeline], []), backfill: undefined }\n}\n\nafterEach(() => {\n globalThis.fetch = originalFetch\n if (originalToken === undefined) delete process.env.GOOGLE_CALENDAR_ACCESS_TOKEN\n else process.env.GOOGLE_CALENDAR_ACCESS_TOKEN = originalToken\n})\n\ntest.serial('Calendar resumes an acknowledged page and promotes only the terminal sync token', async () => {\n const urls: URL[] = []\n setFetch((url) => {\n urls.push(url)\n return Response.json(url.searchParams.has('pageToken')\n ? { items: [{ id: 'cancelled', status: 'cancelled' }], nextSyncToken: 'sync-1' }\n : { items: [{ id: 'series', recurrence: ['RRULE:FREQ=WEEKLY'] }], nextPageToken: 'next' })\n })\n const journal = createMemoryJournal()\n const destination = createMemoryDestination()\n const interrupted = await runIngestion(request(1), { journal, destination })\n expect(interrupted.streams[0]?.outcome).toBe('budget_exhausted')\n expect((await journal.readCheckpoint('google-calendar.events')).envelope?.state).toMatchObject({ pageToken: 'next' })\n const resumed = await runIngestion(request(), { journal, destination })\n expect(resumed.ok).toBe(true)\n expect(urls.map((url) => url.searchParams.get('pageToken'))).toEqual([null, 'next'])\n expect((await journal.readCheckpoint('google-calendar.events')).envelope?.state).toMatchObject({ syncToken: 'sync-1', calendarId: 'calendar@example.com' })\n expect(destination.tables.get('default.google_calendar_events_raw')?.map((row) => row.raw)).toEqual([\n { id: 'series', recurrence: ['RRULE:FREQ=WEEKLY'] }, { id: 'cancelled', status: 'cancelled' },\n ])\n for (const url of urls) {\n expect(url.searchParams.get('singleEvents')).toBe('false')\n expect(url.searchParams.get('showDeleted')).toBe('true')\n expect(url.searchParams.has('timeMin')).toBe(false)\n expect(url.searchParams.has('timeMax')).toBe(false)\n }\n})\n\ntest.serial('Calendar keeps the input sync token across empty delta pages', async () => {\n let bootstrap = true\n const inputs: Array = []\n setFetch((url) => {\n inputs.push(url.searchParams.get('syncToken'))\n if (bootstrap) return Response.json({ items: [], nextSyncToken: 'sync-1' })\n return Response.json(url.searchParams.has('pageToken')\n ? { items: [], nextSyncToken: 'sync-2' }\n : { items: [], nextPageToken: 'delta-next' })\n })\n const journal = createMemoryJournal()\n const destination = createMemoryDestination()\n expect((await runIngestion(request(), { journal, destination })).ok).toBe(true)\n bootstrap = false\n expect((await runIngestion(request(), { journal, destination })).ok).toBe(true)\n expect(inputs).toEqual([null, 'sync-1', 'sync-1'])\n expect((await journal.readCheckpoint('google-calendar.events')).envelope?.state).toMatchObject({ syncToken: 'sync-2' })\n})\n\ntest.serial('Calendar retains its page checkpoint after a lost terminal acknowledgement', async () => {\n setFetch((url) => Response.json(url.searchParams.has('pageToken')\n ? { items: [{ id: 'second' }], nextSyncToken: 'sync-1' }\n : { items: [{ id: 'first' }], nextPageToken: 'next' }))\n const journal = createMemoryJournal()\n const destination = createMemoryDestination()\n const first = await runIngestion(request(), { journal, destination: {\n insert: async (input) => {\n await destination.insert(input)\n if (input.rows.some((row) => JSON.stringify(row.raw).includes('second'))) throw new Error('fixture acknowledgement lost')\n },\n } })\n expect(first.ok).toBe(false)\n expect(destination.tables.get('default.google_calendar_events_raw')).toHaveLength(2)\n expect((await journal.readCheckpoint('google-calendar.events')).envelope?.state).toMatchObject({ pageToken: 'next' })\n expect((await runIngestion(request(), { journal, destination })).ok).toBe(true)\n expect((await journal.readCheckpoint('google-calendar.events')).envelope?.state).toMatchObject({ syncToken: 'sync-1' })\n expect(destination.tables.get('default.google_calendar_events_raw')).toHaveLength(2)\n})\n\ntest.serial('Calendar reboots an expired sync without deleting raw observations', async () => {\n let changed = false\n const inputs: Array = []\n setFetch((url) => {\n inputs.push(url.searchParams.get('syncToken'))\n if (!changed) return Response.json({ items: [{ id: 'old' }], nextSyncToken: 'expired' })\n return url.searchParams.has('syncToken') ? new Response('expired', { status: 410 }) : Response.json({ items: [], nextSyncToken: 'fresh' })\n })\n const journal = createMemoryJournal()\n const destination = createMemoryDestination()\n expect((await runIngestion(request(), { journal, destination })).ok).toBe(true)\n changed = true\n expect((await runIngestion(request(), { journal, destination })).ok).toBe(true)\n expect(inputs).toEqual([null, 'expired', null])\n expect(destination.tables.get('default.google_calendar_events_raw')).toHaveLength(1)\n expect((await journal.readCheckpoint('google-calendar.events')).envelope?.state).toMatchObject({ syncToken: 'fresh' })\n})\n\ntest.serial('Calendar rejects malformed terminal pages and repeated page tokens', async () => {\n setFetch(() => Response.json({ items: [] }))\n expect((await runIngestion(request(), { journal: createMemoryJournal(), destination: createMemoryDestination() })).ok).toBe(false)\n setFetch(() => Response.json({ items: [], nextPageToken: 'same' }))\n expect((await runIngestion(request(), { journal: createMemoryJournal(), destination: createMemoryDestination() })).ok).toBe(false)\n expect(() => google_calendarPipeline.streams[0]?.incremental.parseState({ scope: 'another source' })).toThrow('scope')\n})\n\ntest.serial('Calendar refuses to reuse a checkpoint after primary changes accounts', async () => {\n let calendar = 'first@example.com'\n let eventRequests = 0\n setFetch(() => { eventRequests += 1; return Response.json({ items: [], nextSyncToken: 'sync-1' }) }, () => calendar)\n const journal = createMemoryJournal()\n const destination = createMemoryDestination()\n expect((await runIngestion(request(), { journal, destination })).ok).toBe(true)\n calendar = 'second@example.com'\n const result = await runIngestion(request(), { journal, destination })\n expect(result.ok).toBe(false)\n expect(result.streams[0]?.error).toContain('different authenticated calendar')\n expect(eventRequests).toBe(1)\n})\n\ntest.serial('Calendar safely restarts a rejected page token within the same sync', async () => {\n let reject = true\n const inputs: Array = []\n setFetch((url) => {\n inputs.push(url.searchParams.get('pageToken'))\n if (url.searchParams.has('pageToken') && reject) { reject = false; return new Response('invalid page token', { status: 400 }) }\n return Response.json(url.searchParams.has('pageToken')\n ? { items: [], nextSyncToken: 'sync-1' }\n : { items: [{ id: 'first' }], nextPageToken: 'next' })\n })\n const journal = createMemoryJournal()\n const destination = createMemoryDestination()\n expect((await runIngestion(request(), { journal, destination })).ok).toBe(true)\n expect(inputs).toEqual([null, 'next', null, 'next'])\n expect(new Set(destination.tables.get('default.google_calendar_events_raw')?.map((row) => row.id)).size).toBe(1)\n expect((await journal.readCheckpoint('google-calendar.events')).envelope?.state).toMatchObject({ syncToken: 'sync-1' })\n})\n\ntest.serial('Calendar rejects unsupported explicit occurrence backfill bounds before requesting Google', async () => {\n let requested = false\n setFetch(() => { requested = true; return Response.json({ items: [], nextSyncToken: 'sync-1' }) })\n const result = await runIngestion({ ...request(), backfill: { id: 'dates', from: new Date('2026-10-01T00:00:00Z'), to: new Date('2026-10-05T00:00:00Z') } }, { journal: createMemoryJournal(), destination: createMemoryDestination() })\n expect(result.ok).toBe(false)\n expect(result.streams[0]?.error).toContain('does not accept date bounds')\n expect(requested).toBe(false)\n})\n\ntest.serial('Calendar carries reset debt through serialized fresh runs and successful nonterminal replay', async () => {\n let rejected = 0\n setFetch((url) => {\n if (url.searchParams.has('pageToken')) { rejected += 1; return new Response('expired page token', { status: 400 }) }\n return Response.json({ items: [{ id: 'first' }], nextPageToken: 'next' })\n })\n const destination = createMemoryDestination()\n const initial = createMemoryJournal()\n expect((await runIngestion(request(1), { journal: initial, destination })).streams[0]?.outcome).toBe('budget_exhausted')\n const afterInitial = restoredJournal(initial)\n expect((await runIngestion(request(1), { journal: afterInitial, destination })).streams[0]?.outcome).toBe('budget_exhausted')\n expect((await afterInitial.readCheckpoint('google-calendar.events')).envelope?.state).toMatchObject({ recoveryCount: 1 })\n expect((await afterInitial.readCheckpoint('google-calendar.events')).envelope?.state).not.toHaveProperty('pageToken')\n const afterReset = restoredJournal(afterInitial)\n expect((await runIngestion(request(1), { journal: afterReset, destination })).streams[0]?.outcome).toBe('budget_exhausted')\n const replayCheckpoint = await afterReset.readCheckpoint('google-calendar.events')\n expect(replayCheckpoint.envelope?.state).toMatchObject({ recoveryCount: 1, pageToken: 'next' })\n const afterReplay = restoredJournal(afterReset)\n const stopped = await runIngestion(request(2), { journal: afterReplay, destination })\n expect(stopped.ok).toBe(false)\n expect(stopped.streams[0]?.error).toContain('across resumed runs')\n expect(stopped.streams[0]?.error).toContain('editable stream chunk budget')\n expect((await afterReplay.readCheckpoint('google-calendar.events')).version).toBe(replayCheckpoint.version)\n expect(rejected).toBe(2)\n expect(() => google_calendarPipeline.streams[0]?.incremental.parseState({ ...JSON.parse(JSON.stringify(replayCheckpoint.envelope?.state)), recoveryCount: 2 })).toThrow('recoveryCount')\n})\n\nfunction restoredJournal(previous: ReturnType) {\n const journal = createMemoryJournal()\n journal.events.push(...previous.events.map((event) => ({ ...event, checkpoint: event.checkpoint === undefined ? undefined : JSON.parse(JSON.stringify(event.checkpoint)) })))\n return journal\n}\n" + } + ], + "meta": { + "chkit": { + "formatVersion": 1, + "version": "0.2.0", + "language": "typescript", + "license": "MIT", + "documentation": "https://chkit.obsessiondb.com/integrations/google-calendar/", + "logo": "https://chkit.obsessiondb.com/logos/google-calendar.svg", + "authentication": { + "method": "OAuth access token", + "env": [ + "GOOGLE_CALENDAR_ACCESS_TOKEN" + ], + "setup": [ + "Provide GOOGLE_CALENDAR_ACCESS_TOKEN with read access to the selected resources.", + "Supply a new access token when the existing token expires." + ], + "documentation": "https://developers.google.com/calendar/api/guides/auth" + }, + "chkit": "^0.2.0-beta.10", + "ingest": "^0.2.0-beta.10", + "clickhouse": ">=25.3.0", + "root": "src/integrations/google-calendar", + "entry": "index.ts", + "exports": [ + "google_calendarPipeline", + "google_calendar_eventsRaw" + ], + "resources": [ + { + "name": "events", + "title": "Events", + "table": "google_calendar_events_raw", + "description": "Canonical events, recurring masters, exceptions, and cancellation payloads from the configured calendar", + "scopes": [ + "https://www.googleapis.com/auth/calendar.readonly" + ], + "strategy": "cursor", + "endpoints": [ + { + "method": "GET", + "path": "/calendars/{calendarId}", + "documentation": "https://developers.google.com/workspace/calendar/api/v3/reference/calendars/get" + }, + { + "method": "GET", + "path": "/calendars/{calendarId}/events", + "documentation": "https://developers.google.com/workspace/calendar/api/v3/reference/events/list" + } + ] + } + ], + "views": [], + "sync": { + "description": "Initial canonical event read followed by Google sync-token deltas. Acknowledged pages resume from journaled state.", + "schedule": "Run bunx chkit ingest run --tag provider:google-calendar from a scheduler.", + "deletions": "Cancellation payloads are retained. An expired token reboots provider progress without clearing raw observations; absent records are not reconciled." + }, + "env": { + "GOOGLE_CALENDAR_ACCESS_TOKEN": "" + }, + "fileHashes": { + "src/integrations/google-calendar/README.md": "f33f58dcffc61d9d16ec0e5fa641edac7d7db1269b886ef7aafd727b1b721b45", + "src/integrations/google-calendar/index.ts": "7c280708a9da4ae84ac35d371cfa82a03a2ed90723f3585214e7d92fc9dadb43", + "src/integrations/google-calendar/tests/basic.test.ts": "8276017bca4feb121deb958964d0c7d034dddfd6b68ad4e5e0990738bc47cd75" + } + } + } +} diff --git a/registry/google-calendar/tests/basic.test.ts b/registry/google-calendar/tests/basic.test.ts index 5aeb6bda..1d3cd0ba 100644 --- a/registry/google-calendar/tests/basic.test.ts +++ b/registry/google-calendar/tests/basic.test.ts @@ -7,8 +7,17 @@ import { google_calendarPipeline } from '../index.js' const originalFetch = globalThis.fetch const originalToken = process.env.GOOGLE_CALENDAR_ACCESS_TOKEN -function setFetch(handler: (...args: Parameters) => Promise): void { - globalThis.fetch = Object.assign(handler, { preconnect: originalFetch.preconnect }) +function setFetch(handler: (url: URL) => Response | Promise, calendar = () => 'calendar@example.com'): void { + process.env.GOOGLE_CALENDAR_ACCESS_TOKEN = 'fixture' + globalThis.fetch = Object.assign(async (input: Parameters[0]) => { + const url = new URL(String(input)) + return url.pathname.endsWith('/events') ? handler(url) : Response.json({ id: calendar() }) + }, { preconnect: originalFetch.preconnect }) +} + +function request(maxChunks?: number) { + const pipeline = { ...google_calendarPipeline, retry: { retries: 0 }, streams: google_calendarPipeline.streams.map((stream) => ({ ...stream, ...(maxChunks === undefined ? {} : { budget: { maxChunks } }) })) } + return { selected: selectStreams([pipeline], []), backfill: undefined } } afterEach(() => { @@ -17,21 +26,169 @@ afterEach(() => { else process.env.GOOGLE_CALENDAR_ACCESS_TOKEN = originalToken }) -test.serial('Google Calendar follows page tokens and keeps cancellation payloads', async () => { - process.env.GOOGLE_CALENDAR_ACCESS_TOKEN = 'fixture' - const tokens: Array = [] - setFetch(async (input) => { - const url = new URL(String(input)) - tokens.push(url.searchParams.get('pageToken')) +test.serial('Calendar resumes an acknowledged page and promotes only the terminal sync token', async () => { + const urls: URL[] = [] + setFetch((url) => { + urls.push(url) return Response.json(url.searchParams.has('pageToken') - ? { items: [{ id: 'cancelled', status: 'cancelled' }] } - : { items: [{ id: 'active', summary: 'Call' }], nextPageToken: 'next' }) + ? { items: [{ id: 'cancelled', status: 'cancelled' }], nextSyncToken: 'sync-1' } + : { items: [{ id: 'series', recurrence: ['RRULE:FREQ=WEEKLY'] }], nextPageToken: 'next' }) }) + const journal = createMemoryJournal() const destination = createMemoryDestination() - const result = await runIngestion({ selected: selectStreams([google_calendarPipeline], []), backfill: undefined }, { journal: createMemoryJournal(), destination }) - expect(result.ok).toBe(true) - expect(tokens).toEqual([null, 'next']) + const interrupted = await runIngestion(request(1), { journal, destination }) + expect(interrupted.streams[0]?.outcome).toBe('budget_exhausted') + expect((await journal.readCheckpoint('google-calendar.events')).envelope?.state).toMatchObject({ pageToken: 'next' }) + const resumed = await runIngestion(request(), { journal, destination }) + expect(resumed.ok).toBe(true) + expect(urls.map((url) => url.searchParams.get('pageToken'))).toEqual([null, 'next']) + expect((await journal.readCheckpoint('google-calendar.events')).envelope?.state).toMatchObject({ syncToken: 'sync-1', calendarId: 'calendar@example.com' }) expect(destination.tables.get('default.google_calendar_events_raw')?.map((row) => row.raw)).toEqual([ - { id: 'active', summary: 'Call' }, { id: 'cancelled', status: 'cancelled' }, + { id: 'series', recurrence: ['RRULE:FREQ=WEEKLY'] }, { id: 'cancelled', status: 'cancelled' }, ]) + for (const url of urls) { + expect(url.searchParams.get('singleEvents')).toBe('false') + expect(url.searchParams.get('showDeleted')).toBe('true') + expect(url.searchParams.has('timeMin')).toBe(false) + expect(url.searchParams.has('timeMax')).toBe(false) + } +}) + +test.serial('Calendar keeps the input sync token across empty delta pages', async () => { + let bootstrap = true + const inputs: Array = [] + setFetch((url) => { + inputs.push(url.searchParams.get('syncToken')) + if (bootstrap) return Response.json({ items: [], nextSyncToken: 'sync-1' }) + return Response.json(url.searchParams.has('pageToken') + ? { items: [], nextSyncToken: 'sync-2' } + : { items: [], nextPageToken: 'delta-next' }) + }) + const journal = createMemoryJournal() + const destination = createMemoryDestination() + expect((await runIngestion(request(), { journal, destination })).ok).toBe(true) + bootstrap = false + expect((await runIngestion(request(), { journal, destination })).ok).toBe(true) + expect(inputs).toEqual([null, 'sync-1', 'sync-1']) + expect((await journal.readCheckpoint('google-calendar.events')).envelope?.state).toMatchObject({ syncToken: 'sync-2' }) +}) + +test.serial('Calendar retains its page checkpoint after a lost terminal acknowledgement', async () => { + setFetch((url) => Response.json(url.searchParams.has('pageToken') + ? { items: [{ id: 'second' }], nextSyncToken: 'sync-1' } + : { items: [{ id: 'first' }], nextPageToken: 'next' })) + const journal = createMemoryJournal() + const destination = createMemoryDestination() + const first = await runIngestion(request(), { journal, destination: { + insert: async (input) => { + await destination.insert(input) + if (input.rows.some((row) => JSON.stringify(row.raw).includes('second'))) throw new Error('fixture acknowledgement lost') + }, + } }) + expect(first.ok).toBe(false) + expect(destination.tables.get('default.google_calendar_events_raw')).toHaveLength(2) + expect((await journal.readCheckpoint('google-calendar.events')).envelope?.state).toMatchObject({ pageToken: 'next' }) + expect((await runIngestion(request(), { journal, destination })).ok).toBe(true) + expect((await journal.readCheckpoint('google-calendar.events')).envelope?.state).toMatchObject({ syncToken: 'sync-1' }) + expect(destination.tables.get('default.google_calendar_events_raw')).toHaveLength(2) +}) + +test.serial('Calendar reboots an expired sync without deleting raw observations', async () => { + let changed = false + const inputs: Array = [] + setFetch((url) => { + inputs.push(url.searchParams.get('syncToken')) + if (!changed) return Response.json({ items: [{ id: 'old' }], nextSyncToken: 'expired' }) + return url.searchParams.has('syncToken') ? new Response('expired', { status: 410 }) : Response.json({ items: [], nextSyncToken: 'fresh' }) + }) + const journal = createMemoryJournal() + const destination = createMemoryDestination() + expect((await runIngestion(request(), { journal, destination })).ok).toBe(true) + changed = true + expect((await runIngestion(request(), { journal, destination })).ok).toBe(true) + expect(inputs).toEqual([null, 'expired', null]) + expect(destination.tables.get('default.google_calendar_events_raw')).toHaveLength(1) + expect((await journal.readCheckpoint('google-calendar.events')).envelope?.state).toMatchObject({ syncToken: 'fresh' }) }) + +test.serial('Calendar rejects malformed terminal pages and repeated page tokens', async () => { + setFetch(() => Response.json({ items: [] })) + expect((await runIngestion(request(), { journal: createMemoryJournal(), destination: createMemoryDestination() })).ok).toBe(false) + setFetch(() => Response.json({ items: [], nextPageToken: 'same' })) + expect((await runIngestion(request(), { journal: createMemoryJournal(), destination: createMemoryDestination() })).ok).toBe(false) + expect(() => google_calendarPipeline.streams[0]?.incremental.parseState({ scope: 'another source' })).toThrow('scope') +}) + +test.serial('Calendar refuses to reuse a checkpoint after primary changes accounts', async () => { + let calendar = 'first@example.com' + let eventRequests = 0 + setFetch(() => { eventRequests += 1; return Response.json({ items: [], nextSyncToken: 'sync-1' }) }, () => calendar) + const journal = createMemoryJournal() + const destination = createMemoryDestination() + expect((await runIngestion(request(), { journal, destination })).ok).toBe(true) + calendar = 'second@example.com' + const result = await runIngestion(request(), { journal, destination }) + expect(result.ok).toBe(false) + expect(result.streams[0]?.error).toContain('different authenticated calendar') + expect(eventRequests).toBe(1) +}) + +test.serial('Calendar safely restarts a rejected page token within the same sync', async () => { + let reject = true + const inputs: Array = [] + setFetch((url) => { + inputs.push(url.searchParams.get('pageToken')) + if (url.searchParams.has('pageToken') && reject) { reject = false; return new Response('invalid page token', { status: 400 }) } + return Response.json(url.searchParams.has('pageToken') + ? { items: [], nextSyncToken: 'sync-1' } + : { items: [{ id: 'first' }], nextPageToken: 'next' }) + }) + const journal = createMemoryJournal() + const destination = createMemoryDestination() + expect((await runIngestion(request(), { journal, destination })).ok).toBe(true) + expect(inputs).toEqual([null, 'next', null, 'next']) + expect(new Set(destination.tables.get('default.google_calendar_events_raw')?.map((row) => row.id)).size).toBe(1) + expect((await journal.readCheckpoint('google-calendar.events')).envelope?.state).toMatchObject({ syncToken: 'sync-1' }) +}) + +test.serial('Calendar rejects unsupported explicit occurrence backfill bounds before requesting Google', async () => { + let requested = false + setFetch(() => { requested = true; return Response.json({ items: [], nextSyncToken: 'sync-1' }) }) + const result = await runIngestion({ ...request(), backfill: { id: 'dates', from: new Date('2026-10-01T00:00:00Z'), to: new Date('2026-10-05T00:00:00Z') } }, { journal: createMemoryJournal(), destination: createMemoryDestination() }) + expect(result.ok).toBe(false) + expect(result.streams[0]?.error).toContain('does not accept date bounds') + expect(requested).toBe(false) +}) + +test.serial('Calendar carries reset debt through serialized fresh runs and successful nonterminal replay', async () => { + let rejected = 0 + setFetch((url) => { + if (url.searchParams.has('pageToken')) { rejected += 1; return new Response('expired page token', { status: 400 }) } + return Response.json({ items: [{ id: 'first' }], nextPageToken: 'next' }) + }) + const destination = createMemoryDestination() + const initial = createMemoryJournal() + expect((await runIngestion(request(1), { journal: initial, destination })).streams[0]?.outcome).toBe('budget_exhausted') + const afterInitial = restoredJournal(initial) + expect((await runIngestion(request(1), { journal: afterInitial, destination })).streams[0]?.outcome).toBe('budget_exhausted') + expect((await afterInitial.readCheckpoint('google-calendar.events')).envelope?.state).toMatchObject({ recoveryCount: 1 }) + expect((await afterInitial.readCheckpoint('google-calendar.events')).envelope?.state).not.toHaveProperty('pageToken') + const afterReset = restoredJournal(afterInitial) + expect((await runIngestion(request(1), { journal: afterReset, destination })).streams[0]?.outcome).toBe('budget_exhausted') + const replayCheckpoint = await afterReset.readCheckpoint('google-calendar.events') + expect(replayCheckpoint.envelope?.state).toMatchObject({ recoveryCount: 1, pageToken: 'next' }) + const afterReplay = restoredJournal(afterReset) + const stopped = await runIngestion(request(2), { journal: afterReplay, destination }) + expect(stopped.ok).toBe(false) + expect(stopped.streams[0]?.error).toContain('across resumed runs') + expect(stopped.streams[0]?.error).toContain('editable stream chunk budget') + expect((await afterReplay.readCheckpoint('google-calendar.events')).version).toBe(replayCheckpoint.version) + expect(rejected).toBe(2) + expect(() => google_calendarPipeline.streams[0]?.incremental.parseState({ ...JSON.parse(JSON.stringify(replayCheckpoint.envelope?.state)), recoveryCount: 2 })).toThrow('recoveryCount') +}) + +function restoredJournal(previous: ReturnType) { + const journal = createMemoryJournal() + journal.events.push(...previous.events.map((event) => ({ ...event, checkpoint: event.checkpoint === undefined ? undefined : JSON.parse(JSON.stringify(event.checkpoint)) }))) + return journal +} diff --git a/registry/google-meet/README.md b/registry/google-meet/README.md index 002bf57e..d0479dfa 100644 --- a/registry/google-meet/README.md +++ b/registry/google-meet/README.md @@ -1,8 +1,8 @@ -# Google Meet raw ingestion example +# Google Meet raw ingestion -This is a small editable starting point based on the Google-Meet source in [obsessiondb/brain](https://github.com/obsessiondb/brain/tree/main/scripts/brain-sync). It stores provider responses as native JSON without Brain's memory or sales projections. +This editable reader stores provider responses as native JSON and journals fixed discovery windows plus pending conference and artifact progress. It builds on the overlapping-window pattern in [obsessiondb/brain](https://github.com/obsessiondb/brain/tree/main/scripts/brain-sync) and retains parents so late artifacts survive discovery advancing. -Set `GOOGLE_MEET_ACCESS_TOKEN` in the runtime environment and configure a direct ClickHouse connection. Edit `lookbackDays` in `index.ts` to select the conference window. Native JSON requires ClickHouse 25.3 or later. +Set `GOOGLE_MEET_ACCESS_TOKEN` in the runtime environment and configure a direct ClickHouse connection. Review `sourceId`, `lookbackDays`, `overlapDays`, `windowDays`, and `database` in `index.ts`. Keep `sourceId` and stream IDs tied to one authenticated Google account. The API does not expose a source identity check in these list responses; changing the token's account requires a separate source/checkpoint. Native JSON requires ClickHouse 25.3 or later. Tokens are read at request time and are not refreshed by this reader. ```sh bunx chkit check @@ -10,11 +10,54 @@ bunx chkit generate --name add_google_meet # Review the migration before applying it. bunx chkit migrate --apply bunx chkit ingest run --tag provider:google-meet +bunx chkit ingest status --tag provider:google-meet ``` -## Resources +## Resources and identities - `conferences` → `google_meet_conferences_raw` -- `transcripts` → `google_meet_transcripts_raw` +- `transcripts` → `google_meet_transcripts_raw` (transcript metadata with `conference_name`) +- `transcript-entries` → `google_meet_transcript_entries_raw` (one entry per row with `conference_name` and `transcript_name`) -Each run reads a configured time range. Transcripts generated later require another scan of that range. A failed API request fails the stream. Raw rows use provider IDs and keep the last observed value; there is no deletion reconciliation. Edit the readers, select fewer streams, or add SQL views for a specific use case. Run one ingestion process per ClickHouse target at a time. +Every resource uses `[sourceId, provider.name]` as its row identity. Entries are emitted page by page rather than buffering a full transcript inside one JSON row. Join metadata and entries through their provider resource names. These raw destinations keep the latest observed value; they do not capture later edits to the Google Docs transcript document. + +## Discovery and pending work + +Each stream independently discovers completed conferences through `end_time` bounds and ongoing conferences through `end_time IS NULL`. This includes long-running calls whose start predates the lookback. Initial discovery defaults to the retained 30-day period; later runs overlap seven days. A cycle's bounds are persisted before requesting its first page, so pauses keep the same window. `windowDays` caps forward advancement per cycle. + +Transcript and entry streams retain discovered conference names until `expireTime`, independently of the discovery watermark. Every completed cycle checks those parents again, including parents with no transcripts and transcripts still being generated. This captures late artifacts outside the seven-day discovery overlap. Retention, access, and polling frequency still determine coverage; this is polling, not a provider change feed. Meet exposes no modification/change token and does not support transcript/entry date filters. See [conference filtering](https://developers.google.com/workspace/meet/api/reference/rest/v2/conferenceRecords/list) and [artifact behavior](https://developers.google.com/workspace/meet/api/guides/artifacts). + +## Checkpoints and recovery + +Acknowledged pages checkpoint discovery, parent, transcript-page, and entry-page positions. A failed entry load retains its preceding page position. Rejected page tokens restart their scoped collection once and may replay observations; repeated rejection and malformed pagination fail visibly. Candidate state is complete and advances only after its rows have sink evidence. + +Recovery counts remain saved for an unfinished discovery phase, parent transcript collection, or transcript entry collection until its terminal boundary has destination acknowledgement. A budget pause after recovery or a successful nonterminal replay page keeps its count. A second rejection for that unfinished collection fails even in a fresh scheduled execution. Edit `budget.maxChunks` to change the stream chunk cap, choose sufficient `--max-duration`, and run frequently enough to finish before tokens expire. After fixing those conditions and reviewing coverage, migrate saved state explicitly or use a new stream identity for a deliberate fresh scan. + +Each stream permits 200 chunks per run and retains at most 1,000 conferences. Requests have a 30-second timeout. Budget exhaustion preserves progress and a later invocation continues it. Increase those editable limits deliberately for larger sources. A pending queue over its configured limit fails rather than silently dropping parents. The executor owns request retries, cancellation, and load acknowledgements. Run one ingestion process per ClickHouse target at a time. + +Google deletes conference records and API transcript entries 30 days after a conference ends. The reader reports expiry during unfinished work as a coverage gap; it cannot recover expired API entries. Previously checked parents leave the work queue on expiry, while stored raw rows remain. It does not infer deletion from missing, inaccessible, or expired resources. See [conference retention](https://developers.google.com/workspace/meet/api/reference/rest/v2/conferenceRecords) and [artifact retention](https://developers.google.com/workspace/meet/api/guides/artifacts). + +## Historical ranges + +Explicit bounds apply to conference end times and use an isolated backfill checkpoint. Historical ranges skip ongoing-call discovery. Reuse the same backfill ID and bounds to resume; changing the bounds requires another ID. Artifact lists remain parent-scoped and cannot themselves be date-filtered. Expired history is unavailable even when a backfill range requests it. + +```sh +bunx chkit ingest run --tag provider:google-meet --backfill october --from 2026-10-01T00:00:00Z --to 2026-10-05T00:00:00Z +``` + +Checkpoint validation rejects changed source/window scope and invalid nested positions. Migrate state explicitly or use a new stream identity when changing these settings. + +## Upgrading from 0.1.x + +Version 0.2.0 keeps the existing conference and transcript destinations and adds an entry table. New transcript rows contain metadata; entries live in the separate table. Row identities now include `sourceId`, so upgraded observations do not replace legacy provider-name IDs. Old rows and their nested `entries` remain until an explicit identity/data migration. Keep older tables as archives or migrate identities deliberately before mixing datasets. + +## Fixture tests + +```sh +bunx chkit add google-meet --with-tests +bun test src/integrations/google-meet/tests/basic.test.ts +``` + +Fixtures verify nested resumption, retained parents with late transcripts, end-time and ongoing discovery, rejected tokens, failed loads, isolated backfill bounds, malformed state, and expiry gaps without live credentials. + +The [Google Meet integration guide](https://chkit.obsessiondb.com/integrations/google-meet/) covers installation and sync behavior. diff --git a/registry/google-meet/index.ts b/registry/google-meet/index.ts index d6c89111..7ce4b98e 100644 --- a/registry/google-meet/index.ts +++ b/registry/google-meet/index.ts @@ -1,66 +1,332 @@ -import { definePipeline, defineStream, HttpError, rawRows, rawTable, type FetchContext } from '@chkit/plugin-ingest' +import { definePipeline, defineStream, HttpError, IngestConfigError, rawRows, rawTable, type FetchContext, type ReadContext, type SourceChunk } from '@chkit/plugin-ingest' const database = 'default' -// Pick a range that includes conferences whose transcripts may still be processing. +// Bind this label to one authenticated Google account; use separate stream IDs for another source. +const sourceId = 'google-meet.primary' const lookbackDays = 30 +const overlapDays = 7 +const windowDays = 30 +const maxPendingConferences = 1_000 +const dayMs = 86_400_000 const baseUrl = 'https://meet.googleapis.com/v2' +const scope = JSON.stringify([sourceId, lookbackDays, overlapDays, windowDays]) +type Resource = 'conferences' | 'transcripts' | 'transcript-entries' interface Named { name: string; [key: string]: unknown } +interface PendingConference { name: string; expireTime?: string; checkedAt?: string } +interface Cycle { + from: string + to: string + phase: 'ended' | 'ongoing' | 'parents' + pageToken?: string + /** Recovery debt for the unfinished discovery phase, across executions. */ + recoveryCount?: number + parents: string[] + parentIndex: number +} +interface ActiveParent { + name: string + transcriptPageToken?: string + transcriptRecoveryCount?: number + nextTranscriptPageToken?: string + transcripts?: Named[] + transcriptIndex: number + entryPageToken?: string + entryRecoveryCount?: number +} +interface MeetState { + scope: string + watermark?: string + bounds?: { from?: string; to?: string } + pending: PendingConference[] + cycle?: Cycle + active?: ActiveParent +} +interface Page { items: Named[]; nextPageToken?: string } +type MeetContext = ReadContext<{ from: Date | undefined; to: Date | undefined } | undefined, MeetState> +type MeetChunk = SourceChunk, MeetState> export const google_meet_conferencesRaw = rawTable({ database, name: 'google_meet_conferences_raw' }) export const google_meet_transcriptsRaw = rawTable({ database, name: 'google_meet_transcripts_raw' }) +export const google_meet_transcriptEntriesRaw = rawTable({ database, name: 'google_meet_transcript_entries_raw' }) export const google_meetPipeline = definePipeline({ id: 'google-meet', tags: ['provider:google-meet'], maxStreams: 1, maxFetches: 1, streams: [ - defineStream({ id: 'google-meet.conferences', tags: ['resource:conferences'], destination: google_meet_conferencesRaw, - async *read(context) { - for await (const page of conferences(context)) yield { rows: rawRows(page, (item) => item.name) } - }, - }), - defineStream({ id: 'google-meet.transcripts', tags: ['resource:transcripts'], destination: google_meet_transcriptsRaw, - async *read(context) { - for await (const page of conferences(context)) { - for (const conference of page) { - for await (const transcripts of list(context, `${conference.name}/transcripts`, 'transcripts')) { - for (const transcript of transcripts) { - const entries: unknown[] = [] - for await (const part of list(context, `${transcript.name}/entries`, 'transcriptEntries')) entries.push(...part) - yield { rows: rawRows([{ ...transcript, conference_name: conference.name, entries }], (item) => item.name) } - } - } - } - } - }, - }), + stream('conferences', google_meet_conferencesRaw), + stream('transcripts', google_meet_transcriptsRaw), + stream('transcript-entries', google_meet_transcriptEntriesRaw), ], }) -function conferences(context: FetchContext): AsyncGenerator { - const from = new Date(Date.now() - lookbackDays * 86_400_000).toISOString() - const to = new Date().toISOString() - return list(context, `conferenceRecords?filter=${encodeURIComponent(`start_time>="${from}" AND start_time<="${to}"`)}`, 'conferenceRecords') -} - -async function* list(context: FetchContext, path: string, field: string): AsyncGenerator { - let pageToken: string | undefined - const seen = new Set() - do { - const page = await context.attempt(async (signal) => { - const token = process.env.GOOGLE_MEET_ACCESS_TOKEN - if (!token) throw new Error('Set GOOGLE_MEET_ACCESS_TOKEN') - const url = new URL(`${baseUrl}/${path}`) - url.searchParams.set('pageSize', '100') - if (pageToken) url.searchParams.set('pageToken', pageToken) - const response = await fetch(url, { signal, headers: { Authorization: `Bearer ${token}` } }) - if (!response.ok) throw await HttpError.fromResponse(response) - return await response.json() as Record - }, { label: `GET ${field}` }) - const items = page[field] - if (items !== undefined && !Array.isArray(items)) throw new Error(`Meet ${field} is not an array`) - if (Array.isArray(items) && items.length) yield items as T[] - pageToken = typeof page.nextPageToken === 'string' ? page.nextPageToken : undefined - if (pageToken && seen.has(pageToken)) throw new Error(`Meet repeated ${field} page token`) - if (pageToken) seen.add(pageToken) - } while (pageToken) +function stream(resource: Resource, destination: typeof google_meet_conferencesRaw) { + return defineStream({ + id: `google-meet.${resource}`, tags: [`resource:${resource}`], destination, + incremental: { + id: `google-meet.${resource}.pending`, version: 1, parseState, + plan: ({ range }) => range, + }, + batchSize: 1, + budget: { maxChunks: 200, maxChunkRows: 100 }, + classifyError: (cause) => cause instanceof IngestConfigError ? { kind: 'permanent' } : undefined, + read: (context) => readResource(context, resource), + }) +} + +async function* readResource(context: MeetContext, resource: Resource): AsyncGenerator { + let state: MeetState = context.state ?? { scope, pending: [] } + const requestedBounds = context.selection?.from || context.selection?.to + ? { from: context.selection.from?.toISOString(), to: context.selection.to?.toISOString() } + : undefined + if (requestedBounds && state.bounds && JSON.stringify(requestedBounds) !== JSON.stringify(state.bounds)) throw new IngestConfigError('Meet backfill bounds changed. Use a new backfill ID.') + if (requestedBounds && state.watermark && !state.bounds) throw new IngestConfigError('Meet checkpoint was created without date bounds. Use a new backfill ID.') + state = { ...state, bounds: state.bounds ?? requestedBounds } + const seen = new Map>() + if (!state.cycle) { + const expired = state.pending.filter((parent) => parent.expireTime && Date.parse(parent.expireTime) <= context.cutoff.getTime()) + if (expired.some((parent) => !parent.checkedAt)) throw new IngestConfigError('Meet pending conference expired before a complete artifact read. Historical API coverage cannot be recovered; review and migrate the checkpoint explicitly.') + const upper = state.bounds?.to ? Date.parse(state.bounds.to) : context.cutoff.getTime() + const anchor = state.watermark ? Date.parse(state.watermark) : state.bounds?.from ? Date.parse(state.bounds.from) : upper - lookbackDays * dayMs + const to = Math.min(state.bounds?.to ? Date.parse(state.bounds.to) : context.cutoff.getTime(), anchor + windowDays * dayMs) + if (to < anchor) throw new IngestConfigError('Meet cutoff precedes its committed watermark.') + state = { + ...state, pending: state.pending.filter((parent) => !expired.includes(parent)), + cycle: { from: new Date(state.watermark ? Math.max(state.bounds?.from ? Date.parse(state.bounds.from) : -Infinity, anchor - overlapDays * dayMs) : anchor).toISOString(), to: new Date(to).toISOString(), phase: 'ended', parents: [], parentIndex: 0 }, + } + yield checkpoint(state) + } + while (state.cycle) { + const cycle = state.cycle + if (cycle.phase !== 'parents') { + const filter = cycle.phase === 'ended' + ? `end_time>="${cycle.from}" AND end_time<="${cycle.to}"` + : `end_time IS NULL AND start_time<="${cycle.to}"` + const page = await readPage(context, 'conferenceRecords', 'conferenceRecords', cycle.pageToken, filter) + if ('reset' in page) { + const recoveryCount = nextRecoveryCount(cycle.recoveryCount, `discovery:${cycle.phase}`) + seen.delete(`discovery:${cycle.phase}`) + state = { ...state, cycle: { ...cycle, pageToken: undefined, recoveryCount } } + yield checkpoint(state) + continue + } + checkProgress(seen, `discovery:${cycle.phase}`, cycle.pageToken, page.nextPageToken) + const pending = mergeParents(state.pending, page.items) + state = { ...state, pending, cycle: page.nextPageToken + ? { ...cycle, pageToken: page.nextPageToken } + : cycle.phase === 'ended' && !state.bounds + ? { ...cycle, phase: 'ongoing', pageToken: undefined, recoveryCount: undefined } + : { ...cycle, phase: 'parents', pageToken: undefined, recoveryCount: undefined, parents: pending.map((parent) => parent.name) }, + } + const rows = resource === 'conferences' ? rawRows(page.items, (item) => JSON.stringify([sourceId, item.name])) : [] + yield rows.length ? { rows, state } : checkpoint(state) + continue + } + if (resource === 'conferences' || cycle.parentIndex === cycle.parents.length) { + state = { scope, bounds: state.bounds, watermark: cycle.to, pending: resource === 'conferences' ? [] : state.pending } + yield checkpoint(state) + return + } + const name = cycle.parents[cycle.parentIndex] + if (!name) throw new IngestConfigError('Meet parent position is invalid.') + const parent = state.pending.find((item) => item.name === name) + if (!parent) throw new IngestConfigError('Meet pending parent is missing.') + if (parent.expireTime && Date.parse(parent.expireTime) <= context.cutoff.getTime()) { + throw new IngestConfigError(`Meet ${name} expired during unfinished artifact work. Review the coverage gap and migrate the checkpoint explicitly.`) + } + const active: ActiveParent = state.active ?? { name, transcriptIndex: 0 } + if (active.transcripts === undefined) { + const path = `${name}/transcripts` + const page = await readPage(context, path, 'transcripts', active.transcriptPageToken) + if ('reset' in page) { + const transcriptRecoveryCount = nextRecoveryCount(active.transcriptRecoveryCount, path) + seen.delete(path) + state = { ...state, active: { name, transcriptIndex: 0, transcriptRecoveryCount } } + yield checkpoint(state) + continue + } + checkProgress(seen, path, active.transcriptPageToken, page.nextPageToken) + if (resource === 'transcripts') { + state = page.nextPageToken + ? { ...state, active: { name, transcriptIndex: 0, transcriptPageToken: page.nextPageToken, transcriptRecoveryCount: active.transcriptRecoveryCount } } + : finishParent(state, name, cycle.to) + const rows = rawRows(page.items.map((item) => ({ ...item, conference_name: name })), (item) => JSON.stringify([sourceId, item.name])) + yield rows.length ? { rows, state } : checkpoint(state) + } else { + state = { ...state, active: { ...active, transcripts: page.items, nextTranscriptPageToken: page.nextPageToken } } + yield checkpoint(state) + } + continue + } + const transcript = active.transcripts[active.transcriptIndex] + if (!transcript) { + state = active.nextTranscriptPageToken + ? { ...state, active: { name, transcriptIndex: 0, transcriptPageToken: active.nextTranscriptPageToken, transcriptRecoveryCount: active.transcriptRecoveryCount } } + : finishParent(state, name, cycle.to) + yield checkpoint(state) + continue + } + const path = `${transcript.name}/entries` + const page = await readPage(context, path, 'transcriptEntries', active.entryPageToken) + if ('reset' in page) { + const entryRecoveryCount = nextRecoveryCount(active.entryRecoveryCount, path) + seen.delete(path) + state = { ...state, active: { ...active, entryPageToken: undefined, entryRecoveryCount } } + yield checkpoint(state) + continue + } + checkProgress(seen, path, active.entryPageToken, page.nextPageToken) + state = { ...state, active: page.nextPageToken + ? { ...active, entryPageToken: page.nextPageToken } + // Entry recovery debt clears only with acknowledgement of its terminal page. + : { ...active, entryPageToken: undefined, entryRecoveryCount: undefined, transcriptIndex: active.transcriptIndex + 1 }, + } + const rows = rawRows(page.items.map((item) => ({ ...item, conference_name: name, transcript_name: transcript.name })), (item) => JSON.stringify([sourceId, item.name])) + yield rows.length ? { rows, state } : checkpoint(state) + } +} + +async function readPage(context: FetchContext, path: string, field: string, pageToken?: string, filter?: string): Promise { + return context.attempt(async (signal) => { + const token = process.env.GOOGLE_MEET_ACCESS_TOKEN?.trim() + if (!token) throw new IngestConfigError('Set GOOGLE_MEET_ACCESS_TOKEN.') + const url = new URL(`${baseUrl}/${path}`) + url.searchParams.set('pageSize', '100') + if (filter) url.searchParams.set('filter', filter) + if (pageToken) url.searchParams.set('pageToken', pageToken) + const response = await fetch(url, { signal: AbortSignal.any([signal, AbortSignal.timeout(30_000)]), headers: { Authorization: `Bearer ${token}` }, redirect: 'error' }) + // Replaying the same fixed collection is safe. A second rejection fails visibly. + if (pageToken && (response.status === 400 || response.status === 410)) return { reset: true } + if (!response.ok) throw await HttpError.fromResponse(response) + const payload: unknown = await response.json() + if (!isObject(payload) || (payload[field] !== undefined && !Array.isArray(payload[field]))) throw new IngestConfigError(`Meet ${field} is not an array.`) + const values = payload[field] ?? [] + if (!Array.isArray(values)) throw new IngestConfigError(`Meet ${field} is not an array.`) + if (values.length > 100) throw new IngestConfigError(`Meet ${field} exceeded the requested page size.`) + const items = values.map((item: unknown) => named(item)) + if (items.some((item) => field === 'conferenceRecords' + ? !/^conferenceRecords\/[^/]+$/.test(item.name) + : !item.name.startsWith(`${path}/`) || item.name.slice(path.length + 1).includes('/'))) { + throw new IngestConfigError(`Meet ${field} returned a resource outside its requested parent.`) + } + const nextPageToken = optionalString(payload.nextPageToken, 'nextPageToken') + return { items, nextPageToken } + }, { label: `GET ${field}` }) +} + +function mergeParents(previous: PendingConference[], conferences: Named[]): PendingConference[] { + const parents = new Map(previous.map((parent) => [parent.name, parent])) + for (const conference of conferences) { + parents.set(conference.name, { ...parents.get(conference.name), name: conference.name, expireTime: optionalTimestamp(conference.expireTime, 'expireTime') }) + } + if (parents.size > maxPendingConferences) throw new IngestConfigError('Meet pending conference limit exceeded. Restrict the source or deliberately raise maxPendingConferences.') + return [...parents.values()] +} + +function finishParent(state: MeetState, name: string, checkedAt: string): MeetState { + if (!state.cycle) throw new IngestConfigError('Meet parent completion has no cycle.') + return { ...state, active: undefined, pending: state.pending.map((parent) => parent.name === name ? { ...parent, checkedAt } : parent), cycle: { ...state.cycle, parentIndex: state.cycle.parentIndex + 1 } } +} + +function checkpoint(state: MeetState): MeetChunk { + return { rows: [], state, id: 'meet-progress' } +} + +function checkProgress(seen: Map>, path: string, current: string | undefined, next: string | undefined): void { + if (!next) return + const tokens = seen.get(path) ?? new Set() + if (next === current || tokens.has(next)) throw new IngestConfigError(`Meet repeated ${path} page token.`) + tokens.add(next) + seen.set(path, tokens) +} + +function nextRecoveryCount(previous: number | undefined, path: string): number { + if ((previous ?? 0) >= 1) throw new IngestConfigError(`Meet unfinished ${path} collection repeatedly rejected a token across resumed runs. Adjust the editable stream chunk budget, run duration, or polling frequency, then explicitly migrate the saved state or use a new stream identity after reviewing coverage.`) + return (previous ?? 0) + 1 +} + +function parseState(raw: unknown): MeetState { + if (!isObject(raw) || raw.scope !== scope || !Array.isArray(raw.pending)) throw new IngestConfigError('Meet checkpoint scope changed or pending queue is invalid. Migrate the state explicitly or use a new stream ID.') + const pending = raw.pending.map((item: unknown) => { + if (!isObject(item)) throw new IngestConfigError('Meet pending conference is invalid.') + const name = requiredString(item.name, 'pending name') + if (!/^conferenceRecords\/[^/?#]+$/.test(name)) throw new IngestConfigError('Meet pending parent name is invalid.') + return { name, expireTime: optionalTimestamp(item.expireTime, 'pending expireTime'), checkedAt: optionalTimestamp(item.checkedAt, 'pending checkedAt') } + }) + if (pending.length > maxPendingConferences || new Set(pending.map((item) => item.name)).size !== pending.length) throw new IngestConfigError('Meet pending queue is oversized or contains duplicate parents.') + const state: MeetState = { scope, watermark: optionalTimestamp(raw.watermark, 'watermark'), pending } + if (raw.bounds !== undefined) { + if (!isObject(raw.bounds)) throw new IngestConfigError('Meet backfill bounds are invalid.') + state.bounds = { from: optionalTimestamp(raw.bounds.from, 'backfill from'), to: optionalTimestamp(raw.bounds.to, 'backfill to') } + if (state.bounds.from && state.bounds.to && Date.parse(state.bounds.from) > Date.parse(state.bounds.to)) throw new IngestConfigError('Meet backfill bounds are reversed.') + } + if (raw.cycle !== undefined) { + const cycle = raw.cycle + if (!isObject(cycle) || !['ended', 'ongoing', 'parents'].includes(String(cycle.phase)) || !Array.isArray(cycle.parents)) throw new IngestConfigError('Meet cycle is invalid.') + const phase = cycle.phase + if (phase !== 'ended' && phase !== 'ongoing' && phase !== 'parents') throw new IngestConfigError('Meet cycle phase is invalid.') + const from = timestamp(cycle.from, 'cycle from') + const to = timestamp(cycle.to, 'cycle to') + if (Date.parse(from) > Date.parse(to)) throw new IngestConfigError('Meet cycle bounds are reversed.') + const parents = cycle.parents.map((name: unknown) => requiredString(name, 'cycle parent')) + const parentIndex = position(cycle.parentIndex, parents.length, 'parentIndex') + if (new Set(parents).size !== parents.length || parents.some((name) => !pending.some((parent) => parent.name === name))) throw new IngestConfigError('Meet cycle references an invalid parent.') + const recoveryCount = parseRecoveryCount(cycle.recoveryCount) + if (phase === 'parents' && recoveryCount !== undefined) throw new IngestConfigError('Meet completed discovery still has recovery debt.') + state.cycle = { from, to, phase, parents, parentIndex, pageToken: optionalString(cycle.pageToken, 'cycle pageToken'), recoveryCount } + } + if (raw.active !== undefined) { + const active = raw.active + if (!isObject(active) || state.cycle?.phase !== 'parents') throw new IngestConfigError('Meet active parent has no parent cycle.') + const name = requiredString(active.name, 'active parent') + if (name !== state.cycle.parents[state.cycle.parentIndex]) throw new IngestConfigError('Meet active parent does not match its position.') + if (active.transcripts !== undefined && !Array.isArray(active.transcripts)) throw new IngestConfigError('Meet active transcripts are invalid.') + const transcripts = active.transcripts?.map((item: unknown) => named(item)) + if (transcripts && transcripts.length > 100) throw new IngestConfigError('Meet active transcript page is oversized.') + if (transcripts?.some((item) => !item.name.startsWith(`${name}/transcripts/`) || item.name.slice(`${name}/transcripts/`.length).includes('/'))) throw new IngestConfigError('Meet active transcript belongs to another parent.') + const transcriptIndex = position(active.transcriptIndex, transcripts?.length ?? 0, 'transcriptIndex') + const entryRecoveryCount = parseRecoveryCount(active.entryRecoveryCount) + if (entryRecoveryCount !== undefined && !transcripts?.[transcriptIndex]) throw new IngestConfigError('Meet entry recovery debt has no active transcript.') + state.active = { name, transcripts, transcriptIndex, transcriptPageToken: optionalString(active.transcriptPageToken, 'transcriptPageToken'), nextTranscriptPageToken: optionalString(active.nextTranscriptPageToken, 'nextTranscriptPageToken'), entryPageToken: optionalString(active.entryPageToken, 'entryPageToken'), transcriptRecoveryCount: parseRecoveryCount(active.transcriptRecoveryCount), entryRecoveryCount } + } + return state +} + +function named(value: unknown): Named { + if (!isObject(value)) throw new IngestConfigError('Meet returned an invalid resource.') + const name = requiredString(value.name, 'resource name') + if (!/^conferenceRecords\/[^/?#]+(?:\/transcripts\/[^/?#]+(?:\/entries\/[^/?#]+)?)?$/.test(name)) throw new IngestConfigError('Meet returned an invalid resource name.') + return { ...value, name } +} + +function position(value: unknown, max: number, field: string): number { + if (typeof value !== 'number' || !Number.isSafeInteger(value) || value < 0 || value > max) throw new IngestConfigError(`Meet ${field} is invalid.`) + return value +} + +function parseRecoveryCount(value: unknown): number | undefined { + if (value !== undefined && value !== 0 && value !== 1) throw new IngestConfigError('Meet checkpoint recovery count must be zero or one.') + return value +} + +function optionalTimestamp(value: unknown, field: string): string | undefined { + return value === undefined ? undefined : timestamp(value, field) +} + +function timestamp(value: unknown, field: string): string { + const result = requiredString(value, field) + if (!Number.isFinite(Date.parse(result))) throw new IngestConfigError(`Meet ${field} is not a timestamp.`) + return result +} + +function optionalString(value: unknown, field: string): string | undefined { + return value === undefined ? undefined : requiredString(value, field) +} + +function requiredString(value: unknown, field: string): string { + if (typeof value !== 'string' || value.trim().length === 0) throw new IngestConfigError(`Meet ${field} must be a non-empty string.`) + return value +} + +function isObject(value: unknown): value is Record { + return typeof value === 'object' && value !== null && !Array.isArray(value) } diff --git a/registry/google-meet/manifest.json b/registry/google-meet/manifest.json index 1afe46c8..71f0b302 100644 --- a/registry/google-meet/manifest.json +++ b/registry/google-meet/manifest.json @@ -3,10 +3,10 @@ "name": "google-meet", "type": "registry:item", "title": "Google Meet", - "description": "Small, editable Google Meet API ingestion example with raw ClickHouse tables.", + "description": "Editable Google Meet API ingestion with journaled checkpoints and raw ClickHouse tables.", "dependencies": [ - "@chkit/core@^0.2.0-beta.8", - "@chkit/plugin-ingest@^0.2.0-beta.8" + "@chkit/core@^0.2.0-beta.10", + "@chkit/plugin-ingest@^0.2.0-beta.10" ], "files": [ { @@ -18,25 +18,32 @@ "path": "index.ts", "type": "registry:file", "target": "src/integrations/google-meet/index.ts" + }, + { + "path": "tests/basic.test.ts", + "type": "registry:file", + "target": "src/integrations/google-meet/tests/basic.test.ts", + "role": "test" } ], "meta": { "chkit": { "formatVersion": 1, - "version": "0.1.0", + "version": "0.2.0", "documentation": "https://chkit.obsessiondb.com/integrations/google-meet/", "logo": "https://chkit.obsessiondb.com/logos/google-meet.svg", "language": "typescript", "license": "MIT", - "chkit": "^0.2.0-beta.8", - "ingest": "^0.2.0-beta.8", + "chkit": "^0.2.0-beta.10", + "ingest": "^0.2.0-beta.10", "clickhouse": ">=25.3.0", "root": "src/integrations/google-meet", "entry": "index.ts", "exports": [ "google_meetPipeline", "google_meet_conferencesRaw", - "google_meet_transcriptsRaw" + "google_meet_transcriptsRaw", + "google_meet_transcriptEntriesRaw" ], "resources": [ { @@ -47,7 +54,7 @@ "scopes": [ "https://www.googleapis.com/auth/meetings.space.readonly" ], - "strategy": "full", + "strategy": "cursor", "endpoints": [ { "method": "GET", @@ -60,12 +67,34 @@ "name": "transcripts", "title": "Transcripts", "table": "google_meet_transcripts_raw", - "description": "Transcript metadata and paged entries for each visible conference", + "description": "Transcript metadata for discovered and retained pending conferences", + "scopes": [ + "https://www.googleapis.com/auth/meetings.space.readonly" + ], + "strategy": "cursor", + "endpoints": [ + { + "method": "GET", + "path": "/conferenceRecords/{conferenceRecord}/transcripts", + "documentation": "https://developers.google.com/workspace/meet/api/reference/rest/v2/conferenceRecords.transcripts/list" + } + ] + }, + { + "name": "transcript-entries", + "title": "Transcript entries", + "table": "google_meet_transcript_entries_raw", + "description": "Individually keyed, paged transcript entries for retained conferences", "scopes": [ "https://www.googleapis.com/auth/meetings.space.readonly" ], - "strategy": "full", + "strategy": "cursor", "endpoints": [ + { + "method": "GET", + "path": "/conferenceRecords", + "documentation": "https://developers.google.com/workspace/meet/api/reference/rest/v2/conferenceRecords/list" + }, { "method": "GET", "path": "/conferenceRecords/{conferenceRecord}/transcripts", @@ -95,10 +124,13 @@ }, "views": [], "sync": { - "description": "A full read runs for each enabled stream. Stable IDs keep the latest observed record.", + "description": "Fixed end-time discovery windows and retained conference work queues checkpoint nested transcript and entry pagination.", "schedule": "Run bunx chkit ingest run --tag provider:google-meet from a scheduler.", - "deletions": "Each run reads a configured time range. Transcripts generated later require another scan of that range." + "deletions": "Late artifacts revisit retained parents until provider expiry. API expiry does not delete archived raw observations; unfinished work that expires fails visibly." } } - } + }, + "devDependencies": [ + "@types/bun@^1.2.0" + ] } diff --git a/registry/google-meet/releases/0.2.0.json b/registry/google-meet/releases/0.2.0.json new file mode 100644 index 00000000..37d68e32 --- /dev/null +++ b/registry/google-meet/releases/0.2.0.json @@ -0,0 +1,144 @@ +{ + "$schema": "https://ui.shadcn.com/schema/registry-item.json", + "name": "google-meet", + "type": "registry:item", + "title": "Google Meet", + "description": "Editable Google Meet API ingestion with journaled checkpoints and raw ClickHouse tables.", + "dependencies": [ + "@chkit/core@^0.2.0-beta.10", + "@chkit/plugin-ingest@^0.2.0-beta.10" + ], + "devDependencies": [ + "@types/bun@^1.2.0" + ], + "files": [ + { + "path": "google-meet/README.md", + "type": "registry:file", + "target": "src/integrations/google-meet/README.md", + "content": "# Google Meet raw ingestion\n\nThis editable reader stores provider responses as native JSON and journals fixed discovery windows plus pending conference and artifact progress. It builds on the overlapping-window pattern in [obsessiondb/brain](https://github.com/obsessiondb/brain/tree/main/scripts/brain-sync) and retains parents so late artifacts survive discovery advancing.\n\nSet `GOOGLE_MEET_ACCESS_TOKEN` in the runtime environment and configure a direct ClickHouse connection. Review `sourceId`, `lookbackDays`, `overlapDays`, `windowDays`, and `database` in `index.ts`. Keep `sourceId` and stream IDs tied to one authenticated Google account. The API does not expose a source identity check in these list responses; changing the token's account requires a separate source/checkpoint. Native JSON requires ClickHouse 25.3 or later. Tokens are read at request time and are not refreshed by this reader.\n\n```sh\nbunx chkit check\nbunx chkit generate --name add_google_meet\n# Review the migration before applying it.\nbunx chkit migrate --apply\nbunx chkit ingest run --tag provider:google-meet\nbunx chkit ingest status --tag provider:google-meet\n```\n\n## Resources and identities\n\n- `conferences` → `google_meet_conferences_raw`\n- `transcripts` → `google_meet_transcripts_raw` (transcript metadata with `conference_name`)\n- `transcript-entries` → `google_meet_transcript_entries_raw` (one entry per row with `conference_name` and `transcript_name`)\n\nEvery resource uses `[sourceId, provider.name]` as its row identity. Entries are emitted page by page rather than buffering a full transcript inside one JSON row. Join metadata and entries through their provider resource names. These raw destinations keep the latest observed value; they do not capture later edits to the Google Docs transcript document.\n\n## Discovery and pending work\n\nEach stream independently discovers completed conferences through `end_time` bounds and ongoing conferences through `end_time IS NULL`. This includes long-running calls whose start predates the lookback. Initial discovery defaults to the retained 30-day period; later runs overlap seven days. A cycle's bounds are persisted before requesting its first page, so pauses keep the same window. `windowDays` caps forward advancement per cycle.\n\nTranscript and entry streams retain discovered conference names until `expireTime`, independently of the discovery watermark. Every completed cycle checks those parents again, including parents with no transcripts and transcripts still being generated. This captures late artifacts outside the seven-day discovery overlap. Retention, access, and polling frequency still determine coverage; this is polling, not a provider change feed. Meet exposes no modification/change token and does not support transcript/entry date filters. See [conference filtering](https://developers.google.com/workspace/meet/api/reference/rest/v2/conferenceRecords/list) and [artifact behavior](https://developers.google.com/workspace/meet/api/guides/artifacts).\n\n## Checkpoints and recovery\n\nAcknowledged pages checkpoint discovery, parent, transcript-page, and entry-page positions. A failed entry load retains its preceding page position. Rejected page tokens restart their scoped collection once and may replay observations; repeated rejection and malformed pagination fail visibly. Candidate state is complete and advances only after its rows have sink evidence.\n\nRecovery counts remain saved for an unfinished discovery phase, parent transcript collection, or transcript entry collection until its terminal boundary has destination acknowledgement. A budget pause after recovery or a successful nonterminal replay page keeps its count. A second rejection for that unfinished collection fails even in a fresh scheduled execution. Edit `budget.maxChunks` to change the stream chunk cap, choose sufficient `--max-duration`, and run frequently enough to finish before tokens expire. After fixing those conditions and reviewing coverage, migrate saved state explicitly or use a new stream identity for a deliberate fresh scan.\n\nEach stream permits 200 chunks per run and retains at most 1,000 conferences. Requests have a 30-second timeout. Budget exhaustion preserves progress and a later invocation continues it. Increase those editable limits deliberately for larger sources. A pending queue over its configured limit fails rather than silently dropping parents. The executor owns request retries, cancellation, and load acknowledgements. Run one ingestion process per ClickHouse target at a time.\n\nGoogle deletes conference records and API transcript entries 30 days after a conference ends. The reader reports expiry during unfinished work as a coverage gap; it cannot recover expired API entries. Previously checked parents leave the work queue on expiry, while stored raw rows remain. It does not infer deletion from missing, inaccessible, or expired resources. See [conference retention](https://developers.google.com/workspace/meet/api/reference/rest/v2/conferenceRecords) and [artifact retention](https://developers.google.com/workspace/meet/api/guides/artifacts).\n\n## Historical ranges\n\nExplicit bounds apply to conference end times and use an isolated backfill checkpoint. Historical ranges skip ongoing-call discovery. Reuse the same backfill ID and bounds to resume; changing the bounds requires another ID. Artifact lists remain parent-scoped and cannot themselves be date-filtered. Expired history is unavailable even when a backfill range requests it.\n\n```sh\nbunx chkit ingest run --tag provider:google-meet --backfill october --from 2026-10-01T00:00:00Z --to 2026-10-05T00:00:00Z\n```\n\nCheckpoint validation rejects changed source/window scope and invalid nested positions. Migrate state explicitly or use a new stream identity when changing these settings.\n\n## Upgrading from 0.1.x\n\nVersion 0.2.0 keeps the existing conference and transcript destinations and adds an entry table. New transcript rows contain metadata; entries live in the separate table. Row identities now include `sourceId`, so upgraded observations do not replace legacy provider-name IDs. Old rows and their nested `entries` remain until an explicit identity/data migration. Keep older tables as archives or migrate identities deliberately before mixing datasets.\n\n## Fixture tests\n\n```sh\nbunx chkit add google-meet --with-tests\nbun test src/integrations/google-meet/tests/basic.test.ts\n```\n\nFixtures verify nested resumption, retained parents with late transcripts, end-time and ongoing discovery, rejected tokens, failed loads, isolated backfill bounds, malformed state, and expiry gaps without live credentials.\n\nThe [Google Meet integration guide](https://chkit.obsessiondb.com/integrations/google-meet/) covers installation and sync behavior.\n" + }, + { + "path": "google-meet/index.ts", + "type": "registry:file", + "target": "src/integrations/google-meet/index.ts", + "content": "import { definePipeline, defineStream, HttpError, IngestConfigError, rawRows, rawTable, type FetchContext, type ReadContext, type SourceChunk } from '@chkit/plugin-ingest'\n\nconst database = 'default'\n// Bind this label to one authenticated Google account; use separate stream IDs for another source.\nconst sourceId = 'google-meet.primary'\nconst lookbackDays = 30\nconst overlapDays = 7\nconst windowDays = 30\nconst maxPendingConferences = 1_000\nconst dayMs = 86_400_000\nconst baseUrl = 'https://meet.googleapis.com/v2'\nconst scope = JSON.stringify([sourceId, lookbackDays, overlapDays, windowDays])\n\ntype Resource = 'conferences' | 'transcripts' | 'transcript-entries'\ninterface Named { name: string; [key: string]: unknown }\ninterface PendingConference { name: string; expireTime?: string; checkedAt?: string }\ninterface Cycle {\n from: string\n to: string\n phase: 'ended' | 'ongoing' | 'parents'\n pageToken?: string\n /** Recovery debt for the unfinished discovery phase, across executions. */\n recoveryCount?: number\n parents: string[]\n parentIndex: number\n}\ninterface ActiveParent {\n name: string\n transcriptPageToken?: string\n transcriptRecoveryCount?: number\n nextTranscriptPageToken?: string\n transcripts?: Named[]\n transcriptIndex: number\n entryPageToken?: string\n entryRecoveryCount?: number\n}\ninterface MeetState {\n scope: string\n watermark?: string\n bounds?: { from?: string; to?: string }\n pending: PendingConference[]\n cycle?: Cycle\n active?: ActiveParent\n}\ninterface Page { items: Named[]; nextPageToken?: string }\ntype MeetContext = ReadContext<{ from: Date | undefined; to: Date | undefined } | undefined, MeetState>\ntype MeetChunk = SourceChunk, MeetState>\n\nexport const google_meet_conferencesRaw = rawTable({ database, name: 'google_meet_conferences_raw' })\nexport const google_meet_transcriptsRaw = rawTable({ database, name: 'google_meet_transcripts_raw' })\nexport const google_meet_transcriptEntriesRaw = rawTable({ database, name: 'google_meet_transcript_entries_raw' })\n\nexport const google_meetPipeline = definePipeline({\n id: 'google-meet', tags: ['provider:google-meet'], maxStreams: 1, maxFetches: 1,\n streams: [\n stream('conferences', google_meet_conferencesRaw),\n stream('transcripts', google_meet_transcriptsRaw),\n stream('transcript-entries', google_meet_transcriptEntriesRaw),\n ],\n})\n\nfunction stream(resource: Resource, destination: typeof google_meet_conferencesRaw) {\n return defineStream({\n id: `google-meet.${resource}`, tags: [`resource:${resource}`], destination,\n incremental: {\n id: `google-meet.${resource}.pending`, version: 1, parseState,\n plan: ({ range }) => range,\n },\n batchSize: 1,\n budget: { maxChunks: 200, maxChunkRows: 100 },\n classifyError: (cause) => cause instanceof IngestConfigError ? { kind: 'permanent' } : undefined,\n read: (context) => readResource(context, resource),\n })\n}\n\nasync function* readResource(context: MeetContext, resource: Resource): AsyncGenerator {\n let state: MeetState = context.state ?? { scope, pending: [] }\n const requestedBounds = context.selection?.from || context.selection?.to\n ? { from: context.selection.from?.toISOString(), to: context.selection.to?.toISOString() }\n : undefined\n if (requestedBounds && state.bounds && JSON.stringify(requestedBounds) !== JSON.stringify(state.bounds)) throw new IngestConfigError('Meet backfill bounds changed. Use a new backfill ID.')\n if (requestedBounds && state.watermark && !state.bounds) throw new IngestConfigError('Meet checkpoint was created without date bounds. Use a new backfill ID.')\n state = { ...state, bounds: state.bounds ?? requestedBounds }\n const seen = new Map>()\n if (!state.cycle) {\n const expired = state.pending.filter((parent) => parent.expireTime && Date.parse(parent.expireTime) <= context.cutoff.getTime())\n if (expired.some((parent) => !parent.checkedAt)) throw new IngestConfigError('Meet pending conference expired before a complete artifact read. Historical API coverage cannot be recovered; review and migrate the checkpoint explicitly.')\n const upper = state.bounds?.to ? Date.parse(state.bounds.to) : context.cutoff.getTime()\n const anchor = state.watermark ? Date.parse(state.watermark) : state.bounds?.from ? Date.parse(state.bounds.from) : upper - lookbackDays * dayMs\n const to = Math.min(state.bounds?.to ? Date.parse(state.bounds.to) : context.cutoff.getTime(), anchor + windowDays * dayMs)\n if (to < anchor) throw new IngestConfigError('Meet cutoff precedes its committed watermark.')\n state = {\n ...state, pending: state.pending.filter((parent) => !expired.includes(parent)),\n cycle: { from: new Date(state.watermark ? Math.max(state.bounds?.from ? Date.parse(state.bounds.from) : -Infinity, anchor - overlapDays * dayMs) : anchor).toISOString(), to: new Date(to).toISOString(), phase: 'ended', parents: [], parentIndex: 0 },\n }\n yield checkpoint(state)\n }\n while (state.cycle) {\n const cycle = state.cycle\n if (cycle.phase !== 'parents') {\n const filter = cycle.phase === 'ended'\n ? `end_time>=\"${cycle.from}\" AND end_time<=\"${cycle.to}\"`\n : `end_time IS NULL AND start_time<=\"${cycle.to}\"`\n const page = await readPage(context, 'conferenceRecords', 'conferenceRecords', cycle.pageToken, filter)\n if ('reset' in page) {\n const recoveryCount = nextRecoveryCount(cycle.recoveryCount, `discovery:${cycle.phase}`)\n seen.delete(`discovery:${cycle.phase}`)\n state = { ...state, cycle: { ...cycle, pageToken: undefined, recoveryCount } }\n yield checkpoint(state)\n continue\n }\n checkProgress(seen, `discovery:${cycle.phase}`, cycle.pageToken, page.nextPageToken)\n const pending = mergeParents(state.pending, page.items)\n state = { ...state, pending, cycle: page.nextPageToken\n ? { ...cycle, pageToken: page.nextPageToken }\n : cycle.phase === 'ended' && !state.bounds\n ? { ...cycle, phase: 'ongoing', pageToken: undefined, recoveryCount: undefined }\n : { ...cycle, phase: 'parents', pageToken: undefined, recoveryCount: undefined, parents: pending.map((parent) => parent.name) },\n }\n const rows = resource === 'conferences' ? rawRows(page.items, (item) => JSON.stringify([sourceId, item.name])) : []\n yield rows.length ? { rows, state } : checkpoint(state)\n continue\n }\n if (resource === 'conferences' || cycle.parentIndex === cycle.parents.length) {\n state = { scope, bounds: state.bounds, watermark: cycle.to, pending: resource === 'conferences' ? [] : state.pending }\n yield checkpoint(state)\n return\n }\n const name = cycle.parents[cycle.parentIndex]\n if (!name) throw new IngestConfigError('Meet parent position is invalid.')\n const parent = state.pending.find((item) => item.name === name)\n if (!parent) throw new IngestConfigError('Meet pending parent is missing.')\n if (parent.expireTime && Date.parse(parent.expireTime) <= context.cutoff.getTime()) {\n throw new IngestConfigError(`Meet ${name} expired during unfinished artifact work. Review the coverage gap and migrate the checkpoint explicitly.`)\n }\n const active: ActiveParent = state.active ?? { name, transcriptIndex: 0 }\n if (active.transcripts === undefined) {\n const path = `${name}/transcripts`\n const page = await readPage(context, path, 'transcripts', active.transcriptPageToken)\n if ('reset' in page) {\n const transcriptRecoveryCount = nextRecoveryCount(active.transcriptRecoveryCount, path)\n seen.delete(path)\n state = { ...state, active: { name, transcriptIndex: 0, transcriptRecoveryCount } }\n yield checkpoint(state)\n continue\n }\n checkProgress(seen, path, active.transcriptPageToken, page.nextPageToken)\n if (resource === 'transcripts') {\n state = page.nextPageToken\n ? { ...state, active: { name, transcriptIndex: 0, transcriptPageToken: page.nextPageToken, transcriptRecoveryCount: active.transcriptRecoveryCount } }\n : finishParent(state, name, cycle.to)\n const rows = rawRows(page.items.map((item) => ({ ...item, conference_name: name })), (item) => JSON.stringify([sourceId, item.name]))\n yield rows.length ? { rows, state } : checkpoint(state)\n } else {\n state = { ...state, active: { ...active, transcripts: page.items, nextTranscriptPageToken: page.nextPageToken } }\n yield checkpoint(state)\n }\n continue\n }\n const transcript = active.transcripts[active.transcriptIndex]\n if (!transcript) {\n state = active.nextTranscriptPageToken\n ? { ...state, active: { name, transcriptIndex: 0, transcriptPageToken: active.nextTranscriptPageToken, transcriptRecoveryCount: active.transcriptRecoveryCount } }\n : finishParent(state, name, cycle.to)\n yield checkpoint(state)\n continue\n }\n const path = `${transcript.name}/entries`\n const page = await readPage(context, path, 'transcriptEntries', active.entryPageToken)\n if ('reset' in page) {\n const entryRecoveryCount = nextRecoveryCount(active.entryRecoveryCount, path)\n seen.delete(path)\n state = { ...state, active: { ...active, entryPageToken: undefined, entryRecoveryCount } }\n yield checkpoint(state)\n continue\n }\n checkProgress(seen, path, active.entryPageToken, page.nextPageToken)\n state = { ...state, active: page.nextPageToken\n ? { ...active, entryPageToken: page.nextPageToken }\n // Entry recovery debt clears only with acknowledgement of its terminal page.\n : { ...active, entryPageToken: undefined, entryRecoveryCount: undefined, transcriptIndex: active.transcriptIndex + 1 },\n }\n const rows = rawRows(page.items.map((item) => ({ ...item, conference_name: name, transcript_name: transcript.name })), (item) => JSON.stringify([sourceId, item.name]))\n yield rows.length ? { rows, state } : checkpoint(state)\n }\n}\n\nasync function readPage(context: FetchContext, path: string, field: string, pageToken?: string, filter?: string): Promise {\n return context.attempt(async (signal) => {\n const token = process.env.GOOGLE_MEET_ACCESS_TOKEN?.trim()\n if (!token) throw new IngestConfigError('Set GOOGLE_MEET_ACCESS_TOKEN.')\n const url = new URL(`${baseUrl}/${path}`)\n url.searchParams.set('pageSize', '100')\n if (filter) url.searchParams.set('filter', filter)\n if (pageToken) url.searchParams.set('pageToken', pageToken)\n const response = await fetch(url, { signal: AbortSignal.any([signal, AbortSignal.timeout(30_000)]), headers: { Authorization: `Bearer ${token}` }, redirect: 'error' })\n // Replaying the same fixed collection is safe. A second rejection fails visibly.\n if (pageToken && (response.status === 400 || response.status === 410)) return { reset: true }\n if (!response.ok) throw await HttpError.fromResponse(response)\n const payload: unknown = await response.json()\n if (!isObject(payload) || (payload[field] !== undefined && !Array.isArray(payload[field]))) throw new IngestConfigError(`Meet ${field} is not an array.`)\n const values = payload[field] ?? []\n if (!Array.isArray(values)) throw new IngestConfigError(`Meet ${field} is not an array.`)\n if (values.length > 100) throw new IngestConfigError(`Meet ${field} exceeded the requested page size.`)\n const items = values.map((item: unknown) => named(item))\n if (items.some((item) => field === 'conferenceRecords'\n ? !/^conferenceRecords\\/[^/]+$/.test(item.name)\n : !item.name.startsWith(`${path}/`) || item.name.slice(path.length + 1).includes('/'))) {\n throw new IngestConfigError(`Meet ${field} returned a resource outside its requested parent.`)\n }\n const nextPageToken = optionalString(payload.nextPageToken, 'nextPageToken')\n return { items, nextPageToken }\n }, { label: `GET ${field}` })\n}\n\nfunction mergeParents(previous: PendingConference[], conferences: Named[]): PendingConference[] {\n const parents = new Map(previous.map((parent) => [parent.name, parent]))\n for (const conference of conferences) {\n parents.set(conference.name, { ...parents.get(conference.name), name: conference.name, expireTime: optionalTimestamp(conference.expireTime, 'expireTime') })\n }\n if (parents.size > maxPendingConferences) throw new IngestConfigError('Meet pending conference limit exceeded. Restrict the source or deliberately raise maxPendingConferences.')\n return [...parents.values()]\n}\n\nfunction finishParent(state: MeetState, name: string, checkedAt: string): MeetState {\n if (!state.cycle) throw new IngestConfigError('Meet parent completion has no cycle.')\n return { ...state, active: undefined, pending: state.pending.map((parent) => parent.name === name ? { ...parent, checkedAt } : parent), cycle: { ...state.cycle, parentIndex: state.cycle.parentIndex + 1 } }\n}\n\nfunction checkpoint(state: MeetState): MeetChunk {\n return { rows: [], state, id: 'meet-progress' }\n}\n\nfunction checkProgress(seen: Map>, path: string, current: string | undefined, next: string | undefined): void {\n if (!next) return\n const tokens = seen.get(path) ?? new Set()\n if (next === current || tokens.has(next)) throw new IngestConfigError(`Meet repeated ${path} page token.`)\n tokens.add(next)\n seen.set(path, tokens)\n}\n\nfunction nextRecoveryCount(previous: number | undefined, path: string): number {\n if ((previous ?? 0) >= 1) throw new IngestConfigError(`Meet unfinished ${path} collection repeatedly rejected a token across resumed runs. Adjust the editable stream chunk budget, run duration, or polling frequency, then explicitly migrate the saved state or use a new stream identity after reviewing coverage.`)\n return (previous ?? 0) + 1\n}\n\nfunction parseState(raw: unknown): MeetState {\n if (!isObject(raw) || raw.scope !== scope || !Array.isArray(raw.pending)) throw new IngestConfigError('Meet checkpoint scope changed or pending queue is invalid. Migrate the state explicitly or use a new stream ID.')\n const pending = raw.pending.map((item: unknown) => {\n if (!isObject(item)) throw new IngestConfigError('Meet pending conference is invalid.')\n const name = requiredString(item.name, 'pending name')\n if (!/^conferenceRecords\\/[^/?#]+$/.test(name)) throw new IngestConfigError('Meet pending parent name is invalid.')\n return { name, expireTime: optionalTimestamp(item.expireTime, 'pending expireTime'), checkedAt: optionalTimestamp(item.checkedAt, 'pending checkedAt') }\n })\n if (pending.length > maxPendingConferences || new Set(pending.map((item) => item.name)).size !== pending.length) throw new IngestConfigError('Meet pending queue is oversized or contains duplicate parents.')\n const state: MeetState = { scope, watermark: optionalTimestamp(raw.watermark, 'watermark'), pending }\n if (raw.bounds !== undefined) {\n if (!isObject(raw.bounds)) throw new IngestConfigError('Meet backfill bounds are invalid.')\n state.bounds = { from: optionalTimestamp(raw.bounds.from, 'backfill from'), to: optionalTimestamp(raw.bounds.to, 'backfill to') }\n if (state.bounds.from && state.bounds.to && Date.parse(state.bounds.from) > Date.parse(state.bounds.to)) throw new IngestConfigError('Meet backfill bounds are reversed.')\n }\n if (raw.cycle !== undefined) {\n const cycle = raw.cycle\n if (!isObject(cycle) || !['ended', 'ongoing', 'parents'].includes(String(cycle.phase)) || !Array.isArray(cycle.parents)) throw new IngestConfigError('Meet cycle is invalid.')\n const phase = cycle.phase\n if (phase !== 'ended' && phase !== 'ongoing' && phase !== 'parents') throw new IngestConfigError('Meet cycle phase is invalid.')\n const from = timestamp(cycle.from, 'cycle from')\n const to = timestamp(cycle.to, 'cycle to')\n if (Date.parse(from) > Date.parse(to)) throw new IngestConfigError('Meet cycle bounds are reversed.')\n const parents = cycle.parents.map((name: unknown) => requiredString(name, 'cycle parent'))\n const parentIndex = position(cycle.parentIndex, parents.length, 'parentIndex')\n if (new Set(parents).size !== parents.length || parents.some((name) => !pending.some((parent) => parent.name === name))) throw new IngestConfigError('Meet cycle references an invalid parent.')\n const recoveryCount = parseRecoveryCount(cycle.recoveryCount)\n if (phase === 'parents' && recoveryCount !== undefined) throw new IngestConfigError('Meet completed discovery still has recovery debt.')\n state.cycle = { from, to, phase, parents, parentIndex, pageToken: optionalString(cycle.pageToken, 'cycle pageToken'), recoveryCount }\n }\n if (raw.active !== undefined) {\n const active = raw.active\n if (!isObject(active) || state.cycle?.phase !== 'parents') throw new IngestConfigError('Meet active parent has no parent cycle.')\n const name = requiredString(active.name, 'active parent')\n if (name !== state.cycle.parents[state.cycle.parentIndex]) throw new IngestConfigError('Meet active parent does not match its position.')\n if (active.transcripts !== undefined && !Array.isArray(active.transcripts)) throw new IngestConfigError('Meet active transcripts are invalid.')\n const transcripts = active.transcripts?.map((item: unknown) => named(item))\n if (transcripts && transcripts.length > 100) throw new IngestConfigError('Meet active transcript page is oversized.')\n if (transcripts?.some((item) => !item.name.startsWith(`${name}/transcripts/`) || item.name.slice(`${name}/transcripts/`.length).includes('/'))) throw new IngestConfigError('Meet active transcript belongs to another parent.')\n const transcriptIndex = position(active.transcriptIndex, transcripts?.length ?? 0, 'transcriptIndex')\n const entryRecoveryCount = parseRecoveryCount(active.entryRecoveryCount)\n if (entryRecoveryCount !== undefined && !transcripts?.[transcriptIndex]) throw new IngestConfigError('Meet entry recovery debt has no active transcript.')\n state.active = { name, transcripts, transcriptIndex, transcriptPageToken: optionalString(active.transcriptPageToken, 'transcriptPageToken'), nextTranscriptPageToken: optionalString(active.nextTranscriptPageToken, 'nextTranscriptPageToken'), entryPageToken: optionalString(active.entryPageToken, 'entryPageToken'), transcriptRecoveryCount: parseRecoveryCount(active.transcriptRecoveryCount), entryRecoveryCount }\n }\n return state\n}\n\nfunction named(value: unknown): Named {\n if (!isObject(value)) throw new IngestConfigError('Meet returned an invalid resource.')\n const name = requiredString(value.name, 'resource name')\n if (!/^conferenceRecords\\/[^/?#]+(?:\\/transcripts\\/[^/?#]+(?:\\/entries\\/[^/?#]+)?)?$/.test(name)) throw new IngestConfigError('Meet returned an invalid resource name.')\n return { ...value, name }\n}\n\nfunction position(value: unknown, max: number, field: string): number {\n if (typeof value !== 'number' || !Number.isSafeInteger(value) || value < 0 || value > max) throw new IngestConfigError(`Meet ${field} is invalid.`)\n return value\n}\n\nfunction parseRecoveryCount(value: unknown): number | undefined {\n if (value !== undefined && value !== 0 && value !== 1) throw new IngestConfigError('Meet checkpoint recovery count must be zero or one.')\n return value\n}\n\nfunction optionalTimestamp(value: unknown, field: string): string | undefined {\n return value === undefined ? undefined : timestamp(value, field)\n}\n\nfunction timestamp(value: unknown, field: string): string {\n const result = requiredString(value, field)\n if (!Number.isFinite(Date.parse(result))) throw new IngestConfigError(`Meet ${field} is not a timestamp.`)\n return result\n}\n\nfunction optionalString(value: unknown, field: string): string | undefined {\n return value === undefined ? undefined : requiredString(value, field)\n}\n\nfunction requiredString(value: unknown, field: string): string {\n if (typeof value !== 'string' || value.trim().length === 0) throw new IngestConfigError(`Meet ${field} must be a non-empty string.`)\n return value\n}\n\nfunction isObject(value: unknown): value is Record {\n return typeof value === 'object' && value !== null && !Array.isArray(value)\n}\n" + }, + { + "path": "google-meet/tests/basic.test.ts", + "type": "registry:file", + "target": "src/integrations/google-meet/tests/basic.test.ts", + "role": "test", + "content": "import { afterEach, expect, test } from 'bun:test'\nimport { runIngestion, selectStreams } from '@chkit/plugin-ingest'\nimport { createMemoryDestination, createMemoryJournal } from '@chkit/plugin-ingest/testing'\n\nimport { google_meetPipeline } from '../index.js'\n\nconst originalFetch = globalThis.fetch\nconst originalToken = process.env.GOOGLE_MEET_ACCESS_TOKEN\nconst cutoff = new Date('2026-10-20T00:00:00Z')\nconst conference = { name: 'conferenceRecords/c1', startTime: '2026-08-01T00:00:00Z', endTime: '2026-10-01T00:00:00Z', expireTime: '2026-10-31T00:00:00Z' }\nconst transcript = { name: `${conference.name}/transcripts/t1`, state: 'FILE_GENERATED' }\nconst entry = { name: `${transcript.name}/entries/e1`, text: 'first' }\n\nfunction setFetch(handler?: (url: URL) => Response): void {\n process.env.GOOGLE_MEET_ACCESS_TOKEN = 'fixture'\n globalThis.fetch = Object.assign(async (input: Parameters[0]) => handler ? handler(new URL(String(input))) : fixture(new URL(String(input))), { preconnect: originalFetch.preconnect })\n}\n\nfunction fixture(url: URL): Response {\n if (url.pathname === '/v2/conferenceRecords') return Response.json({ conferenceRecords: url.searchParams.get('filter')?.includes('IS NULL') ? [] : [conference] })\n if (url.pathname.endsWith('/transcripts')) return Response.json({ transcripts: [transcript] })\n if (url.pathname.endsWith('/entries')) return Response.json(url.searchParams.has('pageToken')\n ? { transcriptEntries: [{ ...entry, name: `${transcript.name}/entries/e2`, text: 'second' }] }\n : { transcriptEntries: [entry], nextPageToken: 'entries-next' })\n throw new Error(`Unexpected Meet request ${url}`)\n}\n\nfunction request(resource = 'transcript-entries', maxChunks?: number) {\n const pipeline = { ...google_meetPipeline, retry: { retries: 0 }, streams: google_meetPipeline.streams.map((stream) => ({ ...stream, ...(maxChunks === undefined ? {} : { budget: { ...stream.budget, maxChunks } }) })) }\n return { selected: selectStreams([pipeline], [`resource:${resource}`]), backfill: undefined }\n}\n\nafterEach(() => {\n globalThis.fetch = originalFetch\n if (originalToken === undefined) delete process.env.GOOGLE_MEET_ACCESS_TOKEN\n else process.env.GOOGLE_MEET_ACCESS_TOKEN = originalToken\n})\n\ntest.serial('Meet resumes within an entry collection without losing earlier pages or changing its window', async () => {\n const urls: URL[] = []\n setFetch((url) => { urls.push(url); return fixture(url) })\n const journal = createMemoryJournal()\n const destination = createMemoryDestination()\n const first = await runIngestion(request('transcript-entries', 5), { journal, destination, now: () => cutoff })\n expect(first.streams[0]?.outcome).toBe('budget_exhausted')\n expect((await journal.readCheckpoint('google-meet.transcript-entries')).envelope?.state).toMatchObject({ active: { entryPageToken: 'entries-next' }, cycle: { to: cutoff.toISOString() } })\n const previousRequests = urls.length\n const resumed = await runIngestion(request(), { journal, destination, now: () => new Date('2026-10-21T00:00:00Z') })\n expect(resumed.ok).toBe(true)\n expect(urls.slice(previousRequests).map((url) => [url.pathname, url.searchParams.get('pageToken')])).toEqual([[`/v2/${transcript.name}/entries`, 'entries-next']])\n expect(destination.tables.get('default.google_meet_transcript_entries_raw')?.map((row) => row.raw)).toEqual([\n { ...entry, conference_name: conference.name, transcript_name: transcript.name },\n { ...entry, name: `${transcript.name}/entries/e2`, text: 'second', conference_name: conference.name, transcript_name: transcript.name },\n ])\n expect((await journal.readCheckpoint('google-meet.transcript-entries')).envelope?.state).toMatchObject({ watermark: cutoff.toISOString() })\n})\n\ntest.serial('Meet keeps a parent beyond discovery overlap for a late transcript', async () => {\n let late = false\n const urls: URL[] = []\n setFetch((url) => {\n urls.push(url)\n if (url.pathname === '/v2/conferenceRecords' && late) return Response.json({ conferenceRecords: [] })\n if (url.pathname.endsWith('/transcripts') && !late) return Response.json({ transcripts: [] })\n return fixture(url)\n })\n const journal = createMemoryJournal()\n const destination = createMemoryDestination()\n expect((await runIngestion(request(), { journal, destination, now: () => cutoff })).ok).toBe(true)\n expect(destination.tables.size).toBe(0)\n expect((await journal.readCheckpoint('google-meet.transcript-entries')).envelope?.state).toMatchObject({ pending: [{ name: conference.name, checkedAt: cutoff.toISOString() }] })\n late = true\n expect((await runIngestion(request(), { journal, destination, now: () => new Date('2026-10-28T00:00:00Z') })).ok).toBe(true)\n expect(destination.tables.get('default.google_meet_transcript_entries_raw')).toHaveLength(2)\n const endedFilters = urls.filter((url) => url.pathname === '/v2/conferenceRecords' && !url.searchParams.get('filter')?.includes('IS NULL')).map((url) => url.searchParams.get('filter'))\n expect(endedFilters.at(-1)).toContain('2026-10-13T00:00:00.000Z')\n})\n\ntest.serial('Meet queries end-time windows and ongoing calls instead of excluding long-running conferences', async () => {\n const filters: Array = []\n setFetch((url) => { filters.push(url.searchParams.get('filter')); return fixture(url) })\n const destination = createMemoryDestination()\n expect((await runIngestion(request('conferences'), { journal: createMemoryJournal(), destination, now: () => cutoff })).ok).toBe(true)\n expect(filters).toEqual([\n 'end_time>=\"2026-09-20T00:00:00.000Z\" AND end_time<=\"2026-10-20T00:00:00.000Z\"',\n 'end_time IS NULL AND start_time<=\"2026-10-20T00:00:00.000Z\"',\n ])\n expect(destination.tables.get('default.google_meet_conferences_raw')?.[0]?.raw).toEqual(conference)\n})\n\ntest.serial('Meet replays an expired entry page token without replacing an earlier entry identity', async () => {\n let rejected = false\n setFetch((url) => {\n if (url.pathname.endsWith('/entries') && url.searchParams.has('pageToken') && !rejected) { rejected = true; return new Response('page token expired', { status: 400 }) }\n return fixture(url)\n })\n const destination = createMemoryDestination()\n const result = await runIngestion(request(), { journal: createMemoryJournal(), destination, now: () => cutoff })\n expect(result.ok).toBe(true)\n const rows = destination.tables.get('default.google_meet_transcript_entries_raw') ?? []\n expect(new Set(rows.map((row) => row.id)).size).toBe(2)\n expect(rows.some((row) => JSON.stringify(row.raw).includes('second'))).toBe(true)\n})\n\ntest.serial('Meet preserves entry progress after a terminal page load failure', async () => {\n setFetch()\n const journal = createMemoryJournal()\n const destination = createMemoryDestination()\n const first = await runIngestion(request(), { journal, destination: {\n insert: async (input) => {\n if (input.rows.some((row) => JSON.stringify(row.raw).includes('second'))) throw new Error('fixture sink failure')\n await destination.insert(input)\n },\n }, now: () => cutoff })\n expect(first.ok).toBe(false)\n expect((await journal.readCheckpoint('google-meet.transcript-entries')).envelope?.state).toMatchObject({ active: { entryPageToken: 'entries-next' } })\n expect((await runIngestion(request(), { journal, destination, now: () => cutoff })).ok).toBe(true)\n expect(destination.tables.get('default.google_meet_transcript_entries_raw')).toHaveLength(2)\n})\n\ntest.serial('Meet explicit backfill bounds stay isolated and constrain completed conference discovery', async () => {\n const filters: Array = []\n setFetch((url) => { filters.push(url.searchParams.get('filter')); return Response.json({ conferenceRecords: [] }) })\n const journal = createMemoryJournal()\n const result = await runIngestion({ ...request('conferences'), backfill: { id: 'october', from: new Date('2026-10-01T00:00:00Z'), to: new Date('2026-10-05T00:00:00Z') } }, { journal, destination: createMemoryDestination(), now: () => cutoff })\n expect(result.ok).toBe(true)\n expect(filters).toEqual(['end_time>=\"2026-10-01T00:00:00.000Z\" AND end_time<=\"2026-10-05T00:00:00.000Z\"'])\n expect((await journal.readCheckpoint('google-meet.conferences')).version).toBe(0)\n})\n\ntest.serial('Meet rejects malformed checkpoints and repeated page tokens', async () => {\n setFetch(() => Response.json({ conferenceRecords: [], nextPageToken: 'same' }))\n const result = await runIngestion(request('conferences'), { journal: createMemoryJournal(), destination: createMemoryDestination(), now: () => cutoff })\n expect(result.ok).toBe(false)\n expect(result.streams[0]?.error).toContain('repeated')\n expect(() => google_meetPipeline.streams[0]?.incremental.parseState({ scope: 'wrong', pending: [] })).toThrow('scope')\n})\n\ntest.serial('Meet reports expiry during unfinished artifact work instead of silently advancing', async () => {\n setFetch()\n const journal = createMemoryJournal()\n const destination = createMemoryDestination()\n expect((await runIngestion(request('transcript-entries', 5), { journal, destination, now: () => cutoff })).streams[0]?.outcome).toBe('budget_exhausted')\n const result = await runIngestion(request(), { journal, destination, now: () => new Date('2026-11-01T00:00:00Z') })\n expect(result.ok).toBe(false)\n expect(result.streams[0]?.error).toContain('expired during unfinished artifact work')\n expect(destination.tables.get('default.google_meet_transcript_entries_raw')).toHaveLength(1)\n})\n\ntest.serial('Meet persists an empty discovery page and freezes bounds while resuming its token', async () => {\n const urls: URL[] = []\n setFetch((url) => {\n urls.push(url)\n return Response.json({ conferenceRecords: [], ...(!url.searchParams.has('pageToken') && !url.searchParams.get('filter')?.includes('IS NULL') ? { nextPageToken: 'discover-next' } : {}) })\n })\n const journal = createMemoryJournal()\n const destination = createMemoryDestination()\n expect((await runIngestion(request('conferences', 2), { journal, destination, now: () => cutoff })).streams[0]?.outcome).toBe('budget_exhausted')\n expect((await journal.readCheckpoint('google-meet.conferences')).envelope?.state).toMatchObject({ cycle: { pageToken: 'discover-next', to: cutoff.toISOString() } })\n expect((await runIngestion(request('conferences'), { journal, destination, now: () => new Date('2026-10-21T00:00:00Z') })).ok).toBe(true)\n expect(urls[1]?.searchParams.get('pageToken')).toBe('discover-next')\n expect(urls[1]?.searchParams.get('filter')).toBe(urls[0]?.searchParams.get('filter'))\n expect((await journal.readCheckpoint('google-meet.conferences')).envelope?.state).toMatchObject({ watermark: cutoff.toISOString() })\n})\n\ntest.serial('Meet rejects a changed range under the same backfill checkpoint', async () => {\n setFetch(() => Response.json({ conferenceRecords: [] }))\n const journal = createMemoryJournal()\n const destination = createMemoryDestination()\n const backfill = { id: 'stable', from: new Date('2026-10-01T00:00:00Z'), to: new Date('2026-10-05T00:00:00Z') }\n expect((await runIngestion({ ...request('conferences'), backfill }, { journal, destination, now: () => cutoff })).ok).toBe(true)\n const changed = await runIngestion({ ...request('conferences'), backfill: { ...backfill, to: new Date('2026-10-06T00:00:00Z') } }, { journal, destination, now: () => cutoff })\n expect(changed.ok).toBe(false)\n expect(changed.streams[0]?.error).toContain('backfill bounds changed')\n})\n\ntest.serial('Meet discovery reset debt survives serialized fresh runs and an empty replay page', async () => {\n setFetch((url) => url.searchParams.has('pageToken')\n ? new Response('expired discovery token', { status: 400 })\n : Response.json({ conferenceRecords: [], nextPageToken: 'discover-next' }))\n const destination = createMemoryDestination()\n const initial = createMemoryJournal()\n expect((await runIngestion(request('conferences', 2), { journal: initial, destination, now: () => cutoff })).streams[0]?.outcome).toBe('budget_exhausted')\n const afterInitial = restoredJournal(initial)\n expect((await runIngestion(request('conferences', 1), { journal: afterInitial, destination, now: () => cutoff })).streams[0]?.outcome).toBe('budget_exhausted')\n expect((await afterInitial.readCheckpoint('google-meet.conferences')).envelope?.state).toMatchObject({ cycle: { recoveryCount: 1 } })\n expect((await afterInitial.readCheckpoint('google-meet.conferences')).envelope?.state).toHaveProperty('cycle.pageToken', undefined)\n const afterReset = restoredJournal(afterInitial)\n expect((await runIngestion(request('conferences', 1), { journal: afterReset, destination, now: () => cutoff })).streams[0]?.outcome).toBe('budget_exhausted')\n const checkpoint = await afterReset.readCheckpoint('google-meet.conferences')\n expect(checkpoint.envelope?.state).toMatchObject({ cycle: { recoveryCount: 1, pageToken: 'discover-next' } })\n const afterReplay = restoredJournal(afterReset)\n const stopped = await runIngestion(request('conferences'), { journal: afterReplay, destination, now: () => cutoff })\n expect(stopped.ok).toBe(false)\n expect(stopped.streams[0]?.error).toContain('discovery:ended collection repeatedly rejected')\n expect((await afterReplay.readCheckpoint('google-meet.conferences')).version).toBe(checkpoint.version)\n})\n\ntest.serial('Meet transcript collection retains reset debt across fresh-run parent replay', async () => {\n setFetch((url) => {\n if (url.pathname.endsWith('/transcripts')) return url.searchParams.has('pageToken')\n ? new Response('expired transcript token', { status: 400 })\n : Response.json({ transcripts: [transcript], nextPageToken: 'transcripts-next' })\n return fixture(url)\n })\n const destination = createMemoryDestination()\n const initial = createMemoryJournal()\n expect((await runIngestion(request('transcripts', 4), { journal: initial, destination, now: () => cutoff })).streams[0]?.outcome).toBe('budget_exhausted')\n const afterInitial = restoredJournal(initial)\n expect((await runIngestion(request('transcripts', 1), { journal: afterInitial, destination, now: () => cutoff })).streams[0]?.outcome).toBe('budget_exhausted')\n expect((await afterInitial.readCheckpoint('google-meet.transcripts')).envelope?.state).toMatchObject({ active: { transcriptRecoveryCount: 1 } })\n expect((await afterInitial.readCheckpoint('google-meet.transcripts')).envelope?.state).not.toHaveProperty('active.transcriptPageToken')\n const afterReset = restoredJournal(afterInitial)\n expect((await runIngestion(request('transcripts', 1), { journal: afterReset, destination, now: () => cutoff })).streams[0]?.outcome).toBe('budget_exhausted')\n const checkpoint = await afterReset.readCheckpoint('google-meet.transcripts')\n expect(checkpoint.envelope?.state).toMatchObject({ active: { transcriptRecoveryCount: 1, transcriptPageToken: 'transcripts-next' } })\n const afterReplay = restoredJournal(afterReset)\n const stopped = await runIngestion(request('transcripts'), { journal: afterReplay, destination, now: () => cutoff })\n expect(stopped.ok).toBe(false)\n expect(stopped.streams[0]?.error).toContain('/transcripts collection repeatedly rejected')\n expect((await afterReplay.readCheckpoint('google-meet.transcripts')).version).toBe(checkpoint.version)\n})\n\ntest.serial('Meet entry reset debt survives serialized fresh runs and a loaded nonterminal replay page', async () => {\n setFetch((url) => url.pathname.endsWith('/entries') && url.searchParams.has('pageToken')\n ? new Response('expired entry token', { status: 400 })\n : fixture(url))\n const destination = createMemoryDestination()\n const initial = createMemoryJournal()\n expect((await runIngestion(request('transcript-entries', 5), { journal: initial, destination, now: () => cutoff })).streams[0]?.outcome).toBe('budget_exhausted')\n const afterInitial = restoredJournal(initial)\n expect((await runIngestion(request('transcript-entries', 1), { journal: afterInitial, destination, now: () => cutoff })).streams[0]?.outcome).toBe('budget_exhausted')\n expect((await afterInitial.readCheckpoint('google-meet.transcript-entries')).envelope?.state).toMatchObject({ active: { entryRecoveryCount: 1 } })\n expect((await afterInitial.readCheckpoint('google-meet.transcript-entries')).envelope?.state).toHaveProperty('active.entryPageToken', undefined)\n const afterReset = restoredJournal(afterInitial)\n expect((await runIngestion(request('transcript-entries', 1), { journal: afterReset, destination, now: () => cutoff })).streams[0]?.outcome).toBe('budget_exhausted')\n const checkpoint = await afterReset.readCheckpoint('google-meet.transcript-entries')\n expect(checkpoint.envelope?.state).toMatchObject({ active: { entryRecoveryCount: 1, entryPageToken: 'entries-next' } })\n const afterReplay = restoredJournal(afterReset)\n const stopped = await runIngestion(request(), { journal: afterReplay, destination, now: () => cutoff })\n expect(stopped.ok).toBe(false)\n expect(stopped.streams[0]?.error).toContain('/entries collection repeatedly rejected')\n expect((await afterReplay.readCheckpoint('google-meet.transcript-entries')).version).toBe(checkpoint.version)\n expect(() => google_meetPipeline.streams[2]?.incremental.parseState({ ...JSON.parse(JSON.stringify(checkpoint.envelope?.state)), active: { ...JSON.parse(JSON.stringify(checkpoint.envelope?.state)).active, entryRecoveryCount: 2 } })).toThrow('recovery count')\n})\n\nfunction restoredJournal(previous: ReturnType) {\n const journal = createMemoryJournal()\n journal.events.push(...previous.events.map((event) => ({ ...event, checkpoint: event.checkpoint === undefined ? undefined : JSON.parse(JSON.stringify(event.checkpoint)) })))\n return journal\n}\n" + } + ], + "meta": { + "chkit": { + "formatVersion": 1, + "version": "0.2.0", + "language": "typescript", + "license": "MIT", + "documentation": "https://chkit.obsessiondb.com/integrations/google-meet/", + "logo": "https://chkit.obsessiondb.com/logos/google-meet.svg", + "authentication": { + "method": "OAuth access token", + "env": [ + "GOOGLE_MEET_ACCESS_TOKEN" + ], + "setup": [ + "Provide GOOGLE_MEET_ACCESS_TOKEN with read access to the selected resources.", + "Supply a new access token when the existing token expires." + ], + "documentation": "https://developers.google.com/workspace/meet/api/guides/authenticate-authorize" + }, + "chkit": "^0.2.0-beta.10", + "ingest": "^0.2.0-beta.10", + "clickhouse": ">=25.3.0", + "root": "src/integrations/google-meet", + "entry": "index.ts", + "exports": [ + "google_meetPipeline", + "google_meet_conferencesRaw", + "google_meet_transcriptsRaw", + "google_meet_transcriptEntriesRaw" + ], + "resources": [ + { + "name": "conferences", + "title": "Conferences", + "table": "google_meet_conferences_raw", + "description": "Conference records visible to the access token", + "scopes": [ + "https://www.googleapis.com/auth/meetings.space.readonly" + ], + "strategy": "cursor", + "endpoints": [ + { + "method": "GET", + "path": "/conferenceRecords", + "documentation": "https://developers.google.com/workspace/meet/api/reference/rest/v2/conferenceRecords/list" + } + ] + }, + { + "name": "transcripts", + "title": "Transcripts", + "table": "google_meet_transcripts_raw", + "description": "Transcript metadata for discovered and retained pending conferences", + "scopes": [ + "https://www.googleapis.com/auth/meetings.space.readonly" + ], + "strategy": "cursor", + "endpoints": [ + { + "method": "GET", + "path": "/conferenceRecords/{conferenceRecord}/transcripts", + "documentation": "https://developers.google.com/workspace/meet/api/reference/rest/v2/conferenceRecords.transcripts/list" + } + ] + }, + { + "name": "transcript-entries", + "title": "Transcript entries", + "table": "google_meet_transcript_entries_raw", + "description": "Individually keyed, paged transcript entries for retained conferences", + "scopes": [ + "https://www.googleapis.com/auth/meetings.space.readonly" + ], + "strategy": "cursor", + "endpoints": [ + { + "method": "GET", + "path": "/conferenceRecords", + "documentation": "https://developers.google.com/workspace/meet/api/reference/rest/v2/conferenceRecords/list" + }, + { + "method": "GET", + "path": "/conferenceRecords/{conferenceRecord}/transcripts", + "documentation": "https://developers.google.com/workspace/meet/api/reference/rest/v2/conferenceRecords.transcripts/list" + }, + { + "method": "GET", + "path": "/conferenceRecords/{conferenceRecord}/transcripts/{transcript}/entries", + "documentation": "https://developers.google.com/workspace/meet/api/reference/rest/v2/conferenceRecords.transcripts.entries/list" + } + ] + } + ], + "views": [], + "sync": { + "description": "Fixed end-time discovery windows and retained conference work queues checkpoint nested transcript and entry pagination.", + "schedule": "Run bunx chkit ingest run --tag provider:google-meet from a scheduler.", + "deletions": "Late artifacts revisit retained parents until provider expiry. API expiry does not delete archived raw observations; unfinished work that expires fails visibly." + }, + "env": { + "GOOGLE_MEET_ACCESS_TOKEN": "" + }, + "fileHashes": { + "src/integrations/google-meet/README.md": "0a478abd6b5d3f57deba481ff38a93e70f34bad36cbaa4e57b6a693b27a78dc1", + "src/integrations/google-meet/index.ts": "da7e615e0a68b4bc280999ee2afce84928e04756b2dfe6744a7957dd7083624c", + "src/integrations/google-meet/tests/basic.test.ts": "59408ecd22fba6cf26b3f4d0ff6cb6f035a2d80dd00378fa88b8334beed6c3b7" + } + } + } +} diff --git a/registry/google-meet/tests/basic.test.ts b/registry/google-meet/tests/basic.test.ts new file mode 100644 index 00000000..0f442c3a --- /dev/null +++ b/registry/google-meet/tests/basic.test.ts @@ -0,0 +1,251 @@ +import { afterEach, expect, test } from 'bun:test' +import { runIngestion, selectStreams } from '@chkit/plugin-ingest' +import { createMemoryDestination, createMemoryJournal } from '@chkit/plugin-ingest/testing' + +import { google_meetPipeline } from '../index.js' + +const originalFetch = globalThis.fetch +const originalToken = process.env.GOOGLE_MEET_ACCESS_TOKEN +const cutoff = new Date('2026-10-20T00:00:00Z') +const conference = { name: 'conferenceRecords/c1', startTime: '2026-08-01T00:00:00Z', endTime: '2026-10-01T00:00:00Z', expireTime: '2026-10-31T00:00:00Z' } +const transcript = { name: `${conference.name}/transcripts/t1`, state: 'FILE_GENERATED' } +const entry = { name: `${transcript.name}/entries/e1`, text: 'first' } + +function setFetch(handler?: (url: URL) => Response): void { + process.env.GOOGLE_MEET_ACCESS_TOKEN = 'fixture' + globalThis.fetch = Object.assign(async (input: Parameters[0]) => handler ? handler(new URL(String(input))) : fixture(new URL(String(input))), { preconnect: originalFetch.preconnect }) +} + +function fixture(url: URL): Response { + if (url.pathname === '/v2/conferenceRecords') return Response.json({ conferenceRecords: url.searchParams.get('filter')?.includes('IS NULL') ? [] : [conference] }) + if (url.pathname.endsWith('/transcripts')) return Response.json({ transcripts: [transcript] }) + if (url.pathname.endsWith('/entries')) return Response.json(url.searchParams.has('pageToken') + ? { transcriptEntries: [{ ...entry, name: `${transcript.name}/entries/e2`, text: 'second' }] } + : { transcriptEntries: [entry], nextPageToken: 'entries-next' }) + throw new Error(`Unexpected Meet request ${url}`) +} + +function request(resource = 'transcript-entries', maxChunks?: number) { + const pipeline = { ...google_meetPipeline, retry: { retries: 0 }, streams: google_meetPipeline.streams.map((stream) => ({ ...stream, ...(maxChunks === undefined ? {} : { budget: { ...stream.budget, maxChunks } }) })) } + return { selected: selectStreams([pipeline], [`resource:${resource}`]), backfill: undefined } +} + +afterEach(() => { + globalThis.fetch = originalFetch + if (originalToken === undefined) delete process.env.GOOGLE_MEET_ACCESS_TOKEN + else process.env.GOOGLE_MEET_ACCESS_TOKEN = originalToken +}) + +test.serial('Meet resumes within an entry collection without losing earlier pages or changing its window', async () => { + const urls: URL[] = [] + setFetch((url) => { urls.push(url); return fixture(url) }) + const journal = createMemoryJournal() + const destination = createMemoryDestination() + const first = await runIngestion(request('transcript-entries', 5), { journal, destination, now: () => cutoff }) + expect(first.streams[0]?.outcome).toBe('budget_exhausted') + expect((await journal.readCheckpoint('google-meet.transcript-entries')).envelope?.state).toMatchObject({ active: { entryPageToken: 'entries-next' }, cycle: { to: cutoff.toISOString() } }) + const previousRequests = urls.length + const resumed = await runIngestion(request(), { journal, destination, now: () => new Date('2026-10-21T00:00:00Z') }) + expect(resumed.ok).toBe(true) + expect(urls.slice(previousRequests).map((url) => [url.pathname, url.searchParams.get('pageToken')])).toEqual([[`/v2/${transcript.name}/entries`, 'entries-next']]) + expect(destination.tables.get('default.google_meet_transcript_entries_raw')?.map((row) => row.raw)).toEqual([ + { ...entry, conference_name: conference.name, transcript_name: transcript.name }, + { ...entry, name: `${transcript.name}/entries/e2`, text: 'second', conference_name: conference.name, transcript_name: transcript.name }, + ]) + expect((await journal.readCheckpoint('google-meet.transcript-entries')).envelope?.state).toMatchObject({ watermark: cutoff.toISOString() }) +}) + +test.serial('Meet keeps a parent beyond discovery overlap for a late transcript', async () => { + let late = false + const urls: URL[] = [] + setFetch((url) => { + urls.push(url) + if (url.pathname === '/v2/conferenceRecords' && late) return Response.json({ conferenceRecords: [] }) + if (url.pathname.endsWith('/transcripts') && !late) return Response.json({ transcripts: [] }) + return fixture(url) + }) + const journal = createMemoryJournal() + const destination = createMemoryDestination() + expect((await runIngestion(request(), { journal, destination, now: () => cutoff })).ok).toBe(true) + expect(destination.tables.size).toBe(0) + expect((await journal.readCheckpoint('google-meet.transcript-entries')).envelope?.state).toMatchObject({ pending: [{ name: conference.name, checkedAt: cutoff.toISOString() }] }) + late = true + expect((await runIngestion(request(), { journal, destination, now: () => new Date('2026-10-28T00:00:00Z') })).ok).toBe(true) + expect(destination.tables.get('default.google_meet_transcript_entries_raw')).toHaveLength(2) + const endedFilters = urls.filter((url) => url.pathname === '/v2/conferenceRecords' && !url.searchParams.get('filter')?.includes('IS NULL')).map((url) => url.searchParams.get('filter')) + expect(endedFilters.at(-1)).toContain('2026-10-13T00:00:00.000Z') +}) + +test.serial('Meet queries end-time windows and ongoing calls instead of excluding long-running conferences', async () => { + const filters: Array = [] + setFetch((url) => { filters.push(url.searchParams.get('filter')); return fixture(url) }) + const destination = createMemoryDestination() + expect((await runIngestion(request('conferences'), { journal: createMemoryJournal(), destination, now: () => cutoff })).ok).toBe(true) + expect(filters).toEqual([ + 'end_time>="2026-09-20T00:00:00.000Z" AND end_time<="2026-10-20T00:00:00.000Z"', + 'end_time IS NULL AND start_time<="2026-10-20T00:00:00.000Z"', + ]) + expect(destination.tables.get('default.google_meet_conferences_raw')?.[0]?.raw).toEqual(conference) +}) + +test.serial('Meet replays an expired entry page token without replacing an earlier entry identity', async () => { + let rejected = false + setFetch((url) => { + if (url.pathname.endsWith('/entries') && url.searchParams.has('pageToken') && !rejected) { rejected = true; return new Response('page token expired', { status: 400 }) } + return fixture(url) + }) + const destination = createMemoryDestination() + const result = await runIngestion(request(), { journal: createMemoryJournal(), destination, now: () => cutoff }) + expect(result.ok).toBe(true) + const rows = destination.tables.get('default.google_meet_transcript_entries_raw') ?? [] + expect(new Set(rows.map((row) => row.id)).size).toBe(2) + expect(rows.some((row) => JSON.stringify(row.raw).includes('second'))).toBe(true) +}) + +test.serial('Meet preserves entry progress after a terminal page load failure', async () => { + setFetch() + const journal = createMemoryJournal() + const destination = createMemoryDestination() + const first = await runIngestion(request(), { journal, destination: { + insert: async (input) => { + if (input.rows.some((row) => JSON.stringify(row.raw).includes('second'))) throw new Error('fixture sink failure') + await destination.insert(input) + }, + }, now: () => cutoff }) + expect(first.ok).toBe(false) + expect((await journal.readCheckpoint('google-meet.transcript-entries')).envelope?.state).toMatchObject({ active: { entryPageToken: 'entries-next' } }) + expect((await runIngestion(request(), { journal, destination, now: () => cutoff })).ok).toBe(true) + expect(destination.tables.get('default.google_meet_transcript_entries_raw')).toHaveLength(2) +}) + +test.serial('Meet explicit backfill bounds stay isolated and constrain completed conference discovery', async () => { + const filters: Array = [] + setFetch((url) => { filters.push(url.searchParams.get('filter')); return Response.json({ conferenceRecords: [] }) }) + const journal = createMemoryJournal() + const result = await runIngestion({ ...request('conferences'), backfill: { id: 'october', from: new Date('2026-10-01T00:00:00Z'), to: new Date('2026-10-05T00:00:00Z') } }, { journal, destination: createMemoryDestination(), now: () => cutoff }) + expect(result.ok).toBe(true) + expect(filters).toEqual(['end_time>="2026-10-01T00:00:00.000Z" AND end_time<="2026-10-05T00:00:00.000Z"']) + expect((await journal.readCheckpoint('google-meet.conferences')).version).toBe(0) +}) + +test.serial('Meet rejects malformed checkpoints and repeated page tokens', async () => { + setFetch(() => Response.json({ conferenceRecords: [], nextPageToken: 'same' })) + const result = await runIngestion(request('conferences'), { journal: createMemoryJournal(), destination: createMemoryDestination(), now: () => cutoff }) + expect(result.ok).toBe(false) + expect(result.streams[0]?.error).toContain('repeated') + expect(() => google_meetPipeline.streams[0]?.incremental.parseState({ scope: 'wrong', pending: [] })).toThrow('scope') +}) + +test.serial('Meet reports expiry during unfinished artifact work instead of silently advancing', async () => { + setFetch() + const journal = createMemoryJournal() + const destination = createMemoryDestination() + expect((await runIngestion(request('transcript-entries', 5), { journal, destination, now: () => cutoff })).streams[0]?.outcome).toBe('budget_exhausted') + const result = await runIngestion(request(), { journal, destination, now: () => new Date('2026-11-01T00:00:00Z') }) + expect(result.ok).toBe(false) + expect(result.streams[0]?.error).toContain('expired during unfinished artifact work') + expect(destination.tables.get('default.google_meet_transcript_entries_raw')).toHaveLength(1) +}) + +test.serial('Meet persists an empty discovery page and freezes bounds while resuming its token', async () => { + const urls: URL[] = [] + setFetch((url) => { + urls.push(url) + return Response.json({ conferenceRecords: [], ...(!url.searchParams.has('pageToken') && !url.searchParams.get('filter')?.includes('IS NULL') ? { nextPageToken: 'discover-next' } : {}) }) + }) + const journal = createMemoryJournal() + const destination = createMemoryDestination() + expect((await runIngestion(request('conferences', 2), { journal, destination, now: () => cutoff })).streams[0]?.outcome).toBe('budget_exhausted') + expect((await journal.readCheckpoint('google-meet.conferences')).envelope?.state).toMatchObject({ cycle: { pageToken: 'discover-next', to: cutoff.toISOString() } }) + expect((await runIngestion(request('conferences'), { journal, destination, now: () => new Date('2026-10-21T00:00:00Z') })).ok).toBe(true) + expect(urls[1]?.searchParams.get('pageToken')).toBe('discover-next') + expect(urls[1]?.searchParams.get('filter')).toBe(urls[0]?.searchParams.get('filter')) + expect((await journal.readCheckpoint('google-meet.conferences')).envelope?.state).toMatchObject({ watermark: cutoff.toISOString() }) +}) + +test.serial('Meet rejects a changed range under the same backfill checkpoint', async () => { + setFetch(() => Response.json({ conferenceRecords: [] })) + const journal = createMemoryJournal() + const destination = createMemoryDestination() + const backfill = { id: 'stable', from: new Date('2026-10-01T00:00:00Z'), to: new Date('2026-10-05T00:00:00Z') } + expect((await runIngestion({ ...request('conferences'), backfill }, { journal, destination, now: () => cutoff })).ok).toBe(true) + const changed = await runIngestion({ ...request('conferences'), backfill: { ...backfill, to: new Date('2026-10-06T00:00:00Z') } }, { journal, destination, now: () => cutoff }) + expect(changed.ok).toBe(false) + expect(changed.streams[0]?.error).toContain('backfill bounds changed') +}) + +test.serial('Meet discovery reset debt survives serialized fresh runs and an empty replay page', async () => { + setFetch((url) => url.searchParams.has('pageToken') + ? new Response('expired discovery token', { status: 400 }) + : Response.json({ conferenceRecords: [], nextPageToken: 'discover-next' })) + const destination = createMemoryDestination() + const initial = createMemoryJournal() + expect((await runIngestion(request('conferences', 2), { journal: initial, destination, now: () => cutoff })).streams[0]?.outcome).toBe('budget_exhausted') + const afterInitial = restoredJournal(initial) + expect((await runIngestion(request('conferences', 1), { journal: afterInitial, destination, now: () => cutoff })).streams[0]?.outcome).toBe('budget_exhausted') + expect((await afterInitial.readCheckpoint('google-meet.conferences')).envelope?.state).toMatchObject({ cycle: { recoveryCount: 1 } }) + expect((await afterInitial.readCheckpoint('google-meet.conferences')).envelope?.state).toHaveProperty('cycle.pageToken', undefined) + const afterReset = restoredJournal(afterInitial) + expect((await runIngestion(request('conferences', 1), { journal: afterReset, destination, now: () => cutoff })).streams[0]?.outcome).toBe('budget_exhausted') + const checkpoint = await afterReset.readCheckpoint('google-meet.conferences') + expect(checkpoint.envelope?.state).toMatchObject({ cycle: { recoveryCount: 1, pageToken: 'discover-next' } }) + const afterReplay = restoredJournal(afterReset) + const stopped = await runIngestion(request('conferences'), { journal: afterReplay, destination, now: () => cutoff }) + expect(stopped.ok).toBe(false) + expect(stopped.streams[0]?.error).toContain('discovery:ended collection repeatedly rejected') + expect((await afterReplay.readCheckpoint('google-meet.conferences')).version).toBe(checkpoint.version) +}) + +test.serial('Meet transcript collection retains reset debt across fresh-run parent replay', async () => { + setFetch((url) => { + if (url.pathname.endsWith('/transcripts')) return url.searchParams.has('pageToken') + ? new Response('expired transcript token', { status: 400 }) + : Response.json({ transcripts: [transcript], nextPageToken: 'transcripts-next' }) + return fixture(url) + }) + const destination = createMemoryDestination() + const initial = createMemoryJournal() + expect((await runIngestion(request('transcripts', 4), { journal: initial, destination, now: () => cutoff })).streams[0]?.outcome).toBe('budget_exhausted') + const afterInitial = restoredJournal(initial) + expect((await runIngestion(request('transcripts', 1), { journal: afterInitial, destination, now: () => cutoff })).streams[0]?.outcome).toBe('budget_exhausted') + expect((await afterInitial.readCheckpoint('google-meet.transcripts')).envelope?.state).toMatchObject({ active: { transcriptRecoveryCount: 1 } }) + expect((await afterInitial.readCheckpoint('google-meet.transcripts')).envelope?.state).not.toHaveProperty('active.transcriptPageToken') + const afterReset = restoredJournal(afterInitial) + expect((await runIngestion(request('transcripts', 1), { journal: afterReset, destination, now: () => cutoff })).streams[0]?.outcome).toBe('budget_exhausted') + const checkpoint = await afterReset.readCheckpoint('google-meet.transcripts') + expect(checkpoint.envelope?.state).toMatchObject({ active: { transcriptRecoveryCount: 1, transcriptPageToken: 'transcripts-next' } }) + const afterReplay = restoredJournal(afterReset) + const stopped = await runIngestion(request('transcripts'), { journal: afterReplay, destination, now: () => cutoff }) + expect(stopped.ok).toBe(false) + expect(stopped.streams[0]?.error).toContain('/transcripts collection repeatedly rejected') + expect((await afterReplay.readCheckpoint('google-meet.transcripts')).version).toBe(checkpoint.version) +}) + +test.serial('Meet entry reset debt survives serialized fresh runs and a loaded nonterminal replay page', async () => { + setFetch((url) => url.pathname.endsWith('/entries') && url.searchParams.has('pageToken') + ? new Response('expired entry token', { status: 400 }) + : fixture(url)) + const destination = createMemoryDestination() + const initial = createMemoryJournal() + expect((await runIngestion(request('transcript-entries', 5), { journal: initial, destination, now: () => cutoff })).streams[0]?.outcome).toBe('budget_exhausted') + const afterInitial = restoredJournal(initial) + expect((await runIngestion(request('transcript-entries', 1), { journal: afterInitial, destination, now: () => cutoff })).streams[0]?.outcome).toBe('budget_exhausted') + expect((await afterInitial.readCheckpoint('google-meet.transcript-entries')).envelope?.state).toMatchObject({ active: { entryRecoveryCount: 1 } }) + expect((await afterInitial.readCheckpoint('google-meet.transcript-entries')).envelope?.state).toHaveProperty('active.entryPageToken', undefined) + const afterReset = restoredJournal(afterInitial) + expect((await runIngestion(request('transcript-entries', 1), { journal: afterReset, destination, now: () => cutoff })).streams[0]?.outcome).toBe('budget_exhausted') + const checkpoint = await afterReset.readCheckpoint('google-meet.transcript-entries') + expect(checkpoint.envelope?.state).toMatchObject({ active: { entryRecoveryCount: 1, entryPageToken: 'entries-next' } }) + const afterReplay = restoredJournal(afterReset) + const stopped = await runIngestion(request(), { journal: afterReplay, destination, now: () => cutoff }) + expect(stopped.ok).toBe(false) + expect(stopped.streams[0]?.error).toContain('/entries collection repeatedly rejected') + expect((await afterReplay.readCheckpoint('google-meet.transcript-entries')).version).toBe(checkpoint.version) + expect(() => google_meetPipeline.streams[2]?.incremental.parseState({ ...JSON.parse(JSON.stringify(checkpoint.envelope?.state)), active: { ...JSON.parse(JSON.stringify(checkpoint.envelope?.state)).active, entryRecoveryCount: 2 } })).toThrow('recovery count') +}) + +function restoredJournal(previous: ReturnType) { + const journal = createMemoryJournal() + journal.events.push(...previous.events.map((event) => ({ ...event, checkpoint: event.checkpoint === undefined ? undefined : JSON.parse(JSON.stringify(event.checkpoint)) }))) + return journal +} diff --git a/registry/lemlist/README.md b/registry/lemlist/README.md index 365f2da8..3fe33e2a 100644 --- a/registry/lemlist/README.md +++ b/registry/lemlist/README.md @@ -4,6 +4,8 @@ This is a small editable starting point based on the Lemlist source in [obsessio Set `LEMLIST_API_KEY` in the runtime environment and configure a direct ClickHouse connection. Edit `pageSize` and the stream list in `index.ts` for the resources needed by the project. Native JSON requires ClickHouse 25.3 or later. +Keep each stream ID and destination tied to one Lemlist account. The reader does not resolve the authenticated account identity: when changing the key to another account, use new stream IDs and destinations before ingesting so previous watermarks cannot skip that account's history. + ```sh bunx chkit check bunx chkit generate --name add_lemlist @@ -17,4 +19,26 @@ bunx chkit ingest run --tag provider:lemlist - `activities` → `lemlist_activities_raw` - `campaigns` → `lemlist_campaigns_raw` -Both streams scan all accessible pages each run. Removed campaigns or activity records remain stored. A failed API request fails the stream. Raw rows use provider IDs and keep the last observed value; there is no deletion reconciliation. Edit the readers, select fewer streams, or add SQL views for a specific use case. Run one ingestion process per ClickHouse target at a time. +## Sync and recovery + +Activities use the provider's inclusive `minDate`/`maxDate` filters on `createdAt`. The initial range starts at `2000-01-01`; later runs replay one day before the last committed watermark. A large range is split into intervals of at most thirty days. Offset pagination stays local to each interval; an unfinished interval always starts at offset zero after failure. The checkpoint advances only after every page of an interval has been written successfully, including empty intervals. An explicit backfill resumes completed intervals in its own checkpoint namespace while retaining one day of overlap. + +The upper bound does not freeze the provider dataset. Late arrivals or deletions can still change offset pages, and older activities can receive backfilled metadata. Do not treat activity offsets as change cursors or row chunks as immutable page identities. Periodic historical replay refreshes metadata and catches arrivals beyond the overlap: + +```sh +bunx chkit ingest run --tag provider:lemlist --tag resource:activities --backfill reconcile-2026-10-04 --from 2000-01-01 +bunx chkit ingest status --tag provider:lemlist --json +``` + +Use a new backfill ID for each complete reconciliation; reusing an ID resumes that backfill. A reused ID with an upper bound earlier than its committed frontier is rejected; use a new ID for a narrower historical range. Date intervals intentionally have inclusive shared boundaries; stable `_id` row keys reconcile repeated observations. Empty completion markers carry interval identities, while mutable row chunks do not. + +Campaigns always scan from offset zero, ordered by `createdAt:asc`. Their `completedAt` checkpoint records a successful full scan rather than a modification cursor. Creation-only filtering would miss later campaign state changes. Removed campaigns or activities remain stored. Run one ingestion process per ClickHouse target at a time. + +## Fixture tests + +```sh +bunx chkit add lemlist --with-tests +bun test src/integrations/lemlist/tests/basic.test.ts +``` + +The portable suite checks bounded interval pagination, committed-frontier recovery, explicit backfill isolation, sink failure and empty scan completion without live credentials. diff --git a/registry/lemlist/index.ts b/registry/lemlist/index.ts index 7670c1e6..2a48a684 100644 --- a/registry/lemlist/index.ts +++ b/registry/lemlist/index.ts @@ -1,22 +1,65 @@ -import { definePipeline, defineStream, HttpError, rawRows, rawTable, type FetchContext } from '@chkit/plugin-ingest' +import { definePipeline, defineStream, HttpError, IngestConfigError, rawRows, rawTable, timestampWindow, type FetchContext, type IncrementalStrategy, type TimestampRange } from '@chkit/plugin-ingest' const database = 'default' +// Keep stream IDs and destinations tied to one Lemlist account; use new ones when switching accounts. const pageSize = 100 +const intervalMs = 30 * 24 * 60 * 60 * 1000 +const activityWindow = timestampWindow({ start: new Date('2000-01-01T00:00:00Z'), overlapMs: 24 * 60 * 60 * 1000 }) interface Item { _id: string; [key: string]: unknown } +interface ScanState { completedAt: string } export const lemlist_activitiesRaw = rawTable({ database, name: 'lemlist_activities_raw' }) export const lemlist_campaignsRaw = rawTable({ database, name: 'lemlist_campaigns_raw' }) +const campaignScan: IncrementalStrategy = { + id: 'lemlist.campaigns.full_scan', version: 1, + parseState(raw) { + if (!isObject(raw) || typeof raw.completedAt !== 'string' || !Number.isFinite(Date.parse(raw.completedAt))) { + throw new IngestConfigError('Lemlist campaign checkpoint has no valid completedAt.') + } + return { completedAt: raw.completedAt } + }, + plan: ({ cutoff }) => cutoff.toISOString(), + complete: ({ selection }) => ({ completedAt: selection }), +} + export const lemlistPipeline = definePipeline({ - id: 'lemlist', tags: ['provider:lemlist'], maxFetches: 1, + id: 'lemlist', tags: ['provider:lemlist'], maxStreams: 1, maxFetches: 1, streams: [ - defineStream({ id: 'lemlist.activities', tags: ['resource:activities'], destination: lemlist_activitiesRaw, + defineStream({ + id: 'lemlist.activities', tags: ['resource:activities'], destination: lemlist_activitiesRaw, + incremental: { + ...activityWindow, + plan(input) { + const selection = activityWindow.plan(input) + // Explicit backfills have their own namespace; reuse its completed interval frontier. + if (input.range && input.state) { + if (selection.to.getTime() < Date.parse(input.state.watermark)) { + throw new IngestConfigError('Lemlist backfill upper bound precedes its committed interval frontier; use a new backfill ID for a narrower historical range.') + } + return { ...selection, from: new Date(Math.max(selection.from.getTime(), Date.parse(input.state.watermark) - 24 * 60 * 60 * 1000)) } + } + return selection + }, + }, + batchSize: 500, async *read(context) { - for await (const page of pages(context, 'activities')) yield { rows: rawRows(page, (item) => item._id) } + for (let from = context.selection.from; from < context.selection.to;) { + const to = new Date(Math.min(from.getTime() + intervalMs, context.selection.to.getTime())) + for await (const page of pages(context, 'activities', { from, to })) { + yield { rows: rawRows(page, (item) => item._id) } + } + // Only an exhausted interval is a safe frontier. Offset pages remain mutable. + const watermark = new Date(Math.max(to.getTime(), Date.parse(context.state?.watermark ?? to.toISOString()))).toISOString() + yield { rows: [], state: { watermark }, id: `completed-interval:${from.toISOString()}/${to.toISOString()}` } + from = to + } }, }), - defineStream({ id: 'lemlist.campaigns', tags: ['resource:campaigns'], destination: lemlist_campaignsRaw, + defineStream({ + id: 'lemlist.campaigns', tags: ['resource:campaigns'], destination: lemlist_campaignsRaw, + incremental: campaignScan, async *read(context) { for await (const page of pages(context, 'campaigns')) yield { rows: rawRows(page, (item) => item._id) } }, @@ -24,24 +67,47 @@ export const lemlistPipeline = definePipeline({ ], }) -async function* pages(context: FetchContext, resource: 'activities' | 'campaigns'): AsyncGenerator { +async function* pages(context: FetchContext, resource: 'activities' | 'campaigns', window?: TimestampRange): AsyncGenerator { let offset = 0 + const seen = new Set() while (true) { const url = new URL(`https://api.lemlist.com/api/${resource}`) url.searchParams.set('version', 'v2') url.searchParams.set('limit', String(pageSize)) url.searchParams.set('offset', String(offset)) - const result = await context.attempt(async (signal) => { - const key = process.env.LEMLIST_API_KEY - if (!key) throw new Error('Set LEMLIST_API_KEY') - const response = await fetch(url, { signal, headers: { Authorization: `Basic ${Buffer.from(`:${key}`).toString('base64')}` } }) + if (window) { + url.searchParams.set('minDate', window.from.toISOString()) + url.searchParams.set('maxDate', window.to.toISOString()) + } else { + url.searchParams.set('sortBy', 'createdAt') + url.searchParams.set('sortOrder', 'asc') + } + const items = await context.attempt(async (signal) => { + const key = process.env.LEMLIST_API_KEY?.trim() + if (!key) throw new IngestConfigError('Set LEMLIST_API_KEY.') + const response = await fetch(url, { signal: AbortSignal.any([signal, AbortSignal.timeout(30_000)]), redirect: 'error', + headers: { Authorization: `Basic ${Buffer.from(`:${key}`).toString('base64')}` } }) if (!response.ok) throw await HttpError.fromResponse(response) - return await response.json() as Item[] | { campaigns: Item[] } + const result: unknown = await response.json() + const values = resource === 'campaigns' && isObject(result) ? result.campaigns : result + if (!Array.isArray(values) || !values.every(isItem)) throw new IngestConfigError(`Lemlist ${resource} response has no valid items.`) + return values }, { label: `GET /${resource}` }) - const items = resource === 'campaigns' && !Array.isArray(result) ? result.campaigns : result - if (!Array.isArray(items)) throw new Error(`Lemlist ${resource} response has no items`) - if (items.length) yield items + if (items.length) { + const identity = JSON.stringify(items.map((item) => item._id)) + if (seen.has(identity)) throw new IngestConfigError(`Lemlist ${resource} repeated a page.`) + seen.add(identity) + yield items + } if (items.length < pageSize) return offset += items.length } } + +function isItem(value: unknown): value is Item { + return isObject(value) && typeof value._id === 'string' && value._id.length > 0 +} + +function isObject(value: unknown): value is Record { + return typeof value === 'object' && value !== null && !Array.isArray(value) +} diff --git a/registry/lemlist/manifest.json b/registry/lemlist/manifest.json index add645b0..3cd5ad8f 100644 --- a/registry/lemlist/manifest.json +++ b/registry/lemlist/manifest.json @@ -3,10 +3,10 @@ "name": "lemlist", "type": "registry:item", "title": "Lemlist", - "description": "Small, editable Lemlist API ingestion example with raw ClickHouse tables.", + "description": "Editable Lemlist ingestion with resumable activity date intervals and full campaign scans.", "dependencies": [ - "@chkit/core@^0.2.0-beta.8", - "@chkit/plugin-ingest@^0.2.0-beta.8" + "@chkit/core@^0.2.0-beta.10", + "@chkit/plugin-ingest@^0.2.0-beta.10" ], "files": [ { @@ -18,18 +18,24 @@ "path": "index.ts", "type": "registry:file", "target": "src/integrations/lemlist/index.ts" + }, + { + "path": "tests/basic.test.ts", + "type": "registry:file", + "target": "src/integrations/lemlist/tests/basic.test.ts", + "role": "test" } ], "meta": { "chkit": { "formatVersion": 1, - "version": "0.1.0", + "version": "0.2.0", "documentation": "https://chkit.obsessiondb.com/integrations/lemlist/", "logo": "https://chkit.obsessiondb.com/logos/lemlist.svg", "language": "typescript", "license": "MIT", - "chkit": "^0.2.0-beta.8", - "ingest": "^0.2.0-beta.8", + "chkit": "^0.2.0-beta.10", + "ingest": "^0.2.0-beta.10", "clickhouse": ">=25.3.0", "root": "src/integrations/lemlist", "entry": "index.ts", @@ -45,12 +51,12 @@ "table": "lemlist_activities_raw", "description": "Outreach activity events", "scopes": [], - "strategy": "full", + "strategy": "timestamp", "endpoints": [ { "method": "GET", "path": "/activities", - "documentation": "https://developer.lemlist.com/" + "documentation": "https://developer.lemlist.com/api-reference/endpoints/activities/get-many-activities" } ] }, @@ -65,7 +71,7 @@ { "method": "GET", "path": "/campaigns", - "documentation": "https://developer.lemlist.com/" + "documentation": "https://developer.lemlist.com/api-reference/endpoints/campaigns/get-many-campaigns" } ] } @@ -86,10 +92,13 @@ }, "views": [], "sync": { - "description": "A full read runs for each enabled stream. Stable IDs keep the latest observed record.", + "description": "Activities checkpoint completed thirty-day createdAt intervals with one-day overlap; campaigns record successful full-scan completion.", "schedule": "Run bunx chkit ingest run --tag provider:lemlist from a scheduler.", - "deletions": "Both streams scan all accessible pages each run. Removed campaigns or activity records remain stored." + "deletions": "Removed campaigns and activities remain stored. Historical replay is required for older activity metadata updates outside the overlap." } } - } + }, + "devDependencies": [ + "@types/bun@^1.2.0" + ] } diff --git a/registry/lemlist/releases/0.2.0.json b/registry/lemlist/releases/0.2.0.json new file mode 100644 index 00000000..59b5d175 --- /dev/null +++ b/registry/lemlist/releases/0.2.0.json @@ -0,0 +1,112 @@ +{ + "$schema": "https://ui.shadcn.com/schema/registry-item.json", + "name": "lemlist", + "type": "registry:item", + "title": "Lemlist", + "description": "Editable Lemlist ingestion with resumable activity date intervals and full campaign scans.", + "dependencies": [ + "@chkit/core@^0.2.0-beta.10", + "@chkit/plugin-ingest@^0.2.0-beta.10" + ], + "devDependencies": [ + "@types/bun@^1.2.0" + ], + "files": [ + { + "path": "lemlist/README.md", + "type": "registry:file", + "target": "src/integrations/lemlist/README.md", + "content": "# Lemlist raw ingestion example\n\nThis is a small editable starting point based on the Lemlist source in [obsessiondb/brain](https://github.com/obsessiondb/brain/tree/main/scripts/brain-sync). It stores provider responses as native JSON without Brain's memory or sales projections.\n\nSet `LEMLIST_API_KEY` in the runtime environment and configure a direct ClickHouse connection. Edit `pageSize` and the stream list in `index.ts` for the resources needed by the project. Native JSON requires ClickHouse 25.3 or later.\n\nKeep each stream ID and destination tied to one Lemlist account. The reader does not resolve the authenticated account identity: when changing the key to another account, use new stream IDs and destinations before ingesting so previous watermarks cannot skip that account's history.\n\n```sh\nbunx chkit check\nbunx chkit generate --name add_lemlist\n# Review the migration before applying it.\nbunx chkit migrate --apply\nbunx chkit ingest run --tag provider:lemlist\n```\n\n## Resources\n\n- `activities` → `lemlist_activities_raw`\n- `campaigns` → `lemlist_campaigns_raw`\n\n## Sync and recovery\n\nActivities use the provider's inclusive `minDate`/`maxDate` filters on `createdAt`. The initial range starts at `2000-01-01`; later runs replay one day before the last committed watermark. A large range is split into intervals of at most thirty days. Offset pagination stays local to each interval; an unfinished interval always starts at offset zero after failure. The checkpoint advances only after every page of an interval has been written successfully, including empty intervals. An explicit backfill resumes completed intervals in its own checkpoint namespace while retaining one day of overlap.\n\nThe upper bound does not freeze the provider dataset. Late arrivals or deletions can still change offset pages, and older activities can receive backfilled metadata. Do not treat activity offsets as change cursors or row chunks as immutable page identities. Periodic historical replay refreshes metadata and catches arrivals beyond the overlap:\n\n```sh\nbunx chkit ingest run --tag provider:lemlist --tag resource:activities --backfill reconcile-2026-10-04 --from 2000-01-01\nbunx chkit ingest status --tag provider:lemlist --json\n```\n\nUse a new backfill ID for each complete reconciliation; reusing an ID resumes that backfill. A reused ID with an upper bound earlier than its committed frontier is rejected; use a new ID for a narrower historical range. Date intervals intentionally have inclusive shared boundaries; stable `_id` row keys reconcile repeated observations. Empty completion markers carry interval identities, while mutable row chunks do not.\n\nCampaigns always scan from offset zero, ordered by `createdAt:asc`. Their `completedAt` checkpoint records a successful full scan rather than a modification cursor. Creation-only filtering would miss later campaign state changes. Removed campaigns or activities remain stored. Run one ingestion process per ClickHouse target at a time.\n\n## Fixture tests\n\n```sh\nbunx chkit add lemlist --with-tests\nbun test src/integrations/lemlist/tests/basic.test.ts\n```\n\nThe portable suite checks bounded interval pagination, committed-frontier recovery, explicit backfill isolation, sink failure and empty scan completion without live credentials.\n" + }, + { + "path": "lemlist/index.ts", + "type": "registry:file", + "target": "src/integrations/lemlist/index.ts", + "content": "import { definePipeline, defineStream, HttpError, IngestConfigError, rawRows, rawTable, timestampWindow, type FetchContext, type IncrementalStrategy, type TimestampRange } from '@chkit/plugin-ingest'\n\nconst database = 'default'\n// Keep stream IDs and destinations tied to one Lemlist account; use new ones when switching accounts.\nconst pageSize = 100\nconst intervalMs = 30 * 24 * 60 * 60 * 1000\nconst activityWindow = timestampWindow({ start: new Date('2000-01-01T00:00:00Z'), overlapMs: 24 * 60 * 60 * 1000 })\n\ninterface Item { _id: string; [key: string]: unknown }\ninterface ScanState { completedAt: string }\n\nexport const lemlist_activitiesRaw = rawTable({ database, name: 'lemlist_activities_raw' })\nexport const lemlist_campaignsRaw = rawTable({ database, name: 'lemlist_campaigns_raw' })\n\nconst campaignScan: IncrementalStrategy = {\n id: 'lemlist.campaigns.full_scan', version: 1,\n parseState(raw) {\n if (!isObject(raw) || typeof raw.completedAt !== 'string' || !Number.isFinite(Date.parse(raw.completedAt))) {\n throw new IngestConfigError('Lemlist campaign checkpoint has no valid completedAt.')\n }\n return { completedAt: raw.completedAt }\n },\n plan: ({ cutoff }) => cutoff.toISOString(),\n complete: ({ selection }) => ({ completedAt: selection }),\n}\n\nexport const lemlistPipeline = definePipeline({\n id: 'lemlist', tags: ['provider:lemlist'], maxStreams: 1, maxFetches: 1,\n streams: [\n defineStream({\n id: 'lemlist.activities', tags: ['resource:activities'], destination: lemlist_activitiesRaw,\n incremental: {\n ...activityWindow,\n plan(input) {\n const selection = activityWindow.plan(input)\n // Explicit backfills have their own namespace; reuse its completed interval frontier.\n if (input.range && input.state) {\n if (selection.to.getTime() < Date.parse(input.state.watermark)) {\n throw new IngestConfigError('Lemlist backfill upper bound precedes its committed interval frontier; use a new backfill ID for a narrower historical range.')\n }\n return { ...selection, from: new Date(Math.max(selection.from.getTime(), Date.parse(input.state.watermark) - 24 * 60 * 60 * 1000)) }\n }\n return selection\n },\n },\n batchSize: 500,\n async *read(context) {\n for (let from = context.selection.from; from < context.selection.to;) {\n const to = new Date(Math.min(from.getTime() + intervalMs, context.selection.to.getTime()))\n for await (const page of pages(context, 'activities', { from, to })) {\n yield { rows: rawRows(page, (item) => item._id) }\n }\n // Only an exhausted interval is a safe frontier. Offset pages remain mutable.\n const watermark = new Date(Math.max(to.getTime(), Date.parse(context.state?.watermark ?? to.toISOString()))).toISOString()\n yield { rows: [], state: { watermark }, id: `completed-interval:${from.toISOString()}/${to.toISOString()}` }\n from = to\n }\n },\n }),\n defineStream({\n id: 'lemlist.campaigns', tags: ['resource:campaigns'], destination: lemlist_campaignsRaw,\n incremental: campaignScan,\n async *read(context) {\n for await (const page of pages(context, 'campaigns')) yield { rows: rawRows(page, (item) => item._id) }\n },\n }),\n ],\n})\n\nasync function* pages(context: FetchContext, resource: 'activities' | 'campaigns', window?: TimestampRange): AsyncGenerator {\n let offset = 0\n const seen = new Set()\n while (true) {\n const url = new URL(`https://api.lemlist.com/api/${resource}`)\n url.searchParams.set('version', 'v2')\n url.searchParams.set('limit', String(pageSize))\n url.searchParams.set('offset', String(offset))\n if (window) {\n url.searchParams.set('minDate', window.from.toISOString())\n url.searchParams.set('maxDate', window.to.toISOString())\n } else {\n url.searchParams.set('sortBy', 'createdAt')\n url.searchParams.set('sortOrder', 'asc')\n }\n const items = await context.attempt(async (signal) => {\n const key = process.env.LEMLIST_API_KEY?.trim()\n if (!key) throw new IngestConfigError('Set LEMLIST_API_KEY.')\n const response = await fetch(url, { signal: AbortSignal.any([signal, AbortSignal.timeout(30_000)]), redirect: 'error',\n headers: { Authorization: `Basic ${Buffer.from(`:${key}`).toString('base64')}` } })\n if (!response.ok) throw await HttpError.fromResponse(response)\n const result: unknown = await response.json()\n const values = resource === 'campaigns' && isObject(result) ? result.campaigns : result\n if (!Array.isArray(values) || !values.every(isItem)) throw new IngestConfigError(`Lemlist ${resource} response has no valid items.`)\n return values\n }, { label: `GET /${resource}` })\n if (items.length) {\n const identity = JSON.stringify(items.map((item) => item._id))\n if (seen.has(identity)) throw new IngestConfigError(`Lemlist ${resource} repeated a page.`)\n seen.add(identity)\n yield items\n }\n if (items.length < pageSize) return\n offset += items.length\n }\n}\n\nfunction isItem(value: unknown): value is Item {\n return isObject(value) && typeof value._id === 'string' && value._id.length > 0\n}\n\nfunction isObject(value: unknown): value is Record {\n return typeof value === 'object' && value !== null && !Array.isArray(value)\n}\n" + }, + { + "path": "lemlist/tests/basic.test.ts", + "type": "registry:file", + "target": "src/integrations/lemlist/tests/basic.test.ts", + "role": "test", + "content": "import { afterEach, expect, test } from 'bun:test'\nimport { IngestConfigError, runIngestion, selectStreams } from '@chkit/plugin-ingest'\nimport { createMemoryDestination, createMemoryJournal } from '@chkit/plugin-ingest/testing'\n\nimport { lemlistPipeline } from '../index.js'\n\nconst originalFetch = globalThis.fetch\nconst originalToken = process.env.LEMLIST_API_KEY\nconst cutoff = new Date('2000-03-01T00:00:00Z')\n\nfunction setFetch(handler: (...args: Parameters) => Promise): void {\n globalThis.fetch = Object.assign(handler, { preconnect: originalFetch.preconnect })\n process.env.LEMLIST_API_KEY = 'fixture'\n}\n\nfunction selected(resource = 'activities') {\n return selectStreams([lemlistPipeline], [`resource:${resource}`]).map((item) => ({ ...item, stream: { ...item.stream, retry: { retries: 0 } } }))\n}\n\nafterEach(() => {\n globalThis.fetch = originalFetch\n if (originalToken === undefined) delete process.env.LEMLIST_API_KEY\n else process.env.LEMLIST_API_KEY = originalToken\n})\n\ntest.serial('Lemlist commits only exhausted date intervals and replays the unfinished interval with overlap', async () => {\n let fail = true\n const journal = createMemoryJournal()\n let acknowledgeInterval = () => {}\n const intervalCommitted = new Promise((resolve) => { acknowledgeInterval = resolve })\n const append = journal.append\n journal.append = async (events) => {\n await append(events)\n if (events.some((event) => JSON.stringify(event.checkpoint?.state) === '{\"watermark\":\"2000-01-31T00:00:00.000Z\"}')) acknowledgeInterval()\n }\n const windows: { from: string | null; to: string | null; offset: string | null }[] = []\n setFetch(async (input) => {\n const url = new URL(String(input))\n const from = url.searchParams.get('minDate')\n windows.push({ from, to: url.searchParams.get('maxDate'), offset: url.searchParams.get('offset') })\n if (fail && from !== '2000-01-01T00:00:00.000Z') {\n await intervalCommitted\n return new Response('denied', { status: 403 })\n }\n return Response.json([{ _id: from, custom: { retained: true } }])\n })\n const destination = createMemoryDestination()\n const request = { selected: selected(), backfill: undefined }\n expect((await runIngestion(request, { journal, destination, now: () => cutoff })).ok).toBe(false)\n expect((await journal.readCheckpoint('lemlist.activities')).envelope?.state).toEqual({ watermark: '2000-01-31T00:00:00.000Z' })\n expect(destination.tables.get('default.lemlist_activities_raw')?.[0]?.raw).toMatchObject({ custom: { retained: true } })\n fail = false\n expect((await runIngestion(request, { journal, destination, now: () => cutoff })).ok).toBe(true)\n expect(windows[2]).toEqual({ from: '2000-01-30T00:00:00.000Z', to: '2000-02-29T00:00:00.000Z', offset: '0' })\n expect((await journal.readCheckpoint('lemlist.activities')).envelope?.state).toEqual({ watermark: cutoff.toISOString() })\n})\n\ntest.serial('Lemlist resumes isolated explicit backfills from the completed interval frontier', async () => {\n let fail = true\n const starts: string[] = []\n setFetch(async (input) => {\n const from = new URL(String(input)).searchParams.get('minDate') ?? ''\n starts.push(from)\n if (fail && from !== '2025-01-01T00:00:00.000Z') return new Response('denied', { status: 403 })\n return Response.json([])\n })\n const journal = createMemoryJournal()\n const destination = createMemoryDestination()\n const request = { selected: selected(), backfill: { id: 'history', from: new Date('2025-01-01T00:00:00Z'), to: new Date('2025-03-01T00:00:00Z') } }\n expect((await runIngestion(request, { journal, destination, now: () => new Date('2026-01-01') })).ok).toBe(false)\n expect((await journal.readCheckpoint('lemlist.activities#backfill:history')).envelope?.state).toEqual({ watermark: '2025-01-31T00:00:00.000Z' })\n fail = false\n expect((await runIngestion(request, { journal, destination, now: () => new Date('2026-01-01') })).ok).toBe(true)\n expect(starts[2]).toBe('2025-01-30T00:00:00.000Z')\n expect((await journal.readCheckpoint('lemlist.activities')).envelope).toBeUndefined()\n})\n\ntest.serial('Lemlist rejects a narrowed completed backfill upper bound without regressing its frontier', async () => {\n let requests = 0\n setFetch(async () => { requests++; return Response.json([]) })\n const journal = createMemoryJournal()\n const destination = createMemoryDestination()\n const backfill = { id: 'narrowed', from: new Date('2025-01-01T00:00:00Z'), to: new Date('2025-03-01T00:00:00Z') }\n const request = { selected: selected(), backfill }\n expect((await runIngestion(request, { journal, destination, now: () => new Date('2026-01-01') })).ok).toBe(true)\n const completedRequests = requests\n // This upper bound is still after the overlap's lower bound, but must not move the watermark backwards.\n const result = await runIngestion({ ...request, backfill: { ...backfill, to: new Date('2025-02-28T12:00:00Z') } }, {\n journal, destination, now: () => new Date('2026-01-01'),\n })\n expect(result.ok).toBe(false)\n expect(result.streams[0]?.error).toContain('use a new backfill ID')\n expect(requests).toBe(completedRequests)\n expect((await journal.readCheckpoint('lemlist.activities#backfill:narrowed')).envelope?.state).toEqual({ watermark: '2025-03-01T00:00:00.000Z' })\n})\n\ntest.serial('Lemlist paginates a whole date interval and a rejected sink cannot advance it', async () => {\n const offsets: string[] = []\n setFetch(async (input) => {\n const url = new URL(String(input))\n offsets.push(url.searchParams.get('offset') ?? '')\n expect(url.searchParams.get('minDate')).toBe('2000-01-01T00:00:00.000Z')\n expect(url.searchParams.get('maxDate')).toBe('2000-01-02T00:00:00.000Z')\n return Response.json(url.searchParams.get('offset') === '0' ? Array.from({ length: 100 }, (_, index) => ({ _id: String(index) })) : [{ _id: 'last' }])\n })\n const journal = createMemoryJournal()\n const request = { selected: selected(), backfill: undefined }\n const result = await runIngestion(request, { journal, destination: { insert: async () => { throw new IngestConfigError('sink rejected') } }, now: () => new Date('2000-01-02') })\n expect(result.ok).toBe(false)\n expect((await journal.readCheckpoint('lemlist.activities')).envelope).toBeUndefined()\n expect(offsets).toEqual(['0', '100'])\n})\n\ntest.serial('Lemlist empty activity intervals commit and campaigns always scan from zero', async () => {\n const paths: URL[] = []\n setFetch(async (input) => { paths.push(new URL(String(input))); return Response.json({ campaigns: [] }) })\n const journal = createMemoryJournal()\n const destination = createMemoryDestination()\n const request = { selected: selected('campaigns'), backfill: undefined }\n expect((await runIngestion(request, { journal, destination, now: () => cutoff })).ok).toBe(true)\n expect((await runIngestion(request, { journal, destination, now: () => new Date('2000-03-02') })).ok).toBe(true)\n expect(paths.map((url) => url.searchParams.get('offset'))).toEqual(['0', '0'])\n expect(paths.every((url) => url.searchParams.get('sortBy') === 'createdAt' && url.searchParams.get('sortOrder') === 'asc')).toBe(true)\n expect((await journal.readCheckpoint('lemlist.campaigns')).envelope?.state).toEqual({ completedAt: '2000-03-02T00:00:00.000Z' })\n setFetch(async () => Response.json([]))\n expect((await runIngestion({ selected: selected(), backfill: undefined }, { journal, destination, now: () => cutoff })).ok).toBe(true)\n expect((await journal.readCheckpoint('lemlist.activities')).envelope?.state).toEqual({ watermark: cutoff.toISOString() })\n})\n" + } + ], + "meta": { + "chkit": { + "formatVersion": 1, + "version": "0.2.0", + "language": "typescript", + "license": "MIT", + "documentation": "https://chkit.obsessiondb.com/integrations/lemlist/", + "logo": "https://chkit.obsessiondb.com/logos/lemlist.svg", + "authentication": { + "method": "HTTP Basic API key", + "env": [ + "LEMLIST_API_KEY" + ], + "setup": [ + "Provide LEMLIST_API_KEY with read access to the selected resources.", + "Keep the API key in the runtime environment and rotate it according to the provider policy." + ], + "documentation": "https://developer.lemlist.com/" + }, + "chkit": "^0.2.0-beta.10", + "ingest": "^0.2.0-beta.10", + "clickhouse": ">=25.3.0", + "root": "src/integrations/lemlist", + "entry": "index.ts", + "exports": [ + "lemlistPipeline", + "lemlist_activitiesRaw", + "lemlist_campaignsRaw" + ], + "resources": [ + { + "name": "activities", + "title": "Activities", + "table": "lemlist_activities_raw", + "description": "Outreach activity events", + "scopes": [], + "strategy": "timestamp", + "endpoints": [ + { + "method": "GET", + "path": "/activities", + "documentation": "https://developer.lemlist.com/api-reference/endpoints/activities/get-many-activities" + } + ] + }, + { + "name": "campaigns", + "title": "Campaigns", + "table": "lemlist_campaigns_raw", + "description": "Campaign metadata", + "scopes": [], + "strategy": "full", + "endpoints": [ + { + "method": "GET", + "path": "/campaigns", + "documentation": "https://developer.lemlist.com/api-reference/endpoints/campaigns/get-many-campaigns" + } + ] + } + ], + "views": [], + "sync": { + "description": "Activities checkpoint completed thirty-day createdAt intervals with one-day overlap; campaigns record successful full-scan completion.", + "schedule": "Run bunx chkit ingest run --tag provider:lemlist from a scheduler.", + "deletions": "Removed campaigns and activities remain stored. Historical replay is required for older activity metadata updates outside the overlap." + }, + "env": { + "LEMLIST_API_KEY": "" + }, + "fileHashes": { + "src/integrations/lemlist/README.md": "1a9f7df8a2ab4dc273c8389e137dce0bf8aa9444defd92b90c519457a20fa347", + "src/integrations/lemlist/index.ts": "18052a0a385bc0ab625df895a08fd3ce5aae35b49cb8106391994281cff7847f", + "src/integrations/lemlist/tests/basic.test.ts": "8d9164bb852dd2c76d1d18fd011d2918089d0b2651323cc6b8315bd397288105" + } + } + } +} diff --git a/registry/lemlist/tests/basic.test.ts b/registry/lemlist/tests/basic.test.ts new file mode 100644 index 00000000..a57bd168 --- /dev/null +++ b/registry/lemlist/tests/basic.test.ts @@ -0,0 +1,128 @@ +import { afterEach, expect, test } from 'bun:test' +import { IngestConfigError, runIngestion, selectStreams } from '@chkit/plugin-ingest' +import { createMemoryDestination, createMemoryJournal } from '@chkit/plugin-ingest/testing' + +import { lemlistPipeline } from '../index.js' + +const originalFetch = globalThis.fetch +const originalToken = process.env.LEMLIST_API_KEY +const cutoff = new Date('2000-03-01T00:00:00Z') + +function setFetch(handler: (...args: Parameters) => Promise): void { + globalThis.fetch = Object.assign(handler, { preconnect: originalFetch.preconnect }) + process.env.LEMLIST_API_KEY = 'fixture' +} + +function selected(resource = 'activities') { + return selectStreams([lemlistPipeline], [`resource:${resource}`]).map((item) => ({ ...item, stream: { ...item.stream, retry: { retries: 0 } } })) +} + +afterEach(() => { + globalThis.fetch = originalFetch + if (originalToken === undefined) delete process.env.LEMLIST_API_KEY + else process.env.LEMLIST_API_KEY = originalToken +}) + +test.serial('Lemlist commits only exhausted date intervals and replays the unfinished interval with overlap', async () => { + let fail = true + const journal = createMemoryJournal() + let acknowledgeInterval = () => {} + const intervalCommitted = new Promise((resolve) => { acknowledgeInterval = resolve }) + const append = journal.append + journal.append = async (events) => { + await append(events) + if (events.some((event) => JSON.stringify(event.checkpoint?.state) === '{"watermark":"2000-01-31T00:00:00.000Z"}')) acknowledgeInterval() + } + const windows: { from: string | null; to: string | null; offset: string | null }[] = [] + setFetch(async (input) => { + const url = new URL(String(input)) + const from = url.searchParams.get('minDate') + windows.push({ from, to: url.searchParams.get('maxDate'), offset: url.searchParams.get('offset') }) + if (fail && from !== '2000-01-01T00:00:00.000Z') { + await intervalCommitted + return new Response('denied', { status: 403 }) + } + return Response.json([{ _id: from, custom: { retained: true } }]) + }) + const destination = createMemoryDestination() + const request = { selected: selected(), backfill: undefined } + expect((await runIngestion(request, { journal, destination, now: () => cutoff })).ok).toBe(false) + expect((await journal.readCheckpoint('lemlist.activities')).envelope?.state).toEqual({ watermark: '2000-01-31T00:00:00.000Z' }) + expect(destination.tables.get('default.lemlist_activities_raw')?.[0]?.raw).toMatchObject({ custom: { retained: true } }) + fail = false + expect((await runIngestion(request, { journal, destination, now: () => cutoff })).ok).toBe(true) + expect(windows[2]).toEqual({ from: '2000-01-30T00:00:00.000Z', to: '2000-02-29T00:00:00.000Z', offset: '0' }) + expect((await journal.readCheckpoint('lemlist.activities')).envelope?.state).toEqual({ watermark: cutoff.toISOString() }) +}) + +test.serial('Lemlist resumes isolated explicit backfills from the completed interval frontier', async () => { + let fail = true + const starts: string[] = [] + setFetch(async (input) => { + const from = new URL(String(input)).searchParams.get('minDate') ?? '' + starts.push(from) + if (fail && from !== '2025-01-01T00:00:00.000Z') return new Response('denied', { status: 403 }) + return Response.json([]) + }) + const journal = createMemoryJournal() + const destination = createMemoryDestination() + const request = { selected: selected(), backfill: { id: 'history', from: new Date('2025-01-01T00:00:00Z'), to: new Date('2025-03-01T00:00:00Z') } } + expect((await runIngestion(request, { journal, destination, now: () => new Date('2026-01-01') })).ok).toBe(false) + expect((await journal.readCheckpoint('lemlist.activities#backfill:history')).envelope?.state).toEqual({ watermark: '2025-01-31T00:00:00.000Z' }) + fail = false + expect((await runIngestion(request, { journal, destination, now: () => new Date('2026-01-01') })).ok).toBe(true) + expect(starts[2]).toBe('2025-01-30T00:00:00.000Z') + expect((await journal.readCheckpoint('lemlist.activities')).envelope).toBeUndefined() +}) + +test.serial('Lemlist rejects a narrowed completed backfill upper bound without regressing its frontier', async () => { + let requests = 0 + setFetch(async () => { requests++; return Response.json([]) }) + const journal = createMemoryJournal() + const destination = createMemoryDestination() + const backfill = { id: 'narrowed', from: new Date('2025-01-01T00:00:00Z'), to: new Date('2025-03-01T00:00:00Z') } + const request = { selected: selected(), backfill } + expect((await runIngestion(request, { journal, destination, now: () => new Date('2026-01-01') })).ok).toBe(true) + const completedRequests = requests + // This upper bound is still after the overlap's lower bound, but must not move the watermark backwards. + const result = await runIngestion({ ...request, backfill: { ...backfill, to: new Date('2025-02-28T12:00:00Z') } }, { + journal, destination, now: () => new Date('2026-01-01'), + }) + expect(result.ok).toBe(false) + expect(result.streams[0]?.error).toContain('use a new backfill ID') + expect(requests).toBe(completedRequests) + expect((await journal.readCheckpoint('lemlist.activities#backfill:narrowed')).envelope?.state).toEqual({ watermark: '2025-03-01T00:00:00.000Z' }) +}) + +test.serial('Lemlist paginates a whole date interval and a rejected sink cannot advance it', async () => { + const offsets: string[] = [] + setFetch(async (input) => { + const url = new URL(String(input)) + offsets.push(url.searchParams.get('offset') ?? '') + expect(url.searchParams.get('minDate')).toBe('2000-01-01T00:00:00.000Z') + expect(url.searchParams.get('maxDate')).toBe('2000-01-02T00:00:00.000Z') + return Response.json(url.searchParams.get('offset') === '0' ? Array.from({ length: 100 }, (_, index) => ({ _id: String(index) })) : [{ _id: 'last' }]) + }) + const journal = createMemoryJournal() + const request = { selected: selected(), backfill: undefined } + const result = await runIngestion(request, { journal, destination: { insert: async () => { throw new IngestConfigError('sink rejected') } }, now: () => new Date('2000-01-02') }) + expect(result.ok).toBe(false) + expect((await journal.readCheckpoint('lemlist.activities')).envelope).toBeUndefined() + expect(offsets).toEqual(['0', '100']) +}) + +test.serial('Lemlist empty activity intervals commit and campaigns always scan from zero', async () => { + const paths: URL[] = [] + setFetch(async (input) => { paths.push(new URL(String(input))); return Response.json({ campaigns: [] }) }) + const journal = createMemoryJournal() + const destination = createMemoryDestination() + const request = { selected: selected('campaigns'), backfill: undefined } + expect((await runIngestion(request, { journal, destination, now: () => cutoff })).ok).toBe(true) + expect((await runIngestion(request, { journal, destination, now: () => new Date('2000-03-02') })).ok).toBe(true) + expect(paths.map((url) => url.searchParams.get('offset'))).toEqual(['0', '0']) + expect(paths.every((url) => url.searchParams.get('sortBy') === 'createdAt' && url.searchParams.get('sortOrder') === 'asc')).toBe(true) + expect((await journal.readCheckpoint('lemlist.campaigns')).envelope?.state).toEqual({ completedAt: '2000-03-02T00:00:00.000Z' }) + setFetch(async () => Response.json([])) + expect((await runIngestion({ selected: selected(), backfill: undefined }, { journal, destination, now: () => cutoff })).ok).toBe(true) + expect((await journal.readCheckpoint('lemlist.activities')).envelope?.state).toEqual({ watermark: cutoff.toISOString() }) +}) diff --git a/registry/linear/README.md b/registry/linear/README.md index a56e2864..58d3c7ce 100644 --- a/registry/linear/README.md +++ b/registry/linear/README.md @@ -4,6 +4,8 @@ This is a small editable starting point based on the Linear source in [obsession Set `LINEAR_API_KEY` in the runtime environment and configure a direct ClickHouse connection. Edit the GraphQL `query` in `index.ts` to request the issue fields needed by the project. Native JSON requires ClickHouse 25.3 or later. +Keep each stream ID and destination tied to one Linear workspace. The reader does not resolve the authenticated workspace identity: when changing the token to another workspace, use a new stream ID and destination before ingesting so the old watermark cannot skip that workspace's history. Adding requested fields later requires a historical reconciliation to populate older observations. + ```sh bunx chkit check bunx chkit generate --name add_linear @@ -16,4 +18,26 @@ bunx chkit ingest run --tag provider:linear - `issues` → `linear_issues_raw` -Every run reads all accessible issues. The example requests the first 100 comments per issue; extend the query for complete discussions. A failed API request fails the stream. Raw rows use provider IDs and keep the last observed value; there is no deletion reconciliation. Edit the readers, select fewer streams, or add SQL views for a specific use case. Run one ingestion process per ClickHouse target at a time. +## Sync and recovery + +Issues use a bounded `updatedAt` filter and `orderBy: updatedAt`, with `includeArchived: true`. The initial window starts at the Unix epoch; later windows replay five minutes before the committed watermark. Relay issue cursors and comment cursors are used only during the current read. All accessible comment pages are collected before their parent observation is published, retaining the provider's connection shape and the final `pageInfo`. + +Newest-first pages do not define a safe timestamp frontier. The checkpoint advances to the fixed execution cutoff only after the whole window and all destination writes succeed, including an empty successful window. A source or sink failure replays the window from its lower bound. Mutable observations have no declared chunk ID, so changed provider fields remain eligible for insertion on replay. + +Deleted issues remain stored. Archived issues are retained explicitly. A comment-only change is not assumed to advance the parent issue's `updatedAt`, and cursor pagination is not a guaranteed source snapshot. Schedule periodic reconciliation with a new backfill ID to refresh old discussions, or add a durable webhook companion: + +```sh +bunx chkit ingest run --tag provider:linear --backfill reconcile-2026-10-04 --from 1970-01-01 +bunx chkit ingest status --tag provider:linear --json +``` + +Reconciliation reads current issue observations by their last-update timestamp; it does not reconstruct historical issue versions. Run one ingestion process per ClickHouse target at a time. + +## Fixture tests + +```sh +bunx chkit add linear --with-tests +bun test src/integrations/linear/tests/basic.test.ts +``` + +The portable suite covers bounded archive-inclusive queries, complete comments, failure replay, sink ordering and empty-window checkpoints without live credentials. diff --git a/registry/linear/index.ts b/registry/linear/index.ts index 0d200c07..230b6599 100644 --- a/registry/linear/index.ts +++ b/registry/linear/index.ts @@ -1,50 +1,109 @@ -import { definePipeline, defineStream, HttpError, rawRows, rawTable, type FetchContext } from '@chkit/plugin-ingest' +import { definePipeline, defineStream, HttpError, IngestConfigError, rawRows, rawTable, timestampWindow, type ErrorClassifier, type FetchContext, type TimestampRange } from '@chkit/plugin-ingest' const database = 'default' -const query = `query Issues($after: String) { - issues(first: 100, after: $after) { - nodes { id identifier title description url createdAt updatedAt completedAt canceledAt +// Keep stream IDs and destinations tied to one Linear workspace; use new ones when switching accounts. +const query = `query Issues($after: String, $from: DateTimeOrDuration!, $to: DateTimeOrDuration!) { + issues(first: 100, after: $after, includeArchived: true, orderBy: updatedAt, + filter: { updatedAt: { gte: $from, lte: $to } }) { + nodes { id identifier title description url createdAt updatedAt archivedAt completedAt canceledAt team { id key name } state { name type } assignee { id name email } - comments(first: 100) { nodes { id body createdAt updatedAt } pageInfo { hasNextPage endCursor } } + comments(first: 20, includeArchived: true) { nodes { id body createdAt updatedAt } pageInfo { hasNextPage endCursor } } } pageInfo { hasNextPage endCursor } } }` -interface Issue { id: string; [key: string]: unknown } -interface ResponseBody { data?: { issues: { nodes: Issue[]; pageInfo: { hasNextPage: boolean; endCursor: string | null } } }; errors?: { message: string }[] } +const commentsQuery = `query IssueComments($id: String!, $after: String) { + issue(id: $id) { comments(first: 100, after: $after, includeArchived: true) { + nodes { id body createdAt updatedAt } pageInfo { hasNextPage endCursor } + } } +}` +interface Connection { nodes: T[]; pageInfo: { hasNextPage: boolean; endCursor: string | null } } +interface Issue { id: string; updatedAt: string; comments: Connection>; [key: string]: unknown } +interface ResponseBody { data?: { issues: Connection }; errors?: { message: string }[] } +interface CommentsBody { data?: { issue: { comments: Connection> } | null }; errors?: { message: string }[] } export const linear_issuesRaw = rawTable({ database, name: 'linear_issues_raw' }) export const linearPipeline = definePipeline({ - id: 'linear', tags: ['provider:linear'], maxFetches: 1, - streams: [defineStream({ id: 'linear.issues', tags: ['resource:issues'], destination: linear_issuesRaw, + id: 'linear', tags: ['provider:linear'], maxStreams: 1, maxFetches: 1, + streams: [defineStream({ + id: 'linear.issues', tags: ['resource:issues'], destination: linear_issuesRaw, + // Newest-first pages cannot advance a safe watermark until the whole window loads. + incremental: timestampWindow({ start: new Date(0), overlapMs: 5 * 60 * 1000 }), + classifyError: classifyLinearError, async *read(context) { let after: string | null = null const seen = new Set() while (true) { - const body = await readPage(context, after) - const issues = body.data?.issues - if (body.errors?.length || !issues) throw new Error(`Linear query failed: ${body.errors?.map((item) => item.message).join('; ') ?? 'missing issues'}`) - if (issues.nodes.length) yield { rows: rawRows(issues.nodes, (issue) => issue.id) } + const issues = await readPage(context, after, context.selection) + for (const issue of issues.nodes) { + if (!issue.id || !Number.isFinite(Date.parse(issue.updatedAt))) throw new IngestConfigError('Linear issue has no ID or valid updatedAt.') + const comments = await completeComments(context, issue) + yield { rows: rawRows([{ ...issue, comments }], (item) => item.id) } + } if (!issues.pageInfo.hasNextPage) return - after = issues.pageInfo.endCursor - if (!after || seen.has(after)) throw new Error('Linear returned no new issue cursor') - seen.add(after) + after = continuation(issues.pageInfo.endCursor, seen, 'issue') } }, })], }) -async function readPage(context: FetchContext, after: string | null): Promise { +/** Linear reports GraphQL rate exhaustion as HTTP 400 rather than 429. */ +function classifyLinearError(cause: unknown): ReturnType { + if (!(cause instanceof HttpError) || cause.status !== 400 || !/"code"\s*:\s*"RATELIMITED"/.test(cause.body)) return undefined + const headers = cause.response.headers + const resets = ['requests', 'endpoint-requests', 'complexity'].flatMap((kind) => { + if (headers.get(`x-ratelimit-${kind}-remaining`) !== '0') return [] + const reset = Number(headers.get(`x-ratelimit-${kind}-reset`)) + return Number.isFinite(reset) && reset > 0 ? [Math.max(0, reset - Date.now())] : [] + }) + return { kind: 'rate_limited', retryAfterMs: cause.retryAfterMs ?? (resets.length ? Math.max(...resets) : undefined) } +} + +async function readPage(context: FetchContext, after: string | null, window: TimestampRange): Promise> { + const body = await graphql(context, query, { after, from: window.from.toISOString(), to: window.to.toISOString() }, 'issues') + if (body.errors?.length || !body.data?.issues) throw new IngestConfigError(`Linear query failed: ${body.errors?.map((item) => item.message).join('; ') ?? 'missing issues'}`) + return validateConnection(body.data.issues) +} + +async function completeComments(context: FetchContext, issue: Issue): Promise>> { + let page = validateConnection(issue.comments) + const nodes = [...page.nodes] + const seen = new Set() + while (page.pageInfo.hasNextPage) { + const after = continuation(page.pageInfo.endCursor, seen, 'comment') + const body = await graphql(context, commentsQuery, { id: issue.id, after }, 'issue comments') + if (body.errors?.length || !body.data?.issue) throw new IngestConfigError('Linear comment query failed; the issue window is incomplete.') + page = validateConnection(body.data.issue.comments) + nodes.push(...page.nodes) + } + // Preserve the provider connection shape, with the terminal continuation information. + return { ...issue.comments, nodes, pageInfo: page.pageInfo } +} + +async function graphql(context: FetchContext, document: string, variables: Record, label: string): Promise { return context.attempt(async (signal) => { - const token = process.env.LINEAR_API_KEY - if (!token) throw new Error('Set LINEAR_API_KEY') + const token = process.env.LINEAR_API_KEY?.trim() + if (!token) throw new IngestConfigError('Set LINEAR_API_KEY.') const response = await fetch('https://api.linear.app/graphql', { - method: 'POST', signal, + method: 'POST', signal: AbortSignal.any([signal, AbortSignal.timeout(30_000)]), redirect: 'error', headers: { Authorization: token, 'Content-Type': 'application/json' }, - body: JSON.stringify({ query, variables: { after } }), + body: JSON.stringify({ query: document, variables }), }) if (!response.ok) throw await HttpError.fromResponse(response) - return await response.json() as ResponseBody - }, { label: 'POST /graphql issues' }) + return await response.json() as T + }, { label: `POST /graphql ${label}` }) +} + +function continuation(cursor: string | null, seen: Set, resource: string): string { + if (typeof cursor !== 'string' || !cursor.trim() || seen.has(cursor)) throw new IngestConfigError(`Linear returned no new ${resource} cursor.`) + seen.add(cursor) + return cursor +} + +function validateConnection(value: Connection): Connection { + if (!value || !Array.isArray(value.nodes) || !value.pageInfo || typeof value.pageInfo.hasNextPage !== 'boolean') { + throw new IngestConfigError('Linear returned an invalid connection.') + } + return value } diff --git a/registry/linear/manifest.json b/registry/linear/manifest.json index f0154b1a..2f55c701 100644 --- a/registry/linear/manifest.json +++ b/registry/linear/manifest.json @@ -3,10 +3,10 @@ "name": "linear", "type": "registry:item", "title": "Linear", - "description": "Small, editable Linear API ingestion example with raw ClickHouse tables.", + "description": "Editable Linear timestamp ingestion with archived issues and complete paginated comments.", "dependencies": [ - "@chkit/core@^0.2.0-beta.8", - "@chkit/plugin-ingest@^0.2.0-beta.8" + "@chkit/core@^0.2.0-beta.10", + "@chkit/plugin-ingest@^0.2.0-beta.10" ], "files": [ { @@ -18,18 +18,24 @@ "path": "index.ts", "type": "registry:file", "target": "src/integrations/linear/index.ts" + }, + { + "path": "tests/basic.test.ts", + "type": "registry:file", + "target": "src/integrations/linear/tests/basic.test.ts", + "role": "test" } ], "meta": { "chkit": { "formatVersion": 1, - "version": "0.1.0", + "version": "0.2.0", "documentation": "https://chkit.obsessiondb.com/integrations/linear/", "logo": "https://chkit.obsessiondb.com/logos/linear.svg", "language": "typescript", "license": "MIT", - "chkit": "^0.2.0-beta.8", - "ingest": "^0.2.0-beta.8", + "chkit": "^0.2.0-beta.10", + "ingest": "^0.2.0-beta.10", "clickhouse": ">=25.3.0", "root": "src/integrations/linear", "entry": "index.ts", @@ -42,9 +48,9 @@ "name": "issues", "title": "Issues", "table": "linear_issues_raw", - "description": "Issue fields and the first 100 comments per issue.", + "description": "Archived and active issues updated inside a timestamp window, with all accessible comment pages.", "scopes": [], - "strategy": "full", + "strategy": "timestamp", "endpoints": [ { "method": "POST", @@ -70,10 +76,13 @@ }, "views": [], "sync": { - "description": "A full read runs for each enabled stream. Stable IDs keep the latest observed record.", + "description": "Issues use bounded updatedAt windows, five-minute overlap and whole-window checkpoints after destination acknowledgement.", "schedule": "Run bunx chkit ingest run --tag provider:linear from a scheduler.", - "deletions": "Every run reads all accessible issues. The example requests the first 100 comments per issue; extend the query for complete discussions." + "deletions": "Deleted issues remain stored. Periodic historical reconciliation refreshes comments whose changes may not update the parent issue." } } - } + }, + "devDependencies": [ + "@types/bun@^1.2.0" + ] } diff --git a/registry/linear/releases/0.2.0.json b/registry/linear/releases/0.2.0.json new file mode 100644 index 00000000..6b776272 --- /dev/null +++ b/registry/linear/releases/0.2.0.json @@ -0,0 +1,96 @@ +{ + "$schema": "https://ui.shadcn.com/schema/registry-item.json", + "name": "linear", + "type": "registry:item", + "title": "Linear", + "description": "Editable Linear timestamp ingestion with archived issues and complete paginated comments.", + "dependencies": [ + "@chkit/core@^0.2.0-beta.10", + "@chkit/plugin-ingest@^0.2.0-beta.10" + ], + "devDependencies": [ + "@types/bun@^1.2.0" + ], + "files": [ + { + "path": "linear/README.md", + "type": "registry:file", + "target": "src/integrations/linear/README.md", + "content": "# Linear raw ingestion example\n\nThis is a small editable starting point based on the Linear source in [obsessiondb/brain](https://github.com/obsessiondb/brain/tree/main/scripts/brain-sync). It stores provider responses as native JSON without Brain's memory or sales projections.\n\nSet `LINEAR_API_KEY` in the runtime environment and configure a direct ClickHouse connection. Edit the GraphQL `query` in `index.ts` to request the issue fields needed by the project. Native JSON requires ClickHouse 25.3 or later.\n\nKeep each stream ID and destination tied to one Linear workspace. The reader does not resolve the authenticated workspace identity: when changing the token to another workspace, use a new stream ID and destination before ingesting so the old watermark cannot skip that workspace's history. Adding requested fields later requires a historical reconciliation to populate older observations.\n\n```sh\nbunx chkit check\nbunx chkit generate --name add_linear\n# Review the migration before applying it.\nbunx chkit migrate --apply\nbunx chkit ingest run --tag provider:linear\n```\n\n## Resources\n\n- `issues` → `linear_issues_raw`\n\n## Sync and recovery\n\nIssues use a bounded `updatedAt` filter and `orderBy: updatedAt`, with `includeArchived: true`. The initial window starts at the Unix epoch; later windows replay five minutes before the committed watermark. Relay issue cursors and comment cursors are used only during the current read. All accessible comment pages are collected before their parent observation is published, retaining the provider's connection shape and the final `pageInfo`.\n\nNewest-first pages do not define a safe timestamp frontier. The checkpoint advances to the fixed execution cutoff only after the whole window and all destination writes succeed, including an empty successful window. A source or sink failure replays the window from its lower bound. Mutable observations have no declared chunk ID, so changed provider fields remain eligible for insertion on replay.\n\nDeleted issues remain stored. Archived issues are retained explicitly. A comment-only change is not assumed to advance the parent issue's `updatedAt`, and cursor pagination is not a guaranteed source snapshot. Schedule periodic reconciliation with a new backfill ID to refresh old discussions, or add a durable webhook companion:\n\n```sh\nbunx chkit ingest run --tag provider:linear --backfill reconcile-2026-10-04 --from 1970-01-01\nbunx chkit ingest status --tag provider:linear --json\n```\n\nReconciliation reads current issue observations by their last-update timestamp; it does not reconstruct historical issue versions. Run one ingestion process per ClickHouse target at a time.\n\n## Fixture tests\n\n```sh\nbunx chkit add linear --with-tests\nbun test src/integrations/linear/tests/basic.test.ts\n```\n\nThe portable suite covers bounded archive-inclusive queries, complete comments, failure replay, sink ordering and empty-window checkpoints without live credentials.\n" + }, + { + "path": "linear/index.ts", + "type": "registry:file", + "target": "src/integrations/linear/index.ts", + "content": "import { definePipeline, defineStream, HttpError, IngestConfigError, rawRows, rawTable, timestampWindow, type ErrorClassifier, type FetchContext, type TimestampRange } from '@chkit/plugin-ingest'\n\nconst database = 'default'\n// Keep stream IDs and destinations tied to one Linear workspace; use new ones when switching accounts.\nconst query = `query Issues($after: String, $from: DateTimeOrDuration!, $to: DateTimeOrDuration!) {\n issues(first: 100, after: $after, includeArchived: true, orderBy: updatedAt,\n filter: { updatedAt: { gte: $from, lte: $to } }) {\n nodes { id identifier title description url createdAt updatedAt archivedAt completedAt canceledAt\n team { id key name } state { name type } assignee { id name email }\n comments(first: 20, includeArchived: true) { nodes { id body createdAt updatedAt } pageInfo { hasNextPage endCursor } }\n }\n pageInfo { hasNextPage endCursor }\n }\n}`\nconst commentsQuery = `query IssueComments($id: String!, $after: String) {\n issue(id: $id) { comments(first: 100, after: $after, includeArchived: true) {\n nodes { id body createdAt updatedAt } pageInfo { hasNextPage endCursor }\n } }\n}`\ninterface Connection { nodes: T[]; pageInfo: { hasNextPage: boolean; endCursor: string | null } }\ninterface Issue { id: string; updatedAt: string; comments: Connection>; [key: string]: unknown }\ninterface ResponseBody { data?: { issues: Connection }; errors?: { message: string }[] }\ninterface CommentsBody { data?: { issue: { comments: Connection> } | null }; errors?: { message: string }[] }\n\nexport const linear_issuesRaw = rawTable({ database, name: 'linear_issues_raw' })\n\nexport const linearPipeline = definePipeline({\n id: 'linear', tags: ['provider:linear'], maxStreams: 1, maxFetches: 1,\n streams: [defineStream({\n id: 'linear.issues', tags: ['resource:issues'], destination: linear_issuesRaw,\n // Newest-first pages cannot advance a safe watermark until the whole window loads.\n incremental: timestampWindow({ start: new Date(0), overlapMs: 5 * 60 * 1000 }),\n classifyError: classifyLinearError,\n async *read(context) {\n let after: string | null = null\n const seen = new Set()\n while (true) {\n const issues = await readPage(context, after, context.selection)\n for (const issue of issues.nodes) {\n if (!issue.id || !Number.isFinite(Date.parse(issue.updatedAt))) throw new IngestConfigError('Linear issue has no ID or valid updatedAt.')\n const comments = await completeComments(context, issue)\n yield { rows: rawRows([{ ...issue, comments }], (item) => item.id) }\n }\n if (!issues.pageInfo.hasNextPage) return\n after = continuation(issues.pageInfo.endCursor, seen, 'issue')\n }\n },\n })],\n})\n\n/** Linear reports GraphQL rate exhaustion as HTTP 400 rather than 429. */\nfunction classifyLinearError(cause: unknown): ReturnType {\n if (!(cause instanceof HttpError) || cause.status !== 400 || !/\"code\"\\s*:\\s*\"RATELIMITED\"/.test(cause.body)) return undefined\n const headers = cause.response.headers\n const resets = ['requests', 'endpoint-requests', 'complexity'].flatMap((kind) => {\n if (headers.get(`x-ratelimit-${kind}-remaining`) !== '0') return []\n const reset = Number(headers.get(`x-ratelimit-${kind}-reset`))\n return Number.isFinite(reset) && reset > 0 ? [Math.max(0, reset - Date.now())] : []\n })\n return { kind: 'rate_limited', retryAfterMs: cause.retryAfterMs ?? (resets.length ? Math.max(...resets) : undefined) }\n}\n\nasync function readPage(context: FetchContext, after: string | null, window: TimestampRange): Promise> {\n const body = await graphql(context, query, { after, from: window.from.toISOString(), to: window.to.toISOString() }, 'issues')\n if (body.errors?.length || !body.data?.issues) throw new IngestConfigError(`Linear query failed: ${body.errors?.map((item) => item.message).join('; ') ?? 'missing issues'}`)\n return validateConnection(body.data.issues)\n}\n\nasync function completeComments(context: FetchContext, issue: Issue): Promise>> {\n let page = validateConnection(issue.comments)\n const nodes = [...page.nodes]\n const seen = new Set()\n while (page.pageInfo.hasNextPage) {\n const after = continuation(page.pageInfo.endCursor, seen, 'comment')\n const body = await graphql(context, commentsQuery, { id: issue.id, after }, 'issue comments')\n if (body.errors?.length || !body.data?.issue) throw new IngestConfigError('Linear comment query failed; the issue window is incomplete.')\n page = validateConnection(body.data.issue.comments)\n nodes.push(...page.nodes)\n }\n // Preserve the provider connection shape, with the terminal continuation information.\n return { ...issue.comments, nodes, pageInfo: page.pageInfo }\n}\n\nasync function graphql(context: FetchContext, document: string, variables: Record, label: string): Promise {\n return context.attempt(async (signal) => {\n const token = process.env.LINEAR_API_KEY?.trim()\n if (!token) throw new IngestConfigError('Set LINEAR_API_KEY.')\n const response = await fetch('https://api.linear.app/graphql', {\n method: 'POST', signal: AbortSignal.any([signal, AbortSignal.timeout(30_000)]), redirect: 'error',\n headers: { Authorization: token, 'Content-Type': 'application/json' },\n body: JSON.stringify({ query: document, variables }),\n })\n if (!response.ok) throw await HttpError.fromResponse(response)\n return await response.json() as T\n }, { label: `POST /graphql ${label}` })\n}\n\nfunction continuation(cursor: string | null, seen: Set, resource: string): string {\n if (typeof cursor !== 'string' || !cursor.trim() || seen.has(cursor)) throw new IngestConfigError(`Linear returned no new ${resource} cursor.`)\n seen.add(cursor)\n return cursor\n}\n\nfunction validateConnection(value: Connection): Connection {\n if (!value || !Array.isArray(value.nodes) || !value.pageInfo || typeof value.pageInfo.hasNextPage !== 'boolean') {\n throw new IngestConfigError('Linear returned an invalid connection.')\n }\n return value\n}\n" + }, + { + "path": "linear/tests/basic.test.ts", + "type": "registry:file", + "target": "src/integrations/linear/tests/basic.test.ts", + "role": "test", + "content": "import { afterEach, expect, test } from 'bun:test'\nimport { IngestConfigError, runIngestion, selectStreams } from '@chkit/plugin-ingest'\nimport { createMemoryDestination, createMemoryJournal } from '@chkit/plugin-ingest/testing'\n\nimport { linearPipeline } from '../index.js'\n\nconst originalFetch = globalThis.fetch\nconst originalToken = process.env.LINEAR_API_KEY\nconst cutoff = new Date('2026-01-02T00:00:00Z')\nconst emptyComments = { nodes: [], pageInfo: { hasNextPage: false, endCursor: null } }\n\nfunction setFetch(handler: (...args: Parameters) => Promise): void {\n globalThis.fetch = Object.assign(handler, { preconnect: originalFetch.preconnect })\n process.env.LINEAR_API_KEY = 'fixture'\n}\n\nfunction selected() {\n return selectStreams([linearPipeline], []).map((item) => ({ ...item, stream: { ...item.stream, batchSize: 1, retry: { retries: 0 } } }))\n}\n\nafterEach(() => {\n globalThis.fetch = originalFetch\n if (originalToken === undefined) delete process.env.LINEAR_API_KEY\n else process.env.LINEAR_API_KEY = originalToken\n})\n\ntest.serial('Linear filters bounded archived issues and completes comment and issue connections', async () => {\n const requests: { query: string; variables: Record }[] = []\n setFetch(async (_input, init) => {\n const body = JSON.parse(String(init?.body))\n requests.push(body)\n if (body.query.includes('query IssueComments')) return Response.json({ data: { issue: { comments: {\n nodes: [{ id: 'c2', body: 'continued', custom: true }], pageInfo: { hasNextPage: false, endCursor: 'comments-done' },\n } } } })\n if (body.variables.after) return Response.json({ data: { issues: {\n nodes: [{ id: 'b', updatedAt: '2026-01-01T00:00:00Z', comments: emptyComments }], pageInfo: { hasNextPage: false, endCursor: 'done' },\n } } })\n return Response.json({ data: { issues: {\n nodes: [{ id: 'a', updatedAt: '2026-01-01T00:00:00Z', archivedAt: '2026-01-01T00:00:00Z', comments: {\n nodes: [{ id: 'c1', body: 'first' }], pageInfo: { hasNextPage: true, endCursor: 'comments-next' },\n } }], pageInfo: { hasNextPage: true, endCursor: 'issues-next' },\n } } })\n })\n const journal = createMemoryJournal()\n const destination = createMemoryDestination()\n const result = await runIngestion({ selected: selected(), backfill: undefined }, { journal, destination, now: () => cutoff })\n expect(result.ok).toBe(true)\n expect(requests[0]?.query).toContain('includeArchived: true')\n expect(requests[0]?.query).toContain('gte: $from, lte: $to')\n expect(requests[0]?.variables).toEqual({ after: null, from: '1970-01-01T00:00:00.000Z', to: cutoff.toISOString() })\n expect(requests[1]?.variables).toEqual({ id: 'a', after: 'comments-next' })\n expect(requests[2]?.variables.after).toBe('issues-next')\n const rows = destination.tables.get('default.linear_issues_raw') ?? []\n expect(rows.map((row) => row.id)).toEqual(['a', 'b'])\n expect(rows[0]?.raw).toMatchObject({ archivedAt: '2026-01-01T00:00:00Z', comments: { nodes: [{ id: 'c1' }, { id: 'c2', custom: true }], pageInfo: { hasNextPage: false } } })\n expect((await journal.readCheckpoint('linear.issues')).envelope?.state).toEqual({ watermark: cutoff.toISOString() })\n})\n\ntest.serial('Linear comment failures leave the window uncommitted and replay from the beginning', async () => {\n let fail = true\n const starts: unknown[] = []\n setFetch(async (_input, init) => {\n const body = JSON.parse(String(init?.body))\n if (body.query.includes('query IssueComments')) {\n return fail ? Response.json({ errors: [{ message: 'unavailable' }] }) : Response.json({ data: { issue: { comments: emptyComments } } })\n }\n starts.push(body.variables.after)\n return Response.json({ data: { issues: {\n nodes: [{ id: 'a', updatedAt: '2026-01-01T00:00:00Z', comments: { nodes: [], pageInfo: { hasNextPage: true, endCursor: 'next' } } }],\n pageInfo: { hasNextPage: false, endCursor: null },\n } } })\n })\n const journal = createMemoryJournal()\n const destination = createMemoryDestination()\n const request = { selected: selected(), backfill: undefined }\n expect((await runIngestion(request, { journal, destination, now: () => cutoff })).ok).toBe(false)\n expect((await journal.readCheckpoint('linear.issues')).envelope).toBeUndefined()\n expect(destination.tables.size).toBe(0)\n fail = false\n expect((await runIngestion(request, { journal, destination, now: () => cutoff })).ok).toBe(true)\n expect(starts).toEqual([null, null])\n})\n\ntest.serial('Linear destination failure does not save a timestamp watermark', async () => {\n setFetch(async () => Response.json({ data: { issues: { nodes: [{ id: 'a', updatedAt: '2026-01-01T00:00:00Z', comments: emptyComments }], pageInfo: { hasNextPage: false, endCursor: null } } } }))\n const journal = createMemoryJournal()\n const result = await runIngestion({ selected: selected(), backfill: undefined }, {\n journal, destination: { insert: async () => { throw new IngestConfigError('sink rejected') } }, now: () => cutoff,\n })\n expect(result.ok).toBe(false)\n expect((await journal.readCheckpoint('linear.issues')).envelope).toBeUndefined()\n})\n\ntest.serial('Linear empty windows commit and subsequent runs overlap their cutoff', async () => {\n const starts: unknown[] = []\n setFetch(async (_input, init) => {\n starts.push(JSON.parse(String(init?.body)).variables.from)\n return Response.json({ data: { issues: { nodes: [], pageInfo: { hasNextPage: false, endCursor: null } } } })\n })\n const journal = createMemoryJournal()\n const destination = createMemoryDestination()\n const request = { selected: selected(), backfill: undefined }\n expect((await runIngestion(request, { journal, destination, now: () => cutoff })).ok).toBe(true)\n expect((await runIngestion(request, { journal, destination, now: () => new Date('2026-01-03T00:00:00Z') })).ok).toBe(true)\n expect(starts).toEqual(['1970-01-01T00:00:00.000Z', '2026-01-01T23:55:00.000Z'])\n})\n\ntest.serial('Linear retries documented GraphQL HTTP 400 rate-limit errors', async () => {\n let attempts = 0\n setFetch(async () => {\n attempts++\n if (attempts === 1) return Response.json({ errors: [{ message: 'rate limit', extensions: { code: 'RATELIMITED' } }] }, { status: 400 })\n return Response.json({ data: { issues: { nodes: [], pageInfo: { hasNextPage: false, endCursor: null } } } })\n })\n const selected = selectStreams([linearPipeline], []).map((item) => ({ ...item, stream: { ...item.stream, retry: { retries: 1, minTimeout: 0, maxTimeout: 0 } } }))\n const journal = createMemoryJournal()\n const result = await runIngestion({ selected, backfill: undefined }, { journal, destination: createMemoryDestination(), now: () => cutoff })\n expect(result.ok).toBe(true)\n expect(attempts).toBe(2)\n expect(journal.events.some((event) => event.eventKind === 'retry_scheduled' && event.errorClass === 'rate_limited')).toBe(true)\n})\n\ntest.serial('Linear rejects malformed continuation cursors before requesting another page', async () => {\n for (const endCursor of [7, ' ']) {\n let requests = 0\n setFetch(async () => {\n requests++\n return Response.json({ data: { issues: { nodes: [], pageInfo: { hasNextPage: true, endCursor } } } })\n })\n const journal = createMemoryJournal()\n const result = await runIngestion({ selected: selected(), backfill: undefined }, { journal, destination: createMemoryDestination(), now: () => cutoff })\n expect(result.ok).toBe(false)\n expect(result.streams[0]?.error).toContain('no new issue cursor')\n expect(requests).toBe(1)\n expect((await journal.readCheckpoint('linear.issues')).envelope).toBeUndefined()\n }\n})\n" + } + ], + "meta": { + "chkit": { + "formatVersion": 1, + "version": "0.2.0", + "language": "typescript", + "license": "MIT", + "documentation": "https://chkit.obsessiondb.com/integrations/linear/", + "logo": "https://chkit.obsessiondb.com/logos/linear.svg", + "authentication": { + "method": "Personal API key", + "env": [ + "LINEAR_API_KEY" + ], + "setup": [ + "Provide LINEAR_API_KEY with read access to the selected resources.", + "Keep the API key in the runtime environment and rotate it according to the provider policy." + ], + "documentation": "https://linear.app/developers/graphql" + }, + "chkit": "^0.2.0-beta.10", + "ingest": "^0.2.0-beta.10", + "clickhouse": ">=25.3.0", + "root": "src/integrations/linear", + "entry": "index.ts", + "exports": [ + "linearPipeline", + "linear_issuesRaw" + ], + "resources": [ + { + "name": "issues", + "title": "Issues", + "table": "linear_issues_raw", + "description": "Archived and active issues updated inside a timestamp window, with all accessible comment pages.", + "scopes": [], + "strategy": "timestamp", + "endpoints": [ + { + "method": "POST", + "path": "/graphql", + "documentation": "https://linear.app/developers/graphql" + } + ] + } + ], + "views": [], + "sync": { + "description": "Issues use bounded updatedAt windows, five-minute overlap and whole-window checkpoints after destination acknowledgement.", + "schedule": "Run bunx chkit ingest run --tag provider:linear from a scheduler.", + "deletions": "Deleted issues remain stored. Periodic historical reconciliation refreshes comments whose changes may not update the parent issue." + }, + "env": { + "LINEAR_API_KEY": "" + }, + "fileHashes": { + "src/integrations/linear/README.md": "1999b557fc39f34072f1aab16b38b71a62d5a636eb3e9e08043817adc7f9b7bd", + "src/integrations/linear/index.ts": "212a2af04f0080b83d127cab915e2fb20945fc58e119eb3a42aa7264626fa860", + "src/integrations/linear/tests/basic.test.ts": "8a42f2912f1f0ccf40130db92f8b136117d9ea4027c422bbb842683f9e32fc25" + } + } + } +} diff --git a/registry/linear/tests/basic.test.ts b/registry/linear/tests/basic.test.ts new file mode 100644 index 00000000..905c70f7 --- /dev/null +++ b/registry/linear/tests/basic.test.ts @@ -0,0 +1,137 @@ +import { afterEach, expect, test } from 'bun:test' +import { IngestConfigError, runIngestion, selectStreams } from '@chkit/plugin-ingest' +import { createMemoryDestination, createMemoryJournal } from '@chkit/plugin-ingest/testing' + +import { linearPipeline } from '../index.js' + +const originalFetch = globalThis.fetch +const originalToken = process.env.LINEAR_API_KEY +const cutoff = new Date('2026-01-02T00:00:00Z') +const emptyComments = { nodes: [], pageInfo: { hasNextPage: false, endCursor: null } } + +function setFetch(handler: (...args: Parameters) => Promise): void { + globalThis.fetch = Object.assign(handler, { preconnect: originalFetch.preconnect }) + process.env.LINEAR_API_KEY = 'fixture' +} + +function selected() { + return selectStreams([linearPipeline], []).map((item) => ({ ...item, stream: { ...item.stream, batchSize: 1, retry: { retries: 0 } } })) +} + +afterEach(() => { + globalThis.fetch = originalFetch + if (originalToken === undefined) delete process.env.LINEAR_API_KEY + else process.env.LINEAR_API_KEY = originalToken +}) + +test.serial('Linear filters bounded archived issues and completes comment and issue connections', async () => { + const requests: { query: string; variables: Record }[] = [] + setFetch(async (_input, init) => { + const body = JSON.parse(String(init?.body)) + requests.push(body) + if (body.query.includes('query IssueComments')) return Response.json({ data: { issue: { comments: { + nodes: [{ id: 'c2', body: 'continued', custom: true }], pageInfo: { hasNextPage: false, endCursor: 'comments-done' }, + } } } }) + if (body.variables.after) return Response.json({ data: { issues: { + nodes: [{ id: 'b', updatedAt: '2026-01-01T00:00:00Z', comments: emptyComments }], pageInfo: { hasNextPage: false, endCursor: 'done' }, + } } }) + return Response.json({ data: { issues: { + nodes: [{ id: 'a', updatedAt: '2026-01-01T00:00:00Z', archivedAt: '2026-01-01T00:00:00Z', comments: { + nodes: [{ id: 'c1', body: 'first' }], pageInfo: { hasNextPage: true, endCursor: 'comments-next' }, + } }], pageInfo: { hasNextPage: true, endCursor: 'issues-next' }, + } } }) + }) + const journal = createMemoryJournal() + const destination = createMemoryDestination() + const result = await runIngestion({ selected: selected(), backfill: undefined }, { journal, destination, now: () => cutoff }) + expect(result.ok).toBe(true) + expect(requests[0]?.query).toContain('includeArchived: true') + expect(requests[0]?.query).toContain('gte: $from, lte: $to') + expect(requests[0]?.variables).toEqual({ after: null, from: '1970-01-01T00:00:00.000Z', to: cutoff.toISOString() }) + expect(requests[1]?.variables).toEqual({ id: 'a', after: 'comments-next' }) + expect(requests[2]?.variables.after).toBe('issues-next') + const rows = destination.tables.get('default.linear_issues_raw') ?? [] + expect(rows.map((row) => row.id)).toEqual(['a', 'b']) + expect(rows[0]?.raw).toMatchObject({ archivedAt: '2026-01-01T00:00:00Z', comments: { nodes: [{ id: 'c1' }, { id: 'c2', custom: true }], pageInfo: { hasNextPage: false } } }) + expect((await journal.readCheckpoint('linear.issues')).envelope?.state).toEqual({ watermark: cutoff.toISOString() }) +}) + +test.serial('Linear comment failures leave the window uncommitted and replay from the beginning', async () => { + let fail = true + const starts: unknown[] = [] + setFetch(async (_input, init) => { + const body = JSON.parse(String(init?.body)) + if (body.query.includes('query IssueComments')) { + return fail ? Response.json({ errors: [{ message: 'unavailable' }] }) : Response.json({ data: { issue: { comments: emptyComments } } }) + } + starts.push(body.variables.after) + return Response.json({ data: { issues: { + nodes: [{ id: 'a', updatedAt: '2026-01-01T00:00:00Z', comments: { nodes: [], pageInfo: { hasNextPage: true, endCursor: 'next' } } }], + pageInfo: { hasNextPage: false, endCursor: null }, + } } }) + }) + const journal = createMemoryJournal() + const destination = createMemoryDestination() + const request = { selected: selected(), backfill: undefined } + expect((await runIngestion(request, { journal, destination, now: () => cutoff })).ok).toBe(false) + expect((await journal.readCheckpoint('linear.issues')).envelope).toBeUndefined() + expect(destination.tables.size).toBe(0) + fail = false + expect((await runIngestion(request, { journal, destination, now: () => cutoff })).ok).toBe(true) + expect(starts).toEqual([null, null]) +}) + +test.serial('Linear destination failure does not save a timestamp watermark', async () => { + setFetch(async () => Response.json({ data: { issues: { nodes: [{ id: 'a', updatedAt: '2026-01-01T00:00:00Z', comments: emptyComments }], pageInfo: { hasNextPage: false, endCursor: null } } } })) + const journal = createMemoryJournal() + const result = await runIngestion({ selected: selected(), backfill: undefined }, { + journal, destination: { insert: async () => { throw new IngestConfigError('sink rejected') } }, now: () => cutoff, + }) + expect(result.ok).toBe(false) + expect((await journal.readCheckpoint('linear.issues')).envelope).toBeUndefined() +}) + +test.serial('Linear empty windows commit and subsequent runs overlap their cutoff', async () => { + const starts: unknown[] = [] + setFetch(async (_input, init) => { + starts.push(JSON.parse(String(init?.body)).variables.from) + return Response.json({ data: { issues: { nodes: [], pageInfo: { hasNextPage: false, endCursor: null } } } }) + }) + const journal = createMemoryJournal() + const destination = createMemoryDestination() + const request = { selected: selected(), backfill: undefined } + expect((await runIngestion(request, { journal, destination, now: () => cutoff })).ok).toBe(true) + expect((await runIngestion(request, { journal, destination, now: () => new Date('2026-01-03T00:00:00Z') })).ok).toBe(true) + expect(starts).toEqual(['1970-01-01T00:00:00.000Z', '2026-01-01T23:55:00.000Z']) +}) + +test.serial('Linear retries documented GraphQL HTTP 400 rate-limit errors', async () => { + let attempts = 0 + setFetch(async () => { + attempts++ + if (attempts === 1) return Response.json({ errors: [{ message: 'rate limit', extensions: { code: 'RATELIMITED' } }] }, { status: 400 }) + return Response.json({ data: { issues: { nodes: [], pageInfo: { hasNextPage: false, endCursor: null } } } }) + }) + const selected = selectStreams([linearPipeline], []).map((item) => ({ ...item, stream: { ...item.stream, retry: { retries: 1, minTimeout: 0, maxTimeout: 0 } } })) + const journal = createMemoryJournal() + const result = await runIngestion({ selected, backfill: undefined }, { journal, destination: createMemoryDestination(), now: () => cutoff }) + expect(result.ok).toBe(true) + expect(attempts).toBe(2) + expect(journal.events.some((event) => event.eventKind === 'retry_scheduled' && event.errorClass === 'rate_limited')).toBe(true) +}) + +test.serial('Linear rejects malformed continuation cursors before requesting another page', async () => { + for (const endCursor of [7, ' ']) { + let requests = 0 + setFetch(async () => { + requests++ + return Response.json({ data: { issues: { nodes: [], pageInfo: { hasNextPage: true, endCursor } } } }) + }) + const journal = createMemoryJournal() + const result = await runIngestion({ selected: selected(), backfill: undefined }, { journal, destination: createMemoryDestination(), now: () => cutoff }) + expect(result.ok).toBe(false) + expect(result.streams[0]?.error).toContain('no new issue cursor') + expect(requests).toBe(1) + expect((await journal.readCheckpoint('linear.issues')).envelope).toBeUndefined() + } +}) diff --git a/registry/slack/README.md b/registry/slack/README.md index a1eb328b..5c06c7ca 100644 --- a/registry/slack/README.md +++ b/registry/slack/README.md @@ -30,7 +30,7 @@ Use an external scheduler to repeat ingestion, with one ingestion process per pr ## Included resources -All three streams read their complete selected resource on every run. Each discovers its own authenticated workspace through `auth.test`; the messages stream independently discovers conversations. +Channels and users read their complete selected resource on every run. Messages bootstrap retained history and then read checkpointed time ranges. Each reader discovers its authenticated workspace through `auth.test`; messages independently discover conversations. | Stream tag | Default raw table | Required scopes for defaults | |---|---|---| @@ -65,7 +65,7 @@ Slack documents one request per minute and 15 objects per page for new commercia The pipeline runs one stream, fetch, and load at a time. Requests have a 30-second timeout and use the ingestion executor's retry and cancellation context. The pipeline permits five retries after the initial attempt, with exponential backoff from 1 to 60 seconds. HTTP `429` honors `Retry-After`; Slack rate-limit responses and transient server codes also retry. Authentication and permission errors, including Slack `ok: false` responses, fail visibly instead of becoming empty datasets. -Full history and thread reads can take hours or days at the conservative defaults. Restrict conversations and selected streams, and choose `--max-duration` for the expected scan. The CLI's default duration is 3600 seconds. Interrupted runs restart their scans rather than resuming from a saved cursor. +History and thread reads can take hours or days at conservative defaults. Restrict conversations and selected streams, and choose `--max-duration` for the workload. The CLI default is 3600 seconds. Interrupted message reads resume a saved timestamp boundary and retained thread work; metadata reads restart. To read only messages: @@ -103,17 +103,23 @@ LIMIT 20; There are no packaged typed views. Add SQL projections in the project's schema for the fields needed by the application. Preserve source and workspace identity in joins. -## Full reads and coverage limits +## Incremental messages and recovery -Each run starts at the beginning and reads history without date bounds or subtype filtering. Within each channel, history pagination finishes before threads are read. Old roots are scanned again to capture late replies when Slack still retains and exposes them. Short or empty pages continue when a cursor exists; repeated cursors or `has_more: true` without a cursor fail explicitly. Full reads save no incremental checkpoint. +The first messages run reads from `historyFrom` (`0.000000` by default) to a fixed execution cutoff. Later runs begin at the completed watermark minus `overlapMs` (one day by default). `reconcileIntervalMs` (seven days) periodically triggers a full historical scan to revisit old edits and roots that gained late replies. Newly discovered channels bootstrap their retained history even during an incremental run. + +Checkpoints retain exact timestamp strings, the authenticated workspace and query scope, fixed bounds, channel position, and pending thread work. History progresses newest first; replies progress oldest first. Each loaded history page retains its thread work before proceeding to older history, keeping the pending queue bounded to one page. Restart finishes those threads before continuing. Temporary cursors are used only to cross empty API pages; expiration restarts that unfinished timestamp interval. Empty terminal pages also commit progress. + +`historyFrom`, `overlapMs`, and `reconcileIntervalMs` belong in `config.ts`. Zero reconciliation interval means every completed run starts a full scan. Keep source identity and selection stable; changing the account, channels, types, or reply policy fails against existing state. Use a new source ID for a deliberate separate installation. Version 0.2.0 preserves existing raw tables and row IDs. Earlier full readers had no saved provider state; the first upgraded messages run bootstraps once. The destination stores the latest **observed** payload per identity. Deleted messages, deselected conversations, and data made inaccessible remain in ClickHouse. Loaded batches stay visible when a run fails; there is no atomic snapshot replacement or rollback. Changes during pagination may cause misses or repeated observations. Slack retention, permissions, and token access determine historical coverage. -The integration has no deletion reconciliation, permanent edit log, webhooks, or continuous change capture. Date-range backfill flags do not make these readers bounded or incremental. +The integration has no deletion reconciliation, permanent edit log, webhooks, or continuous change capture. Recent overlap alone does not capture all old edits or late replies; periodic full reconciliation catches those still retained and accessible. Deleted messages no longer returned by Slack remain stored. + +Messages support isolated date-bound backfills with `--backfill --from --to `. Unfinished backfills retain their original bounds; changing those bounds requires another ID. Channels and users remain full reads and do not interpret date ranges. As with scheduled reads, historical raw observations loaded later can replace newer observations because replacement uses ingestion time. ## Customize the source -`sources/channels.ts`, `sources/users.ts`, and `sources/messages.ts` each contain a resource's raw schema and reader. `client.ts` handles HTTP, authentication, validation, pagination, row IDs, and pacing; `pipeline.ts` composes the streams. +`sources/channels.ts`, `sources/users.ts`, and `sources/messages.ts` each contain a resource's raw schema and reader. `client.ts` handles HTTP, authentication, validation, pagination, row IDs, and pacing; `state.ts` validates timestamp checkpoints; `pipeline.ts` composes the streams. Remove a stream from `pipeline.ts` to stop ingestion while retaining its schema export in `index.ts` to preserve the table. Removing schema exports can generate destructive migration operations; review them. Test reader changes and revisit pacing when changing concurrency. @@ -123,13 +129,13 @@ Install the optional portable test suite, then run it: ```sh bunx chkit add slack --with-tests -bun test src/integrations/slack/tests/slack.test.ts +bun test src/integrations/slack/tests/slack.test.ts src/integrations/slack/tests/checkpoints.test.ts ``` For a custom install path, point the command at its `tests/slack.test.ts`. Tests use sanitized responses, mocked HTTP, and in-memory ingestion destinations. They require Bun and no Slack credentials or ClickHouse server. Update fixtures and expectations alongside customized readers. ## Smoke check -After a migration and successful run, inspect `chkit ingest status --tag provider:slack` and query `SELECT count() FROM default.slack_messages_raw FINAL`. Compare a known message and an older thread with their stored payloads. Add a reply to that test thread, rerun ingestion, and verify the reply is collected. Row counts alone do not establish complete live API coverage. +After a migration and successful run, inspect `chkit ingest status --tag provider:slack` and query `SELECT count() FROM default.slack_messages_raw FINAL`. Compare a known message and an older thread with their stored payloads. Add a reply to that old thread, then run a messages-only backfill with a new ID and `--from` before the root's timestamp to verify the reply is collected. A normal incremental rerun revisits old roots during the next scheduled full reconciliation. Row counts alone do not establish complete live API coverage. The [Slack integration guide](https://chkit.obsessiondb.com/integrations/slack/) covers installation, SQL queries, scheduling, and recovery. diff --git a/registry/slack/client.ts b/registry/slack/client.ts index 43d8e78d..dbca5231 100644 --- a/registry/slack/client.ts +++ b/registry/slack/client.ts @@ -18,8 +18,13 @@ export interface CollectionRequest { idField: 'id' | 'ts' query?: Record pageSize?: number + /** Message endpoints support timestamp pagination without an opaque cursor. */ + timePagination?: boolean } +export interface SlackPage { items: SlackEntity[]; cursor: string; hasMore: boolean } +export class SlackCursorExpiredError extends IngestConfigError {} + const defaultDeps: SlackClientDeps = { fetch: (url, init) => fetch(url, init), token: () => process.env.SLACK_API_TOKEN, @@ -32,20 +37,11 @@ export async function* readCollection( request: CollectionRequest, deps: SlackClientDeps = defaultDeps, ): AsyncGenerator { - const messageRequest = request.field === 'messages' - const limit = request.pageSize ?? (messageRequest ? slackConfig.messagePageSize : slackConfig.pageSize) - if (!Number.isSafeInteger(limit) || limit <= 0 || limit >= 1_000) { - throw new IngestConfigError('Slack page sizes must be safe integers between 1 and 999.') - } const seenCursors = new Set() let cursor = '' while (true) { context.signal.throwIfAborted() - const query = { ...request.query, limit: String(limit), ...(cursor ? { cursor } : {}) } - const page = await context.attempt(async (signal) => { - const payload = await requestSlack(request.method, query, signal, deps) - return parsePage(payload, request) - }, { label: `GET /api/${request.method}` }) + const page = await readCollectionPage(context, request, cursor, deps) if (page.items.length > 0) yield page.items // Empty/short pages can still have a cursor. Only cursor exhaustion ends a scan. if (!page.cursor) return @@ -55,6 +51,14 @@ export async function* readCollection( } } +/** Retains empty pages and continuation metadata needed by checkpointed readers. */ +export async function readCollectionPage(context: FetchContext, request: CollectionRequest, cursor = '', deps: SlackClientDeps = defaultDeps): Promise { + const limit = request.pageSize ?? (request.field === 'messages' ? slackConfig.messagePageSize : slackConfig.pageSize) + if (!Number.isSafeInteger(limit) || limit <= 0 || limit >= 1_000) throw new IngestConfigError('Slack page sizes must be safe integers between 1 and 999.') + const query = { ...request.query, limit: String(limit), ...(cursor ? { cursor } : {}) } + return context.attempt(async (signal) => parsePage(await requestSlack(request.method, query, signal, deps), request), { label: `GET /api/${request.method}` }) +} + /** Discover independently in each reader; pipeline stream order is not a dependency. */ export async function discoverChannels(context: FetchContext, deps?: SlackClientDeps): Promise { const allowedTypes = ['public_channel', 'private_channel', 'im', 'mpim'] @@ -151,6 +155,7 @@ async function requestSlack( // Slack reports authentication, permission, and transient errors in HTTP 200 responses. if (!payload.ok) { const code = typeof payload.error === 'string' ? payload.error : 'unknown_error' + if (code === 'invalid_cursor') throw new SlackCursorExpiredError(`Slack ${method} pagination cursor expired.`) if (code === 'ratelimited' || code === 'rate_limited') { const headers = new Headers(response.headers) if (!headers.has('Retry-After')) headers.set('Retry-After', '60') @@ -190,10 +195,10 @@ function parsePage(payload: Record, request: CollectionRequest) if (payload.has_more !== undefined && typeof payload.has_more !== 'boolean') { throw new IngestConfigError(`Slack ${request.method} returned an invalid has_more flag.`) } - if (payload.has_more === true && !cursor) { + if (payload.has_more === true && !cursor && !request.timePagination) { throw new IngestConfigError(`Slack ${request.method} reports more data without a pagination cursor; refusing an incomplete scan.`) } - return { items, cursor } + return { items, cursor, hasMore: payload.has_more === true || cursor.length > 0 } } function isObject(value: unknown): value is Record { diff --git a/registry/slack/config.ts b/registry/slack/config.ts index ab5af1e1..a0b241c8 100644 --- a/registry/slack/config.ts +++ b/registry/slack/config.ts @@ -11,6 +11,10 @@ export interface SlackConfig { messagePageSize: number requestIntervalMs: number messageIntervalMs: number + /** Earliest message to bootstrap/reconcile; retain the exact timestamp string. */ + historyFrom: string + overlapMs: number + reconcileIntervalMs: number } // Credentials are read only in client.ts at request time. Keep these values stable across runs. @@ -27,4 +31,7 @@ export const slackConfig: SlackConfig = { messagePageSize: 15, requestIntervalMs: 3_000, messageIntervalMs: 60_000, + historyFrom: '0.000000', + overlapMs: 86_400_000, + reconcileIntervalMs: 7 * 86_400_000, } diff --git a/registry/slack/manifest.json b/registry/slack/manifest.json index b07d86ac..f1b4d8e8 100644 --- a/registry/slack/manifest.json +++ b/registry/slack/manifest.json @@ -3,71 +3,165 @@ "name": "slack", "type": "registry:item", "title": "Slack", - "description": "Editable Slack API readers and raw ClickHouse tables for conversations, users, messages, and thread replies.", - "dependencies": ["@chkit/core@^0.2.0-beta.8", "@chkit/plugin-ingest@^0.2.0-beta.8"], - "devDependencies": ["@types/bun@^1.2.0"], + "description": "Editable raw Slack readers with resumable message ranges, retained thread work, and historical reconciliation.", + "dependencies": [ + "@chkit/core@^0.2.0-beta.10", + "@chkit/plugin-ingest@^0.2.0-beta.10" + ], + "devDependencies": [ + "@types/bun@^1.2.0" + ], "files": [ - { "path": "README.md", "type": "registry:file", "target": "src/integrations/slack/README.md" }, - { "path": "client.ts", "type": "registry:file", "target": "src/integrations/slack/client.ts" }, - { "path": "config.ts", "type": "registry:file", "target": "src/integrations/slack/config.ts" }, - { "path": "index.ts", "type": "registry:file", "target": "src/integrations/slack/index.ts" }, - { "path": "pipeline.ts", "type": "registry:file", "target": "src/integrations/slack/pipeline.ts" }, - { "path": "sources/channels.ts", "type": "registry:file", "target": "src/integrations/slack/sources/channels.ts" }, - { "path": "sources/users.ts", "type": "registry:file", "target": "src/integrations/slack/sources/users.ts" }, - { "path": "sources/messages.ts", "type": "registry:file", "target": "src/integrations/slack/sources/messages.ts" }, - { "path": "tests/slack.test.ts", "type": "registry:file", "target": "src/integrations/slack/tests/slack.test.ts", "role": "test" }, - { "path": "tests/fixtures.ts", "type": "registry:file", "target": "src/integrations/slack/tests/fixtures.ts", "role": "test" } + { + "path": "README.md", + "type": "registry:file", + "target": "src/integrations/slack/README.md" + }, + { + "path": "client.ts", + "type": "registry:file", + "target": "src/integrations/slack/client.ts" + }, + { + "path": "config.ts", + "type": "registry:file", + "target": "src/integrations/slack/config.ts" + }, + { + "path": "index.ts", + "type": "registry:file", + "target": "src/integrations/slack/index.ts" + }, + { + "path": "pipeline.ts", + "type": "registry:file", + "target": "src/integrations/slack/pipeline.ts" + }, + { + "path": "sources/channels.ts", + "type": "registry:file", + "target": "src/integrations/slack/sources/channels.ts" + }, + { + "path": "sources/users.ts", + "type": "registry:file", + "target": "src/integrations/slack/sources/users.ts" + }, + { + "path": "sources/messages.ts", + "type": "registry:file", + "target": "src/integrations/slack/sources/messages.ts" + }, + { + "path": "tests/slack.test.ts", + "type": "registry:file", + "target": "src/integrations/slack/tests/slack.test.ts", + "role": "test" + }, + { + "path": "tests/fixtures.ts", + "type": "registry:file", + "target": "src/integrations/slack/tests/fixtures.ts", + "role": "test" + }, + { + "path": "state.ts", + "type": "registry:file", + "target": "src/integrations/slack/state.ts" + }, + { + "path": "tests/checkpoints.test.ts", + "type": "registry:file", + "target": "src/integrations/slack/tests/checkpoints.test.ts", + "role": "test" + } ], "meta": { "chkit": { "formatVersion": 1, - "version": "0.1.1", + "version": "0.2.0", "documentation": "https://chkit.obsessiondb.com/integrations/slack/", "logo": "https://chkit.obsessiondb.com/logos/slack.svg", "language": "typescript", "license": "MIT", - "chkit": "^0.2.0-beta.8", - "ingest": "^0.2.0-beta.8", + "chkit": "^0.2.0-beta.10", + "ingest": "^0.2.0-beta.10", "clickhouse": ">=25.3.0", "root": "src/integrations/slack", "entry": "index.ts", - "exports": ["slack", "slackChannelsRaw", "slackUsersRaw", "slackMessagesRaw"], + "exports": [ + "slack", + "slackChannelsRaw", + "slackUsersRaw", + "slackMessagesRaw" + ], "resources": [ { "name": "channels", "title": "Conversations", "table": "slack_channels_raw", "description": "Selected accessible conversations, including archived channels, with complete returned topics, purpose, membership flags, and shared-channel metadata. Public channels by default; private channels, DMs, and group DMs are opt-in.", - "scopes": ["channels:read"], + "scopes": [ + "channels:read" + ], "strategy": "full", - "endpoints": [{ "method": "GET", "path": "/conversations.list", "documentation": "https://docs.slack.dev/reference/methods/conversations.list/" }] + "endpoints": [ + { + "method": "GET", + "path": "/conversations.list", + "documentation": "https://docs.slack.dev/reference/methods/conversations.list/" + } + ] }, { "name": "users", "title": "Users", "table": "slack_users_raw", "description": "Complete returned workspace member and bot profiles, IDs, names, status, timezone, and deactivation flags. Email requires the optional users:read.email scope.", - "scopes": ["users:read"], + "scopes": [ + "users:read" + ], "strategy": "full", - "endpoints": [{ "method": "GET", "path": "/users.list", "documentation": "https://docs.slack.dev/reference/methods/users.list/" }] + "endpoints": [ + { + "method": "GET", + "path": "/users.list", + "documentation": "https://docs.slack.dev/reference/methods/users.list/" + } + ] }, { "name": "messages", "title": "Messages and thread replies", "table": "slack_messages_raw", - "description": "All retained accessible history in selected conversations and replies to every discovered thread, preserving timestamp strings, text, blocks, attachments, file references, reactions, subtypes, and metadata as returned. No file downloads or separate reaction enumeration.", - "scopes": ["channels:read", "channels:history"], - "strategy": "full", + "description": "Raw messages and replies selected by checkpointed timestamp windows, with retained thread work and periodic full historical reconciliation. Slack timestamps remain exact strings.", + "scopes": [ + "channels:read", + "channels:history" + ], + "strategy": "cursor", "endpoints": [ - { "method": "GET", "path": "/conversations.history", "documentation": "https://docs.slack.dev/reference/methods/conversations.history/" }, - { "method": "GET", "path": "/conversations.replies", "documentation": "https://docs.slack.dev/reference/methods/conversations.replies/" } + { + "method": "GET", + "path": "/conversations.history", + "documentation": "https://docs.slack.dev/reference/methods/conversations.history/" + }, + { + "method": "GET", + "path": "/conversations.replies", + "documentation": "https://docs.slack.dev/reference/methods/conversations.replies/" + } ] } ], - "env": { "SLACK_API_TOKEN": "" }, + "env": { + "SLACK_API_TOKEN": "" + }, "authentication": { "method": "Slack OAuth bearer token (user token recommended)", - "env": ["SLACK_API_TOKEN"], + "env": [ + "SLACK_API_TOKEN" + ], "documentation": "https://docs.slack.dev/authentication/tokens/", "setup": [ "Create a Slack app for the workspace at api.slack.com/apps. Open OAuth & Permissions.", @@ -79,8 +173,8 @@ }, "views": [], "sync": { - "description": "Each run fully reads enabled resources. Messages discover their own selected conversations, read retained history, and revisit every discovered thread when includeReplies is true. Conservative history/replies defaults use 15-item pages and a 60-second delay before requests; internal apps can tune these to their Slack tier.", - "schedule": "Run chkit ingest run through cron, CI, or an existing scheduler, with one process per project/target. Select channel IDs and size the execution budget for a complete scan; interrupted full reads restart from the beginning.", + "description": "Channels and users are full reads. Messages bootstrap retained history, then read overlapping time ranges. Each page checkpoints its timestamp frontier and pending threads after writes. Weekly full reconciliation revisits old roots and edits; temporary Slack cursors are never persisted.", + "schedule": "Run chkit ingest run through cron, CI, or an existing scheduler, with one process per project/target. Interrupted message windows resume their fixed bounds and pending threads; users/channels restart full reads. Conservative message pacing remains 15 items and one request per minute.", "deletions": "Later observations replace stable row IDs, but deleted, inaccessible, or deselected records remain. The integration does not consume Slack events, propagate deletions, or maintain a complete change log." } } diff --git a/registry/slack/pipeline.ts b/registry/slack/pipeline.ts index 8229b074..1947fdbe 100644 --- a/registry/slack/pipeline.ts +++ b/registry/slack/pipeline.ts @@ -5,6 +5,7 @@ import { slackConfig } from './config.js' import { readChannels, slackChannelsRaw } from './sources/channels.js' import { readUsers, slackUsersRaw } from './sources/users.js' import { readMessages, slackMessagesRaw } from './sources/messages.js' +import { slackMessageStrategy } from './state.js' const streamOptions = { classifyError: classifySlackError, batchSize: 500 } @@ -19,6 +20,6 @@ export const slack = definePipeline({ streams: [ defineStream({ ...streamOptions, id: `${slackConfig.sourceId}.channels`, tags: ['resource:channels'], destination: slackChannelsRaw, read: readChannels }), defineStream({ ...streamOptions, id: `${slackConfig.sourceId}.users`, tags: ['resource:users'], destination: slackUsersRaw, read: readUsers }), - defineStream({ ...streamOptions, id: `${slackConfig.sourceId}.messages`, tags: ['resource:messages'], destination: slackMessagesRaw, read: readMessages }), + defineStream({ ...streamOptions, batchSize: 1, id: `${slackConfig.sourceId}.messages`, tags: ['resource:messages'], destination: slackMessagesRaw, incremental: slackMessageStrategy, read: readMessages }), ], }) diff --git a/registry/slack/releases/0.2.0.json b/registry/slack/releases/0.2.0.json new file mode 100644 index 00000000..c96c0e20 --- /dev/null +++ b/registry/slack/releases/0.2.0.json @@ -0,0 +1,208 @@ +{ + "$schema": "https://ui.shadcn.com/schema/registry-item.json", + "name": "slack", + "type": "registry:item", + "title": "Slack", + "description": "Editable raw Slack readers with resumable message ranges, retained thread work, and historical reconciliation.", + "dependencies": [ + "@chkit/core@^0.2.0-beta.10", + "@chkit/plugin-ingest@^0.2.0-beta.10" + ], + "devDependencies": [ + "@types/bun@^1.2.0" + ], + "files": [ + { + "path": "slack/README.md", + "type": "registry:file", + "target": "src/integrations/slack/README.md", + "content": "# Slack raw ingestion\n\nThis is editable source code: change the readers, schemas, and `pipeline.ts` directly. It uses the [chkit CLI](https://www.npmjs.com/package/chkit), `@chkit/core`, and `@chkit/plugin-ingest`; subsequent runs execute the installed code without contacting the registry.\n\n## First run\n\nSet `SLACK_API_TOKEN` in the execution environment and configure the project's direct ClickHouse connection. Native JSON requires ClickHouse 25.3 or newer. Review `config.ts` before generating the first migration; the default database is `default` and table names start with `slack_`.\n\n### Create a Slack token\n\n1. Create a Slack app for the intended workspace through [Slack app settings](https://docs.slack.dev/app-management/quickstart-app-settings/).\n2. Open **OAuth & Permissions** and add **User Token Scopes** `channels:read`, `channels:history`, and `users:read` for the default public-channel integration. Add `users:read.email` if profile email addresses are needed.\n3. Install or reinstall the app in the workspace and approve the requested scopes. Copy the **User OAuth Token** from **OAuth & Permissions**.\n4. Set `SLACK_API_TOKEN` in the project's `.env` or the scheduler's secret environment.\n\nUser OAuth tokens are recommended for broad history and thread access. A bot may list public channels it has not joined, so select IDs of joined conversations rather than `channels: undefined`. Thread reads also require matching history scopes and conversation access; see [`conversations.replies`](https://docs.slack.dev/reference/methods/conversations.replies/). An enabled thread permission failure fails the messages stream visibly. Fix token access or explicitly set `includeReplies: false` for history-only ingestion. See [Slack tokens](https://docs.slack.dev/authentication/tokens/).\n\nBun loads a project `.env`. A scheduler or other runtime must explicitly supply the variables. `.env.example` lists names and does not load credentials. Tokens are read when requests run; schema imports, `check`, migration generation, and `ingest list` make no Slack requests and need no token. The integration does not obtain or refresh OAuth credentials.\n\n```sh\nbunx chkit check\nbunx chkit ingest list --tag provider:slack\nbunx chkit generate --name add-slack\n# Review the generated SQL.\nbunx chkit migrate --apply\nbunx chkit ingest run --tag provider:slack\n```\n\nUse an external scheduler to repeat ingestion, with one ingestion process per project and destination at a time.\n\n## Included resources\n\nChannels and users read their complete selected resource on every run. Messages bootstrap retained history and then read checkpointed time ranges. Each reader discovers its authenticated workspace through `auth.test`; messages independently discover conversations.\n\n| Stream tag | Default raw table | Required scopes for defaults |\n|---|---|---|\n| `resource:channels` | `slack_channels_raw` | `channels:read` |\n| `resource:users` | `slack_users_raw` | `users:read`; optional `users:read.email` |\n| `resource:messages` | `slack_messages_raw` | `channels:read`, `channels:history` |\n\n- **Channels:** conversation metadata returned by [`conversations.list`](https://docs.slack.dev/reference/methods/conversations.list/), including archived conversations when returned. No separate `conversations.info` expansion or membership request.\n- **Users:** complete objects returned by [`users.list`](https://docs.slack.dev/reference/methods/users.list/), including profiles, bots, and deleted-user flags.\n- **Messages:** all accessible retained [`conversations.history`](https://docs.slack.dev/reference/methods/conversations.history/) pages for selected conversations. With `includeReplies: true`, roots with `reply_count > 0` also fetch every `conversations.replies` page. Replies and their parents share the raw messages table.\n\nMessages preserve blocks, attachments, reactions, metadata, and file references when returned. File references do not download files. No separate files, reactions, pins, canvases, search, or events reader is included.\n\n## Choose conversations and request pacing\n\nThe default config selects public channels, includes replies, uses metadata pages of 200, and uses history/reply pages of 15. `requestIntervalMs` defaults to 3000; `messageIntervalMs` defaults to 60000.\n\nSet `channels` to an array of Slack conversation IDs to restrict channels and messages. `undefined` selects all discovered conversations; `[]` selects none. Selection uses IDs, not names. A configured ID absent from discovery fails visibly. Users remain workspace-wide.\n\n`conversationTypes` defaults to `['public_channel']`. Additional types require these read scopes and history scopes:\n\n| Type | Discovery | History and replies |\n|---|---|---|\n| `public_channel` | `channels:read` | `channels:history` |\n| `private_channel` | `groups:read` | `groups:history` |\n| `im` | `im:read` | `im:history` |\n| `mpim` | `mpim:read` | `mpim:history` |\n\nKeep `sourceId` stable after ingestion begins. A second workspace should use a distinct source ID and a separate table prefix or database. Changing these values changes stream and storage identities and requires a deliberate migration.\n\nSlack documents one request per minute and 15 objects per page for new commercially distributed apps outside the Slack Marketplace. Marketplace and internal customer-built apps retain Tier 3 limits. See the current [history method](https://docs.slack.dev/reference/methods/conversations.history/) and [rate-limit clarification](https://docs.slack.dev/changelog/2025/06/03/rate-limits-clarity/). For an internal app entitled to Tier 3, edit `messagePageSize` to 200 and `messageIntervalMs` to 1200 after confirming its limits.\n\nThe pipeline runs one stream, fetch, and load at a time. Requests have a 30-second timeout and use the ingestion executor's retry and cancellation context. The pipeline permits five retries after the initial attempt, with exponential backoff from 1 to 60 seconds. HTTP `429` honors `Retry-After`; Slack rate-limit responses and transient server codes also retry. Authentication and permission errors, including Slack `ok: false` responses, fail visibly instead of becoming empty datasets.\n\nHistory and thread reads can take hours or days at conservative defaults. Restrict conversations and selected streams, and choose `--max-duration` for the workload. The CLI default is 3600 seconds. Interrupted message reads resume a saved timestamp boundary and retained thread work; metadata reads restart.\n\nTo read only messages:\n\n```sh\nbunx chkit ingest run --tag provider:slack --tag resource:messages --max-duration 86400\n```\n\n## Stored data and identities\n\nEvery raw table contains stable `id`, native JSON `raw`, and chkit batch/run/ingestion-time columns. `raw` contains `source_id`, authenticated `team_id`, and the complete provider object under `data`. Messages also contain `channel_id`.\n\nThe row IDs are JSON-encoded arrays:\n\n| Resource | Identity |\n|---|---|\n| Channels | `[sourceId, 'channels', teamId, conversation.id]` |\n| Users | `[sourceId, 'users', teamId, user.id]` |\n| Messages | `[sourceId, 'messages', teamId, channelId, message.ts]` |\n\nMessage timestamps remain strings. A parent returned by `conversations.replies` has the same identity as its history observation. Replies use their own timestamps; their `thread_ts` retains the root reference. Channel names, user emails, and message text are never keys.\n\nThe tables use `ReplacingMergeTree(_chkit_ingested_at)`. Later ingestion time wins; there is no Slack source-version ordering. Use `FINAL` for the latest observed row:\n\n```sql\nWITH toJSONString(raw) AS payload\nSELECT\n JSONExtractString(payload, 'channel_id') AS channel_id,\n JSONExtractString(payload, 'data', 'ts') AS ts,\n JSONExtractString(payload, 'data', 'thread_ts') AS thread_ts,\n JSONExtractString(payload, 'data', 'user') AS user_id,\n JSONExtractString(payload, 'data', 'text') AS text\nFROM default.slack_messages_raw FINAL\nLIMIT 20;\n```\n\nThere are no packaged typed views. Add SQL projections in the project's schema for the fields needed by the application. Preserve source and workspace identity in joins.\n\n## Incremental messages and recovery\n\nThe first messages run reads from `historyFrom` (`0.000000` by default) to a fixed execution cutoff. Later runs begin at the completed watermark minus `overlapMs` (one day by default). `reconcileIntervalMs` (seven days) periodically triggers a full historical scan to revisit old edits and roots that gained late replies. Newly discovered channels bootstrap their retained history even during an incremental run.\n\nCheckpoints retain exact timestamp strings, the authenticated workspace and query scope, fixed bounds, channel position, and pending thread work. History progresses newest first; replies progress oldest first. Each loaded history page retains its thread work before proceeding to older history, keeping the pending queue bounded to one page. Restart finishes those threads before continuing. Temporary cursors are used only to cross empty API pages; expiration restarts that unfinished timestamp interval. Empty terminal pages also commit progress.\n\n`historyFrom`, `overlapMs`, and `reconcileIntervalMs` belong in `config.ts`. Zero reconciliation interval means every completed run starts a full scan. Keep source identity and selection stable; changing the account, channels, types, or reply policy fails against existing state. Use a new source ID for a deliberate separate installation. Version 0.2.0 preserves existing raw tables and row IDs. Earlier full readers had no saved provider state; the first upgraded messages run bootstraps once.\n\nThe destination stores the latest **observed** payload per identity. Deleted messages, deselected conversations, and data made inaccessible remain in ClickHouse. Loaded batches stay visible when a run fails; there is no atomic snapshot replacement or rollback. Changes during pagination may cause misses or repeated observations. Slack retention, permissions, and token access determine historical coverage.\n\nThe integration has no deletion reconciliation, permanent edit log, webhooks, or continuous change capture. Recent overlap alone does not capture all old edits or late replies; periodic full reconciliation catches those still retained and accessible. Deleted messages no longer returned by Slack remain stored.\n\nMessages support isolated date-bound backfills with `--backfill --from --to `. Unfinished backfills retain their original bounds; changing those bounds requires another ID. Channels and users remain full reads and do not interpret date ranges. As with scheduled reads, historical raw observations loaded later can replace newer observations because replacement uses ingestion time.\n\n## Customize the source\n\n`sources/channels.ts`, `sources/users.ts`, and `sources/messages.ts` each contain a resource's raw schema and reader. `client.ts` handles HTTP, authentication, validation, pagination, row IDs, and pacing; `state.ts` validates timestamp checkpoints; `pipeline.ts` composes the streams.\n\nRemove a stream from `pipeline.ts` to stop ingestion while retaining its schema export in `index.ts` to preserve the table. Removing schema exports can generate destructive migration operations; review them. Test reader changes and revisit pacing when changing concurrency.\n\n## Fixture tests\n\nInstall the optional portable test suite, then run it:\n\n```sh\nbunx chkit add slack --with-tests\nbun test src/integrations/slack/tests/slack.test.ts src/integrations/slack/tests/checkpoints.test.ts\n```\n\nFor a custom install path, point the command at its `tests/slack.test.ts`. Tests use sanitized responses, mocked HTTP, and in-memory ingestion destinations. They require Bun and no Slack credentials or ClickHouse server. Update fixtures and expectations alongside customized readers.\n\n## Smoke check\n\nAfter a migration and successful run, inspect `chkit ingest status --tag provider:slack` and query `SELECT count() FROM default.slack_messages_raw FINAL`. Compare a known message and an older thread with their stored payloads. Add a reply to that old thread, then run a messages-only backfill with a new ID and `--from` before the root's timestamp to verify the reply is collected. A normal incremental rerun revisits old roots during the next scheduled full reconciliation. Row counts alone do not establish complete live API coverage.\n\nThe [Slack integration guide](https://chkit.obsessiondb.com/integrations/slack/) covers installation, SQL queries, scheduling, and recovery.\n" + }, + { + "path": "slack/client.ts", + "type": "registry:file", + "target": "src/integrations/slack/client.ts", + "content": "import { setTimeout as sleep } from 'node:timers/promises'\n\nimport { HttpError, IngestConfigError, rawRows, type ErrorClassifier, type FetchContext } from '@chkit/plugin-ingest'\n\nimport { slackConfig } from './config.js'\n\nexport type SlackEntity = Record\n\nexport interface SlackClientDeps {\n fetch: (url: string, init: RequestInit) => Promise\n token: () => string | undefined\n wait: (milliseconds: number, signal: AbortSignal) => Promise\n}\n\nexport interface CollectionRequest {\n method: 'conversations.list' | 'users.list' | 'conversations.history' | 'conversations.replies'\n field: 'channels' | 'members' | 'messages'\n idField: 'id' | 'ts'\n query?: Record\n pageSize?: number\n /** Message endpoints support timestamp pagination without an opaque cursor. */\n timePagination?: boolean\n}\n\nexport interface SlackPage { items: SlackEntity[]; cursor: string; hasMore: boolean }\nexport class SlackCursorExpiredError extends IngestConfigError {}\n\nconst defaultDeps: SlackClientDeps = {\n fetch: (url, init) => fetch(url, init),\n token: () => process.env.SLACK_API_TOKEN,\n wait: async (milliseconds, signal) => { await sleep(milliseconds, undefined, { signal }) },\n}\n\n/** Each page is validated inside attempt, so executor retry/cancellation owns every request. */\nexport async function* readCollection(\n context: FetchContext,\n request: CollectionRequest,\n deps: SlackClientDeps = defaultDeps,\n): AsyncGenerator {\n const seenCursors = new Set()\n let cursor = ''\n while (true) {\n context.signal.throwIfAborted()\n const page = await readCollectionPage(context, request, cursor, deps)\n if (page.items.length > 0) yield page.items\n // Empty/short pages can still have a cursor. Only cursor exhaustion ends a scan.\n if (!page.cursor) return\n if (seenCursors.has(page.cursor)) throw new IngestConfigError(`Slack ${request.method} repeated a pagination cursor.`)\n seenCursors.add(page.cursor)\n cursor = page.cursor\n }\n}\n\n/** Retains empty pages and continuation metadata needed by checkpointed readers. */\nexport async function readCollectionPage(context: FetchContext, request: CollectionRequest, cursor = '', deps: SlackClientDeps = defaultDeps): Promise {\n const limit = request.pageSize ?? (request.field === 'messages' ? slackConfig.messagePageSize : slackConfig.pageSize)\n if (!Number.isSafeInteger(limit) || limit <= 0 || limit >= 1_000) throw new IngestConfigError('Slack page sizes must be safe integers between 1 and 999.')\n const query = { ...request.query, limit: String(limit), ...(cursor ? { cursor } : {}) }\n return context.attempt(async (signal) => parsePage(await requestSlack(request.method, query, signal, deps), request), { label: `GET /api/${request.method}` })\n}\n\n/** Discover independently in each reader; pipeline stream order is not a dependency. */\nexport async function discoverChannels(context: FetchContext, deps?: SlackClientDeps): Promise {\n const allowedTypes = ['public_channel', 'private_channel', 'im', 'mpim']\n if (slackConfig.conversationTypes.length === 0 || !slackConfig.conversationTypes.every((type) => allowedTypes.includes(type))) {\n throw new IngestConfigError('Slack conversationTypes must select public_channel, private_channel, im, or mpim.')\n }\n const channels: SlackEntity[] = []\n for await (const page of readCollection(context, {\n method: 'conversations.list', field: 'channels', idField: 'id',\n query: { types: slackConfig.conversationTypes.join(','), exclude_archived: 'false' },\n }, deps)) channels.push(...page)\n const selected = slackConfig.channels\n if (selected === undefined) return channels\n for (const id of selected) {\n if (!channels.some((channel) => entityId(channel, 'id') === id)) {\n throw new IngestConfigError(`Configured Slack channel \"${id}\" was not found or is not accessible for conversationTypes.`)\n }\n }\n return channels.filter((channel) => selected.includes(entityId(channel, 'id')))\n}\n\n/** Scope row identities to the authenticated workspace, never its mutable display name. */\nexport async function getWorkspace(context: FetchContext, deps: SlackClientDeps = defaultDeps): Promise {\n return context.attempt(async (signal) => {\n const payload = await requestSlack('auth.test', {}, signal, deps)\n return entityId(payload, 'team_id')\n }, { label: 'POST /api/auth.test' })\n}\n\n/** Original fields stay under data; provider fields cannot overwrite envelope metadata. */\nexport function toSlackRows(\n items: readonly SlackEntity[],\n resource: string,\n teamId: string,\n idField: 'id' | 'ts',\n channelId?: string,\n) {\n return rawRows(\n items.map((data) => ({\n source_id: slackConfig.sourceId, team_id: teamId,\n ...(channelId === undefined ? {} : { channel_id: channelId }), data,\n })),\n ({ data }) => JSON.stringify([\n slackConfig.sourceId, resource, teamId, ...(channelId === undefined ? [] : [channelId]), entityId(data, idField),\n ]),\n )\n}\n\nexport function entityId(entity: SlackEntity, field: string): string {\n const value = entity[field]\n if (typeof value !== 'string' || value.trim().length === 0) {\n throw new IngestConfigError(`Slack entity is missing a valid ${field} string.`)\n }\n if (field === 'ts' && !/^\\d+\\.\\d+$/.test(value)) {\n throw new IngestConfigError('Slack message ts must be an unmodified decimal timestamp string.')\n }\n return value\n}\n\nexport const classifySlackError: ErrorClassifier = (cause) => {\n if (cause instanceof IngestConfigError || cause instanceof SyntaxError) return { kind: 'permanent' }\n // HTTP errors preserve Retry-After, server retries, and permanent 4xx semantics.\n return undefined\n}\n\nasync function requestSlack(\n method: CollectionRequest['method'] | 'auth.test',\n query: Record,\n signal: AbortSignal,\n deps: SlackClientDeps,\n): Promise> {\n const token = deps.token()?.trim()\n if (!token) throw new IngestConfigError('Set SLACK_API_TOKEN before running Slack ingestion. Schema imports do not need credentials.')\n const interval = method === 'conversations.history' || method === 'conversations.replies'\n ? slackConfig.messageIntervalMs : slackConfig.requestIntervalMs\n if (!Number.isSafeInteger(interval) || interval < 0 || interval > 2_147_483_647) {\n throw new IngestConfigError('Slack request intervals must be non-negative safe integers within the timer range.')\n }\n await deps.wait(interval, signal)\n signal.throwIfAborted()\n const url = new URL(`https://slack.com/api/${method}`)\n for (const [key, value] of Object.entries(query)) url.searchParams.set(key, value)\n const response = await deps.fetch(url.toString(), {\n method: method === 'auth.test' ? 'POST' : 'GET',\n headers: { Authorization: `Bearer ${token}`, Accept: 'application/json' },\n signal: AbortSignal.any([signal, AbortSignal.timeout(30_000)]),\n redirect: 'error',\n })\n if (!response.ok) throw await HttpError.fromResponse(response)\n const payload: unknown = await response.json()\n if (!isObject(payload) || typeof payload.ok !== 'boolean') {\n throw new IngestConfigError(`Slack ${method} returned an invalid response envelope.`)\n }\n // Slack reports authentication, permission, and transient errors in HTTP 200 responses.\n if (!payload.ok) {\n const code = typeof payload.error === 'string' ? payload.error : 'unknown_error'\n if (code === 'invalid_cursor') throw new SlackCursorExpiredError(`Slack ${method} pagination cursor expired.`)\n if (code === 'ratelimited' || code === 'rate_limited') {\n const headers = new Headers(response.headers)\n if (!headers.has('Retry-After')) headers.set('Retry-After', '60')\n throw await HttpError.fromResponse(new Response(JSON.stringify({ error: code }), { status: 429, headers }))\n }\n if (['internal_error', 'fatal_error', 'service_unavailable', 'request_timeout'].includes(code)) {\n throw await HttpError.fromResponse(new Response(JSON.stringify({ error: code }), { status: 503 }))\n }\n const needed = typeof payload.needed === 'string' ? ` Required scopes: ${payload.needed}.` : ''\n const repliesHint = method === 'conversations.replies' ? ' Disable includeReplies explicitly for history-only reads.' : ''\n throw new IngestConfigError(`Slack ${method} failed (${code}).${needed} Check SLACK_API_TOKEN, resource scopes, and conversation access.${repliesHint}`)\n }\n return payload\n}\n\nfunction parsePage(payload: Record, request: CollectionRequest) {\n const values = payload[request.field]\n if (!Array.isArray(values)) throw new IngestConfigError(`Slack ${request.method} returned no ${request.field} array.`)\n const items = values.map((value: unknown) => {\n if (!isObject(value)) throw new IngestConfigError(`Slack ${request.method} returned an invalid entity.`)\n entityId(value, request.idField)\n if (request.field === 'messages' && value.reply_count !== undefined\n && (typeof value.reply_count !== 'number' || !Number.isSafeInteger(value.reply_count) || value.reply_count < 0)) {\n throw new IngestConfigError(`Slack ${request.method} returned an invalid reply_count.`)\n }\n return value\n })\n const metadata = payload.response_metadata\n if (metadata !== undefined && !isObject(metadata)) {\n throw new IngestConfigError(`Slack ${request.method} returned invalid response_metadata.`)\n }\n const next = isObject(metadata) ? metadata.next_cursor : undefined\n if (next !== undefined && next !== null && typeof next !== 'string') {\n throw new IngestConfigError(`Slack ${request.method} returned an invalid next_cursor.`)\n }\n const cursor = typeof next === 'string' ? next.trim() : ''\n if (payload.has_more !== undefined && typeof payload.has_more !== 'boolean') {\n throw new IngestConfigError(`Slack ${request.method} returned an invalid has_more flag.`)\n }\n if (payload.has_more === true && !cursor && !request.timePagination) {\n throw new IngestConfigError(`Slack ${request.method} reports more data without a pagination cursor; refusing an incomplete scan.`)\n }\n return { items, cursor, hasMore: payload.has_more === true || cursor.length > 0 }\n}\n\nfunction isObject(value: unknown): value is Record {\n return typeof value === 'object' && value !== null && !Array.isArray(value)\n}\n" + }, + { + "path": "slack/config.ts", + "type": "registry:file", + "target": "src/integrations/slack/config.ts", + "content": "export interface SlackConfig {\n /** Stable, non-secret installation identity. Changing it creates new stream identities. */\n sourceId: string\n database: string\n tablePrefix: string\n /** Undefined selects every conversation returned for conversationTypes; otherwise use IDs. */\n channels: readonly string[] | undefined\n conversationTypes: readonly ('public_channel' | 'private_channel' | 'im' | 'mpim')[]\n includeReplies: boolean\n pageSize: number\n messagePageSize: number\n requestIntervalMs: number\n messageIntervalMs: number\n /** Earliest message to bootstrap/reconcile; retain the exact timestamp string. */\n historyFrom: string\n overlapMs: number\n reconcileIntervalMs: number\n}\n\n// Credentials are read only in client.ts at request time. Keep these values stable across runs.\nexport const slackConfig: SlackConfig = {\n sourceId: 'slack.primary',\n database: 'default',\n tablePrefix: 'slack',\n channels: undefined,\n conversationTypes: ['public_channel'],\n includeReplies: true,\n pageSize: 200,\n // Conservative defaults for commercially distributed apps outside the Slack Marketplace.\n // Internal customer-built apps can use larger pages and their Tier 3 request rate.\n messagePageSize: 15,\n requestIntervalMs: 3_000,\n messageIntervalMs: 60_000,\n historyFrom: '0.000000',\n overlapMs: 86_400_000,\n reconcileIntervalMs: 7 * 86_400_000,\n}\n" + }, + { + "path": "slack/index.ts", + "type": "registry:file", + "target": "src/integrations/slack/index.ts", + "content": "export { slack } from './pipeline.js'\nexport { slackChannelsRaw } from './sources/channels.js'\nexport { slackUsersRaw } from './sources/users.js'\nexport { slackMessagesRaw } from './sources/messages.js'\n" + }, + { + "path": "slack/pipeline.ts", + "type": "registry:file", + "target": "src/integrations/slack/pipeline.ts", + "content": "import { definePipeline, defineStream } from '@chkit/plugin-ingest'\n\nimport { classifySlackError } from './client.js'\nimport { slackConfig } from './config.js'\nimport { readChannels, slackChannelsRaw } from './sources/channels.js'\nimport { readUsers, slackUsersRaw } from './sources/users.js'\nimport { readMessages, slackMessagesRaw } from './sources/messages.js'\nimport { slackMessageStrategy } from './state.js'\n\nconst streamOptions = { classifyError: classifySlackError, batchSize: 500 }\n\n// Delete a stream entry to stop collecting it; keep its schema export to retain existing data.\nexport const slack = definePipeline({\n id: slackConfig.sourceId,\n tags: ['provider:slack'],\n maxStreams: 1,\n maxFetches: 1,\n maxLoads: 1,\n retry: { retries: 5, minTimeout: 1_000, maxTimeout: 60_000, randomize: true },\n streams: [\n defineStream({ ...streamOptions, id: `${slackConfig.sourceId}.channels`, tags: ['resource:channels'], destination: slackChannelsRaw, read: readChannels }),\n defineStream({ ...streamOptions, id: `${slackConfig.sourceId}.users`, tags: ['resource:users'], destination: slackUsersRaw, read: readUsers }),\n defineStream({ ...streamOptions, batchSize: 1, id: `${slackConfig.sourceId}.messages`, tags: ['resource:messages'], destination: slackMessagesRaw, incremental: slackMessageStrategy, read: readMessages }),\n ],\n})\n" + }, + { + "path": "slack/sources/channels.ts", + "type": "registry:file", + "target": "src/integrations/slack/sources/channels.ts", + "content": "import { rawTable, type FetchContext } from '@chkit/plugin-ingest'\n\nimport { discoverChannels, getWorkspace, toSlackRows, type SlackClientDeps } from '../client.js'\nimport { slackConfig } from '../config.js'\n\nexport const slackChannelsRaw = rawTable({ database: slackConfig.database, name: `${slackConfig.tablePrefix}_channels_raw` })\n\nexport async function* readChannels(context: FetchContext, deps?: SlackClientDeps) {\n const teamId = await getWorkspace(context, deps)\n yield { rows: toSlackRows(await discoverChannels(context, deps), 'channels', teamId, 'id') }\n}\n" + }, + { + "path": "slack/sources/users.ts", + "type": "registry:file", + "target": "src/integrations/slack/sources/users.ts", + "content": "import { rawTable, type FetchContext } from '@chkit/plugin-ingest'\n\nimport { getWorkspace, readCollection, toSlackRows, type SlackClientDeps } from '../client.js'\nimport { slackConfig } from '../config.js'\n\nexport const slackUsersRaw = rawTable({ database: slackConfig.database, name: `${slackConfig.tablePrefix}_users_raw` })\n\nexport async function* readUsers(context: FetchContext, deps?: SlackClientDeps) {\n const teamId = await getWorkspace(context, deps)\n for await (const page of readCollection(context, { method: 'users.list', field: 'members', idField: 'id' }, deps)) {\n yield { rows: toSlackRows(page, 'users', teamId, 'id') }\n }\n}\n" + }, + { + "path": "slack/sources/messages.ts", + "type": "registry:file", + "target": "src/integrations/slack/sources/messages.ts", + "content": "import { IngestConfigError, rawTable, type ReadContext, type SourceChunk, type RawRow } from '@chkit/plugin-ingest'\n\nimport { discoverChannels, entityId, getWorkspace, readCollectionPage, SlackCursorExpiredError, toSlackRows, type SlackClientDeps, type SlackEntity, type SlackPage } from '../client.js'\nimport { slackConfig } from '../config.js'\nimport { formatTimestamp, messageScope, timestampMicros, type SlackMessageSelection, type SlackMessageState } from '../state.js'\n\nexport const slackMessagesRaw = rawTable({ database: slackConfig.database, name: `${slackConfig.tablePrefix}_messages_raw` })\n\n/** Temporary API cursors never enter the checkpoint; exact timestamps survive expiration. */\nexport async function* readMessages(\n context: ReadContext,\n deps?: SlackClientDeps,\n): AsyncGenerator> {\n const teamId = await getWorkspace(context, deps)\n const scope = messageScope(teamId)\n if (context.state && context.state.scope !== scope) throw new IngestConfigError('Slack checkpoint belongs to a different workspace or query. Restore the configuration or use a distinct sourceId.')\n let state: SlackMessageState = context.state ?? { scope, watermark: slackConfig.historyFrom, knownChannels: [] }\n if (!state.active) {\n const channels = (await discoverChannels(context, deps)).map((channel) => entityId(channel, 'id')).sort()\n const to = context.selection.to\n const full = state.lastFullAt === undefined || timestampMicros(to) - timestampMicros(state.lastFullAt) >= BigInt(slackConfig.reconcileIntervalMs) * 1_000n\n const from = context.selection.from ?? (full ? slackConfig.historyFrom : formatTimestamp(max(timestampMicros(slackConfig.historyFrom), timestampMicros(state.watermark) - BigInt(slackConfig.overlapMs) * 1_000n)))\n if (timestampMicros(from) > timestampMicros(to)) throw new IngestConfigError('Slack selection starts after its cutoff.')\n state = { ...state, active: { kind: context.selection.backfill ? 'backfill' : full ? 'full' : 'incremental', from, to, channels, channelIndex: 0,\n latest: formatTimestamp(timestampMicros(to) + 1n), historyDone: false, threads: [],\n requestedFrom: context.selection.from, requestedTo: context.selection.requestedTo } }\n yield marker(state, 'window-open')\n } else if (context.selection.backfill && (context.selection.from !== state.active.requestedFrom || context.selection.requestedTo !== state.active.requestedTo)) {\n throw new IngestConfigError('Slack backfill bounds changed while a window is unfinished. Resume the original range or use a new backfill ID.')\n }\n\n while (state.active && state.active.channelIndex < state.active.channels.length) {\n context.signal.throwIfAborted()\n const work = state.active\n const channelId = work.channels[work.channelIndex]\n if (!channelId) throw new IngestConfigError('Slack checkpoint has no current channel.')\n const thread = work.threads[0]\n if (thread) {\n const after = thread.after ?? formatTimestamp(max(0n, timestampMicros(thread.root) - 1n))\n const page = await nonemptyPage(context, { channel: channelId, ts: thread.root, oldest: after,\n latest: formatTimestamp(timestampMicros(work.to) + 1n), inclusive: 'false', include_all_metadata: 'true' }, 'conversations.replies', deps)\n const replies = page.items.filter((item) => entityId(item, 'ts') !== thread.root)\n assertRange(replies, timestampMicros(after) + 1n, timestampMicros(work.to))\n const frontier = replies.length ? formatTimestamp(replies.reduce((latest, item) => max(latest, timestampMicros(entityId(item, 'ts'))), timestampMicros(after))) : undefined\n if (page.hasMore && frontier === undefined) throw new IngestConfigError('Slack thread pagination has no safe timestamp progress.')\n const threads = page.hasMore ? [{ root: thread.root, after: frontier }, ...work.threads.slice(1)] : work.threads.slice(1)\n state = { ...state, active: { ...work, threads } }\n yield page.items.length ? { rows: toSlackRows(page.items, 'messages', teamId, 'ts', channelId), state } : marker(state, 'thread-complete')\n continue\n }\n if (work.historyDone) {\n state = { ...state, active: { ...work, channelIndex: work.channelIndex + 1, latest: formatTimestamp(timestampMicros(work.to) + 1n), historyDone: false } }\n yield marker(state, 'channel-complete')\n continue\n }\n const from = work.kind === 'incremental' && !state.knownChannels.includes(channelId) ? slackConfig.historyFrom : work.from\n const page = await nonemptyPage(context, { channel: channelId, oldest: formatTimestamp(max(0n, timestampMicros(from) - 1n)), latest: work.latest,\n inclusive: 'false', include_all_metadata: 'true' }, 'conversations.history', deps)\n assertRange(page.items, timestampMicros(from), timestampMicros(work.latest) - 1n)\n const latest = page.items.length ? formatTimestamp(page.items.reduce((oldest, item) => min(oldest, timestampMicros(entityId(item, 'ts'))), timestampMicros(work.latest))) : work.latest\n if (page.hasMore && (page.items.length === 0 || timestampMicros(latest) >= timestampMicros(work.latest))) throw new IngestConfigError('Slack history pagination has no safe timestamp progress.')\n const roots = slackConfig.includeReplies ? page.items.filter((item) => typeof item.reply_count === 'number' && item.reply_count > 0)\n .map((item) => typeof item.thread_ts === 'string' ? entityId(item, 'thread_ts') : entityId(item, 'ts')) : []\n state = { ...state, active: { ...work, latest, historyDone: !page.hasMore, threads: [...new Set(roots)].map((root) => ({ root })) } }\n // The page retains its bounded child queue before the history frontier advances durably.\n yield page.items.length ? { rows: toSlackRows(page.items, 'messages', teamId, 'ts', channelId), state } : marker(state, 'history-complete')\n }\n if (state.active) {\n const { to, kind, channels } = state.active\n state = { scope, watermark: formatTimestamp(max(timestampMicros(state.watermark), timestampMicros(to))),\n knownChannels: [...new Set([...state.knownChannels, ...channels])].sort(), lastFullAt: kind === 'full' ? to : state.lastFullAt }\n yield marker(state, 'window-complete')\n }\n}\n\n/** Empty cursor pages use the live cursor; nonempty pages establish timestamp boundaries. */\nasync function nonemptyPage(\n context: ReadContext, query: Record,\n method: 'conversations.history' | 'conversations.replies', deps?: SlackClientDeps,\n): Promise {\n let cursor = ''\n let restarted = false\n const seen = new Set()\n const parents = new Map()\n while (true) {\n let page: SlackPage\n try {\n page = await readCollectionPage(context, { method, field: 'messages', idField: 'ts', query, timePagination: true }, cursor, deps)\n } catch (error) {\n if (!(error instanceof SlackCursorExpiredError) || !cursor || restarted) throw error\n cursor = ''\n restarted = true\n seen.clear()\n continue\n }\n const parentOnly = method === 'conversations.replies' && page.items.every((item) => entityId(item, 'ts') === query.ts)\n if (page.items.length > 0 && (!parentOnly || !page.hasMore) || !page.hasMore) {\n for (const item of page.items) parents.set(entityId(item, 'ts'), item)\n return { ...page, items: [...parents.values()] }\n }\n for (const item of page.items) parents.set(entityId(item, 'ts'), item)\n if (!page.cursor || seen.has(page.cursor)) throw new IngestConfigError('Slack empty page did not provide a new continuation.')\n seen.add(page.cursor)\n cursor = page.cursor\n }\n}\n\nfunction marker(state: SlackMessageState, id: string): SourceChunk {\n // State-only boundaries flush previous rows without assigning identity to mutable data.\n return { rows: [], state, id }\n}\nfunction assertRange(items: SlackEntity[], from: bigint, to: bigint): void {\n if (items.some((item) => {\n const ts = timestampMicros(entityId(item, 'ts'))\n return ts < from || ts > to\n })) throw new IngestConfigError('Slack returned a message outside the requested timestamp interval.')\n}\nfunction max(left: bigint, right: bigint): bigint { return left > right ? left : right }\nfunction min(left: bigint, right: bigint): bigint { return left < right ? left : right }\n" + }, + { + "path": "slack/tests/slack.test.ts", + "type": "registry:file", + "target": "src/integrations/slack/tests/slack.test.ts", + "role": "test", + "content": "import { afterEach, describe, expect, test } from 'bun:test'\n\nimport { HttpError, definePipeline, defineStream, runIngestion, selectStreams, type FetchContext, type ReadContext, type SourceChunk } from '@chkit/plugin-ingest'\nimport { createMemoryDestination, createMemoryJournal } from '@chkit/plugin-ingest/testing'\n\nimport { classifySlackError, discoverChannels, readCollection, type SlackClientDeps } from '../client.js'\nimport { slackConfig } from '../config.js'\nimport * as definitions from '../index.js'\nimport { slack } from '../pipeline.js'\nimport { readChannels } from '../sources/channels.js'\nimport { readUsers } from '../sources/users.js'\nimport { readMessages } from '../sources/messages.js'\nimport { slackMessageStrategy, type SlackMessageSelection, type SlackMessageState } from '../state.js'\nimport { channel, fixtureDeps, parentMessage, replyMessage, standaloneMessage, user } from './fixtures.js'\n\nconst readers: Record AsyncIterable> = {\n channels: readChannels, users: readUsers,\n}\nconst originalConfig = { ...slackConfig }\nconst fixtureNow = () => new Date('2023-11-15T00:00:00Z')\n\nafterEach(() => { Object.assign(slackConfig, originalConfig) })\n\ndescribe('Slack raw template', () => {\n test('exports three raw destinations and checkpointed messages without credentials', () => {\n const exported = Object.values(definitions)\n expect(exported).toContain(slack)\n expect(slack.streams).toHaveLength(3)\n expect(slack.streams.map((stream) => stream.destination.name)).toEqual(['slack_channels_raw', 'slack_users_raw', 'slack_messages_raw'])\n for (const stream of slack.streams) {\n expect(exported).toContain(stream.destination)\n expect(stream.incremental.id).toBe(stream.id.endsWith('.messages') ? 'slack.message-ranges' : 'chkit.full_sync')\n }\n })\n\n test('every stream preserves provider payloads and reruns publish edits under stable identities', async () => {\n let changed = false\n const defaults = fixtureDeps()\n const deps = fixtureDeps((url, init) => url.pathname === '/api/conversations.history' && changed\n ? Response.json({ ok: true, messages: [{ ...parentMessage, text: 'Edited discussion' }, standaloneMessage], response_metadata: { next_cursor: '' } })\n : defaults.fetch(url.toString(), init))\n const pipeline = fixturePipeline(deps)\n const journal = createMemoryJournal()\n const destination = createMemoryDestination()\n const request = { selected: selectStreams([pipeline], []), backfill: undefined }\n const first = await runIngestion(request, { now: fixtureNow, journal, destination })\n expect(first.ok, JSON.stringify(first)).toBe(true)\n expect(first.streams.map((stream) => stream.rows)).toEqual([1, 2, 4])\n expect(destination.tables.size).toBe(3)\n expect(destination.tables.get('default.slack_users_raw')?.map((row) => row.raw)).toContainEqual({ source_id: 'slack.primary', team_id: 'T1', data: user })\n const messages = destination.tables.get('default.slack_messages_raw') ?? []\n expect(messages.map((row) => row.raw)).toContainEqual({ source_id: 'slack.primary', team_id: 'T1', channel_id: 'C1', data: parentMessage })\n expect(messages.map((row) => row.raw)).toContainEqual({ source_id: 'slack.primary', team_id: 'T1', channel_id: 'C1', data: replyMessage })\n expect(new Set(messages.map((row) => row.id)).size).toBe(3)\n expect(messages.find((row) => JSON.stringify(row.raw).includes('Edited discussion'))).toBeUndefined()\n changed = true\n const second = await runIngestion(request, { now: fixtureNow, journal, destination })\n expect(second.ok, JSON.stringify(second)).toBe(true)\n const repeated = destination.tables.get('default.slack_messages_raw') ?? []\n const edited = repeated.find((row) => JSON.stringify(row.raw).includes('Edited discussion'))\n expect(edited?.id).toBe(JSON.stringify(['slack.primary', 'messages', 'T1', 'C1', parentMessage.ts]))\n expect(new Set(repeated.map((row) => row.id)).size).toBe(3)\n })\n\n test('cursor pagination continues across an empty page and includes all later entities', async () => {\n const cursors: Array = []\n const deps = fixtureDeps((url) => {\n const cursor = url.searchParams.get('cursor')\n cursors.push(cursor)\n if (cursor === null) return Response.json({ ok: true, channels: [], response_metadata: { next_cursor: 'empty-next' } })\n if (cursor === 'empty-next') return Response.json({ ok: true, channels: [channel], response_metadata: { next_cursor: 'final-next' } })\n if (cursor === 'final-next') return Response.json({ ok: true, channels: [{ ...channel, id: 'C2' }], response_metadata: { next_cursor: null } })\n throw new Error(`Unexpected cursor ${cursor}`)\n })\n const pages = await collect(readCollection(context(), { method: 'conversations.list', field: 'channels', idField: 'id', pageSize: 2 }, deps))\n expect(pages.flat().map((value) => value.id)).toEqual(['C1', 'C2'])\n expect(cursors).toEqual([null, 'empty-next', 'final-next'])\n })\n\n test('messages discover channels independently and paginate threads including the parent payload', async () => {\n const requests: URL[] = []\n const defaults = fixtureDeps()\n const deps = fixtureDeps((url, init) => {\n requests.push(url)\n if (url.pathname === '/api/conversations.history') return Response.json({ ok: true, messages: [parentMessage], response_metadata: { next_cursor: '' } })\n if (url.pathname === '/api/conversations.replies') {\n expect(url.searchParams.get('channel')).toBe('C1')\n expect(url.searchParams.get('ts')).toBe(parentMessage.ts)\n return url.searchParams.has('cursor')\n ? Response.json({ ok: true, messages: [replyMessage], response_metadata: { next_cursor: '' } })\n : Response.json({ ok: true, messages: [{ ...parentMessage, text: 'Latest parent' }], response_metadata: { next_cursor: 'reply-next' } })\n }\n return defaults.fetch(url.toString(), init)\n })\n const chunks = await collect(readMessages(context(), deps))\n expect(chunks.flatMap((chunk) => chunk.rows).map((row) => row.raw)).toContainEqual({ source_id: 'slack.primary', team_id: 'T1', channel_id: 'C1', data: { ...parentMessage, text: 'Latest parent' } })\n expect(chunks.flatMap((chunk) => chunk.rows)).toHaveLength(3)\n expect(chunks.every((chunk) => 'state' in chunk)).toBe(true)\n expect(chunks.filter((chunk) => chunk.rows.length > 0).every((chunk) => chunk.id === undefined)).toBe(true)\n const messageRequests = requests.filter((url) => ['conversations.history', 'conversations.replies'].some((method) => url.pathname.endsWith(method)))\n expect(messageRequests.map((url) => url.searchParams.get('limit'))).toEqual(['15', '15', '15'])\n expect(messageRequests.at(-1)?.searchParams.get('cursor')).toBe('reply-next')\n expect(requests.some((url) => url.pathname === '/api/conversations.list')).toBe(true)\n })\n\n test('row identities separate source installations, workspaces, and conversations', async () => {\n const defaults = fixtureDeps()\n const readForTeam = async (team: string) => collect(readMessages(context(), fixtureDeps((url, init) => {\n if (url.pathname === '/api/auth.test') return Response.json({ ok: true, team_id: team })\n if (url.pathname === '/api/conversations.list') return Response.json({ ok: true, channels: [channel, { ...channel, id: 'C2' }], response_metadata: { next_cursor: '' } })\n if (url.pathname === '/api/conversations.history') return Response.json({ ok: true, messages: [standaloneMessage], response_metadata: { next_cursor: '' } })\n return defaults.fetch(url.toString(), init)\n })))\n const first = (await readForTeam('T1')).flatMap((chunk) => chunk.rows)\n const second = (await readForTeam('T2')).flatMap((chunk) => chunk.rows)\n slackConfig.sourceId = 'slack.secondary'\n const separateSource = (await readForTeam('T1')).flatMap((chunk) => chunk.rows)\n expect(new Set([...first, ...second, ...separateSource].map((row) => row.id)).size).toBe(6)\n expect(first[0]?.id).toBe(JSON.stringify(['slack.primary', 'messages', 'T1', 'C1', standaloneMessage.ts]))\n })\n\n test('finishes retained thread work before resuming history from its timestamp frontier', async () => {\n const requests: string[] = []\n const defaults = fixtureDeps()\n const deps = fixtureDeps((url, init) => {\n if (url.pathname === '/api/conversations.history') {\n const latest = url.searchParams.get('latest')\n const resumed = latest === parentMessage.ts\n requests.push(`history:${resumed ? 'resumed' : 'first'}`)\n return Response.json({ ok: true, messages: [resumed ? { ...standaloneMessage, ts: '1699999999.000003' } : parentMessage], has_more: !resumed })\n }\n if (url.pathname === '/api/conversations.replies') {\n requests.push('replies')\n return Response.json({ ok: true, messages: [parentMessage, replyMessage], response_metadata: { next_cursor: null } })\n }\n return defaults.fetch(url.toString(), init)\n })\n const chunks = await collect(readMessages(context(), deps))\n expect(chunks.flatMap((chunk) => chunk.rows)).toHaveLength(4)\n expect(requests).toEqual(['history:first', 'replies', 'history:resumed'])\n })\n\n test('history-only selection avoids thread API requests', async () => {\n slackConfig.includeReplies = false\n const requests: string[] = []\n const defaults = fixtureDeps()\n const deps = fixtureDeps((url, init) => { requests.push(url.pathname); return defaults.fetch(url.toString(), init) })\n const chunks = await collect(readMessages(context(), deps))\n expect(chunks.flatMap((chunk) => chunk.rows)).toHaveLength(2)\n expect(requests).not.toContain('/api/conversations.replies')\n })\n\n test('configured channel IDs narrow discovery and unknown IDs fail visibly', async () => {\n slackConfig.channels = ['C1']\n const defaults = fixtureDeps()\n const deps = fixtureDeps((url, init) => url.pathname === '/api/conversations.list'\n ? Response.json({ ok: true, channels: [channel, { ...channel, id: 'C2' }], response_metadata: { next_cursor: '' } })\n : defaults.fetch(url.toString(), init))\n expect((await discoverChannels(context(), deps)).map((value) => value.id)).toEqual(['C1'])\n slackConfig.channels = ['missing']\n await expect(discoverChannels(context(), deps)).rejects.toThrow('missing')\n slackConfig.channels = []\n expect(await discoverChannels(context(), deps)).toEqual([])\n })\n\n test('cancellation stops before the second history page and preserves configured pacing', async () => {\n const controller = new AbortController()\n const requests: URL[] = []\n const waits: number[] = []\n const defaults = fixtureDeps()\n const deps = fixtureDeps((url, init) => {\n requests.push(url)\n return url.pathname === '/api/conversations.history'\n ? Response.json({ ok: true, messages: [standaloneMessage], response_metadata: { next_cursor: 'history-next' } })\n : defaults.fetch(url.toString(), init)\n })\n deps.wait = async (milliseconds, signal) => { waits.push(milliseconds); signal.throwIfAborted() }\n const iterator = readMessages(context(controller.signal), deps)\n const opening = await iterator.next()\n expect(opening.value?.rows).toHaveLength(0)\n const first = await iterator.next()\n expect(first.value?.rows).toHaveLength(1)\n controller.abort(new Error('fixture cancelled'))\n await expect(iterator.next()).rejects.toThrow('fixture cancelled')\n expect(requests.filter((url) => url.pathname === '/api/conversations.history')).toHaveLength(1)\n expect(waits).toContain(slackConfig.messageIntervalMs)\n expect(waits.filter((milliseconds) => milliseconds === slackConfig.requestIntervalMs).length).toBeGreaterThanOrEqual(1)\n })\n\n test('malformed successful responses fail instead of becoming empty datasets', async () => {\n for (const payload of [{}, { ok: true, members: null }, { ok: true, members: [{ name: 'missing-id' }] }]) {\n const defaults = fixtureDeps()\n const deps = fixtureDeps((url, init) => url.pathname === '/api/users.list' ? Response.json(payload) : defaults.fetch(url.toString(), init))\n await expect(collect(readUsers(context(), deps))).rejects.toThrow('Slack')\n }\n })\n\n test('malformed message timestamps and pagination fail rather than truncate a scan', async () => {\n const request = { method: 'conversations.history', field: 'messages', idField: 'ts' } as const\n for (const payload of [\n { ok: true, messages: [{ ...standaloneMessage, ts: 1_700_000_002.000003 }] },\n { ok: true, messages: [{ ...standaloneMessage, ts: 'invalid' }] },\n { ok: true, messages: [standaloneMessage], has_more: true },\n { ok: true, messages: [standaloneMessage], response_metadata: { next_cursor: 123 } },\n ]) {\n await expect(collect(readCollection(context(), request, fixtureDeps(() => Response.json(payload))))).rejects.toThrow('Slack')\n }\n let calls = 0\n const repeated = fixtureDeps(() => {\n calls += 1\n if (calls > 3) throw new Error('Repeated cursor was not rejected')\n return Response.json({ ok: true, messages: [standaloneMessage], response_metadata: { next_cursor: 'same-cursor' } })\n })\n await expect(collect(readCollection(context(), request, repeated))).rejects.toThrow('repeated')\n expect(calls).toBe(2)\n })\n\n test('HTTP 200 permission errors fail permanently without loading any rows', async () => {\n let calls = 0\n const defaults = fixtureDeps()\n const deps = fixtureDeps((url, init) => {\n if (url.pathname !== '/api/users.list') return defaults.fetch(url.toString(), init)\n calls += 1\n return Response.json({ ok: false, error: 'missing_scope', needed: 'users:read', provided: 'channels:read' })\n })\n const destination = createMemoryDestination()\n const result = await runFixture(deps, ['users'], destination)\n expect(result.ok).toBe(false)\n expect(result.streams[0]?.error).toContain('missing_scope')\n expect(result.streams[0]?.error).toContain('users:read')\n expect(result.streams[0]?.rows).toBe(0)\n expect(calls).toBe(1)\n expect(destination.tables.size).toBe(0)\n })\n\n test('HTTP 200 transient errors retry inside the ingestion executor', async () => {\n let calls = 0\n const defaults = fixtureDeps()\n const deps = fixtureDeps((url, init) => {\n if (url.pathname !== '/api/users.list') return defaults.fetch(url.toString(), init)\n calls += 1\n return calls === 1 ? Response.json({ ok: false, error: 'internal_error' }) : Response.json({ ok: true, members: [user], response_metadata: { next_cursor: '' } })\n })\n const result = await runFixture(deps, ['users'])\n expect(result.ok, JSON.stringify(result)).toBe(true)\n expect(calls).toBe(2)\n })\n\n test('HTTP 200 rate-limit errors retain Retry-After and retry under executor authority', async () => {\n let calls = 0\n const defaults = fixtureDeps()\n const deps = fixtureDeps((url, init) => {\n if (url.pathname !== '/api/users.list') return defaults.fetch(url.toString(), init)\n calls += 1\n return calls === 1\n ? Response.json({ ok: false, error: 'ratelimited' }, { headers: { 'Retry-After': '0' } })\n : Response.json({ ok: true, members: [user], response_metadata: { next_cursor: '' } })\n })\n const journal = createMemoryJournal()\n const pipeline = fixturePipeline(deps, ['users'])\n const result = await runIngestion({ selected: selectStreams([pipeline], []), backfill: undefined }, { now: fixtureNow, journal, destination: createMemoryDestination() })\n expect(result.ok, JSON.stringify(result)).toBe(true)\n expect(calls).toBe(2)\n expect(journal.events.some((event) => event.eventKind === 'retry_scheduled' && event.errorClass === 'rate_limited')).toBe(true)\n })\n\n test('thread permission errors fail the stream and leave already loaded history visible', async () => {\n let replyCalls = 0\n const defaults = fixtureDeps()\n const deps = fixtureDeps((url, init) => {\n if (url.pathname !== '/api/conversations.replies') return defaults.fetch(url.toString(), init)\n replyCalls += 1\n return Response.json({ ok: false, error: 'missing_scope', needed: 'channels:history' })\n })\n const fixture = fixturePipeline(deps, ['messages'])\n const pipeline = definePipeline({ ...fixture, streams: fixture.streams.map((stream) => ({ ...stream, batchSize: 1 })) })\n const destination = createMemoryDestination()\n const result = await runIngestion({ selected: selectStreams([pipeline], []), backfill: undefined }, { now: fixtureNow, journal: createMemoryJournal(), destination })\n expect(result.ok).toBe(false)\n expect(result.streams[0]?.outcome).toBe('failed')\n expect(result.streams[0]?.error).toContain('missing_scope')\n expect(replyCalls).toBe(1)\n expect(destination.tables.get('default.slack_messages_raw')).toHaveLength(2)\n expect(destination.tables.get('default.slack_messages_raw')?.map((row) => row.raw)).toContainEqual({ source_id: 'slack.primary', team_id: 'T1', channel_id: 'C1', data: parentMessage })\n })\n\n test('429 errors preserve Retry-After and retry the current page', async () => {\n let calls = 0\n const defaults = fixtureDeps()\n const deps = fixtureDeps((url, init) => {\n if (url.pathname !== '/api/users.list') return defaults.fetch(url.toString(), init)\n calls += 1\n return calls === 1\n ? Response.json({ ok: false, error: 'ratelimited' }, { status: 429, headers: { 'Retry-After': '0' } })\n : Response.json({ ok: true, members: [user], response_metadata: { next_cursor: '' } })\n })\n const journal = createMemoryJournal()\n const pipeline = fixturePipeline(deps, ['users'])\n const result = await runIngestion({ selected: selectStreams([pipeline], []), backfill: undefined }, { now: fixtureNow, journal, destination: createMemoryDestination() })\n expect(result.ok, JSON.stringify(result)).toBe(true)\n expect(calls).toBe(2)\n expect(journal.events.some((event) => event.eventKind === 'retry_scheduled' && event.errorClass === 'rate_limited')).toBe(true)\n const http = await HttpError.fromResponse(new Response(null, { status: 429, headers: { 'Retry-After': '2' } }))\n expect(http.retryAfterMs).toBe(2_000)\n expect(classifySlackError(http, { kind: 'rate_limited', retryAfterMs: 2_000 })).toBeUndefined()\n })\n\n test('missing credentials fail only when a reader runs', async () => {\n const deps = fixtureDeps(() => { throw new Error('must not fetch') })\n deps.token = () => undefined\n expect(slack.streams).toHaveLength(3)\n await expect(collect(readUsers(context(), deps))).rejects.toThrow('SLACK_API_TOKEN')\n })\n})\n\nfunction fixturePipeline(deps: SlackClientDeps, selectedResources?: readonly string[]) {\n return definePipeline({\n ...slack,\n retry: { retries: 1, minTimeout: 0, maxTimeout: 0, randomize: false },\n streams: slack.streams.filter((stream) => selectedResources === undefined || selectedResources.includes(stream.id.split('.').at(-1) ?? '')).map((stream) => {\n if (stream.id.endsWith('.messages')) return defineStream({ ...stream, incremental: slackMessageStrategy, read: (context) => readMessages(context, deps) })\n const read = readers[stream.id.split('.').at(-1) ?? '']\n if (!read) throw new Error(`No fixture reader for ${stream.id}`)\n return { ...stream, read: (context: FetchContext) => read(context, deps) }\n }),\n })\n}\n\nasync function runFixture(deps: SlackClientDeps, selectedResources: readonly string[], destination = createMemoryDestination()) {\n const pipeline = fixturePipeline(deps, selectedResources)\n return runIngestion({ selected: selectStreams([pipeline], []), backfill: undefined }, { now: fixtureNow, journal: createMemoryJournal(), destination })\n}\n\nfunction context(signal = new AbortController().signal): ReadContext {\n return { signal, attempt: async (operation) => operation(signal), streamId: 'slack.primary.messages', state: undefined,\n cutoff: fixtureNow(), selection: slackMessageStrategy.plan({ cutoff: fixtureNow(), state: undefined, range: undefined }) }\n}\n\nasync function collect(items: AsyncIterable): Promise {\n const result: T[] = []\n for await (const item of items) result.push(item)\n return result\n}\n" + }, + { + "path": "slack/tests/fixtures.ts", + "type": "registry:file", + "target": "src/integrations/slack/tests/fixtures.ts", + "role": "test", + "content": "import type { SlackClientDeps } from '../client.js'\n\nexport const channel = {\n id: 'C1', name: 'project-example', is_channel: true, is_private: false, is_archived: false,\n topic: { value: 'Example project', creator: 'U1', last_set: 1_700_000_000 },\n}\n\nexport const user = {\n id: 'U1', team_id: 'T1', name: 'ada-example', deleted: false,\n profile: { real_name: 'Ada Example', email: 'ada@example.test', fields: { custom_field: { value: 'retained' } } },\n}\n\nexport const parentMessage = {\n type: 'message', user: 'U1', ts: '1700000000.000001', thread_ts: '1700000000.000001',\n text: 'A project discussion', reply_count: 1,\n blocks: [{ type: 'section', text: { type: 'mrkdwn', text: '*Original formatting*' } }],\n reactions: [{ name: 'eyes', users: ['U2'], count: 1 }],\n files: [{ id: 'F1', title: 'Example attachment', mimetype: 'text/plain' }],\n}\n\nexport const replyMessage = {\n type: 'message', user: 'U2', ts: '1700000001.000002', thread_ts: parentMessage.ts,\n text: 'Thread reply', attachments: [{ fallback: 'Original attachment', extra_field: ['one', 'two'] }],\n}\n\nexport const standaloneMessage = { type: 'message', user: 'U1', ts: '1700000002.000003', text: 'Independent message' }\n\nexport function fixtureDeps(\n respond: (url: URL, init: RequestInit) => Response | Promise = (url) => {\n switch (url.pathname) {\n case '/api/auth.test': return Response.json({ ok: true, team_id: 'T1', user_id: 'U1' })\n case '/api/conversations.list': return Response.json({ ok: true, channels: [channel], response_metadata: { next_cursor: '' } })\n case '/api/users.list': return Response.json({ ok: true, members: [user, { id: 'U2', team_id: 'T1', name: 'grace-example', profile: {} }], response_metadata: { next_cursor: '' } })\n case '/api/conversations.history': return Response.json({ ok: true, messages: [parentMessage, standaloneMessage], response_metadata: { next_cursor: '' }, has_more: false })\n case '/api/conversations.replies': return Response.json({ ok: true, messages: [parentMessage, replyMessage], response_metadata: { next_cursor: '' }, has_more: false })\n default: throw new Error(`Unexpected fixture URL: ${url.pathname}`)\n }\n },\n): SlackClientDeps {\n return {\n token: () => 'fixture-token',\n wait: async () => {},\n fetch: (url, init) => Promise.resolve(respond(new URL(url), init)),\n }\n}\n" + }, + { + "path": "slack/state.ts", + "type": "registry:file", + "target": "src/integrations/slack/state.ts", + "content": "import { IngestConfigError, type IncrementalStrategy } from '@chkit/plugin-ingest'\nimport { slackConfig } from './config.js'\n\nexport interface PendingThread { root: string; after?: string }\nexport interface MessageWork {\n kind: 'full' | 'incremental' | 'backfill'\n from: string\n to: string\n channels: string[]\n channelIndex: number\n latest: string\n historyDone: boolean\n threads: PendingThread[]\n requestedFrom?: string\n requestedTo?: string\n}\nexport interface SlackMessageState {\n scope: string\n watermark: string\n knownChannels: string[]\n lastFullAt?: string\n active?: MessageWork\n}\nexport interface SlackMessageSelection { from?: string; to: string; requestedTo?: string; backfill: boolean }\n\nexport const slackMessageStrategy: IncrementalStrategy = {\n id: 'slack.message-ranges', version: 1, parseState: parseMessageState,\n plan({ cutoff, range }) {\n validateMessageConfig()\n return { from: range?.from ? dateTimestamp(range.from) : undefined, to: dateTimestamp(range?.to ?? cutoff),\n requestedTo: range?.to ? dateTimestamp(range.to) : undefined, backfill: range !== undefined }\n },\n}\n\nexport function messageScope(teamId: string): string {\n return JSON.stringify([slackConfig.sourceId, teamId, slackConfig.channels ? [...slackConfig.channels].sort() : null,\n [...slackConfig.conversationTypes].sort(), slackConfig.includeReplies, slackConfig.historyFrom,\n slackConfig.overlapMs, slackConfig.reconcileIntervalMs])\n}\n\nexport function timestampMicros(value: string): bigint {\n if (!/^\\d+\\.\\d{1,6}$/.test(value)) throw new IngestConfigError('Slack timestamps must be exact decimal strings with at most six fractional digits.')\n const [seconds = '', fraction = ''] = value.split('.')\n return BigInt(seconds) * 1_000_000n + BigInt(fraction.padEnd(6, '0'))\n}\n\nexport function formatTimestamp(value: bigint): string {\n if (value < 0n) throw new IngestConfigError('Slack timestamp cannot be negative.')\n return `${value / 1_000_000n}.${(value % 1_000_000n).toString().padStart(6, '0')}`\n}\n\nexport function dateTimestamp(value: Date): string {\n if (!Number.isSafeInteger(value.getTime()) || value.getTime() < 0) throw new IngestConfigError('Slack time bounds must be valid dates after the Unix epoch.')\n return formatTimestamp(BigInt(value.getTime()) * 1_000n)\n}\n\nexport function parseMessageState(raw: unknown): SlackMessageState {\n if (!isObject(raw) || typeof raw.scope !== 'string') throw new IngestConfigError('Invalid Slack checkpoint scope.')\n const watermark = readTimestamp(raw.watermark)\n const knownChannels = readIds(raw.knownChannels)\n const lastFullAt = raw.lastFullAt === undefined ? undefined : readTimestamp(raw.lastFullAt)\n let active: MessageWork | undefined\n if (raw.active !== undefined) {\n const work = raw.active\n if (!isObject(work) || typeof work.channelIndex !== 'number' || !Number.isSafeInteger(work.channelIndex)\n || typeof work.historyDone !== 'boolean' || !Array.isArray(work.threads)) throw new IngestConfigError('Invalid Slack active work.')\n const kind = work.kind\n if (kind !== 'full' && kind !== 'incremental' && kind !== 'backfill') throw new IngestConfigError('Invalid Slack work kind.')\n const channels = readIds(work.channels)\n if (work.channelIndex < 0 || work.channelIndex > channels.length) throw new IngestConfigError('Invalid Slack channel position.')\n const from = readTimestamp(work.from), to = readTimestamp(work.to), latest = readTimestamp(work.latest)\n if (timestampMicros(from) > timestampMicros(to) || timestampMicros(latest) > timestampMicros(to) + 1n) throw new IngestConfigError('Invalid Slack window bounds.')\n const threads = work.threads.map((thread: unknown) => {\n if (!isObject(thread)) throw new IngestConfigError('Invalid Slack pending thread.')\n const root = readTimestamp(thread.root)\n const after = thread.after === undefined ? undefined : readTimestamp(thread.after)\n if (timestampMicros(root) > timestampMicros(to) || (after !== undefined && (timestampMicros(after) < timestampMicros(root) || timestampMicros(after) > timestampMicros(to)))) throw new IngestConfigError('Invalid Slack thread frontier.')\n return { root, after }\n })\n if (new Set(threads.map((thread) => thread.root)).size !== threads.length) throw new IngestConfigError('Duplicate Slack pending threads.')\n active = { kind, from, to, channels, channelIndex: work.channelIndex, latest, historyDone: work.historyDone, threads,\n requestedFrom: work.requestedFrom === undefined ? undefined : readTimestamp(work.requestedFrom),\n requestedTo: work.requestedTo === undefined ? undefined : readTimestamp(work.requestedTo) }\n }\n return { scope: raw.scope, watermark, knownChannels, lastFullAt, active }\n}\n\nfunction validateMessageConfig(): void {\n timestampMicros(slackConfig.historyFrom)\n for (const value of [slackConfig.overlapMs, slackConfig.reconcileIntervalMs]) {\n if (!Number.isSafeInteger(value) || value < 0) throw new IngestConfigError('Slack overlap and reconciliation intervals must be non-negative safe integers.')\n }\n}\nfunction readTimestamp(value: unknown): string {\n if (typeof value !== 'string') throw new IngestConfigError('Invalid Slack checkpoint timestamp.')\n timestampMicros(value)\n return value\n}\nfunction readIds(value: unknown): string[] {\n if (!Array.isArray(value) || !value.every((item: unknown): item is string => typeof item === 'string' && item.length > 0)\n || new Set(value).size !== value.length) throw new IngestConfigError('Invalid Slack checkpoint channel IDs.')\n return [...value]\n}\nfunction isObject(value: unknown): value is Record { return typeof value === 'object' && value !== null && !Array.isArray(value) }\n" + }, + { + "path": "slack/tests/checkpoints.test.ts", + "type": "registry:file", + "target": "src/integrations/slack/tests/checkpoints.test.ts", + "role": "test", + "content": "import { afterEach, expect, test } from 'bun:test'\nimport { definePipeline, defineStream, runIngestion, selectStreams } from '@chkit/plugin-ingest'\nimport { createMemoryDestination, createMemoryJournal } from '@chkit/plugin-ingest/testing'\n\nimport { slackConfig } from '../config.js'\nimport { readMessages, slackMessagesRaw } from '../sources/messages.js'\nimport { dateTimestamp, parseMessageState, slackMessageStrategy, timestampMicros } from '../state.js'\nimport { fixtureDeps, parentMessage, replyMessage, standaloneMessage } from './fixtures.js'\nimport type { SlackClientDeps } from '../client.js'\n\nconst originalConfig = { ...slackConfig }\nconst cutoff = new Date('2023-11-15T00:00:00Z')\nafterEach(() => { Object.assign(slackConfig, originalConfig) })\n\ntest('a fresh process resumes retained threads before advancing the frozen history window', async () => {\n const journal = createMemoryJournal(), destination = createMemoryDestination()\n const calls: string[] = []\n const defaults = fixtureDeps()\n const deps = fixtureDeps((url, init) => { calls.push(url.pathname); return defaults.fetch(url.toString(), init) })\n const limited = pipeline(deps, { maxChunks: 2 })\n const first = await runIngestion(request(limited), { journal, destination, now: () => cutoff })\n expect(first.streams[0]?.outcome).toBe('budget_exhausted')\n const saved = parseMessageState((await journal.readCheckpoint('slack.primary.messages')).envelope?.state)\n expect(saved.watermark).toBe(slackConfig.historyFrom)\n expect(saved.active?.to).toBe(dateTimestamp(cutoff))\n expect(saved.active?.threads).toEqual([{ root: parentMessage.ts }])\n expect(saved.active?.historyDone).toBe(true)\n calls.length = 0\n const second = await runIngestion(request(pipeline(deps)), { journal, destination, now: () => new Date('2023-11-16T00:00:00Z') })\n expect(second.ok, JSON.stringify(second)).toBe(true)\n expect(calls).toEqual(['/api/auth.test', '/api/conversations.replies'])\n const completed = parseMessageState((await journal.readCheckpoint('slack.primary.messages')).envelope?.state)\n expect(completed.active).toBeUndefined()\n expect(completed.watermark).toBe(dateTimestamp(cutoff))\n expect(destination.tables.get('default.slack_messages_raw')?.map((row) => row.raw)).toContainEqual({ source_id: 'slack.primary', team_id: 'T1', channel_id: 'C1', data: replyMessage })\n})\n\ntest('a lost destination acknowledgement cannot move the history frontier past unpublished journal progress', async () => {\n slackConfig.includeReplies = false\n const journal = createMemoryJournal(), destination = createMemoryDestination()\n const controller = new AbortController()\n const first = await runIngestion(request(pipeline(fixtureDeps())), {\n journal, now: () => cutoff, signal: controller.signal,\n destination: { async insert(input) { await destination.insert(input); controller.abort(new Error('lost acknowledgement')) } },\n })\n expect(first.ok).toBe(false)\n const saved = parseMessageState((await journal.readCheckpoint('slack.primary.messages')).envelope?.state)\n expect(saved.active?.latest).toBe('1700006400.000001')\n expect(saved.active?.historyDone).toBe(false)\n const second = await runIngestion(request(pipeline(fixtureDeps())), { journal, destination, now: () => cutoff })\n expect(second.ok, JSON.stringify(second)).toBe(true)\n expect(destination.tables.get('default.slack_messages_raw')).toHaveLength(2)\n expect(parseMessageState((await journal.readCheckpoint('slack.primary.messages')).envelope?.state).active).toBeUndefined()\n})\n\ntest('history resumes newest-first without saving expiring cursors or converting timestamp precision', async () => {\n slackConfig.includeReplies = false\n const newest = { ...standaloneMessage, ts: '1700000002.000002' }\n const older = { ...standaloneMessage, ts: '1700000002.000001' }\n const requests: URL[] = []\n const defaults = fixtureDeps()\n const deps = fixtureDeps((url, init) => {\n if (url.pathname !== '/api/conversations.history') return defaults.fetch(url.toString(), init)\n requests.push(url)\n return Response.json({ ok: true, messages: [url.searchParams.get('latest') === newest.ts ? older : newest], has_more: url.searchParams.get('latest') !== newest.ts })\n })\n const journal = createMemoryJournal(), destination = createMemoryDestination()\n expect((await runIngestion(request(pipeline(deps, { maxChunks: 2 })), { journal, destination, now: () => cutoff })).ok).toBe(false)\n const saved = parseMessageState((await journal.readCheckpoint('slack.primary.messages')).envelope?.state)\n expect(saved.active?.latest).toBe(newest.ts)\n expect(JSON.stringify(saved)).not.toContain('cursor')\n expect((await runIngestion(request(pipeline(deps)), { journal, destination, now: () => cutoff })).ok).toBe(true)\n expect(requests.map((url) => url.searchParams.get('latest'))).toEqual(['1700006400.000001', newest.ts])\n expect(requests.every((url) => !url.searchParams.has('cursor'))).toBe(true)\n expect(new Set(destination.tables.get('default.slack_messages_raw')?.map((row) => row.id)).size).toBe(2)\n})\n\ntest('thread reply pages resume from their ascending timestamp frontier', async () => {\n const secondReply = { ...replyMessage, ts: '1700000001.000003', text: 'Second reply' }\n const oldest: string[] = []\n const defaults = fixtureDeps()\n const deps = fixtureDeps((url, init) => {\n if (url.pathname !== '/api/conversations.replies') return defaults.fetch(url.toString(), init)\n oldest.push(url.searchParams.get('oldest') ?? '')\n const resumed = url.searchParams.get('oldest') === replyMessage.ts\n return Response.json({ ok: true, messages: resumed ? [secondReply] : [parentMessage, replyMessage], has_more: !resumed })\n })\n const journal = createMemoryJournal(), destination = createMemoryDestination()\n const first = await runIngestion(request(pipeline(deps, { maxChunks: 3 })), { journal, destination, now: () => cutoff })\n expect(first.streams[0]?.outcome).toBe('budget_exhausted')\n expect(parseMessageState((await journal.readCheckpoint('slack.primary.messages')).envelope?.state).active?.threads[0]?.after).toBe(replyMessage.ts)\n const second = await runIngestion(request(pipeline(deps)), { journal, destination, now: () => cutoff })\n expect(second.ok, JSON.stringify(second)).toBe(true)\n expect(oldest).toEqual(['1700000000.000000', replyMessage.ts])\n expect(destination.tables.get('default.slack_messages_raw')?.map((row) => row.raw)).toContainEqual({ source_id: 'slack.primary', team_id: 'T1', channel_id: 'C1', data: secondReply })\n})\n\ntest('periodic full reconciliation captures new replies on roots outside recent overlap', async () => {\n let now = cutoff, late = false\n const lateReply = { ...replyMessage, ts: dateTimestamp(new Date('2023-11-16T12:00:00Z')), text: 'Late reply' }\n const historyBounds: string[] = []\n const defaults = fixtureDeps()\n const deps = fixtureDeps((url, init) => {\n if (url.pathname === '/api/conversations.history') {\n const from = timestampMicros(url.searchParams.get('oldest') ?? '0.000000')\n historyBounds.push(url.searchParams.get('oldest') ?? '')\n return Response.json({ ok: true, messages: timestampMicros(parentMessage.ts) > from ? [parentMessage] : [], has_more: false })\n }\n if (url.pathname === '/api/conversations.replies') return Response.json({ ok: true, messages: [parentMessage, replyMessage, ...(late ? [lateReply] : [])], has_more: false })\n return defaults.fetch(url.toString(), init)\n })\n const journal = createMemoryJournal(), destination = createMemoryDestination()\n const execute = () => runIngestion(request(pipeline(deps)), { journal, destination, now: () => now })\n expect((await execute()).ok).toBe(true)\n now = new Date('2023-11-16T00:00:00Z')\n expect((await execute()).ok).toBe(true)\n late = true\n now = new Date('2023-11-17T00:00:00Z')\n expect((await execute()).ok).toBe(true)\n expect(destination.tables.get('default.slack_messages_raw')?.some((row) => JSON.stringify(row.raw).includes('Late reply'))).toBe(false)\n now = new Date('2023-11-23T00:00:00Z')\n expect((await execute()).ok).toBe(true)\n expect(historyBounds[0]).toBe('0.000000')\n expect(historyBounds[1]).not.toBe('0.000000')\n expect(historyBounds[2]).not.toBe('0.000000')\n expect(historyBounds[3]).toBe('0.000000')\n expect(destination.tables.get('default.slack_messages_raw')?.some((row) => JSON.stringify(row.raw).includes('Late reply'))).toBe(true)\n})\n\ntest('empty final pages commit progress and account/query changes reject saved state', async () => {\n const defaults = fixtureDeps()\n const deps = fixtureDeps((url, init) => url.pathname === '/api/conversations.history'\n ? Response.json({ ok: true, messages: [], has_more: false }) : defaults.fetch(url.toString(), init))\n const journal = createMemoryJournal(), destination = createMemoryDestination()\n expect((await runIngestion(request(pipeline(deps)), { journal, destination, now: () => cutoff })).ok).toBe(true)\n expect(parseMessageState((await journal.readCheckpoint('slack.primary.messages')).envelope?.state).watermark).toBe(dateTimestamp(cutoff))\n expect(destination.tables.size).toBe(0)\n slackConfig.includeReplies = false\n const failed = await runIngestion(request(pipeline(deps)), { journal, destination, now: () => cutoff })\n expect(failed.ok).toBe(false)\n expect(failed.streams[0]?.error).toContain('different workspace or query')\n expect(() => parseMessageState({ scope: 'broken', watermark: 1700006400, knownChannels: [] })).toThrow('timestamp')\n})\n\ntest('date-bound backfills isolate normal progress and reject changed unfinished bounds', async () => {\n slackConfig.includeReplies = false\n const journal = createMemoryJournal(), destination = createMemoryDestination()\n const selected = selectStreams([pipeline(fixtureDeps(), { maxChunks: 1 })], [])\n const backfill = { id: 'november', from: new Date('2023-11-14T00:00:00Z'), to: cutoff }\n expect((await runIngestion({ selected, backfill }, { journal, destination, now: () => cutoff })).ok).toBe(false)\n expect((await journal.readCheckpoint('slack.primary.messages')).envelope).toBeUndefined()\n const changed = await runIngestion({ selected: selectStreams([pipeline(fixtureDeps())], []), backfill: { ...backfill, to: new Date('2023-11-16T00:00:00Z') } }, { journal, destination, now: () => cutoff })\n expect(changed.ok).toBe(false)\n expect(changed.streams[0]?.error).toContain('bounds changed')\n expect((await runIngestion({ selected: selectStreams([pipeline(fixtureDeps())], []), backfill }, { journal, destination, now: () => cutoff })).ok).toBe(true)\n})\n\ntest('a backfill without an explicit upper bound retains its cutoff across process restarts', async () => {\n const journal = createMemoryJournal(), destination = createMemoryDestination()\n const backfill = { id: 'open-ended', from: new Date('2023-11-14T00:00:00Z'), to: undefined }\n const first = await runIngestion({ selected: selectStreams([pipeline(fixtureDeps(), { maxChunks: 2 })], []), backfill }, { journal, destination, now: () => cutoff })\n expect(first.streams[0]?.outcome).toBe('budget_exhausted')\n const second = await runIngestion({ selected: selectStreams([pipeline(fixtureDeps())], []), backfill }, { journal, destination, now: () => new Date('2023-11-16T00:00:00Z') })\n expect(second.ok, JSON.stringify(second)).toBe(true)\n expect((await journal.readCheckpoint('slack.primary.messages')).envelope).toBeUndefined()\n expect(new Set(destination.tables.get('default.slack_messages_raw')?.map((row) => row.id)).size).toBe(3)\n})\n\ntest('expired temporary cursors replay thread bounds and keep the latest repeated parent payload', async () => {\n const requests: string[] = []\n const updatedParent = { ...parentMessage, text: 'Updated parent' }\n const defaults = fixtureDeps()\n const deps = fixtureDeps((url, init) => {\n if (url.pathname !== '/api/conversations.replies') return defaults.fetch(url.toString(), init)\n const cursor = url.searchParams.get('cursor') ?? ''\n requests.push(cursor)\n if (cursor === 'expired') return Response.json({ ok: false, error: 'invalid_cursor' })\n if (cursor === 'current') return Response.json({ ok: true, messages: [updatedParent, replyMessage], has_more: false })\n return Response.json({ ok: true, messages: [parentMessage], has_more: true, response_metadata: { next_cursor: requests.length === 1 ? 'expired' : 'current' } })\n })\n const journal = createMemoryJournal(), destination = createMemoryDestination()\n const result = await runIngestion(request(pipeline(deps)), { journal, destination, now: () => cutoff })\n expect(result.ok, JSON.stringify(result)).toBe(true)\n expect(requests).toEqual(['', 'expired', '', 'current'])\n const rows = destination.tables.get('default.slack_messages_raw') ?? []\n // One history observation plus one enriched thread observation; cursor pages do not add an older duplicate.\n expect(rows.filter((row) => (row.raw as { data: { ts: string } }).data.ts === parentMessage.ts)).toHaveLength(2)\n expect(rows.map((row) => row.raw)).toContainEqual({ source_id: 'slack.primary', team_id: 'T1', channel_id: 'C1', data: updatedParent })\n expect(JSON.stringify((await journal.readCheckpoint('slack.primary.messages')).envelope?.state)).not.toContain('cursor')\n})\n\nfunction pipeline(deps: SlackClientDeps, budget?: { maxChunks?: number }) {\n return definePipeline({ id: 'slack.primary', retry: { retries: 0 }, streams: [defineStream({\n id: 'slack.primary.messages', destination: slackMessagesRaw, incremental: slackMessageStrategy, batchSize: 1, budget,\n read: (context) => readMessages(context, deps),\n })] })\n}\nfunction request(value: ReturnType) { return { selected: selectStreams([value], []), backfill: undefined } }\n" + } + ], + "meta": { + "chkit": { + "formatVersion": 1, + "version": "0.2.0", + "language": "typescript", + "license": "MIT", + "documentation": "https://chkit.obsessiondb.com/integrations/slack/", + "logo": "https://chkit.obsessiondb.com/logos/slack.svg", + "authentication": { + "method": "Slack OAuth bearer token (user token recommended)", + "env": [ + "SLACK_API_TOKEN" + ], + "setup": [ + "Create a Slack app for the workspace at api.slack.com/apps. Open OAuth & Permissions.", + "Add channels:read, channels:history, and users:read under User Token Scopes for the default three streams. Add users:read.email only when emails are needed.", + "For private channels add groups:read and groups:history; for DMs add im:read and im:history; for group DMs add mpim:read and mpim:history. Enable the matching conversationTypes in config.ts.", + "Install or reinstall the app to the workspace and copy the User OAuth Token into SLACK_API_TOKEN in the project .env or scheduler secret environment. A bot token only reads conversations it can access; enabled thread replies must also be permitted.", + "Keep tokens outside version control. Supply token refresh separately when using expiring tokens; the integration does not manage OAuth issuance or refresh." + ], + "documentation": "https://docs.slack.dev/authentication/tokens/" + }, + "chkit": "^0.2.0-beta.10", + "ingest": "^0.2.0-beta.10", + "clickhouse": ">=25.3.0", + "root": "src/integrations/slack", + "entry": "index.ts", + "exports": [ + "slack", + "slackChannelsRaw", + "slackUsersRaw", + "slackMessagesRaw" + ], + "resources": [ + { + "name": "channels", + "title": "Conversations", + "table": "slack_channels_raw", + "description": "Selected accessible conversations, including archived channels, with complete returned topics, purpose, membership flags, and shared-channel metadata. Public channels by default; private channels, DMs, and group DMs are opt-in.", + "scopes": [ + "channels:read" + ], + "strategy": "full", + "endpoints": [ + { + "method": "GET", + "path": "/conversations.list", + "documentation": "https://docs.slack.dev/reference/methods/conversations.list/" + } + ] + }, + { + "name": "users", + "title": "Users", + "table": "slack_users_raw", + "description": "Complete returned workspace member and bot profiles, IDs, names, status, timezone, and deactivation flags. Email requires the optional users:read.email scope.", + "scopes": [ + "users:read" + ], + "strategy": "full", + "endpoints": [ + { + "method": "GET", + "path": "/users.list", + "documentation": "https://docs.slack.dev/reference/methods/users.list/" + } + ] + }, + { + "name": "messages", + "title": "Messages and thread replies", + "table": "slack_messages_raw", + "description": "Raw messages and replies selected by checkpointed timestamp windows, with retained thread work and periodic full historical reconciliation. Slack timestamps remain exact strings.", + "scopes": [ + "channels:read", + "channels:history" + ], + "strategy": "cursor", + "endpoints": [ + { + "method": "GET", + "path": "/conversations.history", + "documentation": "https://docs.slack.dev/reference/methods/conversations.history/" + }, + { + "method": "GET", + "path": "/conversations.replies", + "documentation": "https://docs.slack.dev/reference/methods/conversations.replies/" + } + ] + } + ], + "views": [], + "sync": { + "description": "Channels and users are full reads. Messages bootstrap retained history, then read overlapping time ranges. Each page checkpoints its timestamp frontier and pending threads after writes. Weekly full reconciliation revisits old roots and edits; temporary Slack cursors are never persisted.", + "schedule": "Run chkit ingest run through cron, CI, or an existing scheduler, with one process per project/target. Interrupted message windows resume their fixed bounds and pending threads; users/channels restart full reads. Conservative message pacing remains 15 items and one request per minute.", + "deletions": "Later observations replace stable row IDs, but deleted, inaccessible, or deselected records remain. The integration does not consume Slack events, propagate deletions, or maintain a complete change log." + }, + "env": { + "SLACK_API_TOKEN": "" + }, + "fileHashes": { + "src/integrations/slack/README.md": "250e312db3fc889608295bbf1895f7706838b0c8ed573cfda45148662f7fdc10", + "src/integrations/slack/client.ts": "2cb6a3dcef7edb03f872a71299e5e6112d29807f83c8932b53bc6702a01af2ae", + "src/integrations/slack/config.ts": "444bbd721aa36631420279846047b819ecea8a615ff8c8e1434f3ce308d1c02e", + "src/integrations/slack/index.ts": "6ff55043ea0e8bbe99722ffdd1c84b06902a917da224d6c7f95e26b9f7f94626", + "src/integrations/slack/pipeline.ts": "0643cc1084b0c330a2ff4857d9e7a6f5fab1dafe55407ec8cc849789bf4faa70", + "src/integrations/slack/sources/channels.ts": "5f7e61d49d08920fac009c674b4406eec3a0a369caebed06e4aad5ba877e931f", + "src/integrations/slack/sources/users.ts": "cc1ff32193965aff706a9c4b0cc4677f1dfe2a5e517ce3f237f1363f809da0b0", + "src/integrations/slack/sources/messages.ts": "011c5049841e8d9d8e562ae850c8f1ffbd82f5e306ed6be30d697639fc8098f4", + "src/integrations/slack/tests/slack.test.ts": "5c659d8fd435a1d1dc3f598786e8e0b0fff99fd569f53aaa6efbdb706bf79fa8", + "src/integrations/slack/tests/fixtures.ts": "844413afb249bd59d1e01de0c7ef260315667f14dcf6be481edf3703d9c59d7c", + "src/integrations/slack/state.ts": "8abefab79594ba672034a7ef8301cc3919ac62d9606c9c3da65a43169311ed08", + "src/integrations/slack/tests/checkpoints.test.ts": "b94d2e1d5eb6d3687bfd7f51070b2f4766c60f3c692b50151fc05093d5eaeb07" + } + } + } +} diff --git a/registry/slack/sources/messages.ts b/registry/slack/sources/messages.ts index 8e3bf9b7..fb4c9d73 100644 --- a/registry/slack/sources/messages.ts +++ b/registry/slack/sources/messages.ts @@ -1,38 +1,119 @@ -import { rawTable, type FetchContext } from '@chkit/plugin-ingest' +import { IngestConfigError, rawTable, type ReadContext, type SourceChunk, type RawRow } from '@chkit/plugin-ingest' -import { discoverChannels, entityId, getWorkspace, readCollection, toSlackRows, type SlackClientDeps } from '../client.js' +import { discoverChannels, entityId, getWorkspace, readCollectionPage, SlackCursorExpiredError, toSlackRows, type SlackClientDeps, type SlackEntity, type SlackPage } from '../client.js' import { slackConfig } from '../config.js' +import { formatTimestamp, messageScope, timestampMicros, type SlackMessageSelection, type SlackMessageState } from '../state.js' export const slackMessagesRaw = rawTable({ database: slackConfig.database, name: `${slackConfig.tablePrefix}_messages_raw` }) -/** Full scans revisit old roots as well as recent messages, including late thread replies. */ -export async function* readMessages(context: FetchContext, deps?: SlackClientDeps) { +/** Temporary API cursors never enter the checkpoint; exact timestamps survive expiration. */ +export async function* readMessages( + context: ReadContext, + deps?: SlackClientDeps, +): AsyncGenerator> { const teamId = await getWorkspace(context, deps) - for (const channel of await discoverChannels(context, deps)) { - const channelId = entityId(channel, 'id') - const threads = new Set() - for await (const page of readCollection(context, { - method: 'conversations.history', field: 'messages', idField: 'ts', - query: { channel: channelId, include_all_metadata: 'true' }, - }, deps)) { - yield { rows: toSlackRows(page, 'messages', teamId, 'ts', channelId) } - if (!slackConfig.includeReplies) continue - for (const message of page) { - if (typeof message.reply_count !== 'number' || message.reply_count === 0) continue - const root = typeof message.thread_ts === 'string' ? entityId(message, 'thread_ts') : entityId(message, 'ts') - threads.add(root) - } + const scope = messageScope(teamId) + if (context.state && context.state.scope !== scope) throw new IngestConfigError('Slack checkpoint belongs to a different workspace or query. Restore the configuration or use a distinct sourceId.') + let state: SlackMessageState = context.state ?? { scope, watermark: slackConfig.historyFrom, knownChannels: [] } + if (!state.active) { + const channels = (await discoverChannels(context, deps)).map((channel) => entityId(channel, 'id')).sort() + const to = context.selection.to + const full = state.lastFullAt === undefined || timestampMicros(to) - timestampMicros(state.lastFullAt) >= BigInt(slackConfig.reconcileIntervalMs) * 1_000n + const from = context.selection.from ?? (full ? slackConfig.historyFrom : formatTimestamp(max(timestampMicros(slackConfig.historyFrom), timestampMicros(state.watermark) - BigInt(slackConfig.overlapMs) * 1_000n))) + if (timestampMicros(from) > timestampMicros(to)) throw new IngestConfigError('Slack selection starts after its cutoff.') + state = { ...state, active: { kind: context.selection.backfill ? 'backfill' : full ? 'full' : 'incremental', from, to, channels, channelIndex: 0, + latest: formatTimestamp(timestampMicros(to) + 1n), historyDone: false, threads: [], + requestedFrom: context.selection.from, requestedTo: context.selection.requestedTo } } + yield marker(state, 'window-open') + } else if (context.selection.backfill && (context.selection.from !== state.active.requestedFrom || context.selection.requestedTo !== state.active.requestedTo)) { + throw new IngestConfigError('Slack backfill bounds changed while a window is unfinished. Resume the original range or use a new backfill ID.') + } + + while (state.active && state.active.channelIndex < state.active.channels.length) { + context.signal.throwIfAborted() + const work = state.active + const channelId = work.channels[work.channelIndex] + if (!channelId) throw new IngestConfigError('Slack checkpoint has no current channel.') + const thread = work.threads[0] + if (thread) { + const after = thread.after ?? formatTimestamp(max(0n, timestampMicros(thread.root) - 1n)) + const page = await nonemptyPage(context, { channel: channelId, ts: thread.root, oldest: after, + latest: formatTimestamp(timestampMicros(work.to) + 1n), inclusive: 'false', include_all_metadata: 'true' }, 'conversations.replies', deps) + const replies = page.items.filter((item) => entityId(item, 'ts') !== thread.root) + assertRange(replies, timestampMicros(after) + 1n, timestampMicros(work.to)) + const frontier = replies.length ? formatTimestamp(replies.reduce((latest, item) => max(latest, timestampMicros(entityId(item, 'ts'))), timestampMicros(after))) : undefined + if (page.hasMore && frontier === undefined) throw new IngestConfigError('Slack thread pagination has no safe timestamp progress.') + const threads = page.hasMore ? [{ root: thread.root, after: frontier }, ...work.threads.slice(1)] : work.threads.slice(1) + state = { ...state, active: { ...work, threads } } + yield page.items.length ? { rows: toSlackRows(page.items, 'messages', teamId, 'ts', channelId), state } : marker(state, 'thread-complete') + continue } - // Finish history pagination before scanning threads: a long thread must not leave - // the history cursor idle for hours, since Slack cursors expire. - for (const root of threads) { - for await (const replies of readCollection(context, { - method: 'conversations.replies', field: 'messages', idField: 'ts', - query: { channel: channelId, ts: root, include_all_metadata: 'true' }, - }, deps)) { - // Slack repeats the parent; the shared channel/ts identity reconciles observations. - yield { rows: toSlackRows(replies, 'messages', teamId, 'ts', channelId) } - } + if (work.historyDone) { + state = { ...state, active: { ...work, channelIndex: work.channelIndex + 1, latest: formatTimestamp(timestampMicros(work.to) + 1n), historyDone: false } } + yield marker(state, 'channel-complete') + continue } + const from = work.kind === 'incremental' && !state.knownChannels.includes(channelId) ? slackConfig.historyFrom : work.from + const page = await nonemptyPage(context, { channel: channelId, oldest: formatTimestamp(max(0n, timestampMicros(from) - 1n)), latest: work.latest, + inclusive: 'false', include_all_metadata: 'true' }, 'conversations.history', deps) + assertRange(page.items, timestampMicros(from), timestampMicros(work.latest) - 1n) + const latest = page.items.length ? formatTimestamp(page.items.reduce((oldest, item) => min(oldest, timestampMicros(entityId(item, 'ts'))), timestampMicros(work.latest))) : work.latest + if (page.hasMore && (page.items.length === 0 || timestampMicros(latest) >= timestampMicros(work.latest))) throw new IngestConfigError('Slack history pagination has no safe timestamp progress.') + const roots = slackConfig.includeReplies ? page.items.filter((item) => typeof item.reply_count === 'number' && item.reply_count > 0) + .map((item) => typeof item.thread_ts === 'string' ? entityId(item, 'thread_ts') : entityId(item, 'ts')) : [] + state = { ...state, active: { ...work, latest, historyDone: !page.hasMore, threads: [...new Set(roots)].map((root) => ({ root })) } } + // The page retains its bounded child queue before the history frontier advances durably. + yield page.items.length ? { rows: toSlackRows(page.items, 'messages', teamId, 'ts', channelId), state } : marker(state, 'history-complete') + } + if (state.active) { + const { to, kind, channels } = state.active + state = { scope, watermark: formatTimestamp(max(timestampMicros(state.watermark), timestampMicros(to))), + knownChannels: [...new Set([...state.knownChannels, ...channels])].sort(), lastFullAt: kind === 'full' ? to : state.lastFullAt } + yield marker(state, 'window-complete') } } + +/** Empty cursor pages use the live cursor; nonempty pages establish timestamp boundaries. */ +async function nonemptyPage( + context: ReadContext, query: Record, + method: 'conversations.history' | 'conversations.replies', deps?: SlackClientDeps, +): Promise { + let cursor = '' + let restarted = false + const seen = new Set() + const parents = new Map() + while (true) { + let page: SlackPage + try { + page = await readCollectionPage(context, { method, field: 'messages', idField: 'ts', query, timePagination: true }, cursor, deps) + } catch (error) { + if (!(error instanceof SlackCursorExpiredError) || !cursor || restarted) throw error + cursor = '' + restarted = true + seen.clear() + continue + } + const parentOnly = method === 'conversations.replies' && page.items.every((item) => entityId(item, 'ts') === query.ts) + if (page.items.length > 0 && (!parentOnly || !page.hasMore) || !page.hasMore) { + for (const item of page.items) parents.set(entityId(item, 'ts'), item) + return { ...page, items: [...parents.values()] } + } + for (const item of page.items) parents.set(entityId(item, 'ts'), item) + if (!page.cursor || seen.has(page.cursor)) throw new IngestConfigError('Slack empty page did not provide a new continuation.') + seen.add(page.cursor) + cursor = page.cursor + } +} + +function marker(state: SlackMessageState, id: string): SourceChunk { + // State-only boundaries flush previous rows without assigning identity to mutable data. + return { rows: [], state, id } +} +function assertRange(items: SlackEntity[], from: bigint, to: bigint): void { + if (items.some((item) => { + const ts = timestampMicros(entityId(item, 'ts')) + return ts < from || ts > to + })) throw new IngestConfigError('Slack returned a message outside the requested timestamp interval.') +} +function max(left: bigint, right: bigint): bigint { return left > right ? left : right } +function min(left: bigint, right: bigint): bigint { return left < right ? left : right } diff --git a/registry/slack/state.ts b/registry/slack/state.ts new file mode 100644 index 00000000..12e36d23 --- /dev/null +++ b/registry/slack/state.ts @@ -0,0 +1,104 @@ +import { IngestConfigError, type IncrementalStrategy } from '@chkit/plugin-ingest' +import { slackConfig } from './config.js' + +export interface PendingThread { root: string; after?: string } +export interface MessageWork { + kind: 'full' | 'incremental' | 'backfill' + from: string + to: string + channels: string[] + channelIndex: number + latest: string + historyDone: boolean + threads: PendingThread[] + requestedFrom?: string + requestedTo?: string +} +export interface SlackMessageState { + scope: string + watermark: string + knownChannels: string[] + lastFullAt?: string + active?: MessageWork +} +export interface SlackMessageSelection { from?: string; to: string; requestedTo?: string; backfill: boolean } + +export const slackMessageStrategy: IncrementalStrategy = { + id: 'slack.message-ranges', version: 1, parseState: parseMessageState, + plan({ cutoff, range }) { + validateMessageConfig() + return { from: range?.from ? dateTimestamp(range.from) : undefined, to: dateTimestamp(range?.to ?? cutoff), + requestedTo: range?.to ? dateTimestamp(range.to) : undefined, backfill: range !== undefined } + }, +} + +export function messageScope(teamId: string): string { + return JSON.stringify([slackConfig.sourceId, teamId, slackConfig.channels ? [...slackConfig.channels].sort() : null, + [...slackConfig.conversationTypes].sort(), slackConfig.includeReplies, slackConfig.historyFrom, + slackConfig.overlapMs, slackConfig.reconcileIntervalMs]) +} + +export function timestampMicros(value: string): bigint { + if (!/^\d+\.\d{1,6}$/.test(value)) throw new IngestConfigError('Slack timestamps must be exact decimal strings with at most six fractional digits.') + const [seconds = '', fraction = ''] = value.split('.') + return BigInt(seconds) * 1_000_000n + BigInt(fraction.padEnd(6, '0')) +} + +export function formatTimestamp(value: bigint): string { + if (value < 0n) throw new IngestConfigError('Slack timestamp cannot be negative.') + return `${value / 1_000_000n}.${(value % 1_000_000n).toString().padStart(6, '0')}` +} + +export function dateTimestamp(value: Date): string { + if (!Number.isSafeInteger(value.getTime()) || value.getTime() < 0) throw new IngestConfigError('Slack time bounds must be valid dates after the Unix epoch.') + return formatTimestamp(BigInt(value.getTime()) * 1_000n) +} + +export function parseMessageState(raw: unknown): SlackMessageState { + if (!isObject(raw) || typeof raw.scope !== 'string') throw new IngestConfigError('Invalid Slack checkpoint scope.') + const watermark = readTimestamp(raw.watermark) + const knownChannels = readIds(raw.knownChannels) + const lastFullAt = raw.lastFullAt === undefined ? undefined : readTimestamp(raw.lastFullAt) + let active: MessageWork | undefined + if (raw.active !== undefined) { + const work = raw.active + if (!isObject(work) || typeof work.channelIndex !== 'number' || !Number.isSafeInteger(work.channelIndex) + || typeof work.historyDone !== 'boolean' || !Array.isArray(work.threads)) throw new IngestConfigError('Invalid Slack active work.') + const kind = work.kind + if (kind !== 'full' && kind !== 'incremental' && kind !== 'backfill') throw new IngestConfigError('Invalid Slack work kind.') + const channels = readIds(work.channels) + if (work.channelIndex < 0 || work.channelIndex > channels.length) throw new IngestConfigError('Invalid Slack channel position.') + const from = readTimestamp(work.from), to = readTimestamp(work.to), latest = readTimestamp(work.latest) + if (timestampMicros(from) > timestampMicros(to) || timestampMicros(latest) > timestampMicros(to) + 1n) throw new IngestConfigError('Invalid Slack window bounds.') + const threads = work.threads.map((thread: unknown) => { + if (!isObject(thread)) throw new IngestConfigError('Invalid Slack pending thread.') + const root = readTimestamp(thread.root) + const after = thread.after === undefined ? undefined : readTimestamp(thread.after) + if (timestampMicros(root) > timestampMicros(to) || (after !== undefined && (timestampMicros(after) < timestampMicros(root) || timestampMicros(after) > timestampMicros(to)))) throw new IngestConfigError('Invalid Slack thread frontier.') + return { root, after } + }) + if (new Set(threads.map((thread) => thread.root)).size !== threads.length) throw new IngestConfigError('Duplicate Slack pending threads.') + active = { kind, from, to, channels, channelIndex: work.channelIndex, latest, historyDone: work.historyDone, threads, + requestedFrom: work.requestedFrom === undefined ? undefined : readTimestamp(work.requestedFrom), + requestedTo: work.requestedTo === undefined ? undefined : readTimestamp(work.requestedTo) } + } + return { scope: raw.scope, watermark, knownChannels, lastFullAt, active } +} + +function validateMessageConfig(): void { + timestampMicros(slackConfig.historyFrom) + for (const value of [slackConfig.overlapMs, slackConfig.reconcileIntervalMs]) { + if (!Number.isSafeInteger(value) || value < 0) throw new IngestConfigError('Slack overlap and reconciliation intervals must be non-negative safe integers.') + } +} +function readTimestamp(value: unknown): string { + if (typeof value !== 'string') throw new IngestConfigError('Invalid Slack checkpoint timestamp.') + timestampMicros(value) + return value +} +function readIds(value: unknown): string[] { + if (!Array.isArray(value) || !value.every((item: unknown): item is string => typeof item === 'string' && item.length > 0) + || new Set(value).size !== value.length) throw new IngestConfigError('Invalid Slack checkpoint channel IDs.') + return [...value] +} +function isObject(value: unknown): value is Record { return typeof value === 'object' && value !== null && !Array.isArray(value) } diff --git a/registry/slack/tests/checkpoints.test.ts b/registry/slack/tests/checkpoints.test.ts new file mode 100644 index 00000000..a53f5f35 --- /dev/null +++ b/registry/slack/tests/checkpoints.test.ts @@ -0,0 +1,198 @@ +import { afterEach, expect, test } from 'bun:test' +import { definePipeline, defineStream, runIngestion, selectStreams } from '@chkit/plugin-ingest' +import { createMemoryDestination, createMemoryJournal } from '@chkit/plugin-ingest/testing' + +import { slackConfig } from '../config.js' +import { readMessages, slackMessagesRaw } from '../sources/messages.js' +import { dateTimestamp, parseMessageState, slackMessageStrategy, timestampMicros } from '../state.js' +import { fixtureDeps, parentMessage, replyMessage, standaloneMessage } from './fixtures.js' +import type { SlackClientDeps } from '../client.js' + +const originalConfig = { ...slackConfig } +const cutoff = new Date('2023-11-15T00:00:00Z') +afterEach(() => { Object.assign(slackConfig, originalConfig) }) + +test('a fresh process resumes retained threads before advancing the frozen history window', async () => { + const journal = createMemoryJournal(), destination = createMemoryDestination() + const calls: string[] = [] + const defaults = fixtureDeps() + const deps = fixtureDeps((url, init) => { calls.push(url.pathname); return defaults.fetch(url.toString(), init) }) + const limited = pipeline(deps, { maxChunks: 2 }) + const first = await runIngestion(request(limited), { journal, destination, now: () => cutoff }) + expect(first.streams[0]?.outcome).toBe('budget_exhausted') + const saved = parseMessageState((await journal.readCheckpoint('slack.primary.messages')).envelope?.state) + expect(saved.watermark).toBe(slackConfig.historyFrom) + expect(saved.active?.to).toBe(dateTimestamp(cutoff)) + expect(saved.active?.threads).toEqual([{ root: parentMessage.ts }]) + expect(saved.active?.historyDone).toBe(true) + calls.length = 0 + const second = await runIngestion(request(pipeline(deps)), { journal, destination, now: () => new Date('2023-11-16T00:00:00Z') }) + expect(second.ok, JSON.stringify(second)).toBe(true) + expect(calls).toEqual(['/api/auth.test', '/api/conversations.replies']) + const completed = parseMessageState((await journal.readCheckpoint('slack.primary.messages')).envelope?.state) + expect(completed.active).toBeUndefined() + expect(completed.watermark).toBe(dateTimestamp(cutoff)) + expect(destination.tables.get('default.slack_messages_raw')?.map((row) => row.raw)).toContainEqual({ source_id: 'slack.primary', team_id: 'T1', channel_id: 'C1', data: replyMessage }) +}) + +test('a lost destination acknowledgement cannot move the history frontier past unpublished journal progress', async () => { + slackConfig.includeReplies = false + const journal = createMemoryJournal(), destination = createMemoryDestination() + const controller = new AbortController() + const first = await runIngestion(request(pipeline(fixtureDeps())), { + journal, now: () => cutoff, signal: controller.signal, + destination: { async insert(input) { await destination.insert(input); controller.abort(new Error('lost acknowledgement')) } }, + }) + expect(first.ok).toBe(false) + const saved = parseMessageState((await journal.readCheckpoint('slack.primary.messages')).envelope?.state) + expect(saved.active?.latest).toBe('1700006400.000001') + expect(saved.active?.historyDone).toBe(false) + const second = await runIngestion(request(pipeline(fixtureDeps())), { journal, destination, now: () => cutoff }) + expect(second.ok, JSON.stringify(second)).toBe(true) + expect(destination.tables.get('default.slack_messages_raw')).toHaveLength(2) + expect(parseMessageState((await journal.readCheckpoint('slack.primary.messages')).envelope?.state).active).toBeUndefined() +}) + +test('history resumes newest-first without saving expiring cursors or converting timestamp precision', async () => { + slackConfig.includeReplies = false + const newest = { ...standaloneMessage, ts: '1700000002.000002' } + const older = { ...standaloneMessage, ts: '1700000002.000001' } + const requests: URL[] = [] + const defaults = fixtureDeps() + const deps = fixtureDeps((url, init) => { + if (url.pathname !== '/api/conversations.history') return defaults.fetch(url.toString(), init) + requests.push(url) + return Response.json({ ok: true, messages: [url.searchParams.get('latest') === newest.ts ? older : newest], has_more: url.searchParams.get('latest') !== newest.ts }) + }) + const journal = createMemoryJournal(), destination = createMemoryDestination() + expect((await runIngestion(request(pipeline(deps, { maxChunks: 2 })), { journal, destination, now: () => cutoff })).ok).toBe(false) + const saved = parseMessageState((await journal.readCheckpoint('slack.primary.messages')).envelope?.state) + expect(saved.active?.latest).toBe(newest.ts) + expect(JSON.stringify(saved)).not.toContain('cursor') + expect((await runIngestion(request(pipeline(deps)), { journal, destination, now: () => cutoff })).ok).toBe(true) + expect(requests.map((url) => url.searchParams.get('latest'))).toEqual(['1700006400.000001', newest.ts]) + expect(requests.every((url) => !url.searchParams.has('cursor'))).toBe(true) + expect(new Set(destination.tables.get('default.slack_messages_raw')?.map((row) => row.id)).size).toBe(2) +}) + +test('thread reply pages resume from their ascending timestamp frontier', async () => { + const secondReply = { ...replyMessage, ts: '1700000001.000003', text: 'Second reply' } + const oldest: string[] = [] + const defaults = fixtureDeps() + const deps = fixtureDeps((url, init) => { + if (url.pathname !== '/api/conversations.replies') return defaults.fetch(url.toString(), init) + oldest.push(url.searchParams.get('oldest') ?? '') + const resumed = url.searchParams.get('oldest') === replyMessage.ts + return Response.json({ ok: true, messages: resumed ? [secondReply] : [parentMessage, replyMessage], has_more: !resumed }) + }) + const journal = createMemoryJournal(), destination = createMemoryDestination() + const first = await runIngestion(request(pipeline(deps, { maxChunks: 3 })), { journal, destination, now: () => cutoff }) + expect(first.streams[0]?.outcome).toBe('budget_exhausted') + expect(parseMessageState((await journal.readCheckpoint('slack.primary.messages')).envelope?.state).active?.threads[0]?.after).toBe(replyMessage.ts) + const second = await runIngestion(request(pipeline(deps)), { journal, destination, now: () => cutoff }) + expect(second.ok, JSON.stringify(second)).toBe(true) + expect(oldest).toEqual(['1700000000.000000', replyMessage.ts]) + expect(destination.tables.get('default.slack_messages_raw')?.map((row) => row.raw)).toContainEqual({ source_id: 'slack.primary', team_id: 'T1', channel_id: 'C1', data: secondReply }) +}) + +test('periodic full reconciliation captures new replies on roots outside recent overlap', async () => { + let now = cutoff, late = false + const lateReply = { ...replyMessage, ts: dateTimestamp(new Date('2023-11-16T12:00:00Z')), text: 'Late reply' } + const historyBounds: string[] = [] + const defaults = fixtureDeps() + const deps = fixtureDeps((url, init) => { + if (url.pathname === '/api/conversations.history') { + const from = timestampMicros(url.searchParams.get('oldest') ?? '0.000000') + historyBounds.push(url.searchParams.get('oldest') ?? '') + return Response.json({ ok: true, messages: timestampMicros(parentMessage.ts) > from ? [parentMessage] : [], has_more: false }) + } + if (url.pathname === '/api/conversations.replies') return Response.json({ ok: true, messages: [parentMessage, replyMessage, ...(late ? [lateReply] : [])], has_more: false }) + return defaults.fetch(url.toString(), init) + }) + const journal = createMemoryJournal(), destination = createMemoryDestination() + const execute = () => runIngestion(request(pipeline(deps)), { journal, destination, now: () => now }) + expect((await execute()).ok).toBe(true) + now = new Date('2023-11-16T00:00:00Z') + expect((await execute()).ok).toBe(true) + late = true + now = new Date('2023-11-17T00:00:00Z') + expect((await execute()).ok).toBe(true) + expect(destination.tables.get('default.slack_messages_raw')?.some((row) => JSON.stringify(row.raw).includes('Late reply'))).toBe(false) + now = new Date('2023-11-23T00:00:00Z') + expect((await execute()).ok).toBe(true) + expect(historyBounds[0]).toBe('0.000000') + expect(historyBounds[1]).not.toBe('0.000000') + expect(historyBounds[2]).not.toBe('0.000000') + expect(historyBounds[3]).toBe('0.000000') + expect(destination.tables.get('default.slack_messages_raw')?.some((row) => JSON.stringify(row.raw).includes('Late reply'))).toBe(true) +}) + +test('empty final pages commit progress and account/query changes reject saved state', async () => { + const defaults = fixtureDeps() + const deps = fixtureDeps((url, init) => url.pathname === '/api/conversations.history' + ? Response.json({ ok: true, messages: [], has_more: false }) : defaults.fetch(url.toString(), init)) + const journal = createMemoryJournal(), destination = createMemoryDestination() + expect((await runIngestion(request(pipeline(deps)), { journal, destination, now: () => cutoff })).ok).toBe(true) + expect(parseMessageState((await journal.readCheckpoint('slack.primary.messages')).envelope?.state).watermark).toBe(dateTimestamp(cutoff)) + expect(destination.tables.size).toBe(0) + slackConfig.includeReplies = false + const failed = await runIngestion(request(pipeline(deps)), { journal, destination, now: () => cutoff }) + expect(failed.ok).toBe(false) + expect(failed.streams[0]?.error).toContain('different workspace or query') + expect(() => parseMessageState({ scope: 'broken', watermark: 1700006400, knownChannels: [] })).toThrow('timestamp') +}) + +test('date-bound backfills isolate normal progress and reject changed unfinished bounds', async () => { + slackConfig.includeReplies = false + const journal = createMemoryJournal(), destination = createMemoryDestination() + const selected = selectStreams([pipeline(fixtureDeps(), { maxChunks: 1 })], []) + const backfill = { id: 'november', from: new Date('2023-11-14T00:00:00Z'), to: cutoff } + expect((await runIngestion({ selected, backfill }, { journal, destination, now: () => cutoff })).ok).toBe(false) + expect((await journal.readCheckpoint('slack.primary.messages')).envelope).toBeUndefined() + const changed = await runIngestion({ selected: selectStreams([pipeline(fixtureDeps())], []), backfill: { ...backfill, to: new Date('2023-11-16T00:00:00Z') } }, { journal, destination, now: () => cutoff }) + expect(changed.ok).toBe(false) + expect(changed.streams[0]?.error).toContain('bounds changed') + expect((await runIngestion({ selected: selectStreams([pipeline(fixtureDeps())], []), backfill }, { journal, destination, now: () => cutoff })).ok).toBe(true) +}) + +test('a backfill without an explicit upper bound retains its cutoff across process restarts', async () => { + const journal = createMemoryJournal(), destination = createMemoryDestination() + const backfill = { id: 'open-ended', from: new Date('2023-11-14T00:00:00Z'), to: undefined } + const first = await runIngestion({ selected: selectStreams([pipeline(fixtureDeps(), { maxChunks: 2 })], []), backfill }, { journal, destination, now: () => cutoff }) + expect(first.streams[0]?.outcome).toBe('budget_exhausted') + const second = await runIngestion({ selected: selectStreams([pipeline(fixtureDeps())], []), backfill }, { journal, destination, now: () => new Date('2023-11-16T00:00:00Z') }) + expect(second.ok, JSON.stringify(second)).toBe(true) + expect((await journal.readCheckpoint('slack.primary.messages')).envelope).toBeUndefined() + expect(new Set(destination.tables.get('default.slack_messages_raw')?.map((row) => row.id)).size).toBe(3) +}) + +test('expired temporary cursors replay thread bounds and keep the latest repeated parent payload', async () => { + const requests: string[] = [] + const updatedParent = { ...parentMessage, text: 'Updated parent' } + const defaults = fixtureDeps() + const deps = fixtureDeps((url, init) => { + if (url.pathname !== '/api/conversations.replies') return defaults.fetch(url.toString(), init) + const cursor = url.searchParams.get('cursor') ?? '' + requests.push(cursor) + if (cursor === 'expired') return Response.json({ ok: false, error: 'invalid_cursor' }) + if (cursor === 'current') return Response.json({ ok: true, messages: [updatedParent, replyMessage], has_more: false }) + return Response.json({ ok: true, messages: [parentMessage], has_more: true, response_metadata: { next_cursor: requests.length === 1 ? 'expired' : 'current' } }) + }) + const journal = createMemoryJournal(), destination = createMemoryDestination() + const result = await runIngestion(request(pipeline(deps)), { journal, destination, now: () => cutoff }) + expect(result.ok, JSON.stringify(result)).toBe(true) + expect(requests).toEqual(['', 'expired', '', 'current']) + const rows = destination.tables.get('default.slack_messages_raw') ?? [] + // One history observation plus one enriched thread observation; cursor pages do not add an older duplicate. + expect(rows.filter((row) => (row.raw as { data: { ts: string } }).data.ts === parentMessage.ts)).toHaveLength(2) + expect(rows.map((row) => row.raw)).toContainEqual({ source_id: 'slack.primary', team_id: 'T1', channel_id: 'C1', data: updatedParent }) + expect(JSON.stringify((await journal.readCheckpoint('slack.primary.messages')).envelope?.state)).not.toContain('cursor') +}) + +function pipeline(deps: SlackClientDeps, budget?: { maxChunks?: number }) { + return definePipeline({ id: 'slack.primary', retry: { retries: 0 }, streams: [defineStream({ + id: 'slack.primary.messages', destination: slackMessagesRaw, incremental: slackMessageStrategy, batchSize: 1, budget, + read: (context) => readMessages(context, deps), + })] }) +} +function request(value: ReturnType) { return { selected: selectStreams([value], []), backfill: undefined } } diff --git a/registry/slack/tests/install.e2e.test.ts b/registry/slack/tests/install.e2e.test.ts index 21514e42..ac85b7a7 100644 --- a/registry/slack/tests/install.e2e.test.ts +++ b/registry/slack/tests/install.e2e.test.ts @@ -5,12 +5,13 @@ import { join, resolve } from 'node:path' import { pathToFileURL } from 'node:url' import { createStatelessLiveExecutor, createPrefix, getLiveEnv, quoteIdent, waitForTable } from '@chkit/clickhouse/e2e-testkit' -import { createClickHouseDestination, definePipeline, runIngestion, selectStreams, type FetchContext, type SourceChunk } from '@chkit/plugin-ingest' +import { createClickHouseDestination, definePipeline, defineStream, runIngestion, selectStreams, type FetchContext, type SourceChunk } from '@chkit/plugin-ingest' import { createMemoryJournal } from '@chkit/plugin-ingest/testing' import { buildRegistryCatalog } from '../../../packages/cli/src/registry/build.js' import { formatTestDiagnostic, runCli } from '../../../packages/cli/src/test/e2e-testkit.js' import { readRegistrySourceCatalog } from '../../../scripts/registry-catalog.js' +import { CLI_VERSION } from '../../../packages/cli/src/runtime/version.js' import { channel, fixtureDeps, parentMessage, replyMessage, standaloneMessage, user } from './fixtures.js' describe.serial('built Slack template installed into a consumer', () => { @@ -33,7 +34,13 @@ describe.serial('built Slack template installed into a consumer', () => { const directory = await mkdtemp(join(tmpdir(), 'chkit-slack-installed-')) temporary.push(directory) const outputDir = join(directory, 'registry') - await buildRegistryCatalog({ catalog: await readRegistrySourceCatalog(), sourceRoot: join(root, 'registry'), outputDir }) + // Source targets the next release; exercise it with this checkout's toolchain. + const sourceCatalog = await readRegistrySourceCatalog() + const catalog = { ...sourceCatalog, items: sourceCatalog.items.map((item) => ({ ...item, + dependencies: [`@chkit/core@${CLI_VERSION}`, `@chkit/plugin-ingest@${CLI_VERSION}`], + meta: { chkit: { ...item.meta.chkit, version: `${item.meta.chkit.version}-workspace`, chkit: CLI_VERSION, ingest: CLI_VERSION } }, + })) } + await buildRegistryCatalog({ catalog, sourceRoot: join(root, 'registry'), outputDir }) const project = join(directory, 'project') await writeProject(project, root, env.clickhouseDatabase, prefix) const cliEnv = { @@ -55,7 +62,7 @@ describe.serial('built Slack template installed into a consumer', () => { const add = cli(['add', 'slack', '--registry', outputDir, '--with-tests', '--no-install', '--json']) expect(add.exitCode, formatTestDiagnostic('install', add)).toBe(0) expect(JSON.parse(add.stdout).template.name).toBe('slack') - const fixtureTests = Bun.spawnSync(['bun', 'test', 'src/integrations/slack/tests/slack.test.ts'], { + const fixtureTests = Bun.spawnSync(['bun', 'test', 'src/integrations/slack/tests/slack.test.ts', 'src/integrations/slack/tests/checkpoints.test.ts'], { cwd: project, env: { ...process.env, SLACK_API_TOKEN: '' }, }) expect(fixtureTests.exitCode, fixtureTests.stderr.toString()).toBe(0) @@ -76,6 +83,7 @@ describe.serial('built Slack template installed into a consumer', () => { const channels: typeof import('../sources/channels.js') = await import(pathToFileURL(join(project, 'src/integrations/slack/sources/channels.ts')).href) const users: typeof import('../sources/users.js') = await import(pathToFileURL(join(project, 'src/integrations/slack/sources/users.ts')).href) const messages: typeof import('../sources/messages.js') = await import(pathToFileURL(join(project, 'src/integrations/slack/sources/messages.ts')).href) + const state: typeof import('../state.js') = await import(pathToFileURL(join(project, 'src/integrations/slack/state.ts')).href) let revision = 1 const defaults = fixtureDeps() const deps = fixtureDeps((url, init) => { @@ -87,11 +95,11 @@ describe.serial('built Slack template installed into a consumer', () => { const readers: Record AsyncIterable> = { channels: (context) => channels.readChannels(context, deps), users: (context) => users.readUsers(context, deps), - messages: (context) => messages.readMessages(context, deps), } const pipeline = definePipeline({ ...installed.slack, retry: { retries: 0 }, streams: installed.slack.streams.map((stream) => { + if (stream.id.endsWith('.messages')) return defineStream({ ...stream, incremental: state.slackMessageStrategy, read: (context) => messages.readMessages(context, deps) }) const read = readers[stream.id.split('.').at(-1) ?? ''] if (!read) throw new Error(`Installed fixture reader missing for ${stream.id}`) return { ...stream, read } @@ -100,7 +108,7 @@ describe.serial('built Slack template installed into a consumer', () => { const journal = createMemoryJournal() for (const next of [1, 2]) { revision = next - const result = await runIngestion({ selected: selectStreams([pipeline], []), backfill: undefined }, { journal, destination: createClickHouseDestination(executor) }) + const result = await runIngestion({ selected: selectStreams([pipeline], []), backfill: undefined }, { journal, destination: createClickHouseDestination(executor), now: () => new Date('2023-11-15T00:00:00Z') }) expect(result.ok, JSON.stringify(result)).toBe(true) } const table = (resource: string) => `${quoteIdent(env.clickhouseDatabase)}.${quoteIdent(`${prefix}_${resource}_raw`)}` diff --git a/registry/slack/tests/slack.test.ts b/registry/slack/tests/slack.test.ts index f3f4aab3..b2fdca90 100644 --- a/registry/slack/tests/slack.test.ts +++ b/registry/slack/tests/slack.test.ts @@ -1,6 +1,6 @@ import { afterEach, describe, expect, test } from 'bun:test' -import { HttpError, definePipeline, runIngestion, selectStreams, type FetchContext, type SourceChunk } from '@chkit/plugin-ingest' +import { HttpError, definePipeline, defineStream, runIngestion, selectStreams, type FetchContext, type ReadContext, type SourceChunk } from '@chkit/plugin-ingest' import { createMemoryDestination, createMemoryJournal } from '@chkit/plugin-ingest/testing' import { classifySlackError, discoverChannels, readCollection, type SlackClientDeps } from '../client.js' @@ -10,24 +10,26 @@ import { slack } from '../pipeline.js' import { readChannels } from '../sources/channels.js' import { readUsers } from '../sources/users.js' import { readMessages } from '../sources/messages.js' +import { slackMessageStrategy, type SlackMessageSelection, type SlackMessageState } from '../state.js' import { channel, fixtureDeps, parentMessage, replyMessage, standaloneMessage, user } from './fixtures.js' const readers: Record AsyncIterable> = { - channels: readChannels, users: readUsers, messages: readMessages, + channels: readChannels, users: readUsers, } const originalConfig = { ...slackConfig } +const fixtureNow = () => new Date('2023-11-15T00:00:00Z') afterEach(() => { Object.assign(slackConfig, originalConfig) }) describe('Slack raw template', () => { - test('exports three raw destinations and full-read streams without credentials', () => { + test('exports three raw destinations and checkpointed messages without credentials', () => { const exported = Object.values(definitions) expect(exported).toContain(slack) expect(slack.streams).toHaveLength(3) expect(slack.streams.map((stream) => stream.destination.name)).toEqual(['slack_channels_raw', 'slack_users_raw', 'slack_messages_raw']) for (const stream of slack.streams) { expect(exported).toContain(stream.destination) - expect(stream.incremental.id).toBe('chkit.full_sync') + expect(stream.incremental.id).toBe(stream.id.endsWith('.messages') ? 'slack.message-ranges' : 'chkit.full_sync') } }) @@ -41,7 +43,7 @@ describe('Slack raw template', () => { const journal = createMemoryJournal() const destination = createMemoryDestination() const request = { selected: selectStreams([pipeline], []), backfill: undefined } - const first = await runIngestion(request, { journal, destination }) + const first = await runIngestion(request, { now: fixtureNow, journal, destination }) expect(first.ok, JSON.stringify(first)).toBe(true) expect(first.streams.map((stream) => stream.rows)).toEqual([1, 2, 4]) expect(destination.tables.size).toBe(3) @@ -52,7 +54,7 @@ describe('Slack raw template', () => { expect(new Set(messages.map((row) => row.id)).size).toBe(3) expect(messages.find((row) => JSON.stringify(row.raw).includes('Edited discussion'))).toBeUndefined() changed = true - const second = await runIngestion(request, { journal, destination }) + const second = await runIngestion(request, { now: fixtureNow, journal, destination }) expect(second.ok, JSON.stringify(second)).toBe(true) const repeated = destination.tables.get('default.slack_messages_raw') ?? [] const edited = repeated.find((row) => JSON.stringify(row.raw).includes('Edited discussion')) @@ -93,7 +95,8 @@ describe('Slack raw template', () => { const chunks = await collect(readMessages(context(), deps)) expect(chunks.flatMap((chunk) => chunk.rows).map((row) => row.raw)).toContainEqual({ source_id: 'slack.primary', team_id: 'T1', channel_id: 'C1', data: { ...parentMessage, text: 'Latest parent' } }) expect(chunks.flatMap((chunk) => chunk.rows)).toHaveLength(3) - expect(chunks.every((chunk) => !('state' in chunk) && !('id' in chunk))).toBe(true) + expect(chunks.every((chunk) => 'state' in chunk)).toBe(true) + expect(chunks.filter((chunk) => chunk.rows.length > 0).every((chunk) => chunk.id === undefined)).toBe(true) const messageRequests = requests.filter((url) => ['conversations.history', 'conversations.replies'].some((method) => url.pathname.endsWith(method))) expect(messageRequests.map((url) => url.searchParams.get('limit'))).toEqual(['15', '15', '15']) expect(messageRequests.at(-1)?.searchParams.get('cursor')).toBe('reply-next') @@ -116,14 +119,15 @@ describe('Slack raw template', () => { expect(first[0]?.id).toBe(JSON.stringify(['slack.primary', 'messages', 'T1', 'C1', standaloneMessage.ts])) }) - test('finishes every history page before opening threads and scans each root once', async () => { + test('finishes retained thread work before resuming history from its timestamp frontier', async () => { const requests: string[] = [] const defaults = fixtureDeps() const deps = fixtureDeps((url, init) => { if (url.pathname === '/api/conversations.history') { - const cursor = url.searchParams.get('cursor') - requests.push(`history:${cursor ?? 'first'}`) - return Response.json({ ok: true, messages: [parentMessage], response_metadata: { next_cursor: cursor === null ? 'history-final' : '' } }) + const latest = url.searchParams.get('latest') + const resumed = latest === parentMessage.ts + requests.push(`history:${resumed ? 'resumed' : 'first'}`) + return Response.json({ ok: true, messages: [resumed ? { ...standaloneMessage, ts: '1699999999.000003' } : parentMessage], has_more: !resumed }) } if (url.pathname === '/api/conversations.replies') { requests.push('replies') @@ -133,7 +137,7 @@ describe('Slack raw template', () => { }) const chunks = await collect(readMessages(context(), deps)) expect(chunks.flatMap((chunk) => chunk.rows)).toHaveLength(4) - expect(requests).toEqual(['history:first', 'history:history-final', 'replies']) + expect(requests).toEqual(['history:first', 'replies', 'history:resumed']) }) test('history-only selection avoids thread API requests', async () => { @@ -172,6 +176,8 @@ describe('Slack raw template', () => { }) deps.wait = async (milliseconds, signal) => { waits.push(milliseconds); signal.throwIfAborted() } const iterator = readMessages(context(controller.signal), deps) + const opening = await iterator.next() + expect(opening.value?.rows).toHaveLength(0) const first = await iterator.next() expect(first.value?.rows).toHaveLength(1) controller.abort(new Error('fixture cancelled')) @@ -252,7 +258,7 @@ describe('Slack raw template', () => { }) const journal = createMemoryJournal() const pipeline = fixturePipeline(deps, ['users']) - const result = await runIngestion({ selected: selectStreams([pipeline], []), backfill: undefined }, { journal, destination: createMemoryDestination() }) + const result = await runIngestion({ selected: selectStreams([pipeline], []), backfill: undefined }, { now: fixtureNow, journal, destination: createMemoryDestination() }) expect(result.ok, JSON.stringify(result)).toBe(true) expect(calls).toBe(2) expect(journal.events.some((event) => event.eventKind === 'retry_scheduled' && event.errorClass === 'rate_limited')).toBe(true) @@ -269,7 +275,7 @@ describe('Slack raw template', () => { const fixture = fixturePipeline(deps, ['messages']) const pipeline = definePipeline({ ...fixture, streams: fixture.streams.map((stream) => ({ ...stream, batchSize: 1 })) }) const destination = createMemoryDestination() - const result = await runIngestion({ selected: selectStreams([pipeline], []), backfill: undefined }, { journal: createMemoryJournal(), destination }) + const result = await runIngestion({ selected: selectStreams([pipeline], []), backfill: undefined }, { now: fixtureNow, journal: createMemoryJournal(), destination }) expect(result.ok).toBe(false) expect(result.streams[0]?.outcome).toBe('failed') expect(result.streams[0]?.error).toContain('missing_scope') @@ -290,7 +296,7 @@ describe('Slack raw template', () => { }) const journal = createMemoryJournal() const pipeline = fixturePipeline(deps, ['users']) - const result = await runIngestion({ selected: selectStreams([pipeline], []), backfill: undefined }, { journal, destination: createMemoryDestination() }) + const result = await runIngestion({ selected: selectStreams([pipeline], []), backfill: undefined }, { now: fixtureNow, journal, destination: createMemoryDestination() }) expect(result.ok, JSON.stringify(result)).toBe(true) expect(calls).toBe(2) expect(journal.events.some((event) => event.eventKind === 'retry_scheduled' && event.errorClass === 'rate_limited')).toBe(true) @@ -312,6 +318,7 @@ function fixturePipeline(deps: SlackClientDeps, selectedResources?: readonly str ...slack, retry: { retries: 1, minTimeout: 0, maxTimeout: 0, randomize: false }, streams: slack.streams.filter((stream) => selectedResources === undefined || selectedResources.includes(stream.id.split('.').at(-1) ?? '')).map((stream) => { + if (stream.id.endsWith('.messages')) return defineStream({ ...stream, incremental: slackMessageStrategy, read: (context) => readMessages(context, deps) }) const read = readers[stream.id.split('.').at(-1) ?? ''] if (!read) throw new Error(`No fixture reader for ${stream.id}`) return { ...stream, read: (context: FetchContext) => read(context, deps) } @@ -321,11 +328,12 @@ function fixturePipeline(deps: SlackClientDeps, selectedResources?: readonly str async function runFixture(deps: SlackClientDeps, selectedResources: readonly string[], destination = createMemoryDestination()) { const pipeline = fixturePipeline(deps, selectedResources) - return runIngestion({ selected: selectStreams([pipeline], []), backfill: undefined }, { journal: createMemoryJournal(), destination }) + return runIngestion({ selected: selectStreams([pipeline], []), backfill: undefined }, { now: fixtureNow, journal: createMemoryJournal(), destination }) } -function context(signal = new AbortController().signal): FetchContext { - return { signal, attempt: async (operation) => operation(signal) } +function context(signal = new AbortController().signal): ReadContext { + return { signal, attempt: async (operation) => operation(signal), streamId: 'slack.primary.messages', state: undefined, + cutoff: fixtureNow(), selection: slackMessageStrategy.plan({ cutoff: fixtureNow(), state: undefined, range: undefined }) } } async function collect(items: AsyncIterable): Promise { diff --git a/skills/chkit-ingestion/SKILL.md b/skills/chkit-ingestion/SKILL.md index 945abe70..a79533f5 100644 --- a/skills/chkit-ingestion/SKILL.md +++ b/skills/chkit-ingestion/SKILL.md @@ -1,11 +1,11 @@ --- name: chkit-ingestion -description: Author or adapt API ingestion sources with @chkit/plugin-ingest. Use for chkit streams, readers, pagination, incremental checkpoints, destinations, and loader choices; excludes generated insert helpers from plugin-codegen. +description: Author or improve API ingestion sources and registry integrations with @chkit/plugin-ingest. Use for provider-specific sync, pagination, durable checkpoints, recovery, raw destinations, and loader choices; excludes generated insert helpers from plugin-codegen. --- # chkit ingestion authoring -Implement a source reader with finite runs. Inspect the installed plugin version, config, and existing streams; preserve the user's choices. +Implement a recoverable source reader with finite runs. Inspect the installed plugin version, config, and existing streams; preserve the user's choices. ## Decide before coding @@ -13,7 +13,15 @@ Establish auth, response shape, pagination, record IDs, change filters/cursors, Treat stored shape and mapping together: prefer raw objects plus a SQL view when the shape may evolve and storage is affordable; map inside `read` for a known schema. Model warehouse entities separately; embed bounded child collections when consumers need complete documents. A unified document table is a specialized retrieval model. -Start with the default loader and a full sync for small datasets; add provider-supported incremental reads when needed. Explain consequential choices at their implementation step, introducing tuning and custom interfaces only as required. +Choose the strongest appropriate sync mechanism the provider actually supports: a change feed or sync token, reliable modification windows, or checkpointed full reconciliation. A simple full read remains appropriate for small bounded catalogs or explicitly requested prototypes. Use the default loader unless another publication contract is needed. Explain consequential choices and coverage limits. + +## Registry quality bar + +Before creating or materially changing an official registry integration, read [Stable sync design and verification](references/stable-sync.md). Registry providers are reusable implementations of supported capabilities: deliver robust bootstrap, incremental selection where available, durable recovery, and reconciliation for changes polling cannot capture. Do not stop at a happy-path pagination example. + +For each resource, establish its provider contract and choose a safe completion boundary before coding. Keep provider payloads raw; application-specific projections are optional and separate. Compare existing registry providers and established implementations such as Brain when accessible, then verify the API contract rather than copying their assumptions. Private repository access is not a dependency of the installed skill. + +The release gate is evidence that interruptions resume or safely replay, progress follows destination acknowledgement, empty reads commit completion, and late changes have a reconciliation plan. Full scans are valid when justified by the API; completion checkpoints do not make them change feeds. Match manifest strategy labels to actual behavior, document limitations and identity/state migrations, and add portable failure/restart fixtures. ## Read only the relevant docs @@ -32,15 +40,16 @@ Use local `apps/docs/src/content/docs/api-sync/` or these URLs. Match the instal - TypeScript and direct ClickHouse config are required. Preserve existing schema discovery: add the provider entry to `schema` paths, or re-export its schema and active pipelines from the project `entry`. Migrations create destinations. - Stream IDs own checkpoints: keep them stable and account-scoped. Pipelines group streams without dependency ordering. - Use `context.attempt`/`paginate`, forward cancellation, and throw `HttpError.fromResponse` for HTTP failures. Yield bounded chunks. -- Enforce both time bounds. Persist only provider-guaranteed, complete resume state after all preceding rows are represented; the executor commits it after writes. -- Size `overlapMs` to the latest arrival: minutes for indexing delay, days when records or children appear long after their timestamp (recordings, transcripts). Newest-first pages have no safe intermediate `state`; resume only from a completed window. +- Support or explicitly reject historical bounds. When supported, enforce both bounds and freeze unfinished selections across restarts. Persist complete validated resume state; the executor commits it after writes. +- Separate the completed watermark from unfinished traversal state. Newest-first pages cannot alone advance a completed timestamp watermark; documented fixed-window timestamp frontiers or scoped page positions can still resume unfinished reads. Preserve timestamp precision and equal-timestamp coverage. +- Choose overlap from indexing/arrival behavior. For children that appear or change without updating their parent, retain bounded pending parents or reconcile old roots; a recent overlap alone is insufficient. - Custom tables need `ingestionColumns`. Prefer `simpleLoader`; custom loaders must preserve write identity and acknowledge publication before returning receipts. -- Delivery is at-least-once: choose reconciliation keys/versioning. Declare chunk `id` only for stable logical intervals; avoid volatile mapped fields. -- Prefer tombstones for logical deletion. Fetch deletion signals explicitly; full sync does not reconcile missing IDs. Never infer deletions from partial scans. Complete required child reads before yielding a root. +- Delivery is at-least-once: choose reconciliation keys/versioning. Leave explicit chunk `id` unset for mutable observations. A stable page, cursor, or window label is insufficient: explicit data-bearing IDs require replay-immutable content. State-only markers can flush progress. +- Fetch deletion signals explicitly and retain raw cancellation/tombstone payloads. Full sync does not reconcile missing IDs; never infer deletions from partial or inaccessible scans. Complete required child reads before publishing a complete embedded document, or use independent raw child streams/durable pending work that gates completion. - Use an external scheduler and run one process per project/target, including backfills. ## Deliver and verify For integrations published in the official app registry, include the provider's official logo for the documentation listing. Store the asset in `apps/docs/public/logos/.svg` (or another supported image format), record its official source in `apps/docs/public/logos/README.md`, and set `meta.chkit.logo` to `https://chkit.obsessiondb.com/logos/.svg`. Use the provider's artwork without redrawing or altering it. The docs build requires the logo URL and a matching local asset. Read `apps/docs/src/content/docs/api-sync/registry-authoring.md` for registry metadata, integration guides, and immutable release publication. -Deliver definitions, config, and run/query commands. Inspect exports with `chkit ingest list`; generate and review migrations, then use `chkit check` against the configured development target after applying them. `check` has no `--offline` flag. Test empty input, pagination, checkpoint boundaries, and failure replay with fixtures. Keep live writes within authorization. Report verified behavior and limitations. +Deliver definitions, config, and run/query commands. Inspect exports with `chkit ingest list`; generate and review migrations, then use `chkit check` against the configured development target after applying them. `check` has no `--offline` flag. Test checkpoint behavior through `runIngestion` and memory adapters, not just reader iteration; use the applicable recovery cases in the stable-sync reference for registry sources. Keep live writes within authorization. Report verified behavior and limitations. diff --git a/skills/chkit-ingestion/references/stable-sync.md b/skills/chkit-ingestion/references/stable-sync.md new file mode 100644 index 00000000..10ccdef5 --- /dev/null +++ b/skills/chkit-ingestion/references/stable-sync.md @@ -0,0 +1,58 @@ +# Stable registry syncs + +Apply this contract when authoring or improving registry providers. Select mechanisms from official provider documentation and observed response shapes; do not invent change filters or durable cursors. Preserve explicit user constraints. A provider limitation calls for safe replay and an honest coverage statement. + +## Establish the source contract + +For each resource, record the account/query scope, stable record identity, bootstrap coverage, filter timestamp meaning, ordering, boundary inclusivity, pagination guarantees, cursor lifetime, and retention. Check whether edits, deletions, and child-only changes update the selected timestamp. Determine whether pages are snapshots, live mutable lists, or a change feed. Verify filter/token incompatibilities and provider caps on child collections. + +Choose the strongest appropriate mechanism, independently for each resource: + +| Provider contract | Selection and durable progress | +| --- | --- | +| Change feed or sync token | Bootstrap completely, retain the input token throughout pagination, and promote a terminal replacement token only after all corresponding writes | +| Reliable modification filter | Fixed upper cutoff and overlapping lower bound; checkpoint a fully consumed window or independently complete bounded interval | +| Creation-time filter | Use for proven immutable events or parent discovery; distinguish discovery progress from freshness, and retain delayed-child work/reconcile later edits separately | +| Full mutable listing | Record scan completion; for expensive reads, freeze parent selection and checkpoint completed parents, replaying unfinished mutable collections from their beginning | + +A small catalog can remain a simple full read if its complete cost is acceptable. For a registry resource, explain that choice. A creation timestamp is not a modification cursor; a pagination cursor is not automatically a change token. Runtime `cursorState()` can implement a full scan, so the helper alone does not determine the manifest's strategy label. + +In a chkit checkout, inspect relevant `registry//` implementations and tests. Calendar demonstrates terminal sync-token promotion; Slack fixed timestamp frontiers and retained threads; Meet late artifacts and nested cursors; Attio/Circleback completed-parent scans; GitHub/Linear complete discussion windows; Lemlist completed intervals. Reuse the invariant, not provider-specific dates, budgets, or account assumptions. Consult Brain's provider implementations when available while keeping its application projections out of raw registry schemas. + +## Keep progress recoverable + +- Bind stream IDs and saved state to stable non-secret source, account, and query identities. Check authenticated identity when the provider exposes it; otherwise require new stream/destination identities when switching accounts. Credentials themselves must not become checkpoint scope. Validate saved types, bounds, parent positions, and scope before reuse. Version custom strategies and document migrations. +- Separate completed coverage from unfinished work. Persist fixed window bounds and enough scoped page/parent/child position to resume them; recomputing a cutoff must not move an unfinished selection. Preserve exact provider timestamp precision and handle equal timestamps without skipping records. +- Yield fresh, complete candidate state. The executor owns its durable commit after destination evidence. Never advance a watermark past rows or required work that is neither loaded nor durably retained. Empty terminal pages and successful empty scans must also checkpoint completion. Use state-only markers or batching that flushes safe boundaries. +- Persist offsets or opaque page tokens only under a documented resumption contract. Expiring page tokens can be saved while valid if state also retains the original input and a safe recovery path. For mutable offset lists, restart the unfinished parent/interval at zero; skip acknowledged parents only within the saved scan cycle. Frozen selection means unchanged predicates and retained discovered parent IDs, not a frozen source dataset. Coverage still depends on provider ordering, snapshots, or eventual stability plus reconciliation. A completed cycle resets parent progress and discovers current parents again. +- On an expired token, replay the same scoped collection from its committed input or rebuild a baseline as the provider requires. Bound protocol resets across the unfinished selection, not just one process: retain reset debt or detect repeated recovery without useful durable progress. Repeated rejection/stalled recovery fails visibly. Keep existing raw observations during baseline recovery. Absence from a rebuilt baseline is not automatically deletion. +- Treat explicit backfills as separate checkpoint namespaces. Honor both bounds or reject unsupported bounds explicitly. Retain the original unfinished bounds, reject incompatible reuse of a backfill ID, and prevent interval watermarks from regressing. A bundled helper ignoring `range` does not establish a source's historical-range behavior. + +## Preserve raw data and delayed children + +Store complete returned objects as raw JSON with stable source/parent/provider keys; preserve sparse cancellation payloads. Keep application projections separate. For large child collections, prefer independently paged raw child rows over unbounded nested buffering. Each stream must discover its own parents; stream order is not a dependency contract. + +A child failure must not publish an incomplete embedded document as complete. Either finish the required children first, or emit raw parent/child observations with durable pending work and explicit completion semantics. Retain relevant parents for delayed recordings/transcripts, and periodically revisit old roots when edits or replies do not update the discovery timestamp. Choose overlap and reconciliation cadence from provider retention, delay, volume, and freshness requirements. + +Bound pending queues and in-memory work; fail visibly rather than silently dropping entries. Pending work that expires before collection is a coverage gap. Distinguish unavailable, forbidden, not-found, and still-processing outcomes using provider evidence. Diagnostics are not a retry queue unless the reader actually executes them. + +Mutable row chunks should normally omit explicit chunk IDs so changed observations remain eligible for publication. Stable page numbers, offsets, cursor strings, and interval labels can suppress changed replay content. Explicit data-bearing IDs require immutable, repeatable content; state-only progress markers may use logical boundary IDs. Stable row IDs handle repeated observations, separately from batch deduplication. + +Raw destinations retain latest observed records unless a stronger current-state protocol exists. Consume explicit deletion signals when supported. Do not infer deletion from partial, failed, inaccessible, or expired scans. Removing unseen records requires a separately designed completed-generation reconciliation. + +## Bound execution and prove recovery + +Use executor request/retry/cancellation authority with finite request timeouts. Validate repeated or malformed continuations and absent required terminal tokens. Classify provider-specific rate limits, permanent permission failures, and recoverable protocol resets correctly. Bound chunk sizes and expensive runs; ensure a run budget can reach a useful checkpoint, or partition the selection rather than perpetually replaying an oversized bootstrap. + +Use portable fixtures through `runIngestion` with memory journal/destination adapters. Exercise the applicable contracts: + +- Interrupt or exhaust a budget mid-window, page, parent, or child; deserialize saved state in a fresh run and verify complete eventual coverage under the original bounds. +- Accept destination rows, then lose the load acknowledgement or fail journal publication; verify replay without skipped rows or an advanced completed frontier. Reader-only tests cannot prove this ordering. +- Commit empty input and empty terminal progress; continue across empty pages that still have a cursor; reject repeated/malformed pagination and malformed or scope-mismatched saved state. +- Expire provider tokens, retain the correct input token across pages, and promote terminal tokens only after writes. Verify bounded recovery instead of an endless reset. +- Cover timestamp ties, exact precision, inclusive/exclusive bounds, overlapping runs, late children/old edits, parent discovery changes, and explicit backfill isolation where supported. +- Replay mutable observations with changed content and unchanged provider IDs. Test child failure/availability/retention behavior instead of silently truncating a document or queue. + +Mark distributed fixture files with `role: "test"`, include their dependencies, and keep them free of repository-only imports and live credentials. For new packaging or destination changes, verify installed consumers and real ClickHouse JSON/replay behavior in an authorized isolated target; memory adapters do not establish merge or deduplication-window behavior. + +Before delivery, align manifest labels, README, and integration guide with implemented coverage, bootstrap, normal sync, resumption, reconciliation, deletion semantics, account changes, backfill support, and tuning. Preserve immutable historical artifacts; bump template versions for changed source and document row/state identity changes. Report the evidence and remaining API limits. From e51ddb0aa7039012ae40f0b5afbdb7f81ac606a6 Mon Sep 17 00:00:00 2001 From: "replicas-connector[bot]" Date: Sun, 4 Oct 2026 19:47:39 +0000 Subject: [PATCH 02/18] Split Attio ingestion into independent entity streams MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Give configured object types and lists separate streams using the existing full-sync and pagination primitives. Remove the custom scan checkpoint ledger, preserve raw tables and row identities, and document the explicit configuration and stream identity migration in a new immutable template release. Co-Authored-By: Marc Höffl Co-Authored-By: replicas-connector[bot] --- .changeset/checkpointed-registry-ingestion.md | 4 +- .../src/content/docs/integrations/attio.mdx | 61 +-- registry/attio/README.md | 32 +- registry/attio/checkpoints.ts | 60 --- registry/attio/client.ts | 164 +++----- registry/attio/config.ts | 11 +- registry/attio/manifest.json | 33 +- registry/attio/pipeline.ts | 56 +-- registry/attio/releases/0.2.0.json | 382 ++++++++++++++++++ registry/attio/sources/entries.ts | 15 +- registry/attio/sources/list-attributes.ts | 15 +- registry/attio/sources/lists.ts | 12 +- registry/attio/sources/members.ts | 13 +- registry/attio/sources/notes.ts | 15 +- registry/attio/sources/object-attributes.ts | 15 +- registry/attio/sources/objects.ts | 12 +- registry/attio/sources/records.ts | 15 +- registry/attio/sources/tasks.ts | 15 +- registry/attio/tests/attio.test.ts | 359 +++++++--------- registry/attio/tests/fixtures.ts | 30 +- registry/attio/tests/install.e2e.test.ts | 24 +- skills/chkit-ingestion/SKILL.md | 8 +- .../chkit-ingestion/references/stable-sync.md | 16 +- 23 files changed, 832 insertions(+), 535 deletions(-) delete mode 100644 registry/attio/checkpoints.ts create mode 100644 registry/attio/releases/0.2.0.json diff --git a/.changeset/checkpointed-registry-ingestion.md b/.changeset/checkpointed-registry-ingestion.md index 041b1da8..21035a00 100644 --- a/.changeset/checkpointed-registry-ingestion.md +++ b/.changeset/checkpointed-registry-ingestion.md @@ -2,6 +2,6 @@ "chkit": patch --- -Allow registry resources to declare cursor checkpoints and timestamp windows alongside full scans. Upgrade the eight raw API integrations with provider-specific sync tokens, durable scan progress, overlapping incremental ranges, and restart recovery, while preserving historical registry releases. +Allow registry resources to declare cursor checkpoints and timestamp windows alongside full scans. Upgrade the eight raw API integrations with provider-specific sync tokens, overlapping incremental ranges, and safe restart recovery, while preserving historical registry releases. Attio now creates independent configured object-type and list streams using existing full-sync pagination, with People and Companies enabled by default. -Update the ingestion authoring skill with the registry sync quality bar, provider capability selection, durable recovery rules, and required failure/restart verification. +Update the ingestion authoring skill with the registry sync quality bar, provider capability selection, meaningful stream boundaries, existing pagination/checkpoint primitives, durable recovery rules, and required failure/restart verification. diff --git a/apps/docs/src/content/docs/integrations/attio.mdx b/apps/docs/src/content/docs/integrations/attio.mdx index bcf7bd76..9a4406aa 100644 --- a/apps/docs/src/content/docs/integrations/attio.mdx +++ b/apps/docs/src/content/docs/integrations/attio.mdx @@ -12,7 +12,7 @@ import RegistryReference from '../../../components/RegistryReference.astro'; Attio logo -The [app registry](/integrations/) installs the Attio integration as editable TypeScript source. It includes nine raw tables, nine checkpointed full-scan streams, and three SQL views for people, companies, and deals. Each complete cycle reads the selected resources through the Attio REST API and updates their latest observed versions in ClickHouse. +The [app registry](/integrations/) installs the Attio integration as editable TypeScript source. It includes nine raw tables, separate streams for each configured object type or list, workspace-wide resource streams, and three SQL views for people, companies, and deals. The default configuration creates nine streams writing to seven resource tables; entries and list-attribute readers require configured lists. Every run performs full reads through the Attio REST API and updates the latest observed versions in ClickHouse. @@ -61,7 +61,7 @@ A workspace admin creates a single-workspace access token in Attio. The followin -Grant read access to the categories required by the enabled streams. The full default integration uses all the scopes below. No write scopes are needed. +Grant read access to the categories required by the enabled streams. The scope table covers all supported resources; list-entry and list-attribute readers run only when `lists` is configured. No write scopes are needed. @@ -88,8 +88,8 @@ export interface AttioConfig { sourceId: string database: string tablePrefix: string - objects: readonly string[] | undefined - lists: readonly string[] | undefined + objects: readonly string[] + lists: readonly string[] pageSize: number notesPageSize: number } @@ -98,8 +98,8 @@ export const attioConfig: AttioConfig = { sourceId: 'attio.primary', database: 'default', tablePrefix: 'attio', - objects: undefined, - lists: undefined, + objects: ['people', 'companies'], + lists: [], pageSize: 500, notesPageSize: 50, } @@ -110,12 +110,14 @@ export const attioConfig: AttioConfig = { | `sourceId` | Stable, non-secret identity used in stream IDs and row IDs | | `database` | Database containing the nine raw tables and three views | | `tablePrefix` | Prefix used before `_records_raw`, `_people`, and the other suffixes | -| `objects` | Object API slugs or UUIDs; `undefined` discovers all accessible objects; `[]` selects none | -| `lists` | List API slugs or UUIDs; `undefined` discovers all accessible lists; `[]` selects none | +| `objects` | Explicit object API slugs or UUIDs; default `['people', 'companies']`; creates records and attribute streams per type | +| `lists` | Explicit list API slugs or UUIDs; default `[]`; creates entries and attribute streams per list | | `pageSize` | Positive integer, default `500`, for attributes, records, entries, and tasks | | `notesPageSize` | Positive integer up to `50`, default `50`, for notes | -For example, set `objects: ['people', 'companies']` to restrict object metadata, object attributes, and records. List selection similarly controls lists, list attributes, and entries. An unknown or inaccessible configured slug or ID fails visibly. Notes, tasks, and members remain workspace-wide; their readers need separate changes to narrow their coverage. +Add `deals` or a custom slug to `objects` to collect that type, and add list slugs or IDs to `lists` to collect their entries. Empty arrays disable the corresponding child streams. The objects and lists catalog streams retain all accessible metadata independently of these selections. An unknown or inaccessible configured slug or ID fails its stream visibly. Notes, tasks, and members remain workspace-wide; their readers need separate changes to narrow their coverage. + +The pipeline creates its stream graph from these explicit arrays without API discovery. People and Companies have independent readers and journal progress, while both write to `attio_records_raw`. An object stream's ID ends in `.records.` or `.object_attributes.`; a list stream's ID ends in `.entries.` or `.list_attributes.`. Keep each configured reference stable: switching a slug to an equivalent UUID changes the stream ID. Keep `sourceId` stable after ingestion starts. For a second workspace, use a distinct source identity and a separate table prefix or database. Copying the directory alone does not isolate pipeline IDs or destination tables. @@ -141,11 +143,11 @@ Queries in this guide use the default database and prefix. Substitute the config ## All synced resources and records -Each resource has its own stream and raw table. This table lists the returned entity data retained in `raw.data`; the complete API entity is preserved, including additional fields returned by Attio. Fields may be absent or null depending on the resource and workspace configuration. +Each object type has its own records and attribute streams, and each configured list has its own entries and attribute streams. Catalogs, notes, tasks, and members have workspace-wide streams. Streams for the same resource share a raw table. This table lists the returned entity data retained in `raw.data`; the complete API entity is preserved, including additional fields returned by Attio. Fields may be absent or null depending on the resource and workspace configuration. -The records stream discovers objects through the [objects API](https://docs.attio.com/rest-api/endpoint-reference/objects/list-objects). It is not restricted to the three objects with packaged views. People, companies, deals, and custom objects all land in the same records table when accessible and selected. A `users` object, when present, contains CRM records; workspace members are collected separately. +The default records streams read People and Companies separately. Deals and custom objects require explicit slugs or IDs in `objects`; configured types all land in the same records table. A `users` object, when configured, contains CRM records; workspace members are collected separately. The object catalog exposes accessible types without automatically adding readers for them. Record and entry attributes remain arrays, including multivalued custom fields and relationship references. Returned attribute values keep their accompanying metadata. The integration does not issue separate requests for historical attribute values, select-option catalogs, or status catalogs. See the [record API](https://docs.attio.com/rest-api/endpoint-reference/records/list-records), [entry API](https://docs.attio.com/rest-api/endpoint-reference/entries/list-entries), and [attribute API](https://docs.attio.com/rest-api/endpoint-reference/attributes/list-attributes) for the provider's response shapes. @@ -153,40 +155,40 @@ Notes retain the content and references in the [notes response](https://docs.att ### API requests and pagination -Every API path in the resource table is relative to `https://api.attio.com/v2`. Object and list placeholders are resolved to IDs by discovery before the child request runs. +Every API path in the resource table is relative to `https://api.attio.com/v2`. An object/list child reader first resolves its single configured slug or ID with an individual parent request, then fetches that parent's collection. | Resource | Pagination and filters | |---|---| -| `objects` | Single collection request, then configured object selection | +| `objects` | Single collection request retaining all accessible objects | | `object_attributes` | `limit` and `offset` query parameters; `show_archived=true` | | `records` | `limit` and `offset` in JSON body; no record filter or custom sort | -| `lists` | Single collection request, then configured list selection | +| `lists` | Single collection request retaining all accessible lists | | `list_attributes` | `limit` and `offset` query parameters; `show_archived=true` | | `entries` | `limit` and `offset` in JSON body; no entry filter or custom sort | | `notes` | `limit` and `offset` query parameters; no parent filter | | `tasks` | `limit` and `offset` query parameters; `sort=created_at:asc`; no completion filter | | `members` | Single collection request | -Paginated reads start at offset zero, advance by the number of returned entities, and stop when a page is shorter than the requested limit. Each stream discovers its own parents, so running records alone does not depend on the objects stream having run first. The pipeline's stream array does not establish dependencies. +Paginated reads use the ingestion plugin's `paginate()` helper: they start at offset zero, advance by the number of returned entities, and stop when a page is shorter than the requested limit. Each reader resolves only its configured object/list, so running People records does not depend on the objects catalog having run first. The pipeline's stream array does not establish dependencies. ## How the sync works 1. The CLI loads the installed pipeline and selects streams by their tags. -2. Each reader discovers accessible parents where needed, applies the configured selection, and fetches the collection pages. +2. Each selected reader resolves its one configured object/list where needed and fetches that collection's pages. 3. The client validates the response's `data` array and required entity IDs. Record `values` and entry `entry_values` must contain arrays. 4. The reader wraps each entity with its source identity and relevant parent slug, then assigns a stable composite row ID. 5. The runtime batches rows and writes them to ClickHouse. The journal records execution and load progress. 6. SQL views project the raw records and reconcile repeated row IDs at query time with `FINAL`. -### Checkpointed full scans and freshness +### Full reads and independent recovery -All nine streams save validated scan-cycle checkpoints. A fresh cycle discovers and freezes the selected object/list IDs and slugs. After every parent is fully fetched and loaded, an empty completion marker flushes preceding rows and commits that parent to the frontier. After interruption, the reader skips completed parents and restarts the unfinished parent at offset zero. It resumes the frozen set before performing new discovery. New parents are included in the next full cycle. +Every stream uses `fullSync()` and begins its next execution at offset zero. Attio's offsets describe positions in mutable collections and are not durable change cursors. A failed Companies reader therefore replays Companies independently of People; it does not maintain an outer loop over object types or a completed-parent ledger. The executor journal records execution and load completion, including successful empty reads, without custom provider checkpoint state. -Notes, tasks, and members use workspace-wide scans. Their unfinished collections restart at offset zero; terminal completion is persisted even when a collection is empty. Objects and lists likewise record completion after their returned rows load. A completed cycle starts a fresh full read on the next run, so edits remain observable. These checkpoints are scan boundaries, not a modification watermark. +Child response workspace and parent IDs are checked against the reader's resolved object/list before rows are published. Adding a configured type creates a new stream; removing it stops future reads without removing existing rows. Every scheduled run rereads the selected collections, so edits remain observable. -`ingest status` exposes the cycle, phase, frozen parent set, and completed parents. A selection change rejects the saved scope rather than reusing a frontier for different objects/lists. Restore the configuration or use a new source identity for a deliberate fresh scan. A removed or inaccessible frozen parent fails visibly. Child response workspace and parent IDs are checked against frozen discovery identity before any rows are published. Existing installations that have no checkpoint begin the first checkpointed cycle without a state migration. +Version `0.2.0` changes the former combined records, object-attributes, entries, and list-attributes stream IDs to per-type/per-list IDs. Those streams begin with fresh journal progress; the raw table schemas, row IDs for the same `sourceId`, and views are unchanged. The `0.1.3` release artifact remains available. When upgrading from its checkpointed-scan preview, use a new `sourceId` to avoid reusing the old custom strategy for unchanged workspace stream IDs. A new source ID also changes raw row IDs, so review retained observations as part of that migration. Installed files are not automatically overwritten by a newer registry release. The destination stores the latest **observed** entity. New observations replace earlier observations for the same row identity when ClickHouse reconciles versions. Freshness depends on the external schedule, scan duration, and successful API responses. There are no webhooks or continuous change capture in this integration. @@ -267,7 +269,7 @@ The original entity lives under `data`, so custom provider fields cannot overwri ## Included people, company, and deal views -The integration creates ordinary SQL views over `attio_records_raw FINAL`, filtered by `object_slug`. These views reuse the records stream; they do not make additional API requests. +The integration creates ordinary SQL views over `attio_records_raw FINAL`, filtered by `object_slug`. These views project the collected record rows; they do not make additional API requests. @@ -370,7 +372,7 @@ bunx chkit ingest run --tag provider:attio --max-duration 3600 --json The scheduler supplies the project directory, credentials, runtime, cadence, and concurrency control. Run at most one ingestion process per project and ClickHouse target at a time, including runs that select different tags. Pipeline limits apply within one process and do not lock out another process. -For a narrower run, select the records stream: +For a narrower run, select all configured records streams: ```sh bunx chkit ingest list --tag provider:attio --tag resource:records @@ -378,11 +380,11 @@ bunx chkit ingest run --tag provider:attio --tag resource:records --json bunx chkit ingest status --tag provider:attio --json ``` -Repeated tags use AND matching. With the default source ID, `--tag stream:attio.primary.records` selects the same records stream; a custom `sourceId` changes that derived tag. +Repeated tags use AND matching. Add `--tag object:people` to select only People records. Its record stream carries `resource:records` and `object:people`; its attribute stream carries `resource:object_attributes` and `object:people`. With the default source ID, `--tag stream:attio.primary.records.people` selects the People record stream. List readers carry `list:` together with `resource:entries` or `resource:list_attributes`. -Inspect the run's per-stream outcomes and error messages. `status` reports committed scan-cycle checkpoints rather than complete run history. Keep the ingestion journal as durable operational state. After fixing a token, permission, or destination issue, rerun `ingest run`; the Attio reader resumes its frozen parent set and restarts any unfinished offset collection. +Inspect the run's per-stream outcomes and error messages. `status` reports committed execution state rather than complete run history. Keep the ingestion journal as durable operational state. After fixing a token, permission, or destination issue, rerun the affected stream; its collection starts at offset zero. -Verify a known Attio record against its retained `raw.data`, update that record in a test workspace, rerun the sync, and confirm its view reflects the new observation. A row count alone does not establish full API coverage. Date-range backfill flags do not turn these full-read readers into historical or incremental readers; implementing those behaviors requires a source-specific strategy. +Verify a known Attio record against its retained `raw.data`, update that record in a test workspace, rerun the sync, and confirm its view reflects the new observation. A row count alone does not establish full API coverage. These full-sync readers ignore date bounds and do not support date-range backfills; implementing historical selection requires a source-specific strategy. ## Coverage and consistency limits @@ -391,7 +393,7 @@ Verify a known Attio record against its retained `raw.data`, update that record | Source record deleted or merged | Previously observed row remains; no deletion or merge reconciliation | | Token loses access or a parent is deselected | Existing rows remain in ClickHouse | | Source changes during offset pagination | A scan may miss or repeat entities; it is not a provider snapshot | -| Run stops after some batches | Loaded batches remain visible; completed parents are skipped and unfinished offset collections restart | +| Run stops after some batches | Loaded batches remain visible; the affected stream restarts its collection at offset zero | | Older source data loaded later | Later ingestion time wins; no source-version ordering | | Complete change history needed | Raw tables reconcile observations and do not form a permanent change log | | Meetings, call recordings, transcripts, emails, files, comments, or sequences needed | No readers are included for these resources | @@ -404,15 +406,14 @@ Deleting or deselecting a source entity does not remove it from a view. Applicat | File | Responsibility | |---|---| -| `config.ts` | Source identity, destination names, parent selection, page sizes | -| `client.ts` | Authentication, response validation, pagination, parent-completion boundaries, row identity, request pacing | -| `checkpoints.ts` | Validated scan cycles, frozen parent sets, and selection scope | +| `config.ts` | Source identity, destination names, explicit object/list references, page sizes | +| `client.ts` | Authentication, response validation, single-parent lookup, pagination, row identity, request pacing | | `sources/objects.ts`, `sources/object-attributes.ts` | Object and attribute readers with their raw table schemas | | `sources/records.ts` | Record reader, raw records table, people/company/deal views | | `sources/lists.ts`, `sources/list-attributes.ts`, `sources/entries.ts` | List resource readers with their raw table schemas | | `sources/notes.ts`, `sources/tasks.ts`, `sources/members.ts` | One reader and raw table schema per resource | | `tests/attio.test.ts`, `tests/fixtures.ts` | Optional installed tests and mocked API responses | -| `pipeline.ts` | Enabled streams, resource tags, retries, batching, concurrency | +| `pipeline.ts` | Per-type/per-list streams, resource tags, retries, batching, concurrency | | `index.ts` | Exports that make the pipeline and schema objects discoverable | Remove a stream entry from `pipeline.ts` to stop collecting a resource. Keep its schema export to preserve existing tables under migration management. Removing a schema export changes the desired database schema and can generate a drop operation; review that migration separately. diff --git a/registry/attio/README.md b/registry/attio/README.md index ef952477..cdf859c4 100644 --- a/registry/attio/README.md +++ b/registry/attio/README.md @@ -32,7 +32,9 @@ The schema uses native JSON (ClickHouse 25.3 or newer). A direct ClickHouse conn ## Included resources -All nine streams perform checkpointed full scan cycles. Every stream runs independently; a new cycle discovers its own parent objects/lists. After interruption, parent-scoped streams keep the frozen parent set and resume after the last completely loaded parent. The unfinished parent restarts at offset zero because offsets into mutable collections are not durable change cursors. +Each configured object type has its own records and object-attributes streams. The defaults create separate People and Companies readers; tasks, notes, members, and the object/list catalogs have workspace-wide streams. Each configured list similarly gets independent entries and list-attributes streams. The number of streams depends on `objects` and `lists`; the default configuration has nine streams writing to seven resource tables. All nine raw tables are defined, including entries and list attributes, whose readers are enabled by configuring `lists`. + +Every stream uses `fullSync()` and the existing pagination helper. A records reader fetches one configured object type, such as People, rather than looping over all object types. An interrupted stream restarts its collection at offset zero because Attio's mutable offsets are not durable cursors. Each stream has its own execution journal; no parent-completion ledger or custom scan state is needed. | Stream tag | Raw table | Required Attio scopes | | --- | --- | --- | @@ -46,7 +48,7 @@ All nine streams perform checkpointed full scan cycles. Every stream runs indepe | `resource:tasks` | `attio_tasks_raw` | `task:read`, `object_configuration:read`, `record_permission:read`, `user_management:read` | | `resource:members` | `attio_members_raw` | `user_management:read` | -Records include all accessible standard and custom objects returned by the [objects API](https://docs.attio.com/rest-api/endpoint-reference/objects/list-objects). Standard `users` records are distinct from [workspace members](https://docs.attio.com/rest-api/endpoint-reference/workspace-members/list-workspace-members). Attribute streams include archived attribute definitions; separate select-option/status catalogs and historical attribute values are not fetched. Active record/entry payloads retain their returned select/status values. +The objects and lists catalog streams retain all accessible metadata. Records and attributes are fetched only for the object types configured in `objects`; entries and list attributes require configured `lists`. Standard `users` records, when explicitly configured, are distinct from [workspace members](https://docs.attio.com/rest-api/endpoint-reference/workspace-members/list-workspace-members). Attribute streams include archived attribute definitions; separate select-option/status catalogs and historical attribute values are not fetched. Active record/entry payloads retain their returned select/status values. The template does not collect meetings, call recordings, transcripts, emails, files, comments, or historical attribute values. It does not receive webhooks or detect source deletions/merges. Coverage is limited to data the token can access. A permission error fails that stream with a diagnostic; it is never treated as an empty dataset. @@ -54,18 +56,28 @@ API references: [attributes](https://docs.attio.com/rest-api/endpoint-reference/ ## Customize the source -Each file in `sources/` contains one resource's raw table and reader, such as `sources/notes.ts`, `sources/tasks.ts`, or `sources/members.ts`. `sources/records.ts` also defines the people, company, and deal views. `client.ts` shares HTTP and pagination logic; `checkpoints.ts` validates scan state and selection scope; `pipeline.ts` composes the nine streams. +Each file in `sources/` contains one resource's raw table and reader, such as `sources/notes.ts`, `sources/tasks.ts`, or `sources/members.ts`. `sources/records.ts` also defines the people, company, and deal views. `client.ts` shares HTTP, validation, and pagination logic; `pipeline.ts` creates streams from the explicit object/list configuration. Remove an entry from `pipeline.ts` to stop collecting that resource. Keep its schema export in `index.ts` to preserve already collected data. Removing schema exports can generate destructive migration operations, which should be reviewed. After removing a resource's stream and schema export, its local files can also be deleted. -Set `objects` and `lists` in `config.ts` to arrays of API slugs or IDs to restrict discovery. `undefined` means all accessible parents; `[]` means none. Unknown configured parents fail visibly. Object selection affects objects, object attributes, and records. List selection affects lists, list attributes, and entries. Notes, tasks, and members remain workspace-wide; customize their readers separately if narrower scope is needed. +Set `objects` and `lists` in `config.ts` to explicit arrays of API slugs or IDs. Defaults are `objects: ['people', 'companies']` and `lists: []`. Add `deals` or a custom object slug to collect that type; add list slugs or IDs to collect entries. `[]` disables the corresponding object/list child streams. The stream graph is built from these arrays without API discovery, so schema imports and `ingest list` remain offline. A reader resolves its one configured slug or ID through Attio and fails visibly if it is unknown or inaccessible. The objects/lists catalogs still collect all accessible metadata; notes, tasks, and members remain workspace-wide. + +Object streams use `.records.` and `.object_attributes.`. List streams use `.entries.` and `.list_attributes.`. Each has independent journal progress even though streams of the same resource write to the same raw table. Keep configured references stable: changing from a slug to its equivalent ID changes the stream ID. -To run only records: +To run all configured records streams: ```sh bunx chkit ingest run --tag provider:attio --tag resource:records ``` +To run only People records: + +```sh +bunx chkit ingest run --tag provider:attio --tag resource:records --tag object:people +``` + +Record streams carry `resource:records` and `object:`. Object-attribute streams carry `resource:object_attributes` and `object:`; entries/list-attribute streams carry their resource tag and `list:`. + `pageSize` defaults to 500. Notes have a separate `notesPageSize` of 50, the documented maximum. Requests go through the ingestion executor's retry/cancellation context, with a 30-second request timeout. The serial pipeline leaves a 25ms gap before each request and a 125ms gap for notes; raising concurrency requires revisiting rate control. `429` responses retain `Retry-After` for executor backoff. Attio also limits query complexity; a permanently expensive query may need simpler filters/sorts rather than repeated retries. See [rate limits](https://docs.attio.com/rest-api/guides/rate-limiting). ## Stored data and editable views @@ -87,11 +99,13 @@ FROM default.attio_records_raw FINAL; | `attio_companies` | name, domain array, description | | `attio_deals` | name, stage, nullable numeric value, currency code, company/people record ID arrays | -All include the stable row `id`, `source_id`, `workspace_id`, `object_id`, `record_id`, nullable `created_at`, `web_url`, and `_chkit_ingested_at`. Missing optional strings yield empty strings, absent arrays yield empty arrays, and absent/unparseable dates or deal amounts yield null. A workspace without deals simply has an empty deals view. Edit these SQL projections to expose your custom fields, then generate and review a migration. The original payload remains available for further projections. +All include the stable row `id`, `source_id`, `workspace_id`, `object_id`, `record_id`, nullable `created_at`, `web_url`, and `_chkit_ingested_at`. Missing optional strings yield empty strings, absent arrays yield empty arrays, and absent/unparseable dates or deal amounts yield null. The deals view remains empty until `deals` is configured and observed; previously loaded deal rows remain when it is later deselected. Edit these SQL projections to expose your custom fields, then generate and review a migration. The original payload remains available for further projections. + +The executor journal records per-stream execution and load completion, including successful empty reads. Every scheduled run reads each selected stream from the beginning. An interruption in Companies affects that stream's completion independently of People; the next Companies attempt starts at offset zero. The reader validates returned child workspace and parent IDs against its resolved object/list before publishing rows. The journal stores no Attio-specific cycle, parent inventory, or offset checkpoint. These full-sync readers ignore date bounds; date-range backfills are not supported. -The journal stores scan cycles, selected parent IDs/slugs, completed parent IDs, and terminal completion. Parent-completion markers flush all preceding rows before advancing state. Workspace-wide notes/tasks/members restart an interrupted collection at offset zero; a completed cycle starts a fresh full read on the next run. New parents discovered during an interrupted cycle wait until that next cycle. An empty scan still records completion. Changing the object/list selection rejects the saved scope; restore it or use a new source identity for a deliberate fresh scan. A frozen parent that disappears or loses access fails visibly instead of being marked complete. Returned child workspace and parent IDs must match the frozen discovery identity; switching account tokens cannot publish rows from a different workspace under that frontier. Existing installations without a checkpoint start their first checkpointed cycle normally. +Version `0.2.0` changes records, object-attributes, entries, and list-attributes stream IDs to the per-type/per-list forms above. Those new stream IDs begin with fresh journal progress. Raw table schemas, row identities for the same `sourceId`, and all three views remain unchanged. The previous `0.1.3` artifact remains available. If upgrading from its checkpointed-scan preview, the unchanged workspace stream IDs would have saved a different strategy; use a new `sourceId` and review the existing rows because source identity is also part of every raw row ID. Installed source is editable and is not automatically replaced by a registry update. -These views show the latest **observed** version per ID, not a guaranteed snapshot of Attio now. Full reads update returned IDs without clearing tables. Deleted, merged, inaccessible, or deselected entities remain until you implement reconciliation. Data is published batch by batch, so a failed run may have loaded some updates. Offset pagination over changing source data can miss or duplicate entities within a scan. There is no universal modification cursor, atomic snapshot replacement, source-version ordering, or complete change history in this template. Checkpoints reduce repeated parent work; they do not turn offset scans into provider snapshots. No data-bearing chunk uses a page offset as its identity. +These views show the latest **observed** version per ID, not a guaranteed snapshot of Attio now. Full reads update returned IDs without clearing tables. Deleted, merged, inaccessible, or deselected entities remain until you implement reconciliation. Data is published batch by batch, so a failed run may have loaded some updates. Offset pagination over changing source data can miss or duplicate entities within a read. There is no universal modification cursor, atomic snapshot replacement, source-version ordering, or complete change history in this template. No data-bearing chunk uses a page offset as its identity. ## Fixture tests @@ -101,7 +115,7 @@ Install the optional test suite with `bunx chkit add attio --with-tests`, then r bun test src/integrations/attio/tests/attio.test.ts ``` -For a custom install path, point the command at that folder's `tests/attio.test.ts`. Tests mock Attio HTTP responses and use in-memory ingestion destinations and journals. They require Bun but no Attio credentials or ClickHouse server. The fixture suite checks resource coverage, pagination, preserved custom fields, repeat runs, cancellation, errors, serialized checkpoint resumption, frozen parents, safe restart of unfinished offsets, empty completion, and rows loaded before a journal failure. Adjust the fixtures and expectations when customizing the source. +For a custom install path, point the command at that folder's `tests/attio.test.ts`. Tests mock Attio HTTP responses and use in-memory ingestion destinations and journals. They require Bun but no Attio credentials or ClickHouse server. The fixture suite checks per-type stream selection, independent pagination, preserved custom fields, repeat runs, cancellation, errors, safe offset-zero restart, successful empty reads, and rows loaded before a journal failure. Adjust the fixtures and expectations when customizing the source. ## Smoke check diff --git a/registry/attio/checkpoints.ts b/registry/attio/checkpoints.ts deleted file mode 100644 index 28b06d79..00000000 --- a/registry/attio/checkpoints.ts +++ /dev/null @@ -1,60 +0,0 @@ -import { cursorState, IngestConfigError, type ReadContext } from '@chkit/plugin-ingest' - -import { attioConfig } from './config.js' - -export interface ScanParent { - workspaceId: string - id: string - slug: string -} - -export interface AttioScanState { - scope: string - cycle: number - phase: 'scanning' | 'complete' - parents: readonly ScanParent[] - completedParents: readonly string[] -} - -export type AttioReadContext = ReadContext - -/** Full scans have completion checkpoints, never an inferred modification watermark. */ -export function scanCheckpoint(resource: string) { - return cursorState({ id: `attio.scan.${resource}`, version: 1, parse: parseScanState }) -} - -export function beginScan(context: AttioReadContext, resource: string): AttioScanState { - const parents = ['objects', 'object_attributes', 'records'].includes(resource) ? attioConfig.objects - : ['lists', 'list_attributes', 'entries'].includes(resource) ? attioConfig.lists : undefined - const scope = JSON.stringify([attioConfig.sourceId, resource, parents === undefined ? null : [...new Set(parents)].sort()]) - if (context.state && context.state.scope !== scope) { - throw new IngestConfigError('Attio checkpoint scope changed. Restore the parent selection or use a new sourceId for a deliberate fresh scan.') - } - if (context.state?.phase === 'scanning') return context.state - const cycle = (context.state?.cycle ?? 0) + 1 - if (!Number.isSafeInteger(cycle)) throw new IngestConfigError('Attio scan cycle exceeded the safe integer range.') - return { scope, cycle, phase: 'scanning', parents: [], completedParents: [] } -} - -function parseScanState(raw: unknown): AttioScanState { - if (!isObject(raw) || typeof raw.scope !== 'string' || typeof raw.cycle !== 'number' - || !Number.isSafeInteger(raw.cycle) || raw.cycle < 1 || (raw.phase !== 'scanning' && raw.phase !== 'complete') - || !Array.isArray(raw.parents) || !raw.parents.every(isParent) - || !Array.isArray(raw.completedParents) || !raw.completedParents.every((id): id is string => typeof id === 'string')) { - throw new IngestConfigError('Invalid Attio scan checkpoint.') - } - const ids = raw.parents.map((parent) => parent.id) - if (new Set(ids).size !== ids.length || new Set(raw.completedParents).size !== raw.completedParents.length - || raw.completedParents.some((id) => !ids.includes(id))) { - throw new IngestConfigError('Invalid Attio scan checkpoint parent frontier.') - } - return { scope: raw.scope, cycle: raw.cycle, phase: raw.phase, parents: raw.parents, completedParents: raw.completedParents } -} - -function isParent(value: unknown): value is ScanParent { - return isObject(value) && [value.workspaceId, value.id, value.slug].every((part) => typeof part === 'string' && part.length > 0) -} - -function isObject(value: unknown): value is Record { - return typeof value === 'object' && value !== null && !Array.isArray(value) -} diff --git a/registry/attio/client.ts b/registry/attio/client.ts index d6396425..7f854208 100644 --- a/registry/attio/client.ts +++ b/registry/attio/client.ts @@ -1,13 +1,13 @@ import { setTimeout as sleep } from 'node:timers/promises' -import { HttpError, IngestConfigError, rawRows, type ErrorClassifier, type FetchContext } from '@chkit/plugin-ingest' +import { HttpError, IngestConfigError, paginate, rawRows, type ErrorClassifier, type FetchContext } from '@chkit/plugin-ingest' -import { attioConfig } from './config.js' -import { beginScan, type AttioReadContext, type ScanParent } from './checkpoints.js' +import { attioConfig, type AttioConfig } from './config.js' export type AttioEntity = Record & { id: Record } export interface AttioClientDeps { + config: AttioConfig fetch: (url: string, init: RequestInit) => Promise token: () => string | undefined wait: (milliseconds: number, signal: AbortSignal) => Promise @@ -25,101 +25,80 @@ interface CollectionRequest { parent?: { field: 'object_id' | 'list_id'; id: string; workspaceId: string } } -const defaultDeps: AttioClientDeps = { +export const defaultAttioClientDeps: AttioClientDeps = { + config: attioConfig, fetch: (url, init) => fetch(url, init), token: () => process.env.ATTIO_API_TOKEN, wait: async (milliseconds, signal) => { await sleep(milliseconds, undefined, { signal }) }, } -/** Offset pages are mutable: resume only after a completely loaded parent. */ -export async function* readParentCollections( - context: AttioReadContext, +/** Resolve one configured object/list; each stream owns one collection. */ +export async function* readParentCollection( + context: FetchContext, input: { resource: string - parents: 'objects' | 'lists' - idFields: readonly string[] - request: (parent: ScanParent) => CollectionRequest + parent: { kind: 'objects' | 'lists'; ref: string } + request: (parent: AttioEntity) => CollectionRequest }, - deps?: AttioClientDeps, + deps: AttioClientDeps = defaultAttioClientDeps, ) { - let state = beginScan(context, input.resource) - if (context.state?.phase !== 'scanning') { - const parents = input.parents === 'objects' ? await discoverObjects(context, deps) : await discoverLists(context, deps) - state = { ...state, parents: parents.map((parent) => ({ - workspaceId: entityId(parent, 'workspace_id'), - id: entityId(parent, input.parents === 'objects' ? 'object_id' : 'list_id'), - slug: entitySlug(parent), - })) } - yield { rows: [], state, id: `scan:${state.cycle}:start` } - } - for (const parent of state.parents) { - if (state.completedParents.includes(parent.id)) continue - const request = { ...input.request(parent), parent: { - field: input.parents === 'objects' || input.resource === 'list_attributes' ? 'object_id' as const : 'list_id' as const, - id: parent.id, workspaceId: parent.workspaceId, - } } - for await (const page of readCollection(context, request, deps)) { - yield { rows: toAttioRows(page, input.resource, input.idFields, - input.parents === 'objects' ? { object_slug: parent.slug } : { list_slug: parent.slug }) } + const idField = input.parent.kind === 'objects' ? 'object_id' : 'list_id' + const path = `/${input.parent.kind}/${encodeURIComponent(input.parent.ref)}` + const parent = await context.attempt(async (signal) => { + const entity = parseEntity(await requestData({ path, idFields: ['workspace_id', idField] }, 0, signal, deps), { + path, idFields: ['workspace_id', idField], + }) + entitySlug(entity) + if (input.parent.ref !== entityId(entity, idField) && input.parent.ref !== entitySlug(entity)) { + throw new IngestConfigError(`Attio ${path} returned a different configured parent.`) } - state = { ...state, completedParents: [...state.completedParents, parent.id] } - // This empty marker flushes preceding rows before publishing the parent frontier. - yield { rows: [], state, id: `scan:${state.cycle}:parent:${parent.id}` } - } - yield { rows: [], state: { ...state, phase: 'complete' as const }, id: `scan:${state.cycle}:complete` } -} - -/** Workspace collections restart offset zero after interruption and record completion. */ -export async function* readCollectionScan( - context: AttioReadContext, - resource: string, - request: CollectionRequest, - deps?: AttioClientDeps, + return entity + }, { label: `GET /v2${path}` }) + const request = { ...input.request(parent), parent: { + field: input.parent.kind === 'objects' || input.resource === 'list_attributes' ? 'object_id' as const : 'list_id' as const, + id: entityId(parent, idField), workspaceId: entityId(parent, 'workspace_id'), + } } + yield* readRows(context, { + resource: input.resource, request, + metadata: input.parent.kind === 'objects' ? { object_slug: entitySlug(parent) } : { list_slug: entitySlug(parent) }, + }, deps) +} + +export async function* readRows( + context: FetchContext, + input: { resource: string; request: CollectionRequest; metadata?: { object_slug?: string; list_slug?: string } }, + deps: AttioClientDeps = defaultAttioClientDeps, ) { - const state = beginScan(context, resource) - if (context.state?.phase !== 'scanning') yield { rows: [], state, id: `scan:${state.cycle}:start` } - for await (const page of readCollection(context, request, deps)) { - yield { rows: toAttioRows(page, resource, request.idFields) } + for await (const page of readCollection(context, input.request, deps)) { + yield { rows: toAttioRows(page, input.resource, input.request.idFields, input.metadata, deps.config.sourceId) } } - yield { rows: [], state: { ...state, phase: 'complete' as const }, id: `scan:${state.cycle}:complete` } } /** Each page is validated inside attempt, so executor retry/cancellation owns every request. */ -export async function* readCollection( +export function readCollection( context: FetchContext, request: CollectionRequest, - deps: AttioClientDeps = defaultDeps, -): AsyncGenerator { + deps: AttioClientDeps = defaultAttioClientDeps, +) { if (request.pageSize !== undefined && (!Number.isSafeInteger(request.pageSize) || request.pageSize <= 0)) { throw new IngestConfigError('Attio pageSize must be a positive safe integer.') } - let offset = 0 - while (true) { - context.signal.throwIfAborted() - const currentOffset = offset - const page = await context.attempt( - (signal) => requestPage(request, currentOffset, signal, deps), - { label: `${request.method ?? 'GET'} /v2${request.path}` }, - ) - if (page.length > 0) yield page - if (request.pageSize === undefined || page.length < request.pageSize) return - offset += page.length - if (!Number.isSafeInteger(offset)) throw new IngestConfigError('Attio pagination offset exceeded the safe integer range.') - } -} - -export async function discoverObjects(context: FetchContext, deps?: AttioClientDeps): Promise { - for await (const objects of readCollection(context, { path: '/objects', idFields: ['workspace_id', 'object_id'] }, deps)) { - return selectEntities(objects, attioConfig.objects, 'object_id') - } - return selectEntities([], attioConfig.objects, 'object_id') -} - -export async function discoverLists(context: FetchContext, deps?: AttioClientDeps): Promise { - for await (const lists of readCollection(context, { path: '/lists', idFields: ['workspace_id', 'list_id'] }, deps)) { - return selectEntities(lists, attioConfig.lists, 'list_id') - } - return selectEntities([], attioConfig.lists, 'list_id') + return paginate({ + context, initial: 0, label: `${request.method ?? 'GET'} /v2${request.path}`, + fetchPage: async (offset = 0, signal) => { + const data = await requestData(request, offset, signal, deps) + if (!Array.isArray(data)) throw new IngestConfigError(`Attio ${request.path} returned no data array.`) + const items = data.map((value: unknown) => parseEntity(value, request)) + if (request.pageSize !== undefined && items.length > request.pageSize) { + throw new IngestConfigError(`Attio ${request.path} returned more rows than the requested page size.`) + } + const next = request.pageSize !== undefined && items.length === request.pageSize ? offset + items.length : undefined + if (next !== undefined && !Number.isSafeInteger(next)) { + throw new IngestConfigError('Attio pagination offset exceeded the safe integer range.') + } + return { items, next } + }, + }) } /** Original provider fields stay under data; metadata does not overwrite custom attributes. */ @@ -128,10 +107,11 @@ export function toAttioRows( resource: string, idFields: readonly string[], metadata: { object_slug?: string; list_slug?: string } = {}, + sourceId = attioConfig.sourceId, ) { return rawRows( - items.map((data) => ({ source_id: attioConfig.sourceId, ...metadata, data })), - ({ data }) => JSON.stringify([attioConfig.sourceId, resource, ...idFields.map((field) => entityId(data, field))]), + items.map((data) => ({ source_id: sourceId, ...metadata, data })), + ({ data }) => JSON.stringify([sourceId, resource, ...idFields.map((field) => entityId(data, field))]), ) } @@ -155,12 +135,12 @@ export const classifyAttioError: ErrorClassifier = (cause) => { return undefined } -async function requestPage( +async function requestData( request: CollectionRequest, offset: number, signal: AbortSignal, deps: AttioClientDeps, -): Promise { +): Promise { const token = deps.token()?.trim() if (!token) throw new IngestConfigError('Set ATTIO_API_TOKEN before running Attio ingestion. Schema imports do not need credentials.') const url = new URL(`https://api.attio.com/v2${request.path}`) @@ -189,22 +169,10 @@ async function requestPage( throw error } const payload: unknown = await response.json() - if (!isObject(payload) || !Array.isArray(payload.data)) { - throw new IngestConfigError(`Attio ${request.path} returned no data array.`) - } - return payload.data.map((value: unknown) => parseEntity(value, request)) -} - -function selectEntities(entities: AttioEntity[], selection: readonly string[] | undefined, idField: string): AttioEntity[] { - // Validate slugs even when all resources are selected: records/views rely on this metadata. - for (const entity of entities) entitySlug(entity) - if (selection === undefined) return entities - for (const wanted of selection) { - if (!entities.some((entity) => entityId(entity, idField) === wanted || entitySlug(entity) === wanted)) { - throw new IngestConfigError(`Configured Attio ${idField} or slug "${wanted}" was not found or is not accessible.`) - } + if (!isObject(payload) || !('data' in payload)) { + throw new IngestConfigError(`Attio ${request.path} returned no data field.`) } - return entities.filter((entity) => selection.includes(entityId(entity, idField)) || selection.includes(entitySlug(entity))) + return payload.data } function parseEntity(value: unknown, request: CollectionRequest): AttioEntity { @@ -212,7 +180,7 @@ function parseEntity(value: unknown, request: CollectionRequest): AttioEntity { for (const field of request.idFields) entityId(value, field) if (request.parent && (entityId(value, 'workspace_id') !== request.parent.workspaceId || entityId(value, request.parent.field) !== request.parent.id)) { - throw new IngestConfigError(`Attio ${request.path} returned an entity from a different workspace or parent. Restore the original account token before resuming this scan.`) + throw new IngestConfigError(`Attio ${request.path} returned an entity from a different workspace or parent than the resolved collection.`) } if (request.valuesField !== undefined) { const values = value[request.valuesField] diff --git a/registry/attio/config.ts b/registry/attio/config.ts index 24dda4f6..0e558136 100644 --- a/registry/attio/config.ts +++ b/registry/attio/config.ts @@ -3,9 +3,10 @@ export interface AttioConfig { sourceId: string database: string tablePrefix: string - /** Undefined reads every accessible object/list; otherwise use UUIDs or API slugs. */ - objects: readonly string[] | undefined - lists: readonly string[] | undefined + /** Each UUID or API slug creates its own records and attributes streams. */ + objects: readonly string[] + /** Each UUID or API slug creates its own entries and attributes streams. */ + lists: readonly string[] pageSize: number notesPageSize: number } @@ -15,8 +16,8 @@ export const attioConfig: AttioConfig = { sourceId: 'attio.primary', database: 'default', tablePrefix: 'attio', - objects: undefined, - lists: undefined, + objects: ['people', 'companies'], + lists: [], pageSize: 500, notesPageSize: 50, } diff --git a/registry/attio/manifest.json b/registry/attio/manifest.json index 147fa290..5c72271f 100644 --- a/registry/attio/manifest.json +++ b/registry/attio/manifest.json @@ -3,7 +3,7 @@ "name": "attio", "type": "registry:item", "title": "Attio", - "description": "Editable Attio CRM schemas, raw data readers, typed views, and checkpointed full-scan ingestion pipelines.", + "description": "Editable Attio CRM schemas, raw data readers, typed views, and independent streams for People, Companies, configured custom types, and workspace resources.", "dependencies": [ "@chkit/core@^0.2.0-beta.10", "@chkit/plugin-ingest@^0.2.0-beta.10" @@ -19,11 +19,6 @@ "type": "registry:file", "target": "src/integrations/attio/client.ts" }, - { - "path": "checkpoints.ts", - "type": "registry:file", - "target": "src/integrations/attio/checkpoints.ts" - }, { "path": "config.ts", "type": "registry:file", @@ -100,7 +95,7 @@ "meta": { "chkit": { "formatVersion": 1, - "version": "0.1.3", + "version": "0.2.0", "documentation": "https://chkit.obsessiondb.com/integrations/attio/", "logo": "https://chkit.obsessiondb.com/logos/attio.svg", "language": "typescript", @@ -157,12 +152,17 @@ "method": "GET", "path": "/objects/{object_id}/attributes", "documentation": "https://docs.attio.com/rest-api/endpoint-reference/attributes/list-attributes" + }, + { + "method": "GET", + "path": "/objects/{object_id}", + "documentation": "https://docs.attio.com/rest-api/endpoint-reference/objects/get-an-object" } ] }, { "name": "records", - "description": "Current records for every selected object, IDs, creation time, Attio URL, and the complete values object with all returned attribute arrays", + "description": "Independent records streams for configured object types (People and Companies by default), IDs, creation time, Attio URL, and the complete values object with all returned attribute arrays", "scopes": [ "object_configuration:read", "record_permission:read" @@ -175,6 +175,11 @@ "method": "POST", "path": "/objects/{object_id}/records/query", "documentation": "https://docs.attio.com/rest-api/endpoint-reference/records/list-records" + }, + { + "method": "GET", + "path": "/objects/{object_id}", + "documentation": "https://docs.attio.com/rest-api/endpoint-reference/objects/get-an-object" } ] }, @@ -209,6 +214,11 @@ "method": "GET", "path": "/lists/{list_id}/attributes", "documentation": "https://docs.attio.com/rest-api/endpoint-reference/attributes/list-attributes" + }, + { + "method": "GET", + "path": "/lists/{list_id}", + "documentation": "https://docs.attio.com/rest-api/endpoint-reference/lists/get-a-list" } ] }, @@ -227,6 +237,11 @@ "method": "POST", "path": "/lists/{list_id}/entries/query", "documentation": "https://docs.attio.com/rest-api/endpoint-reference/entries/list-entries" + }, + { + "method": "GET", + "path": "/lists/{list_id}", + "documentation": "https://docs.attio.com/rest-api/endpoint-reference/lists/get-a-list" } ] }, @@ -321,7 +336,7 @@ } ], "sync": { - "description": "Full scan cycles retain the latest observed raw entities. Interrupted parent collections resume from the frozen object/list set after the last completely loaded parent; unfinished offset scans restart at zero. A completed cycle starts fresh discovery and full reads on the next run.", + "description": "People, Companies, each configured custom object/list, and workspace resources have independent full-sync streams. Each reader paginates one collection through chkit-ingest; interrupted mutable offset reads replay that collection from zero. Completed and empty reads are recorded by the executor journal without a custom scan checkpoint. Custom types and lists must be configured explicitly.", "schedule": "Run chkit ingest run through cron, CI, or an existing scheduler; the template does not start a background worker.", "deletions": "Records absent from a later API response remain in ClickHouse. The template does not propagate deletions or consume webhooks." } diff --git a/registry/attio/pipeline.ts b/registry/attio/pipeline.ts index 759805fa..9cddb91d 100644 --- a/registry/attio/pipeline.ts +++ b/registry/attio/pipeline.ts @@ -1,8 +1,7 @@ import { definePipeline, defineStream } from '@chkit/plugin-ingest' -import { classifyAttioError } from './client.js' -import { attioConfig } from './config.js' -import { scanCheckpoint } from './checkpoints.js' +import { classifyAttioError, defaultAttioClientDeps, type AttioClientDeps } from './client.js' +import { attioConfig, type AttioConfig } from './config.js' import { attioObjectsRaw, readObjects } from './sources/objects.js' import { attioObjectAttributesRaw, readObjectAttributes } from './sources/object-attributes.js' import { attioRecordsRaw, readRecords } from './sources/records.js' @@ -15,24 +14,33 @@ import { attioMembersRaw, readMembers } from './sources/members.js' const streamOptions = { classifyError: classifyAttioError, batchSize: 500 } -// Delete a stream entry to stop collecting it. Keep its schema export to retain existing data. -// Every reader discovers its own parents; this array does not establish execution dependencies. -export const attio = definePipeline({ - id: attioConfig.sourceId, - tags: ['provider:attio'], - maxStreams: 1, - maxFetches: 1, - maxLoads: 1, - retry: { retries: 5, minTimeout: 1_000, maxTimeout: 30_000, randomize: true }, - streams: [ - defineStream({ ...streamOptions, id: `${attioConfig.sourceId}.objects`, tags: ['resource:objects'], incremental: scanCheckpoint('objects'), destination: attioObjectsRaw, read: readObjects }), - defineStream({ ...streamOptions, id: `${attioConfig.sourceId}.object_attributes`, tags: ['resource:object_attributes'], incremental: scanCheckpoint('object_attributes'), destination: attioObjectAttributesRaw, read: readObjectAttributes }), - defineStream({ ...streamOptions, id: `${attioConfig.sourceId}.records`, tags: ['resource:records'], incremental: scanCheckpoint('records'), destination: attioRecordsRaw, read: readRecords }), - defineStream({ ...streamOptions, id: `${attioConfig.sourceId}.lists`, tags: ['resource:lists'], incremental: scanCheckpoint('lists'), destination: attioListsRaw, read: readLists }), - defineStream({ ...streamOptions, id: `${attioConfig.sourceId}.list_attributes`, tags: ['resource:list_attributes'], incremental: scanCheckpoint('list_attributes'), destination: attioListAttributesRaw, read: readListAttributes }), - defineStream({ ...streamOptions, id: `${attioConfig.sourceId}.entries`, tags: ['resource:entries'], incremental: scanCheckpoint('entries'), destination: attioEntriesRaw, read: readEntries }), - defineStream({ ...streamOptions, id: `${attioConfig.sourceId}.notes`, tags: ['resource:notes'], incremental: scanCheckpoint('notes'), destination: attioNotesRaw, read: readNotes }), - defineStream({ ...streamOptions, id: `${attioConfig.sourceId}.tasks`, tags: ['resource:tasks'], incremental: scanCheckpoint('tasks'), destination: attioTasksRaw, read: readTasks }), - defineStream({ ...streamOptions, id: `${attioConfig.sourceId}.members`, tags: ['resource:members'], incremental: scanCheckpoint('members'), destination: attioMembersRaw, read: readMembers }), - ], -}) +// One records stream per object type; no reader traverses other objects. +// Keep schema exports when removing streams to retain existing stored data. +export function createAttioPipeline(config: AttioConfig = attioConfig, deps: AttioClientDeps = defaultAttioClientDeps) { + const client = { ...deps, config } + return definePipeline({ + id: config.sourceId, + tags: ['provider:attio'], + maxStreams: 1, + maxFetches: 1, + maxLoads: 1, + retry: { retries: 5, minTimeout: 1_000, maxTimeout: 30_000, randomize: true }, + streams: [ + defineStream({ ...streamOptions, id: `${config.sourceId}.objects`, tags: ['resource:objects'], destination: attioObjectsRaw, read: (context) => readObjects(context, client) }), + ...config.objects.flatMap((object) => [ + defineStream({ ...streamOptions, id: `${config.sourceId}.records.${object}`, tags: ['resource:records', `object:${object}`], destination: attioRecordsRaw, read: (context) => readRecords(context, object, client) }), + defineStream({ ...streamOptions, id: `${config.sourceId}.object_attributes.${object}`, tags: ['resource:object_attributes', `object:${object}`], destination: attioObjectAttributesRaw, read: (context) => readObjectAttributes(context, object, client) }), + ]), + defineStream({ ...streamOptions, id: `${config.sourceId}.lists`, tags: ['resource:lists'], destination: attioListsRaw, read: (context) => readLists(context, client) }), + ...config.lists.flatMap((list) => [ + defineStream({ ...streamOptions, id: `${config.sourceId}.entries.${list}`, tags: ['resource:entries', `list:${list}`], destination: attioEntriesRaw, read: (context) => readEntries(context, list, client) }), + defineStream({ ...streamOptions, id: `${config.sourceId}.list_attributes.${list}`, tags: ['resource:list_attributes', `list:${list}`], destination: attioListAttributesRaw, read: (context) => readListAttributes(context, list, client) }), + ]), + defineStream({ ...streamOptions, id: `${config.sourceId}.notes`, tags: ['resource:notes'], destination: attioNotesRaw, read: (context) => readNotes(context, client) }), + defineStream({ ...streamOptions, id: `${config.sourceId}.tasks`, tags: ['resource:tasks'], destination: attioTasksRaw, read: (context) => readTasks(context, client) }), + defineStream({ ...streamOptions, id: `${config.sourceId}.members`, tags: ['resource:members'], destination: attioMembersRaw, read: (context) => readMembers(context, client) }), + ], + }) +} + +export const attio = createAttioPipeline() diff --git a/registry/attio/releases/0.2.0.json b/registry/attio/releases/0.2.0.json new file mode 100644 index 00000000..c75fd809 --- /dev/null +++ b/registry/attio/releases/0.2.0.json @@ -0,0 +1,382 @@ +{ + "$schema": "https://ui.shadcn.com/schema/registry-item.json", + "name": "attio", + "type": "registry:item", + "title": "Attio", + "description": "Editable Attio CRM schemas, raw data readers, typed views, and independent streams for People, Companies, configured custom types, and workspace resources.", + "dependencies": [ + "@chkit/core@^0.2.0-beta.10", + "@chkit/plugin-ingest@^0.2.0-beta.10" + ], + "devDependencies": [ + "@types/bun@^1.2.0" + ], + "files": [ + { + "path": "attio/README.md", + "type": "registry:file", + "target": "src/integrations/attio/README.md", + "content": "# Attio CRM ingestion\n\nThis is your source code: edit the API readers, schemas, views, and `pipeline.ts` directly. It uses the [chkit CLI](https://www.npmjs.com/package/chkit), `@chkit/core`, and `@chkit/plugin-ingest`; there is no runtime connection to the template registry.\n\n## First run\n\nSet `ATTIO_API_TOKEN` in your execution environment and configure the project's direct ClickHouse connection. A single-workspace API token works; an OAuth access token also works, but this template does not implement OAuth issuance or refresh. See [Attio authentication](https://docs.attio.com/rest-api/guides/authentication).\n\n### Create an Attio API key\n\n1. As a workspace admin, open the workspace-name dropdown in Attio and select **Workspace settings**.\n2. Open **Developers**, select **+ New access token**, and name the token, for example **chkit ClickHouse**.\n3. Grant the read scopes in the resource table below for the streams to run.\n4. Click the token on the Developers page to copy it. Set `ATTIO_API_TOKEN` in the project's `.env` file or the scheduler's secret environment.\n\nSee Attio's [API key instructions](https://attio.com/help/reference/apps/generating-an-api-key). Use a single-workspace API token for this setup; OAuth token creation and refresh require a separate application flow.\n\nBun loads a project `.env` automatically. With Node or a scheduler, explicitly supply environment variables using that runtime's environment-loading mechanism. `.env.example` only lists required variable names; it does not load values. Schema imports, `check`, and `ingest list` do not read the token or make API requests.\n\nReview `config.ts` before the first migration. The default target database is `default`; raw table and view names begin with `attio_`. Use a stable, non-secret `sourceId` for this source instance. For another installation, give it a distinct source ID and table prefix or database before exporting its pipeline. Changing these after ingestion changes stream/storage identities and requires a deliberate migration.\n\n```sh\nbunx chkit check\nbunx chkit ingest list --tag provider:attio\nbunx chkit generate --name add_attio\n# Review the generated SQL.\nbunx chkit migrate --apply\nbunx chkit ingest run --tag provider:attio\n```\n\nThe schema uses native JSON (ClickHouse 25.3 or newer). A direct ClickHouse connection is required; workbench-only execution is not supported. Use an external scheduler to repeat the run command, with one ingestion process per project/target at a time.\n\n## Included resources\n\nEach configured object type has its own records and object-attributes streams. The defaults create separate People and Companies readers; tasks, notes, members, and the object/list catalogs have workspace-wide streams. Each configured list similarly gets independent entries and list-attributes streams. The number of streams depends on `objects` and `lists`; the default configuration has nine streams writing to seven resource tables. All nine raw tables are defined, including entries and list attributes, whose readers are enabled by configuring `lists`.\n\nEvery stream uses `fullSync()` and the existing pagination helper. A records reader fetches one configured object type, such as People, rather than looping over all object types. An interrupted stream restarts its collection at offset zero because Attio's mutable offsets are not durable cursors. Each stream has its own execution journal; no parent-completion ledger or custom scan state is needed.\n\n| Stream tag | Raw table | Required Attio scopes |\n| --- | --- | --- |\n| `resource:objects` | `attio_objects_raw` | `object_configuration:read` |\n| `resource:object_attributes` | `attio_object_attributes_raw` | `object_configuration:read` |\n| `resource:records` | `attio_records_raw` | `object_configuration:read`, `record_permission:read` |\n| `resource:lists` | `attio_lists_raw` | `list_configuration:read` |\n| `resource:list_attributes` | `attio_list_attributes_raw` | `list_configuration:read` |\n| `resource:entries` | `attio_entries_raw` | `list_configuration:read`, `list_entry:read` |\n| `resource:notes` | `attio_notes_raw` | `note:read`, `object_configuration:read`, `record_permission:read` |\n| `resource:tasks` | `attio_tasks_raw` | `task:read`, `object_configuration:read`, `record_permission:read`, `user_management:read` |\n| `resource:members` | `attio_members_raw` | `user_management:read` |\n\nThe objects and lists catalog streams retain all accessible metadata. Records and attributes are fetched only for the object types configured in `objects`; entries and list attributes require configured `lists`. Standard `users` records, when explicitly configured, are distinct from [workspace members](https://docs.attio.com/rest-api/endpoint-reference/workspace-members/list-workspace-members). Attribute streams include archived attribute definitions; separate select-option/status catalogs and historical attribute values are not fetched. Active record/entry payloads retain their returned select/status values.\n\nThe template does not collect meetings, call recordings, transcripts, emails, files, comments, or historical attribute values. It does not receive webhooks or detect source deletions/merges. Coverage is limited to data the token can access. A permission error fails that stream with a diagnostic; it is never treated as an empty dataset.\n\nAPI references: [attributes](https://docs.attio.com/rest-api/endpoint-reference/attributes/list-attributes), [records](https://docs.attio.com/rest-api/endpoint-reference/records/list-records), [lists](https://docs.attio.com/rest-api/endpoint-reference/lists/list-all-lists), [entries](https://docs.attio.com/rest-api/endpoint-reference/entries/list-entries), [notes](https://docs.attio.com/rest-api/endpoint-reference/notes/list-notes), [tasks](https://docs.attio.com/rest-api/endpoint-reference/tasks/list-tasks).\n\n## Customize the source\n\nEach file in `sources/` contains one resource's raw table and reader, such as `sources/notes.ts`, `sources/tasks.ts`, or `sources/members.ts`. `sources/records.ts` also defines the people, company, and deal views. `client.ts` shares HTTP, validation, and pagination logic; `pipeline.ts` creates streams from the explicit object/list configuration.\n\nRemove an entry from `pipeline.ts` to stop collecting that resource. Keep its schema export in `index.ts` to preserve already collected data. Removing schema exports can generate destructive migration operations, which should be reviewed. After removing a resource's stream and schema export, its local files can also be deleted.\n\nSet `objects` and `lists` in `config.ts` to explicit arrays of API slugs or IDs. Defaults are `objects: ['people', 'companies']` and `lists: []`. Add `deals` or a custom object slug to collect that type; add list slugs or IDs to collect entries. `[]` disables the corresponding object/list child streams. The stream graph is built from these arrays without API discovery, so schema imports and `ingest list` remain offline. A reader resolves its one configured slug or ID through Attio and fails visibly if it is unknown or inaccessible. The objects/lists catalogs still collect all accessible metadata; notes, tasks, and members remain workspace-wide.\n\nObject streams use `.records.` and `.object_attributes.`. List streams use `.entries.` and `.list_attributes.`. Each has independent journal progress even though streams of the same resource write to the same raw table. Keep configured references stable: changing from a slug to its equivalent ID changes the stream ID.\n\nTo run all configured records streams:\n\n```sh\nbunx chkit ingest run --tag provider:attio --tag resource:records\n```\n\nTo run only People records:\n\n```sh\nbunx chkit ingest run --tag provider:attio --tag resource:records --tag object:people\n```\n\nRecord streams carry `resource:records` and `object:`. Object-attribute streams carry `resource:object_attributes` and `object:`; entries/list-attribute streams carry their resource tag and `list:`.\n\n`pageSize` defaults to 500. Notes have a separate `notesPageSize` of 50, the documented maximum. Requests go through the ingestion executor's retry/cancellation context, with a 30-second request timeout. The serial pipeline leaves a 25ms gap before each request and a 125ms gap for notes; raising concurrency requires revisiting rate control. `429` responses retain `Retry-After` for executor backoff. Attio also limits query complexity; a permanently expensive query may need simpler filters/sorts rather than repeated retries. See [rate limits](https://docs.attio.com/rest-api/guides/rate-limiting).\n\n## Stored data and editable views\n\nEach raw table contains a stable `id`, native JSON `raw`, and chkit batch/run/ingestion-time metadata. The key includes source identity, resource kind, workspace ID, and every relevant object/list/entity ID. Names, emails, and domains are never keys.\n\nThe complete returned entity is retained under `raw.data`. The envelope adds `source_id`; record and object-attribute rows also include `object_slug`, and list-entry/list-attribute rows include `list_slug`. No custom attributes or multivalued arrays are flattened away. For example:\n\n```sql\nSELECT id, raw.data, _chkit_ingested_at\nFROM default.attio_records_raw FINAL;\n```\n\n`sources/records.ts` defines three ordinary `FINAL` views:\n\n| View | Editable projection |\n| --- | --- |\n| `attio_people` | name, email address array, company record ID array |\n| `attio_companies` | name, domain array, description |\n| `attio_deals` | name, stage, nullable numeric value, currency code, company/people record ID arrays |\n\nAll include the stable row `id`, `source_id`, `workspace_id`, `object_id`, `record_id`, nullable `created_at`, `web_url`, and `_chkit_ingested_at`. Missing optional strings yield empty strings, absent arrays yield empty arrays, and absent/unparseable dates or deal amounts yield null. The deals view remains empty until `deals` is configured and observed; previously loaded deal rows remain when it is later deselected. Edit these SQL projections to expose your custom fields, then generate and review a migration. The original payload remains available for further projections.\n\nThe executor journal records per-stream execution and load completion, including successful empty reads. Every scheduled run reads each selected stream from the beginning. An interruption in Companies affects that stream's completion independently of People; the next Companies attempt starts at offset zero. The reader validates returned child workspace and parent IDs against its resolved object/list before publishing rows. The journal stores no Attio-specific cycle, parent inventory, or offset checkpoint. These full-sync readers ignore date bounds; date-range backfills are not supported.\n\nVersion `0.2.0` changes records, object-attributes, entries, and list-attributes stream IDs to the per-type/per-list forms above. Those new stream IDs begin with fresh journal progress. Raw table schemas, row identities for the same `sourceId`, and all three views remain unchanged. The previous `0.1.3` artifact remains available. If upgrading from its checkpointed-scan preview, the unchanged workspace stream IDs would have saved a different strategy; use a new `sourceId` and review the existing rows because source identity is also part of every raw row ID. Installed source is editable and is not automatically replaced by a registry update.\n\nThese views show the latest **observed** version per ID, not a guaranteed snapshot of Attio now. Full reads update returned IDs without clearing tables. Deleted, merged, inaccessible, or deselected entities remain until you implement reconciliation. Data is published batch by batch, so a failed run may have loaded some updates. Offset pagination over changing source data can miss or duplicate entities within a read. There is no universal modification cursor, atomic snapshot replacement, source-version ordering, or complete change history in this template. No data-bearing chunk uses a page offset as its identity.\n\n## Fixture tests\n\nInstall the optional test suite with `bunx chkit add attio --with-tests`, then run:\n\n```sh\nbun test src/integrations/attio/tests/attio.test.ts\n```\n\nFor a custom install path, point the command at that folder's `tests/attio.test.ts`. Tests mock Attio HTTP responses and use in-memory ingestion destinations and journals. They require Bun but no Attio credentials or ClickHouse server. The fixture suite checks per-type stream selection, independent pagination, preserved custom fields, repeat runs, cancellation, errors, safe offset-zero restart, successful empty reads, and rows loaded before a journal failure. Adjust the fixtures and expectations when customizing the source.\n\n## Smoke check\n\nAfter a migration and one successful run, inspect `chkit ingest status --tag provider:attio` and query `SELECT count() FROM default.attio_records_raw FINAL`. Compare a known record's retained `raw.data` with its Attio API response. Run again after changing that record and verify its view reflects the new observation. This smoke check needs a real test workspace and database; the registry's sanitized fixture tests do not assert complete live API coverage.\n" + }, + { + "path": "attio/client.ts", + "type": "registry:file", + "target": "src/integrations/attio/client.ts", + "content": "import { setTimeout as sleep } from 'node:timers/promises'\n\nimport { HttpError, IngestConfigError, paginate, rawRows, type ErrorClassifier, type FetchContext } from '@chkit/plugin-ingest'\n\nimport { attioConfig, type AttioConfig } from './config.js'\n\nexport type AttioEntity = Record & { id: Record }\n\nexport interface AttioClientDeps {\n config: AttioConfig\n fetch: (url: string, init: RequestInit) => Promise\n token: () => string | undefined\n wait: (milliseconds: number, signal: AbortSignal) => Promise\n}\n\ninterface CollectionRequest {\n path: string\n idFields: readonly string[]\n method?: 'GET' | 'POST'\n body?: Record\n query?: Record\n /** Omit for the unpaginated objects, lists, and workspace_members endpoints. */\n pageSize?: number\n valuesField?: 'values' | 'entry_values'\n parent?: { field: 'object_id' | 'list_id'; id: string; workspaceId: string }\n}\n\nexport const defaultAttioClientDeps: AttioClientDeps = {\n config: attioConfig,\n fetch: (url, init) => fetch(url, init),\n token: () => process.env.ATTIO_API_TOKEN,\n wait: async (milliseconds, signal) => { await sleep(milliseconds, undefined, { signal }) },\n}\n\n/** Resolve one configured object/list; each stream owns one collection. */\nexport async function* readParentCollection(\n context: FetchContext,\n input: {\n resource: string\n parent: { kind: 'objects' | 'lists'; ref: string }\n request: (parent: AttioEntity) => CollectionRequest\n },\n deps: AttioClientDeps = defaultAttioClientDeps,\n) {\n const idField = input.parent.kind === 'objects' ? 'object_id' : 'list_id'\n const path = `/${input.parent.kind}/${encodeURIComponent(input.parent.ref)}`\n const parent = await context.attempt(async (signal) => {\n const entity = parseEntity(await requestData({ path, idFields: ['workspace_id', idField] }, 0, signal, deps), {\n path, idFields: ['workspace_id', idField],\n })\n entitySlug(entity)\n if (input.parent.ref !== entityId(entity, idField) && input.parent.ref !== entitySlug(entity)) {\n throw new IngestConfigError(`Attio ${path} returned a different configured parent.`)\n }\n return entity\n }, { label: `GET /v2${path}` })\n const request = { ...input.request(parent), parent: {\n field: input.parent.kind === 'objects' || input.resource === 'list_attributes' ? 'object_id' as const : 'list_id' as const,\n id: entityId(parent, idField), workspaceId: entityId(parent, 'workspace_id'),\n } }\n yield* readRows(context, {\n resource: input.resource, request,\n metadata: input.parent.kind === 'objects' ? { object_slug: entitySlug(parent) } : { list_slug: entitySlug(parent) },\n }, deps)\n}\n\nexport async function* readRows(\n context: FetchContext,\n input: { resource: string; request: CollectionRequest; metadata?: { object_slug?: string; list_slug?: string } },\n deps: AttioClientDeps = defaultAttioClientDeps,\n) {\n for await (const page of readCollection(context, input.request, deps)) {\n yield { rows: toAttioRows(page, input.resource, input.request.idFields, input.metadata, deps.config.sourceId) }\n }\n}\n\n/** Each page is validated inside attempt, so executor retry/cancellation owns every request. */\nexport function readCollection(\n context: FetchContext,\n request: CollectionRequest,\n deps: AttioClientDeps = defaultAttioClientDeps,\n) {\n if (request.pageSize !== undefined && (!Number.isSafeInteger(request.pageSize) || request.pageSize <= 0)) {\n throw new IngestConfigError('Attio pageSize must be a positive safe integer.')\n }\n return paginate({\n context, initial: 0, label: `${request.method ?? 'GET'} /v2${request.path}`,\n fetchPage: async (offset = 0, signal) => {\n const data = await requestData(request, offset, signal, deps)\n if (!Array.isArray(data)) throw new IngestConfigError(`Attio ${request.path} returned no data array.`)\n const items = data.map((value: unknown) => parseEntity(value, request))\n if (request.pageSize !== undefined && items.length > request.pageSize) {\n throw new IngestConfigError(`Attio ${request.path} returned more rows than the requested page size.`)\n }\n const next = request.pageSize !== undefined && items.length === request.pageSize ? offset + items.length : undefined\n if (next !== undefined && !Number.isSafeInteger(next)) {\n throw new IngestConfigError('Attio pagination offset exceeded the safe integer range.')\n }\n return { items, next }\n },\n })\n}\n\n/** Original provider fields stay under data; metadata does not overwrite custom attributes. */\nexport function toAttioRows(\n items: readonly AttioEntity[],\n resource: string,\n idFields: readonly string[],\n metadata: { object_slug?: string; list_slug?: string } = {},\n sourceId = attioConfig.sourceId,\n) {\n return rawRows(\n items.map((data) => ({ source_id: sourceId, ...metadata, data })),\n ({ data }) => JSON.stringify([sourceId, resource, ...idFields.map((field) => entityId(data, field))]),\n )\n}\n\nexport function entityId(entity: AttioEntity, field: string): string {\n const value = entity.id[field]\n if (!value) throw new IngestConfigError(`Attio entity is missing id.${field}.`)\n return value\n}\n\nexport function entitySlug(entity: AttioEntity): string {\n if (typeof entity.api_slug !== 'string' || entity.api_slug.length === 0) {\n throw new IngestConfigError('Attio object/list is missing api_slug.')\n }\n return entity.api_slug\n}\n\n/** Invalid successful responses/configurations cannot be fixed by retrying. */\nexport const classifyAttioError: ErrorClassifier = (cause) => {\n if (cause instanceof IngestConfigError || cause instanceof SyntaxError) return { kind: 'permanent' }\n // HttpError preserves 429 Retry-After dates, 5xx retries, and permanent 4xx semantics.\n return undefined\n}\n\nasync function requestData(\n request: CollectionRequest,\n offset: number,\n signal: AbortSignal,\n deps: AttioClientDeps,\n): Promise {\n const token = deps.token()?.trim()\n if (!token) throw new IngestConfigError('Set ATTIO_API_TOKEN before running Attio ingestion. Schema imports do not need credentials.')\n const url = new URL(`https://api.attio.com/v2${request.path}`)\n for (const [key, value] of Object.entries(request.query ?? {})) url.searchParams.set(key, value)\n const method = request.method ?? 'GET'\n const pagination = request.pageSize === undefined ? {} : { limit: request.pageSize, offset }\n if (method === 'GET') {\n for (const [key, value] of Object.entries(pagination)) url.searchParams.set(key, String(value))\n }\n // The default pipeline runs one request at a time: <=40 reads/s and <=8 notes/s.\n // Attio may lower limits and applies query-complexity limits; 429s still use executor backoff.\n await deps.wait(request.path === '/notes' ? 125 : 25, signal)\n signal.throwIfAborted()\n const response = await deps.fetch(url.toString(), {\n method,\n headers: { Authorization: `Bearer ${token}`, Accept: 'application/json', 'Content-Type': 'application/json' },\n ...(method === 'POST' ? { body: JSON.stringify({ ...request.body, ...pagination }) } : {}),\n signal: AbortSignal.any([signal, AbortSignal.timeout(30_000)]),\n redirect: 'error',\n })\n if (!response.ok) {\n const error = await HttpError.fromResponse(response)\n if (response.status === 401 || response.status === 403) {\n error.message += ' Check ATTIO_API_TOKEN and the resource scopes in the Attio README, or disable this stream.'\n }\n throw error\n }\n const payload: unknown = await response.json()\n if (!isObject(payload) || !('data' in payload)) {\n throw new IngestConfigError(`Attio ${request.path} returned no data field.`)\n }\n return payload.data\n}\n\nfunction parseEntity(value: unknown, request: CollectionRequest): AttioEntity {\n if (!isEntity(value)) throw new IngestConfigError(`Attio ${request.path} returned an entity without a valid id object.`)\n for (const field of request.idFields) entityId(value, field)\n if (request.parent && (entityId(value, 'workspace_id') !== request.parent.workspaceId\n || entityId(value, request.parent.field) !== request.parent.id)) {\n throw new IngestConfigError(`Attio ${request.path} returned an entity from a different workspace or parent than the resolved collection.`)\n }\n if (request.valuesField !== undefined) {\n const values = value[request.valuesField]\n if (!isObject(values) || !Object.values(values).every(Array.isArray)) {\n throw new IngestConfigError(`Attio ${request.path} returned invalid ${request.valuesField}; expected arrays of attribute values.`)\n }\n }\n return value\n}\n\nfunction isEntity(value: unknown): value is AttioEntity {\n return isObject(value) && isObject(value.id) && Object.values(value.id).every((part) => typeof part === 'string')\n}\n\nfunction isObject(value: unknown): value is Record {\n return typeof value === 'object' && value !== null && !Array.isArray(value)\n}\n" + }, + { + "path": "attio/config.ts", + "type": "registry:file", + "target": "src/integrations/attio/config.ts", + "content": "export interface AttioConfig {\n /** Stable, non-secret name for this installation. Changing it creates new stream identities. */\n sourceId: string\n database: string\n tablePrefix: string\n /** Each UUID or API slug creates its own records and attributes streams. */\n objects: readonly string[]\n /** Each UUID or API slug creates its own entries and attributes streams. */\n lists: readonly string[]\n pageSize: number\n notesPageSize: number\n}\n\n// Keep these values stable across runs. Credentials are read only in client.ts at request time.\nexport const attioConfig: AttioConfig = {\n sourceId: 'attio.primary',\n database: 'default',\n tablePrefix: 'attio',\n objects: ['people', 'companies'],\n lists: [],\n pageSize: 500,\n notesPageSize: 50,\n}\n" + }, + { + "path": "attio/index.ts", + "type": "registry:file", + "target": "src/integrations/attio/index.ts", + "content": "export { attio } from './pipeline.js'\nexport { attioObjectsRaw } from './sources/objects.js'\nexport { attioObjectAttributesRaw } from './sources/object-attributes.js'\nexport { attioRecordsRaw, attioPeople, attioCompanies, attioDeals } from './sources/records.js'\nexport { attioListsRaw } from './sources/lists.js'\nexport { attioListAttributesRaw } from './sources/list-attributes.js'\nexport { attioEntriesRaw } from './sources/entries.js'\nexport { attioNotesRaw } from './sources/notes.js'\nexport { attioTasksRaw } from './sources/tasks.js'\nexport { attioMembersRaw } from './sources/members.js'\n" + }, + { + "path": "attio/pipeline.ts", + "type": "registry:file", + "target": "src/integrations/attio/pipeline.ts", + "content": "import { definePipeline, defineStream } from '@chkit/plugin-ingest'\n\nimport { classifyAttioError, defaultAttioClientDeps, type AttioClientDeps } from './client.js'\nimport { attioConfig, type AttioConfig } from './config.js'\nimport { attioObjectsRaw, readObjects } from './sources/objects.js'\nimport { attioObjectAttributesRaw, readObjectAttributes } from './sources/object-attributes.js'\nimport { attioRecordsRaw, readRecords } from './sources/records.js'\nimport { attioListsRaw, readLists } from './sources/lists.js'\nimport { attioListAttributesRaw, readListAttributes } from './sources/list-attributes.js'\nimport { attioEntriesRaw, readEntries } from './sources/entries.js'\nimport { attioNotesRaw, readNotes } from './sources/notes.js'\nimport { attioTasksRaw, readTasks } from './sources/tasks.js'\nimport { attioMembersRaw, readMembers } from './sources/members.js'\n\nconst streamOptions = { classifyError: classifyAttioError, batchSize: 500 }\n\n// One records stream per object type; no reader traverses other objects.\n// Keep schema exports when removing streams to retain existing stored data.\nexport function createAttioPipeline(config: AttioConfig = attioConfig, deps: AttioClientDeps = defaultAttioClientDeps) {\n const client = { ...deps, config }\n return definePipeline({\n id: config.sourceId,\n tags: ['provider:attio'],\n maxStreams: 1,\n maxFetches: 1,\n maxLoads: 1,\n retry: { retries: 5, minTimeout: 1_000, maxTimeout: 30_000, randomize: true },\n streams: [\n defineStream({ ...streamOptions, id: `${config.sourceId}.objects`, tags: ['resource:objects'], destination: attioObjectsRaw, read: (context) => readObjects(context, client) }),\n ...config.objects.flatMap((object) => [\n defineStream({ ...streamOptions, id: `${config.sourceId}.records.${object}`, tags: ['resource:records', `object:${object}`], destination: attioRecordsRaw, read: (context) => readRecords(context, object, client) }),\n defineStream({ ...streamOptions, id: `${config.sourceId}.object_attributes.${object}`, tags: ['resource:object_attributes', `object:${object}`], destination: attioObjectAttributesRaw, read: (context) => readObjectAttributes(context, object, client) }),\n ]),\n defineStream({ ...streamOptions, id: `${config.sourceId}.lists`, tags: ['resource:lists'], destination: attioListsRaw, read: (context) => readLists(context, client) }),\n ...config.lists.flatMap((list) => [\n defineStream({ ...streamOptions, id: `${config.sourceId}.entries.${list}`, tags: ['resource:entries', `list:${list}`], destination: attioEntriesRaw, read: (context) => readEntries(context, list, client) }),\n defineStream({ ...streamOptions, id: `${config.sourceId}.list_attributes.${list}`, tags: ['resource:list_attributes', `list:${list}`], destination: attioListAttributesRaw, read: (context) => readListAttributes(context, list, client) }),\n ]),\n defineStream({ ...streamOptions, id: `${config.sourceId}.notes`, tags: ['resource:notes'], destination: attioNotesRaw, read: (context) => readNotes(context, client) }),\n defineStream({ ...streamOptions, id: `${config.sourceId}.tasks`, tags: ['resource:tasks'], destination: attioTasksRaw, read: (context) => readTasks(context, client) }),\n defineStream({ ...streamOptions, id: `${config.sourceId}.members`, tags: ['resource:members'], destination: attioMembersRaw, read: (context) => readMembers(context, client) }),\n ],\n })\n}\n\nexport const attio = createAttioPipeline()\n" + }, + { + "path": "attio/sources/objects.ts", + "type": "registry:file", + "target": "src/integrations/attio/sources/objects.ts", + "content": "import { rawTable, type FetchContext } from '@chkit/plugin-ingest'\n\nimport { readRows, type AttioClientDeps } from '../client.js'\nimport { attioConfig } from '../config.js'\n\nexport const attioObjectsRaw = rawTable({ database: attioConfig.database, name: `${attioConfig.tablePrefix}_objects_raw` })\n\nexport function readObjects(context: FetchContext, deps?: AttioClientDeps) {\n return readRows(context, {\n resource: 'objects', request: { path: '/objects', idFields: ['workspace_id', 'object_id'] },\n }, deps)\n}\n" + }, + { + "path": "attio/sources/object-attributes.ts", + "type": "registry:file", + "target": "src/integrations/attio/sources/object-attributes.ts", + "content": "import { rawTable, type FetchContext } from '@chkit/plugin-ingest'\n\nimport { defaultAttioClientDeps, entityId, readParentCollection, type AttioClientDeps } from '../client.js'\nimport { attioConfig } from '../config.js'\n\nexport const attioObjectAttributesRaw = rawTable({ database: attioConfig.database, name: `${attioConfig.tablePrefix}_object_attributes_raw` })\n\nexport function readObjectAttributes(context: FetchContext, object: string, deps: AttioClientDeps = defaultAttioClientDeps) {\n return readParentCollection(context, {\n resource: 'object_attributes', parent: { kind: 'objects', ref: object },\n request: (parent) => ({\n path: `/objects/${encodeURIComponent(entityId(parent, 'object_id'))}/attributes`,\n idFields: ['workspace_id', 'object_id', 'attribute_id'],\n query: { show_archived: 'true' },\n pageSize: deps.config.pageSize,\n }),\n }, deps)\n}\n" + }, + { + "path": "attio/sources/records.ts", + "type": "registry:file", + "target": "src/integrations/attio/sources/records.ts", + "content": "import { view } from '@chkit/core'\nimport { rawTable, type FetchContext } from '@chkit/plugin-ingest'\n\nimport { defaultAttioClientDeps, entityId, readParentCollection, type AttioClientDeps } from '../client.js'\nimport { attioConfig } from '../config.js'\n\nexport const attioRecordsRaw = rawTable({ database: attioConfig.database, name: `${attioConfig.tablePrefix}_records_raw` })\n\n// Each view depends only on records. JSON extraction tolerates absent optional attributes.\nexport const attioPeople = recordView('people', `\n JSONExtractString(values, 'name', 1, 'full_name') AS name,\n arrayMap(value -> JSONExtractString(value, 'email_address'), JSONExtractArrayRaw(values, 'email_addresses')) AS email_addresses,\n arrayMap(value -> JSONExtractString(value, 'target_record_id'), JSONExtractArrayRaw(values, 'company')) AS company_record_ids`)\n\nexport const attioCompanies = recordView('companies', `\n JSONExtractString(values, 'name', 1, 'value') AS name,\n arrayMap(value -> JSONExtractString(value, 'domain'), JSONExtractArrayRaw(values, 'domains')) AS domains,\n JSONExtractString(values, 'description', 1, 'value') AS description`)\n\nexport const attioDeals = recordView('deals', `\n JSONExtractString(values, 'name', 1, 'value') AS name,\n JSONExtractString(values, 'stage', 1, 'status', 'title') AS stage,\n JSONExtract(values, 'value', 1, 'currency_value', 'Nullable(Float64)') AS value,\n JSONExtractString(values, 'value', 1, 'currency_code') AS currency_code,\n arrayMap(value -> JSONExtractString(value, 'target_record_id'), JSONExtractArrayRaw(values, 'associated_company')) AS company_record_ids,\n arrayMap(value -> JSONExtractString(value, 'target_record_id'), JSONExtractArrayRaw(values, 'associated_people')) AS people_record_ids`)\n\nexport function readRecords(context: FetchContext, object: string, deps: AttioClientDeps = defaultAttioClientDeps) {\n return readParentCollection(context, {\n resource: 'records', parent: { kind: 'objects', ref: object },\n request: (parent) => ({\n path: `/objects/${encodeURIComponent(entityId(parent, 'object_id'))}/records/query`,\n idFields: ['workspace_id', 'object_id', 'record_id'],\n method: 'POST',\n valuesField: 'values',\n pageSize: deps.config.pageSize,\n }),\n }, deps)\n}\n\nfunction recordView(objectSlug: string, columns: string) {\n return view({\n database: attioConfig.database,\n name: `${attioConfig.tablePrefix}_${objectSlug}`,\n as: `WITH toJSONString(raw) AS payload,\n JSONExtractRaw(payload, 'data') AS record,\n JSONExtractRaw(record, 'values') AS values\nSELECT id,\n JSONExtractString(payload, 'source_id') AS source_id,\n JSONExtractString(record, 'id', 'workspace_id') AS workspace_id,\n JSONExtractString(record, 'id', 'object_id') AS object_id,\n JSONExtractString(record, 'id', 'record_id') AS record_id,\n parseDateTime64BestEffortOrNull(JSONExtractString(record, 'created_at'), 6, 'UTC') AS created_at,\n JSONExtractString(record, 'web_url') AS web_url,${columns},\n _chkit_ingested_at\nFROM ${quoteIdentifier(attioRecordsRaw.database)}.${quoteIdentifier(attioRecordsRaw.name)} FINAL\nWHERE JSONExtractString(payload, 'object_slug') = '${objectSlug}'`,\n })\n}\n\nfunction quoteIdentifier(value: string): string {\n return `\\`${value.replaceAll('\\\\', '\\\\\\\\').replaceAll('`', '\\\\`')}\\``\n}\n" + }, + { + "path": "attio/sources/lists.ts", + "type": "registry:file", + "target": "src/integrations/attio/sources/lists.ts", + "content": "import { rawTable, type FetchContext } from '@chkit/plugin-ingest'\n\nimport { readRows, type AttioClientDeps } from '../client.js'\nimport { attioConfig } from '../config.js'\n\nexport const attioListsRaw = rawTable({ database: attioConfig.database, name: `${attioConfig.tablePrefix}_lists_raw` })\n\nexport function readLists(context: FetchContext, deps?: AttioClientDeps) {\n return readRows(context, {\n resource: 'lists', request: { path: '/lists', idFields: ['workspace_id', 'list_id'] },\n }, deps)\n}\n" + }, + { + "path": "attio/sources/list-attributes.ts", + "type": "registry:file", + "target": "src/integrations/attio/sources/list-attributes.ts", + "content": "import { rawTable, type FetchContext } from '@chkit/plugin-ingest'\n\nimport { defaultAttioClientDeps, entityId, readParentCollection, type AttioClientDeps } from '../client.js'\nimport { attioConfig } from '../config.js'\n\nexport const attioListAttributesRaw = rawTable({ database: attioConfig.database, name: `${attioConfig.tablePrefix}_list_attributes_raw` })\n\nexport function readListAttributes(context: FetchContext, list: string, deps: AttioClientDeps = defaultAttioClientDeps) {\n return readParentCollection(context, {\n resource: 'list_attributes', parent: { kind: 'lists', ref: list },\n request: (parent) => ({\n path: `/lists/${encodeURIComponent(entityId(parent, 'list_id'))}/attributes`,\n idFields: ['workspace_id', 'object_id', 'attribute_id'],\n // Attio's shared attribute schema calls the parent list ID object_id too.\n query: { show_archived: 'true' },\n pageSize: deps.config.pageSize,\n }),\n }, deps)\n}\n" + }, + { + "path": "attio/sources/entries.ts", + "type": "registry:file", + "target": "src/integrations/attio/sources/entries.ts", + "content": "import { rawTable, type FetchContext } from '@chkit/plugin-ingest'\n\nimport { defaultAttioClientDeps, entityId, readParentCollection, type AttioClientDeps } from '../client.js'\nimport { attioConfig } from '../config.js'\n\nexport const attioEntriesRaw = rawTable({ database: attioConfig.database, name: `${attioConfig.tablePrefix}_entries_raw` })\n\nexport function readEntries(context: FetchContext, list: string, deps: AttioClientDeps = defaultAttioClientDeps) {\n return readParentCollection(context, {\n resource: 'entries', parent: { kind: 'lists', ref: list },\n request: (parent) => ({\n path: `/lists/${encodeURIComponent(entityId(parent, 'list_id'))}/entries/query`,\n idFields: ['workspace_id', 'list_id', 'entry_id'],\n method: 'POST',\n valuesField: 'entry_values',\n pageSize: deps.config.pageSize,\n }),\n }, deps)\n}\n" + }, + { + "path": "attio/sources/notes.ts", + "type": "registry:file", + "target": "src/integrations/attio/sources/notes.ts", + "content": "import { IngestConfigError, rawTable, type FetchContext } from '@chkit/plugin-ingest'\n\nimport { defaultAttioClientDeps, readRows, type AttioClientDeps } from '../client.js'\nimport { attioConfig } from '../config.js'\n\nexport const attioNotesRaw = rawTable({ database: attioConfig.database, name: `${attioConfig.tablePrefix}_notes_raw` })\n\nexport function readNotes(context: FetchContext, deps: AttioClientDeps = defaultAttioClientDeps) {\n if (deps.config.notesPageSize > 50) throw new IngestConfigError('Attio notesPageSize cannot exceed 50.')\n return readRows(context, {\n resource: 'notes', request: {\n path: '/notes', idFields: ['workspace_id', 'note_id'], pageSize: deps.config.notesPageSize,\n },\n }, deps)\n}\n" + }, + { + "path": "attio/sources/tasks.ts", + "type": "registry:file", + "target": "src/integrations/attio/sources/tasks.ts", + "content": "import { rawTable, type FetchContext } from '@chkit/plugin-ingest'\n\nimport { defaultAttioClientDeps, readRows, type AttioClientDeps } from '../client.js'\nimport { attioConfig } from '../config.js'\n\nexport const attioTasksRaw = rawTable({ database: attioConfig.database, name: `${attioConfig.tablePrefix}_tasks_raw` })\n\nexport function readTasks(context: FetchContext, deps: AttioClientDeps = defaultAttioClientDeps) {\n return readRows(context, {\n resource: 'tasks', request: {\n path: '/tasks', idFields: ['workspace_id', 'task_id'], pageSize: deps.config.pageSize,\n query: { sort: 'created_at:asc' },\n },\n }, deps)\n}\n" + }, + { + "path": "attio/sources/members.ts", + "type": "registry:file", + "target": "src/integrations/attio/sources/members.ts", + "content": "import { rawTable, type FetchContext } from '@chkit/plugin-ingest'\n\nimport { readRows, type AttioClientDeps } from '../client.js'\nimport { attioConfig } from '../config.js'\n\nexport const attioMembersRaw = rawTable({ database: attioConfig.database, name: `${attioConfig.tablePrefix}_members_raw` })\n\nexport function readMembers(context: FetchContext, deps?: AttioClientDeps) {\n return readRows(context, {\n resource: 'members', request: {\n path: '/workspace_members', idFields: ['workspace_id', 'workspace_member_id'],\n },\n }, deps)\n}\n" + }, + { + "path": "attio/tests/attio.test.ts", + "type": "registry:file", + "target": "src/integrations/attio/tests/attio.test.ts", + "role": "test", + "content": "import { describe, expect, test } from 'bun:test'\n\nimport { HttpError, definePipeline, runIngestion, selectStreams, type FetchContext } from '@chkit/plugin-ingest'\nimport { createMemoryDestination, createMemoryJournal } from '@chkit/plugin-ingest/testing'\n\nimport { classifyAttioError, readCollection, type AttioClientDeps } from '../client.js'\nimport { attioConfig } from '../config.js'\nimport * as definitions from '../index.js'\nimport { attio, createAttioPipeline } from '../pipeline.js'\nimport { readEntries } from '../sources/entries.js'\nimport { readMembers } from '../sources/members.js'\nimport { readNotes } from '../sources/notes.js'\nimport { readRecords } from '../sources/records.js'\nimport { readTasks } from '../sources/tasks.js'\nimport { fixtureDeps, fixtureResponse, peopleObject, person } from './fixtures.js'\n\ndescribe('Attio template', () => {\n test('exports independently selectable object streams and raw destinations without credentials', () => {\n const exported = Object.values(definitions)\n expect(exported).toContain(attio)\n expect(attio.streams).toHaveLength(9)\n expect(attio.streams.filter((stream) => stream.tags.includes('resource:records')).map((stream) => stream.id))\n .toEqual(['attio.primary.records.people', 'attio.primary.records.companies'])\n expect(selectStreams([attio], ['resource:records', 'object:people']).map(({ stream }) => stream.id)).toEqual(['attio.primary.records.people'])\n expect(selectStreams([attio], ['resource:records', 'object:companies']).map(({ stream }) => stream.id))\n .toEqual(['attio.primary.records.companies'])\n for (const stream of attio.streams) {\n expect(exported).toContain(stream.destination)\n expect(stream.incremental.id).toBe('chkit.full_sync')\n }\n expect([definitions.attioPeople, definitions.attioCompanies, definitions.attioDeals].map((view) => view.name))\n .toEqual(['attio_people', 'attio_companies', 'attio_deals'])\n })\n\n test('an object named tasks does not overlap the workspace tasks resource tag', () => {\n const pipeline = createAttioPipeline({ ...attioConfig, objects: ['tasks'], lists: [] }, fixtureDeps())\n expect(selectStreams([pipeline], ['resource:tasks']).map(({ stream }) => stream.id)).toEqual(['attio.primary.tasks'])\n expect(selectStreams([pipeline], ['resource:records', 'object:tasks']).map(({ stream }) => stream.id))\n .toEqual(['attio.primary.records.tasks'])\n })\n\n test('configured custom objects and lists have their own streams sharing raw tables', async () => {\n const pipeline = fixturePipeline(fixtureDeps())\n const journal = createMemoryJournal()\n const destination = createMemoryDestination()\n const result = await runIngestion({ selected: selectStreams([pipeline], []), backfill: undefined }, { journal, destination })\n expect(result.ok, JSON.stringify(result)).toBe(true)\n expect(pipeline.streams).toHaveLength(11)\n expect(result.streams.filter((stream) => stream.streamId.includes('.records.')).map((stream) => stream.rows)).toEqual([1, 1])\n expect(destination.tables.size).toBe(9)\n expect(destination.tables.get('default.attio_records_raw')?.map((row) => row.raw)).toContainEqual({\n source_id: 'attio.primary', object_slug: 'people', data: person,\n })\n expect(destination.tables.get('default.attio_entries_raw')?.[0]?.raw).toMatchObject({ list_slug: 'sales' })\n expect(selectStreams([pipeline], ['resource:records', 'object:subscriptions']).map(({ stream }) => stream.id))\n .toEqual(['attio.primary.records.subscriptions'])\n expect(selectStreams([pipeline], ['resource:entries']).map(({ stream }) => stream.id))\n .toEqual(['attio.primary.entries.sales'])\n })\n\n test('one object reader paginates only that object and preserves raw custom values and IDs', async () => {\n const requests: Array<{ path: string; body: unknown }> = []\n const firstPage = Array.from({ length: 500 }, (_, index) => ({ ...person, id: { ...person.id, record_id: `record-${index}` } }))\n const deps = fixtureDeps((url, init) => {\n const body = typeof init.body === 'string' ? JSON.parse(init.body) : undefined\n requests.push({ path: url.pathname, body })\n if (url.pathname === '/v2/objects/people') return Response.json({ data: peopleObject })\n if (url.pathname === '/v2/objects/object-people/records/query') {\n return Response.json({ data: body.offset === 0 ? firstPage : [{ ...person, id: { ...person.id, record_id: 'last' } }] })\n }\n throw new Error(`Unexpected URL ${url.pathname}`)\n })\n const chunks = await collect(readRecords(context(), 'people', deps))\n expect(chunks.map((chunk) => chunk.rows.length)).toEqual([500, 1])\n expect(requests).toEqual([\n { path: '/v2/objects/people', body: undefined },\n { path: '/v2/objects/object-people/records/query', body: { limit: 500, offset: 0 } },\n { path: '/v2/objects/object-people/records/query', body: { limit: 500, offset: 500 } },\n ])\n expect(chunks.every((chunk) => !('state' in chunk) && !('id' in chunk))).toBe(true)\n expect(chunks[0]?.rows[0]?.id).toBe(JSON.stringify(['attio.primary', 'records', 'workspace-1', 'object-people', 'record-0']))\n expect(chunks[0]?.rows[0]?.raw).toMatchObject({ data: { values: { custom_multi: [{ value: 'one' }, { value: 'two' }] } } })\n const custom = await collect(readRecords(context(), 'subscriptions', fixtureDeps()))\n expect(custom[0]?.rows[0]?.raw).toMatchObject({ object_slug: 'subscriptions' })\n expect(custom[0]?.rows[0]?.id).not.toBe(chunks[0]?.rows[0]?.id)\n })\n\n test('a source identity supplied to the pipeline owns stream IDs and raw row IDs', async () => {\n const pipeline = createAttioPipeline({ ...attioConfig, sourceId: 'attio.secondary', objects: ['people'], lists: [] }, fixtureDeps())\n const destination = createMemoryDestination()\n const result = await runIngestion({ selected: selectStreams([pipeline], ['resource:records', 'object:people']), backfill: undefined }, {\n journal: createMemoryJournal(), destination,\n })\n expect(result.ok).toBe(true)\n expect(result.streams[0]?.streamId).toBe('attio.secondary.records.people')\n const row = destination.tables.get('default.attio_records_raw')?.[0]\n expect(row?.id).toBe(JSON.stringify(['attio.secondary', 'records', 'workspace-1', 'object-people', 'record-1']))\n expect(row?.raw).toMatchObject({ source_id: 'attio.secondary' })\n })\n\n test('notes use their 50-item limit and stop fetching after cancellation', async () => {\n const controller = new AbortController()\n const waits: number[] = []\n const urls: string[] = []\n const page = Array.from({ length: 50 }, (_, index) => ({ id: { workspace_id: 'workspace-1', note_id: `note-${index}` } }))\n const deps = fixtureDeps((url) => { urls.push(url.toString()); return Response.json({ data: page }) })\n deps.wait = async (milliseconds, signal) => { waits.push(milliseconds); signal.throwIfAborted() }\n const iterator = readNotes(context(controller.signal), deps)\n expect((await iterator.next()).value?.rows).toHaveLength(50)\n controller.abort(new Error('fixture cancelled'))\n await expect(iterator.next()).rejects.toThrow('fixture cancelled')\n expect(urls).toHaveLength(1)\n expect(new URL(urls[0] ?? '').searchParams.get('limit')).toBe('50')\n expect(waits).toEqual([125])\n })\n\n test('GET offset pagination requests the terminating empty page for an exact multiple', async () => {\n const offsets: string[] = []\n const deps = fixtureDeps((url) => {\n const offset = url.searchParams.get('offset') ?? ''\n offsets.push(offset)\n return Response.json({ data: offset === '0' ? [{ id: { task_id: 'a' } }, { id: { task_id: 'b' } }] : [] })\n })\n const pages = await collect(readCollection(context(), { path: '/tasks', idFields: ['task_id'], pageSize: 2 }, deps))\n expect(pages.map((page) => page.length)).toEqual([2])\n expect(offsets).toEqual(['0', '2'])\n })\n\n test('malformed successful responses and missing identities fail visibly', async () => {\n for (const payload of [{}, { data: null }, { data: [{ id: { task_id: 'task-1' } }] }]) {\n await expect(collect(readTasks(context(), fixtureDeps(() => Response.json(payload))))).rejects.toThrow('Attio')\n }\n const malformed = fixtureDeps((url) => url.pathname === '/v2/objects/people'\n ? fixtureResponse(url) : Response.json({ data: [{ ...person, values: { name: 'not-an-array' } }] }))\n await expect(collect(readRecords(context(), 'people', malformed))).rejects.toThrow('invalid values')\n await expect(collect(readRecords(context(), 'people', fixtureDeps(() => Response.json({ data: [peopleObject] }))))).rejects.toThrow('Attio')\n })\n\n test.each(['workspace_id', 'object_id'])('object readers reject children with a different %s', async (field) => {\n const deps = fixtureDeps((url) => url.pathname === '/v2/objects/people'\n ? fixtureResponse(url) : Response.json({ data: [{ ...person, id: { ...person.id, [field]: 'foreign' } }] }))\n await expect(collect(readRecords(context(), 'people', deps))).rejects.toThrow('different workspace or parent')\n })\n\n test('list readers resolve one list and reject entries belonging to another list', async () => {\n const requests: string[] = []\n const deps = fixtureDeps((url) => {\n requests.push(url.pathname)\n return fixtureResponse(url)\n })\n const chunks = await collect(readEntries(context(), 'sales', deps))\n expect(requests).toEqual(['/v2/lists/sales', '/v2/lists/list-sales/entries/query'])\n expect(chunks[0]?.rows[0]?.raw).toMatchObject({ list_slug: 'sales' })\n const invalid = fixtureDeps((url) => url.pathname === '/v2/lists/sales'\n ? fixtureResponse(url) : Response.json({ data: [{ id: { workspace_id: 'workspace-1', list_id: 'foreign', entry_id: 'one' }, entry_values: {} }] }))\n await expect(collect(readEntries(context(), 'sales', invalid))).rejects.toThrow('different workspace or parent')\n })\n\n test('permission failures fail one object stream while another remains independent', async () => {\n const requests: string[] = []\n const deps = fixtureDeps((url) => {\n requests.push(url.pathname)\n return url.pathname === '/v2/objects/object-custom/records/query'\n ? Response.json({ code: 'insufficient_scope' }, { status: 403 }) : fixtureResponse(url)\n })\n const pipeline = fixturePipeline(deps)\n const destination = createMemoryDestination()\n const journal = createMemoryJournal()\n const result = await runIngestion({ selected: selectStreams([pipeline], ['resource:records']), backfill: undefined }, { journal, destination })\n expect(result.ok).toBe(false)\n expect(result.streams.map((stream) => [stream.streamId, stream.outcome, stream.rows])).toEqual([\n ['attio.primary.records.people', 'succeeded', 1], ['attio.primary.records.subscriptions', 'failed', 0],\n ])\n expect(result.streams[1]?.error).toContain('resource scopes')\n expect(requests.filter((path) => path === '/v2/objects/object-custom/records/query')).toHaveLength(1)\n expect(destination.tables.get('default.attio_records_raw')).toHaveLength(1)\n expect((await journal.readCheckpoint('attio.primary.records.people')).lastSuccessSeq).toBeGreaterThan(0)\n expect((await journal.readCheckpoint('attio.primary.records.subscriptions')).lastSuccessSeq).toBe(0)\n })\n\n test('429 retries remain under executor authority and retain Retry-After', async () => {\n let calls = 0\n const deps = fixtureDeps((url) => {\n calls += 1\n return calls === 1 ? Response.json({ code: 'rate_limit_exceeded' }, { status: 429, headers: { 'Retry-After': '0' } }) : fixtureResponse(url)\n })\n const journal = createMemoryJournal()\n const result = await runIngestion({ selected: selectStreams([fixturePipeline(deps)], ['resource:notes']), backfill: undefined }, {\n journal, destination: createMemoryDestination(),\n })\n expect(result.ok).toBe(true)\n expect(calls).toBe(2)\n expect(journal.events.some((event) => event.eventKind === 'retry_scheduled' && event.errorClass === 'rate_limited')).toBe(true)\n const http = await HttpError.fromResponse(new Response(null, { status: 429, headers: { 'Retry-After': '2' } }))\n expect(http.retryAfterMs).toBe(2_000)\n expect(classifyAttioError(http, { kind: 'rate_limited', retryAfterMs: 2_000 })).toBeUndefined()\n })\n\n test('a missing token fails only when a reader runs', async () => {\n const deps = fixtureDeps(() => { throw new Error('must not fetch') })\n deps.token = () => undefined\n expect(attio.streams).toHaveLength(9)\n await expect(collect(readMembers(context(), deps))).rejects.toThrow('Set ATTIO_API_TOKEN')\n })\n\n test('an interrupted object restarts offset zero without requesting other objects', async () => {\n let fail = true\n const requests: string[] = []\n const offsets: number[] = []\n const page = Array.from({ length: 500 }, (_, index) => ({ ...person, id: { ...person.id, record_id: `record-${index}` } }))\n const deps = fixtureDeps((url, init) => {\n requests.push(url.pathname)\n if (url.pathname === '/v2/objects/people') return fixtureResponse(url)\n if (url.pathname !== '/v2/objects/object-people/records/query') throw new Error(`Unexpected object request ${url.pathname}`)\n const { offset } = JSON.parse(String(init.body))\n offsets.push(offset)\n if (offset === 0) return Response.json({ data: page })\n return fail ? Response.json({ code: 'insufficient_scope' }, { status: 403 }) : Response.json({ data: [] })\n })\n const journal = createMemoryJournal()\n const destination = createMemoryDestination()\n const first = await runIngestion({ selected: selectStreams([fixturePipeline(deps)], ['resource:records', 'object:people']), backfill: undefined }, { journal, destination })\n expect(first.ok).toBe(false)\n expect((await journal.readCheckpoint('attio.primary.records.people')).envelope).toBeUndefined()\n fail = false\n const restored = await restoredJournal(journal)\n const resumed = await runIngestion({ selected: selectStreams([fixturePipeline(deps)], ['resource:records', 'object:people']), backfill: undefined }, { journal: restored, destination })\n expect(resumed.ok).toBe(true)\n expect(offsets).toEqual([0, 500, 0, 500])\n expect(requests).not.toContain('/v2/objects')\n expect(destination.tables.get('default.attio_records_raw')).toHaveLength(500)\n })\n\n test('reruns publish changed observations with the same stable raw ID', async () => {\n let changed = false\n const deps = fixtureDeps((url) => url.pathname === '/v2/objects/object-people/records/query'\n ? Response.json({ data: [{ ...person, values: { ...person.values, name: [{ full_name: changed ? 'Ada Updated' : 'Ada Example' }] } }] }) : fixtureResponse(url))\n const journal = createMemoryJournal()\n const destination = createMemoryDestination()\n const request = { selected: selectStreams([fixturePipeline(deps)], ['resource:records', 'object:people']), backfill: undefined }\n expect((await runIngestion(request, { journal, destination })).ok).toBe(true)\n changed = true\n expect((await runIngestion(request, { journal, destination })).ok).toBe(true)\n const rows = destination.tables.get('default.attio_records_raw') ?? []\n expect(rows).toHaveLength(2)\n expect(rows[0]?.id).toBe(rows[1]?.id)\n expect(rows[1]?.raw).toMatchObject({ data: { values: { name: [{ full_name: 'Ada Updated' }] } } })\n })\n\n test('a fresh runtime after a lost journal acknowledgement replays changed rows safely', async () => {\n let changed = false\n const deps = fixtureDeps((url) => url.pathname === '/v2/objects/object-people/records/query'\n ? Response.json({ data: [{ ...person, values: { ...person.values, revision: [{ value: changed ? 'new' : 'old' }] } }] }) : fixtureResponse(url))\n const journal = createMemoryJournal()\n const destination = createMemoryDestination()\n const failingJournal = {\n ...journal,\n async append(events: Parameters[0]) {\n if (events.some((event) => event.eventKind === 'batch_committed' && Number(event.detail.rows) > 0)) {\n throw new Error('fixture journal unavailable after rows loaded')\n }\n await journal.append(events)\n },\n }\n const failed = await runIngestion({ selected: selectStreams([fixturePipeline(deps)], ['resource:records', 'object:people']), backfill: undefined }, {\n journal: failingJournal, destination,\n })\n expect(failed.ok).toBe(false)\n expect(destination.tables.get('default.attio_records_raw')).toHaveLength(1)\n changed = true\n const resumed = await runIngestion({ selected: selectStreams([fixturePipeline(deps)], ['resource:records', 'object:people']), backfill: undefined }, {\n journal: await restoredJournal(journal), destination,\n })\n expect(resumed.ok).toBe(true)\n const rows = destination.tables.get('default.attio_records_raw') ?? []\n expect(rows).toHaveLength(2)\n expect(rows[0]?.id).toBe(rows[1]?.id)\n expect(rows[1]?.raw).toMatchObject({ data: { values: { revision: [{ value: 'new' }] } } })\n }, 10_000)\n})\n\nfunction fixturePipeline(deps: AttioClientDeps) {\n return definePipeline({\n ...createAttioPipeline({ ...attioConfig, objects: ['people', 'subscriptions'], lists: ['sales'] }, deps),\n retry: { retries: 1, minTimeout: 0, maxTimeout: 0, randomize: false },\n })\n}\n\nfunction context(signal = new AbortController().signal): FetchContext {\n return { signal, attempt: async (operation) => operation(signal) }\n}\n\nasync function collect(items: AsyncIterable): Promise {\n const result: T[] = []\n for await (const item of items) result.push(item)\n return result\n}\n\nasync function restoredJournal(previous: ReturnType) {\n const journal = createMemoryJournal()\n await journal.append(previous.events.map((event) => ({\n ...event, checkpoint: event.checkpoint === undefined ? undefined : JSON.parse(JSON.stringify(event.checkpoint)),\n })))\n return journal\n}\n" + }, + { + "path": "attio/tests/fixtures.ts", + "type": "registry:file", + "target": "src/integrations/attio/tests/fixtures.ts", + "role": "test", + "content": "import type { AttioClientDeps } from '../client.js'\nimport { attioConfig } from '../config.js'\n\nexport const peopleObject = { id: { workspace_id: 'workspace-1', object_id: 'object-people' }, api_slug: 'people' }\nexport const companiesObject = { id: { workspace_id: 'workspace-1', object_id: 'object-companies' }, api_slug: 'companies' }\nexport const customObject = { id: { workspace_id: 'workspace-1', object_id: 'object-custom' }, api_slug: 'subscriptions' }\nexport const salesList = { id: { workspace_id: 'workspace-1', list_id: 'list-sales' }, api_slug: 'sales' }\n\nconst fixtureParents: Record = {\n '/v2/objects/people': peopleObject,\n '/v2/objects/object-people': peopleObject,\n '/v2/objects/companies': companiesObject,\n '/v2/objects/object-companies': companiesObject,\n '/v2/objects/subscriptions': customObject,\n '/v2/objects/object-custom': customObject,\n '/v2/lists/sales': salesList,\n '/v2/lists/list-sales': salesList,\n}\n\nexport const person = {\n id: { workspace_id: 'workspace-1', object_id: 'object-people', record_id: 'record-1' },\n created_at: '2026-01-01T00:00:00Z',\n web_url: 'https://app.attio.com/example/person/record-1',\n values: {\n name: [{ full_name: 'Ada Example', first_name: 'Ada', last_name: 'Example', attribute_type: 'personal-name', active_until: null }],\n email_addresses: [{ email_address: 'ada@example.test', attribute_type: 'email-address' }],\n custom_multi: [{ value: 'one' }, { value: 'two' }],\n },\n}\n\nexport const fixtureCollections: Record = {\n '/v2/objects': [peopleObject, customObject],\n '/v2/objects/object-people/records/query': [person],\n '/v2/objects/object-custom/records/query': [{ id: { ...person.id, object_id: 'object-custom' }, values: { custom_number: [{ value: 7 }] } }],\n '/v2/objects/object-companies/records/query': [],\n '/v2/objects/object-people/attributes': [{ id: { ...peopleObject.id, attribute_id: 'attribute-1' }, api_slug: 'name', type: 'personal-name' }],\n '/v2/objects/object-custom/attributes': [{ id: { ...customObject.id, attribute_id: 'attribute-1' }, api_slug: 'custom_number', type: 'number' }],\n '/v2/objects/object-companies/attributes': [],\n '/v2/lists': [salesList],\n '/v2/lists/list-sales/attributes': [{ id: { workspace_id: 'workspace-1', object_id: 'list-sales', attribute_id: 'attribute-1' }, api_slug: 'stage', type: 'status' }],\n '/v2/lists/list-sales/entries/query': [{ id: { ...salesList.id, entry_id: 'entry-1' }, parent_object: 'people', parent_record_id: 'record-1', entry_values: { stage: [{ status: { title: 'Prospect' } }] } }],\n '/v2/notes': [{ id: { workspace_id: 'workspace-1', note_id: 'note-1' }, parent_object: 'people', parent_record_id: 'record-1', content_markdown: '# First call' }],\n '/v2/tasks': [{ id: { workspace_id: 'workspace-1', task_id: 'task-1' }, content_plaintext: 'Follow up', is_completed: false }],\n '/v2/workspace_members': [{ id: { workspace_id: 'workspace-1', workspace_member_id: 'member-1' }, first_name: 'Grace' }],\n}\n\nexport function fixtureDeps(\n respond: (url: URL, init: RequestInit) => Response | Promise = fixtureResponse,\n): AttioClientDeps {\n return {\n config: attioConfig,\n token: () => 'fixture-token',\n wait: async () => {},\n fetch: (url, init) => Promise.resolve(respond(new URL(url), init)),\n }\n}\n\nexport function fixtureResponse(url: URL): Response {\n const data = fixtureParents[url.pathname] ?? fixtureCollections[url.pathname]\n if (data === undefined) throw new Error(`Unexpected fixture URL: ${url.pathname}`)\n return Response.json({ data })\n}\n" + } + ], + "meta": { + "chkit": { + "formatVersion": 1, + "version": "0.2.0", + "language": "typescript", + "license": "MIT", + "documentation": "https://chkit.obsessiondb.com/integrations/attio/", + "logo": "https://chkit.obsessiondb.com/logos/attio.svg", + "authentication": { + "method": "Bearer token", + "env": [ + "ATTIO_API_TOKEN" + ], + "setup": [ + "As a workspace admin, open the workspace-name dropdown in Attio and select Workspace settings.", + "Open Developers, select + New access token, and name the token (for example, chkit ClickHouse).", + "Grant the read scopes listed for the resources to sync.", + "Click the token on the Developers page to copy it. Set ATTIO_API_TOKEN in the project .env file or the scheduler secret environment." + ], + "documentation": "https://attio.com/help/reference/apps/generating-an-api-key" + }, + "chkit": "^0.2.0-beta.10", + "ingest": "^0.2.0-beta.10", + "clickhouse": ">=25.3.0", + "root": "src/integrations/attio", + "entry": "index.ts", + "exports": [ + "attio", + "attioObjectsRaw", + "attioObjectAttributesRaw", + "attioRecordsRaw", + "attioPeople", + "attioCompanies", + "attioDeals", + "attioListsRaw", + "attioListAttributesRaw", + "attioEntriesRaw", + "attioNotesRaw", + "attioTasksRaw", + "attioMembersRaw" + ], + "resources": [ + { + "name": "objects", + "title": "Objects", + "table": "attio_objects_raw", + "description": "Accessible standard and custom object definitions, IDs, API slug, singular and plural names, creation time", + "scopes": [ + "object_configuration:read" + ], + "strategy": "full", + "endpoints": [ + { + "method": "GET", + "path": "/objects", + "documentation": "https://docs.attio.com/rest-api/endpoint-reference/objects/list-objects" + } + ] + }, + { + "name": "object_attributes", + "title": "Object attributes", + "table": "attio_object_attributes_raw", + "description": "Attribute IDs, slug, title, description, type, flags, defaults, relationship definitions, configuration, creation time; includes archived definitions", + "scopes": [ + "object_configuration:read" + ], + "strategy": "full", + "endpoints": [ + { + "method": "GET", + "path": "/objects/{object_id}/attributes", + "documentation": "https://docs.attio.com/rest-api/endpoint-reference/attributes/list-attributes" + }, + { + "method": "GET", + "path": "/objects/{object_id}", + "documentation": "https://docs.attio.com/rest-api/endpoint-reference/objects/get-an-object" + } + ] + }, + { + "name": "records", + "title": "Records", + "table": "attio_records_raw", + "description": "Independent records streams for configured object types (People and Companies by default), IDs, creation time, Attio URL, and the complete values object with all returned attribute arrays", + "scopes": [ + "object_configuration:read", + "record_permission:read" + ], + "strategy": "full", + "endpoints": [ + { + "method": "POST", + "path": "/objects/{object_id}/records/query", + "documentation": "https://docs.attio.com/rest-api/endpoint-reference/records/list-records" + }, + { + "method": "GET", + "path": "/objects/{object_id}", + "documentation": "https://docs.attio.com/rest-api/endpoint-reference/objects/get-an-object" + } + ] + }, + { + "name": "lists", + "title": "Lists", + "table": "attio_lists_raw", + "description": "List IDs, slug, name, parent objects, workspace and member access configuration, creator, creation time", + "scopes": [ + "list_configuration:read" + ], + "strategy": "full", + "endpoints": [ + { + "method": "GET", + "path": "/lists", + "documentation": "https://docs.attio.com/rest-api/endpoint-reference/lists/list-all-lists" + } + ] + }, + { + "name": "list_attributes", + "title": "List attributes", + "table": "attio_list_attributes_raw", + "description": "Attribute definitions for selected lists, including types, configuration, flags, defaults, relationships, and archived definitions", + "scopes": [ + "list_configuration:read" + ], + "strategy": "full", + "endpoints": [ + { + "method": "GET", + "path": "/lists/{list_id}/attributes", + "documentation": "https://docs.attio.com/rest-api/endpoint-reference/attributes/list-attributes" + }, + { + "method": "GET", + "path": "/lists/{list_id}", + "documentation": "https://docs.attio.com/rest-api/endpoint-reference/lists/get-a-list" + } + ] + }, + { + "name": "entries", + "title": "List entries", + "table": "attio_entries_raw", + "description": "Entry IDs, parent object and record ID, creation time, and the complete entry_values object", + "scopes": [ + "list_configuration:read", + "list_entry:read" + ], + "strategy": "full", + "endpoints": [ + { + "method": "POST", + "path": "/lists/{list_id}/entries/query", + "documentation": "https://docs.attio.com/rest-api/endpoint-reference/entries/list-entries" + }, + { + "method": "GET", + "path": "/lists/{list_id}", + "documentation": "https://docs.attio.com/rest-api/endpoint-reference/lists/get-a-list" + } + ] + }, + { + "name": "notes", + "title": "Notes", + "table": "attio_notes_raw", + "description": "Note IDs, parent record links, title, plaintext and Markdown content, tags, creator, creation time, and meeting reference when returned", + "scopes": [ + "note:read", + "object_configuration:read", + "record_permission:read" + ], + "strategy": "full", + "endpoints": [ + { + "method": "GET", + "path": "/notes", + "documentation": "https://docs.attio.com/rest-api/endpoint-reference/notes/list-notes" + } + ] + }, + { + "name": "tasks", + "title": "Tasks", + "table": "attio_tasks_raw", + "description": "Task IDs, plaintext content, deadline, completion state and time, linked records, assignees, creator, creation time", + "scopes": [ + "task:read", + "object_configuration:read", + "record_permission:read", + "user_management:read" + ], + "strategy": "full", + "endpoints": [ + { + "method": "GET", + "path": "/tasks", + "documentation": "https://docs.attio.com/rest-api/endpoint-reference/tasks/list-tasks" + } + ] + }, + { + "name": "members", + "title": "Workspace members", + "table": "attio_members_raw", + "description": "Member IDs, first and last name, email address, avatar URL, access level, creation time", + "scopes": [ + "user_management:read" + ], + "strategy": "full", + "endpoints": [ + { + "method": "GET", + "path": "/workspace_members", + "documentation": "https://docs.attio.com/rest-api/endpoint-reference/workspace-members/list-workspace-members" + } + ] + } + ], + "views": [ + { + "name": "attio_people", + "source": "records", + "description": "Names, email addresses, and linked companies for people records." + }, + { + "name": "attio_companies", + "source": "records", + "description": "Names, domains, and descriptions for company records." + }, + { + "name": "attio_deals", + "source": "records", + "description": "Deal names, stages, values, currencies, and linked people and companies." + } + ], + "sync": { + "description": "People, Companies, each configured custom object/list, and workspace resources have independent full-sync streams. Each reader paginates one collection through chkit-ingest; interrupted mutable offset reads replay that collection from zero. Completed and empty reads are recorded by the executor journal without a custom scan checkpoint. Custom types and lists must be configured explicitly.", + "schedule": "Run chkit ingest run through cron, CI, or an existing scheduler; the template does not start a background worker.", + "deletions": "Records absent from a later API response remain in ClickHouse. The template does not propagate deletions or consume webhooks." + }, + "env": { + "ATTIO_API_TOKEN": "" + }, + "fileHashes": { + "src/integrations/attio/README.md": "0d4472f92632ed1b12eea5f05622deccf027ae9d488998ec193f01f542397514", + "src/integrations/attio/client.ts": "ae90b76cfa969ea871e589e48a1f22efbf8488a4000ea25af22ac271609cc5bf", + "src/integrations/attio/config.ts": "08688e8b38ceab1c30573cb8ab32e620b19219d080c7e2aa56546d6e4395aed3", + "src/integrations/attio/index.ts": "74be5d3684988a95cef41a53ad44071b3d06000b8adb49bebbf30c3b2698a663", + "src/integrations/attio/pipeline.ts": "338be6e5b14f155d4d67c53daa47a61d844caf98016287f7e66cfd82acd5d436", + "src/integrations/attio/sources/objects.ts": "3262f82c51f3367b169548af28e6e98ca859fb85dc60cfe20816a0352365293d", + "src/integrations/attio/sources/object-attributes.ts": "4c6e543f1f13a1ac9afb10e10be8524efbc817df23f5dab6c3ba47742f3b7178", + "src/integrations/attio/sources/records.ts": "2dd116f2ea2e3476c278153e762adbeecbf6d61952c2e5f69332ab3674fe75b7", + "src/integrations/attio/sources/lists.ts": "89cd338931fb8959fb16c88b872b02229be9737cbf9987ee9d0aa025ea647a88", + "src/integrations/attio/sources/list-attributes.ts": "95f4e2aaac531e05115916c26d8b97973daa358e7679970aec6ac1db16060d93", + "src/integrations/attio/sources/entries.ts": "9329630f494d8cdc0ea2711a49845f09cfc9ab1775a7e39cb381a85e4c5960aa", + "src/integrations/attio/sources/notes.ts": "ca5b71bf24d432ff1967eefa9a1b5feef938ee6b102a06c6a2cc8058d23fb743", + "src/integrations/attio/sources/tasks.ts": "a2d64a19efe7bdeedf0925d7883feb6e931b1b62264f83532c1932c893625e97", + "src/integrations/attio/sources/members.ts": "7ed50b334465e4060ab3ecc4d47c6863c1c03b9940026f8e7089603504b72462", + "src/integrations/attio/tests/attio.test.ts": "9200421e0876f99147b8e7d7a8f7d0e0b2bc4784460d3ba0f67211ca7a38119e", + "src/integrations/attio/tests/fixtures.ts": "22d332ae81948450a9323c18159406dc7c44856c74efbcf2e0b09e6980c03274" + } + } + } +} diff --git a/registry/attio/sources/entries.ts b/registry/attio/sources/entries.ts index 2352168e..dfc3893e 100644 --- a/registry/attio/sources/entries.ts +++ b/registry/attio/sources/entries.ts @@ -1,20 +1,19 @@ -import { rawTable } from '@chkit/plugin-ingest' +import { rawTable, type FetchContext } from '@chkit/plugin-ingest' -import { readParentCollections, type AttioClientDeps } from '../client.js' +import { defaultAttioClientDeps, entityId, readParentCollection, type AttioClientDeps } from '../client.js' import { attioConfig } from '../config.js' -import type { AttioReadContext } from '../checkpoints.js' export const attioEntriesRaw = rawTable({ database: attioConfig.database, name: `${attioConfig.tablePrefix}_entries_raw` }) -export async function* readEntries(context: AttioReadContext, deps?: AttioClientDeps) { - yield* readParentCollections(context, { - resource: 'entries', parents: 'lists', idFields: ['workspace_id', 'list_id', 'entry_id'], +export function readEntries(context: FetchContext, list: string, deps: AttioClientDeps = defaultAttioClientDeps) { + return readParentCollection(context, { + resource: 'entries', parent: { kind: 'lists', ref: list }, request: (parent) => ({ - path: `/lists/${encodeURIComponent(parent.id)}/entries/query`, + path: `/lists/${encodeURIComponent(entityId(parent, 'list_id'))}/entries/query`, idFields: ['workspace_id', 'list_id', 'entry_id'], method: 'POST', valuesField: 'entry_values', - pageSize: attioConfig.pageSize, + pageSize: deps.config.pageSize, }), }, deps) } diff --git a/registry/attio/sources/list-attributes.ts b/registry/attio/sources/list-attributes.ts index 44464ca3..f113d6c4 100644 --- a/registry/attio/sources/list-attributes.ts +++ b/registry/attio/sources/list-attributes.ts @@ -1,20 +1,19 @@ -import { rawTable } from '@chkit/plugin-ingest' +import { rawTable, type FetchContext } from '@chkit/plugin-ingest' -import { readParentCollections, type AttioClientDeps } from '../client.js' +import { defaultAttioClientDeps, entityId, readParentCollection, type AttioClientDeps } from '../client.js' import { attioConfig } from '../config.js' -import type { AttioReadContext } from '../checkpoints.js' export const attioListAttributesRaw = rawTable({ database: attioConfig.database, name: `${attioConfig.tablePrefix}_list_attributes_raw` }) -export async function* readListAttributes(context: AttioReadContext, deps?: AttioClientDeps) { - yield* readParentCollections(context, { - resource: 'list_attributes', parents: 'lists', idFields: ['workspace_id', 'object_id', 'attribute_id'], +export function readListAttributes(context: FetchContext, list: string, deps: AttioClientDeps = defaultAttioClientDeps) { + return readParentCollection(context, { + resource: 'list_attributes', parent: { kind: 'lists', ref: list }, request: (parent) => ({ - path: `/lists/${encodeURIComponent(parent.id)}/attributes`, + path: `/lists/${encodeURIComponent(entityId(parent, 'list_id'))}/attributes`, idFields: ['workspace_id', 'object_id', 'attribute_id'], // Attio's shared attribute schema calls the parent list ID object_id too. query: { show_archived: 'true' }, - pageSize: attioConfig.pageSize, + pageSize: deps.config.pageSize, }), }, deps) } diff --git a/registry/attio/sources/lists.ts b/registry/attio/sources/lists.ts index 70880ff5..80c9017f 100644 --- a/registry/attio/sources/lists.ts +++ b/registry/attio/sources/lists.ts @@ -1,12 +1,12 @@ -import { rawTable } from '@chkit/plugin-ingest' +import { rawTable, type FetchContext } from '@chkit/plugin-ingest' -import { discoverLists, toAttioRows, type AttioClientDeps } from '../client.js' +import { readRows, type AttioClientDeps } from '../client.js' import { attioConfig } from '../config.js' -import { beginScan, type AttioReadContext } from '../checkpoints.js' export const attioListsRaw = rawTable({ database: attioConfig.database, name: `${attioConfig.tablePrefix}_lists_raw` }) -export async function* readLists(context: AttioReadContext, deps?: AttioClientDeps) { - const state = beginScan(context, 'lists') - yield { rows: toAttioRows(await discoverLists(context, deps), 'lists', ['workspace_id', 'list_id']), state: { ...state, phase: 'complete' as const } } +export function readLists(context: FetchContext, deps?: AttioClientDeps) { + return readRows(context, { + resource: 'lists', request: { path: '/lists', idFields: ['workspace_id', 'list_id'] }, + }, deps) } diff --git a/registry/attio/sources/members.ts b/registry/attio/sources/members.ts index 7f07e5b5..c3b1b970 100644 --- a/registry/attio/sources/members.ts +++ b/registry/attio/sources/members.ts @@ -1,13 +1,14 @@ -import { rawTable } from '@chkit/plugin-ingest' +import { rawTable, type FetchContext } from '@chkit/plugin-ingest' -import { readCollectionScan, type AttioClientDeps } from '../client.js' +import { readRows, type AttioClientDeps } from '../client.js' import { attioConfig } from '../config.js' -import type { AttioReadContext } from '../checkpoints.js' export const attioMembersRaw = rawTable({ database: attioConfig.database, name: `${attioConfig.tablePrefix}_members_raw` }) -export async function* readMembers(context: AttioReadContext, deps?: AttioClientDeps) { - yield* readCollectionScan(context, 'members', { - path: '/workspace_members', idFields: ['workspace_id', 'workspace_member_id'], +export function readMembers(context: FetchContext, deps?: AttioClientDeps) { + return readRows(context, { + resource: 'members', request: { + path: '/workspace_members', idFields: ['workspace_id', 'workspace_member_id'], + }, }, deps) } diff --git a/registry/attio/sources/notes.ts b/registry/attio/sources/notes.ts index dc0fba5a..9e54c497 100644 --- a/registry/attio/sources/notes.ts +++ b/registry/attio/sources/notes.ts @@ -1,14 +1,15 @@ -import { IngestConfigError, rawTable } from '@chkit/plugin-ingest' +import { IngestConfigError, rawTable, type FetchContext } from '@chkit/plugin-ingest' -import { readCollectionScan, type AttioClientDeps } from '../client.js' +import { defaultAttioClientDeps, readRows, type AttioClientDeps } from '../client.js' import { attioConfig } from '../config.js' -import type { AttioReadContext } from '../checkpoints.js' export const attioNotesRaw = rawTable({ database: attioConfig.database, name: `${attioConfig.tablePrefix}_notes_raw` }) -export async function* readNotes(context: AttioReadContext, deps?: AttioClientDeps) { - if (attioConfig.notesPageSize > 50) throw new IngestConfigError('Attio notesPageSize cannot exceed 50.') - yield* readCollectionScan(context, 'notes', { - path: '/notes', idFields: ['workspace_id', 'note_id'], pageSize: attioConfig.notesPageSize, +export function readNotes(context: FetchContext, deps: AttioClientDeps = defaultAttioClientDeps) { + if (deps.config.notesPageSize > 50) throw new IngestConfigError('Attio notesPageSize cannot exceed 50.') + return readRows(context, { + resource: 'notes', request: { + path: '/notes', idFields: ['workspace_id', 'note_id'], pageSize: deps.config.notesPageSize, + }, }, deps) } diff --git a/registry/attio/sources/object-attributes.ts b/registry/attio/sources/object-attributes.ts index 2793a2a6..08458061 100644 --- a/registry/attio/sources/object-attributes.ts +++ b/registry/attio/sources/object-attributes.ts @@ -1,19 +1,18 @@ -import { rawTable } from '@chkit/plugin-ingest' +import { rawTable, type FetchContext } from '@chkit/plugin-ingest' -import { readParentCollections, type AttioClientDeps } from '../client.js' +import { defaultAttioClientDeps, entityId, readParentCollection, type AttioClientDeps } from '../client.js' import { attioConfig } from '../config.js' -import type { AttioReadContext } from '../checkpoints.js' export const attioObjectAttributesRaw = rawTable({ database: attioConfig.database, name: `${attioConfig.tablePrefix}_object_attributes_raw` }) -export async function* readObjectAttributes(context: AttioReadContext, deps?: AttioClientDeps) { - yield* readParentCollections(context, { - resource: 'object_attributes', parents: 'objects', idFields: ['workspace_id', 'object_id', 'attribute_id'], +export function readObjectAttributes(context: FetchContext, object: string, deps: AttioClientDeps = defaultAttioClientDeps) { + return readParentCollection(context, { + resource: 'object_attributes', parent: { kind: 'objects', ref: object }, request: (parent) => ({ - path: `/objects/${encodeURIComponent(parent.id)}/attributes`, + path: `/objects/${encodeURIComponent(entityId(parent, 'object_id'))}/attributes`, idFields: ['workspace_id', 'object_id', 'attribute_id'], query: { show_archived: 'true' }, - pageSize: attioConfig.pageSize, + pageSize: deps.config.pageSize, }), }, deps) } diff --git a/registry/attio/sources/objects.ts b/registry/attio/sources/objects.ts index 6a4fba5d..0491cce1 100644 --- a/registry/attio/sources/objects.ts +++ b/registry/attio/sources/objects.ts @@ -1,12 +1,12 @@ -import { rawTable } from '@chkit/plugin-ingest' +import { rawTable, type FetchContext } from '@chkit/plugin-ingest' -import { discoverObjects, toAttioRows, type AttioClientDeps } from '../client.js' +import { readRows, type AttioClientDeps } from '../client.js' import { attioConfig } from '../config.js' -import { beginScan, type AttioReadContext } from '../checkpoints.js' export const attioObjectsRaw = rawTable({ database: attioConfig.database, name: `${attioConfig.tablePrefix}_objects_raw` }) -export async function* readObjects(context: AttioReadContext, deps?: AttioClientDeps) { - const state = beginScan(context, 'objects') - yield { rows: toAttioRows(await discoverObjects(context, deps), 'objects', ['workspace_id', 'object_id']), state: { ...state, phase: 'complete' as const } } +export function readObjects(context: FetchContext, deps?: AttioClientDeps) { + return readRows(context, { + resource: 'objects', request: { path: '/objects', idFields: ['workspace_id', 'object_id'] }, + }, deps) } diff --git a/registry/attio/sources/records.ts b/registry/attio/sources/records.ts index 5133f0b9..142a5885 100644 --- a/registry/attio/sources/records.ts +++ b/registry/attio/sources/records.ts @@ -1,9 +1,8 @@ import { view } from '@chkit/core' -import { rawTable } from '@chkit/plugin-ingest' +import { rawTable, type FetchContext } from '@chkit/plugin-ingest' -import { readParentCollections, type AttioClientDeps } from '../client.js' +import { defaultAttioClientDeps, entityId, readParentCollection, type AttioClientDeps } from '../client.js' import { attioConfig } from '../config.js' -import type { AttioReadContext } from '../checkpoints.js' export const attioRecordsRaw = rawTable({ database: attioConfig.database, name: `${attioConfig.tablePrefix}_records_raw` }) @@ -26,15 +25,15 @@ export const attioDeals = recordView('deals', ` arrayMap(value -> JSONExtractString(value, 'target_record_id'), JSONExtractArrayRaw(values, 'associated_company')) AS company_record_ids, arrayMap(value -> JSONExtractString(value, 'target_record_id'), JSONExtractArrayRaw(values, 'associated_people')) AS people_record_ids`) -export async function* readRecords(context: AttioReadContext, deps?: AttioClientDeps) { - yield* readParentCollections(context, { - resource: 'records', parents: 'objects', idFields: ['workspace_id', 'object_id', 'record_id'], +export function readRecords(context: FetchContext, object: string, deps: AttioClientDeps = defaultAttioClientDeps) { + return readParentCollection(context, { + resource: 'records', parent: { kind: 'objects', ref: object }, request: (parent) => ({ - path: `/objects/${encodeURIComponent(parent.id)}/records/query`, + path: `/objects/${encodeURIComponent(entityId(parent, 'object_id'))}/records/query`, idFields: ['workspace_id', 'object_id', 'record_id'], method: 'POST', valuesField: 'values', - pageSize: attioConfig.pageSize, + pageSize: deps.config.pageSize, }), }, deps) } diff --git a/registry/attio/sources/tasks.ts b/registry/attio/sources/tasks.ts index 8ed6c60d..b8b80ac8 100644 --- a/registry/attio/sources/tasks.ts +++ b/registry/attio/sources/tasks.ts @@ -1,14 +1,15 @@ -import { rawTable } from '@chkit/plugin-ingest' +import { rawTable, type FetchContext } from '@chkit/plugin-ingest' -import { readCollectionScan, type AttioClientDeps } from '../client.js' +import { defaultAttioClientDeps, readRows, type AttioClientDeps } from '../client.js' import { attioConfig } from '../config.js' -import type { AttioReadContext } from '../checkpoints.js' export const attioTasksRaw = rawTable({ database: attioConfig.database, name: `${attioConfig.tablePrefix}_tasks_raw` }) -export async function* readTasks(context: AttioReadContext, deps?: AttioClientDeps) { - yield* readCollectionScan(context, 'tasks', { - path: '/tasks', idFields: ['workspace_id', 'task_id'], pageSize: attioConfig.pageSize, - query: { sort: 'created_at:asc' }, +export function readTasks(context: FetchContext, deps: AttioClientDeps = defaultAttioClientDeps) { + return readRows(context, { + resource: 'tasks', request: { + path: '/tasks', idFields: ['workspace_id', 'task_id'], pageSize: deps.config.pageSize, + query: { sort: 'created_at:asc' }, + }, }, deps) } diff --git a/registry/attio/tests/attio.test.ts b/registry/attio/tests/attio.test.ts index f43d7c9c..4e5a4803 100644 --- a/registry/attio/tests/attio.test.ts +++ b/registry/attio/tests/attio.test.ts @@ -1,96 +1,104 @@ import { describe, expect, test } from 'bun:test' -import { HttpError, definePipeline, runIngestion, selectStreams, type SourceChunk } from '@chkit/plugin-ingest' +import { HttpError, definePipeline, runIngestion, selectStreams, type FetchContext } from '@chkit/plugin-ingest' import { createMemoryDestination, createMemoryJournal } from '@chkit/plugin-ingest/testing' import { classifyAttioError, readCollection, type AttioClientDeps } from '../client.js' -import { scanCheckpoint, type AttioReadContext } from '../checkpoints.js' +import { attioConfig } from '../config.js' import * as definitions from '../index.js' -import { attio } from '../pipeline.js' -import { readObjects } from '../sources/objects.js' -import { readObjectAttributes } from '../sources/object-attributes.js' -import { readRecords } from '../sources/records.js' -import { readLists } from '../sources/lists.js' -import { readListAttributes } from '../sources/list-attributes.js' +import { attio, createAttioPipeline } from '../pipeline.js' import { readEntries } from '../sources/entries.js' +import { readMembers } from '../sources/members.js' import { readNotes } from '../sources/notes.js' +import { readRecords } from '../sources/records.js' import { readTasks } from '../sources/tasks.js' -import { readMembers } from '../sources/members.js' -import { customObject, fixtureCollections, fixtureDeps, peopleObject, person } from './fixtures.js' - -const readers: Record AsyncIterable> = { - objects: readObjects, object_attributes: readObjectAttributes, records: readRecords, - lists: readLists, list_attributes: readListAttributes, entries: readEntries, - notes: readNotes, tasks: readTasks, members: readMembers, -} +import { fixtureDeps, fixtureResponse, peopleObject, person } from './fixtures.js' describe('Attio template', () => { - test('exports every stream destination and all three views without reading credentials', () => { + test('exports independently selectable object streams and raw destinations without credentials', () => { const exported = Object.values(definitions) expect(exported).toContain(attio) expect(attio.streams).toHaveLength(9) + expect(attio.streams.filter((stream) => stream.tags.includes('resource:records')).map((stream) => stream.id)) + .toEqual(['attio.primary.records.people', 'attio.primary.records.companies']) + expect(selectStreams([attio], ['resource:records', 'object:people']).map(({ stream }) => stream.id)).toEqual(['attio.primary.records.people']) + expect(selectStreams([attio], ['resource:records', 'object:companies']).map(({ stream }) => stream.id)) + .toEqual(['attio.primary.records.companies']) for (const stream of attio.streams) { expect(exported).toContain(stream.destination) - expect(stream.incremental.id).toBe(`attio.scan.${stream.id.split('.').at(-1)}`) + expect(stream.incremental.id).toBe('chkit.full_sync') } expect([definitions.attioPeople, definitions.attioCompanies, definitions.attioDeals].map((view) => view.name)) .toEqual(['attio_people', 'attio_companies', 'attio_deals']) }) - test('all streams ingest sanitized fixtures and reruns publish changed observations', async () => { - let changed = false - const deps = fixtureDeps((url) => { - const data = fixtureCollections[url.pathname] - if (data === undefined) throw new Error(`Unexpected fixture URL: ${url.pathname}`) - return Response.json({ data: changed && url.pathname === '/v2/notes' - ? [{ id: { workspace_id: 'workspace-1', note_id: 'note-1' }, content_markdown: '# Revised call' }] - : data }) - }) - const pipeline = fixturePipeline(deps) + test('an object named tasks does not overlap the workspace tasks resource tag', () => { + const pipeline = createAttioPipeline({ ...attioConfig, objects: ['tasks'], lists: [] }, fixtureDeps()) + expect(selectStreams([pipeline], ['resource:tasks']).map(({ stream }) => stream.id)).toEqual(['attio.primary.tasks']) + expect(selectStreams([pipeline], ['resource:records', 'object:tasks']).map(({ stream }) => stream.id)) + .toEqual(['attio.primary.records.tasks']) + }) + + test('configured custom objects and lists have their own streams sharing raw tables', async () => { + const pipeline = fixturePipeline(fixtureDeps()) const journal = createMemoryJournal() const destination = createMemoryDestination() - const request = { selected: selectStreams([pipeline], []), backfill: undefined } - const first = await runIngestion(request, { journal, destination }) - expect(first.ok).toBe(true) - expect(first.streams.map((stream) => stream.rows)).toEqual([2, 2, 2, 1, 1, 1, 1, 1, 1]) + const result = await runIngestion({ selected: selectStreams([pipeline], []), backfill: undefined }, { journal, destination }) + expect(result.ok, JSON.stringify(result)).toBe(true) + expect(pipeline.streams).toHaveLength(11) + expect(result.streams.filter((stream) => stream.streamId.includes('.records.')).map((stream) => stream.rows)).toEqual([1, 1]) expect(destination.tables.size).toBe(9) expect(destination.tables.get('default.attio_records_raw')?.map((row) => row.raw)).toContainEqual({ source_id: 'attio.primary', object_slug: 'people', data: person, }) - changed = true - const second = await runIngestion(request, { journal, destination }) - expect(second.ok).toBe(true) - const notes = destination.tables.get('default.attio_notes_raw') ?? [] - expect(notes).toHaveLength(2) - expect(notes[0]?.id).toBe(notes[1]?.id) - expect(notes[1]?.raw).toMatchObject({ data: { content_markdown: '# Revised call' } }) + expect(destination.tables.get('default.attio_entries_raw')?.[0]?.raw).toMatchObject({ list_slug: 'sales' }) + expect(selectStreams([pipeline], ['resource:records', 'object:subscriptions']).map(({ stream }) => stream.id)) + .toEqual(['attio.primary.records.subscriptions']) + expect(selectStreams([pipeline], ['resource:entries']).map(({ stream }) => stream.id)) + .toEqual(['attio.primary.entries.sales']) }) - test('records discover parents independently and read all offset pages including custom objects', async () => { + test('one object reader paginates only that object and preserves raw custom values and IDs', async () => { const requests: Array<{ path: string; body: unknown }> = [] const firstPage = Array.from({ length: 500 }, (_, index) => ({ ...person, id: { ...person.id, record_id: `record-${index}` } })) const deps = fixtureDeps((url, init) => { const body = typeof init.body === 'string' ? JSON.parse(init.body) : undefined requests.push({ path: url.pathname, body }) - if (url.pathname === '/v2/objects') return Response.json({ data: [peopleObject, customObject] }) + if (url.pathname === '/v2/objects/people') return Response.json({ data: peopleObject }) if (url.pathname === '/v2/objects/object-people/records/query') { return Response.json({ data: body.offset === 0 ? firstPage : [{ ...person, id: { ...person.id, record_id: 'last' } }] }) } - if (url.pathname === '/v2/objects/object-custom/records/query') { - return Response.json({ data: [{ id: { ...person.id, object_id: 'object-custom' }, values: {} }] }) - } throw new Error(`Unexpected URL ${url.pathname}`) }) - const chunks = await collect(readRecords(context(), deps)) - const rowChunks = chunks.filter((chunk) => chunk.rows.length > 0) - expect(rowChunks.map((chunk) => chunk.rows.length)).toEqual([500, 1, 1]) - expect(requests.map((request) => request.body)).toEqual([undefined, { limit: 500, offset: 0 }, { limit: 500, offset: 500 }, { limit: 500, offset: 0 }]) - expect(rowChunks.every((chunk) => !('state' in chunk) && !('id' in chunk))).toBe(true) - expect(rowChunks[0]?.rows[1]?.id).not.toBe(rowChunks[2]?.rows[0]?.id) - expect(chunks.at(-1)?.state).toMatchObject({ phase: 'complete', completedParents: ['object-people', 'object-custom'] }) + const chunks = await collect(readRecords(context(), 'people', deps)) + expect(chunks.map((chunk) => chunk.rows.length)).toEqual([500, 1]) + expect(requests).toEqual([ + { path: '/v2/objects/people', body: undefined }, + { path: '/v2/objects/object-people/records/query', body: { limit: 500, offset: 0 } }, + { path: '/v2/objects/object-people/records/query', body: { limit: 500, offset: 500 } }, + ]) + expect(chunks.every((chunk) => !('state' in chunk) && !('id' in chunk))).toBe(true) + expect(chunks[0]?.rows[0]?.id).toBe(JSON.stringify(['attio.primary', 'records', 'workspace-1', 'object-people', 'record-0'])) + expect(chunks[0]?.rows[0]?.raw).toMatchObject({ data: { values: { custom_multi: [{ value: 'one' }, { value: 'two' }] } } }) + const custom = await collect(readRecords(context(), 'subscriptions', fixtureDeps())) + expect(custom[0]?.rows[0]?.raw).toMatchObject({ object_slug: 'subscriptions' }) + expect(custom[0]?.rows[0]?.id).not.toBe(chunks[0]?.rows[0]?.id) }) - test('notes use their 50-item page limit, honor cancellation, and never fetch a second page after abort', async () => { + test('a source identity supplied to the pipeline owns stream IDs and raw row IDs', async () => { + const pipeline = createAttioPipeline({ ...attioConfig, sourceId: 'attio.secondary', objects: ['people'], lists: [] }, fixtureDeps()) + const destination = createMemoryDestination() + const result = await runIngestion({ selected: selectStreams([pipeline], ['resource:records', 'object:people']), backfill: undefined }, { + journal: createMemoryJournal(), destination, + }) + expect(result.ok).toBe(true) + expect(result.streams[0]?.streamId).toBe('attio.secondary.records.people') + const row = destination.tables.get('default.attio_records_raw')?.[0] + expect(row?.id).toBe(JSON.stringify(['attio.secondary', 'records', 'workspace-1', 'object-people', 'record-1'])) + expect(row?.raw).toMatchObject({ source_id: 'attio.secondary' }) + }) + + test('notes use their 50-item limit and stop fetching after cancellation', async () => { const controller = new AbortController() const waits: number[] = [] const urls: string[] = [] @@ -98,9 +106,7 @@ describe('Attio template', () => { const deps = fixtureDeps((url) => { urls.push(url.toString()); return Response.json({ data: page }) }) deps.wait = async (milliseconds, signal) => { waits.push(milliseconds); signal.throwIfAborted() } const iterator = readNotes(context(controller.signal), deps) - expect((await iterator.next()).value?.rows).toEqual([]) - const first = await iterator.next() - expect(first.value?.rows).toHaveLength(50) + expect((await iterator.next()).value?.rows).toHaveLength(50) controller.abort(new Error('fixture cancelled')) await expect(iterator.next()).rejects.toThrow('fixture cancelled') expect(urls).toHaveLength(1) @@ -108,7 +114,7 @@ describe('Attio template', () => { expect(waits).toEqual([125]) }) - test('GET pagination includes the terminating empty page for an exact multiple', async () => { + test('GET offset pagination requests the terminating empty page for an exact multiple', async () => { const offsets: string[] = [] const deps = fixtureDeps((url) => { const offset = url.searchParams.get('offset') ?? '' @@ -120,42 +126,66 @@ describe('Attio template', () => { expect(offsets).toEqual(['0', '2']) }) - test('malformed successful responses and missing identity fail rather than become empty data', async () => { + test('malformed successful responses and missing identities fail visibly', async () => { for (const payload of [{}, { data: null }, { data: [{ id: { task_id: 'task-1' } }] }]) { - const deps = fixtureDeps(() => Response.json(payload)) - await expect(collect(readTasks(context(), deps))).rejects.toThrow('Attio') + await expect(collect(readTasks(context(), fixtureDeps(() => Response.json(payload))))).rejects.toThrow('Attio') } - const malformedRecord = fixtureDeps((url) => Response.json({ data: url.pathname === '/v2/objects' - ? [peopleObject] : [{ ...person, values: { name: 'not-an-array' } }] })) - await expect(collect(readRecords(context(), malformedRecord))).rejects.toThrow('invalid values') + const malformed = fixtureDeps((url) => url.pathname === '/v2/objects/people' + ? fixtureResponse(url) : Response.json({ data: [{ ...person, values: { name: 'not-an-array' } }] })) + await expect(collect(readRecords(context(), 'people', malformed))).rejects.toThrow('invalid values') + await expect(collect(readRecords(context(), 'people', fixtureDeps(() => Response.json({ data: [peopleObject] }))))).rejects.toThrow('Attio') }) - test('permission failures are permanent, actionable, and produce no rows', async () => { - let calls = 0 - const deps = fixtureDeps(() => { calls += 1; return Response.json({ code: 'insufficient_scope' }, { status: 403 }) }) - const pipeline = fixturePipeline(deps, ['notes']) - const destination = createMemoryDestination() - const result = await runIngestion({ selected: selectStreams([pipeline], []), backfill: undefined }, { - journal: createMemoryJournal(), destination, + test.each(['workspace_id', 'object_id'])('object readers reject children with a different %s', async (field) => { + const deps = fixtureDeps((url) => url.pathname === '/v2/objects/people' + ? fixtureResponse(url) : Response.json({ data: [{ ...person, id: { ...person.id, [field]: 'foreign' } }] })) + await expect(collect(readRecords(context(), 'people', deps))).rejects.toThrow('different workspace or parent') + }) + + test('list readers resolve one list and reject entries belonging to another list', async () => { + const requests: string[] = [] + const deps = fixtureDeps((url) => { + requests.push(url.pathname) + return fixtureResponse(url) + }) + const chunks = await collect(readEntries(context(), 'sales', deps)) + expect(requests).toEqual(['/v2/lists/sales', '/v2/lists/list-sales/entries/query']) + expect(chunks[0]?.rows[0]?.raw).toMatchObject({ list_slug: 'sales' }) + const invalid = fixtureDeps((url) => url.pathname === '/v2/lists/sales' + ? fixtureResponse(url) : Response.json({ data: [{ id: { workspace_id: 'workspace-1', list_id: 'foreign', entry_id: 'one' }, entry_values: {} }] })) + await expect(collect(readEntries(context(), 'sales', invalid))).rejects.toThrow('different workspace or parent') + }) + + test('permission failures fail one object stream while another remains independent', async () => { + const requests: string[] = [] + const deps = fixtureDeps((url) => { + requests.push(url.pathname) + return url.pathname === '/v2/objects/object-custom/records/query' + ? Response.json({ code: 'insufficient_scope' }, { status: 403 }) : fixtureResponse(url) }) + const pipeline = fixturePipeline(deps) + const destination = createMemoryDestination() + const journal = createMemoryJournal() + const result = await runIngestion({ selected: selectStreams([pipeline], ['resource:records']), backfill: undefined }, { journal, destination }) expect(result.ok).toBe(false) - expect(result.streams[0]?.error).toContain('resource scopes') - expect(result.streams[0]?.rows).toBe(0) - expect(calls).toBe(1) - expect(destination.tables.size).toBe(0) + expect(result.streams.map((stream) => [stream.streamId, stream.outcome, stream.rows])).toEqual([ + ['attio.primary.records.people', 'succeeded', 1], ['attio.primary.records.subscriptions', 'failed', 0], + ]) + expect(result.streams[1]?.error).toContain('resource scopes') + expect(requests.filter((path) => path === '/v2/objects/object-custom/records/query')).toHaveLength(1) + expect(destination.tables.get('default.attio_records_raw')).toHaveLength(1) + expect((await journal.readCheckpoint('attio.primary.records.people')).lastSuccessSeq).toBeGreaterThan(0) + expect((await journal.readCheckpoint('attio.primary.records.subscriptions')).lastSuccessSeq).toBe(0) }) - test('429 requests retry under executor authority and preserve Retry-After', async () => { + test('429 retries remain under executor authority and retain Retry-After', async () => { let calls = 0 - const deps = fixtureDeps(() => { + const deps = fixtureDeps((url) => { calls += 1 - return calls === 1 - ? Response.json({ code: 'rate_limit_exceeded' }, { status: 429, headers: { 'Retry-After': '0' } }) - : Response.json({ data: fixtureCollections['/v2/notes'] }) + return calls === 1 ? Response.json({ code: 'rate_limit_exceeded' }, { status: 429, headers: { 'Retry-After': '0' } }) : fixtureResponse(url) }) - const pipeline = fixturePipeline(deps, ['notes']) const journal = createMemoryJournal() - const result = await runIngestion({ selected: selectStreams([pipeline], []), backfill: undefined }, { + const result = await runIngestion({ selected: selectStreams([fixturePipeline(deps)], ['resource:notes']), backfill: undefined }, { journal, destination: createMemoryDestination(), }) expect(result.ok).toBe(true) @@ -166,57 +196,22 @@ describe('Attio template', () => { expect(classifyAttioError(http, { kind: 'rate_limited', retryAfterMs: 2_000 })).toBeUndefined() }) - test('a missing token fails only when the reader runs', async () => { + test('a missing token fails only when a reader runs', async () => { const deps = fixtureDeps(() => { throw new Error('must not fetch') }) deps.token = () => undefined expect(attio.streams).toHaveLength(9) await expect(collect(readMembers(context(), deps))).rejects.toThrow('Set ATTIO_API_TOKEN') }) - test('a fresh runtime resumes the frozen parent set after the completed parent', async () => { - const requests: string[] = [] - const deps = fixtureDeps((url) => { - requests.push(url.pathname) - const data = fixtureCollections[url.pathname] - if (data === undefined) throw new Error(`Unexpected fixture URL ${url.pathname}`) - return Response.json({ data }) - }) - const pipeline = fixturePipeline(deps, ['records']) - const firstPipeline = definePipeline({ ...pipeline, streams: pipeline.streams.map((stream) => ({ ...stream, budget: { maxChunks: 3 } })) }) - const journal = createMemoryJournal() - const destination = createMemoryDestination() - const first = await runIngestion({ selected: selectStreams([firstPipeline], []), backfill: undefined }, { journal, destination }) - expect(first.streams[0]?.outcome).toBe('budget_exhausted') - expect((await journal.readCheckpoint('attio.primary.records')).envelope?.state).toMatchObject({ - phase: 'scanning', cycle: 1, completedParents: ['object-people'], - parents: [{ id: 'object-people', slug: 'people' }, { id: 'object-custom', slug: 'subscriptions' }], - }) - - requests.length = 0 - const restored = await restoredJournal(journal) - const resumed = await runIngestion({ selected: selectStreams([fixturePipeline(deps, ['records'])], []), backfill: undefined }, { - journal: restored, destination, - }) - expect(resumed.ok).toBe(true) - expect(requests).toEqual(['/v2/objects/object-custom/records/query']) - expect((await restored.readCheckpoint('attio.primary.records')).envelope?.state).toMatchObject({ phase: 'complete', cycle: 1 }) - expect(destination.tables.get('default.attio_records_raw')).toHaveLength(2) - - requests.length = 0 - const next = await runIngestion({ selected: selectStreams([fixturePipeline(deps, ['records'])], []), backfill: undefined }, { - journal: restored, destination, - }) - expect(next.ok).toBe(true) - expect(requests[0]).toBe('/v2/objects') - expect((await restored.readCheckpoint('attio.primary.records')).envelope?.state).toMatchObject({ phase: 'complete', cycle: 2 }) - }) - - test('an interrupted mutable parent restarts offset zero instead of bookmarking an offset', async () => { + test('an interrupted object restarts offset zero without requesting other objects', async () => { let fail = true + const requests: string[] = [] const offsets: number[] = [] const page = Array.from({ length: 500 }, (_, index) => ({ ...person, id: { ...person.id, record_id: `record-${index}` } })) const deps = fixtureDeps((url, init) => { - if (url.pathname === '/v2/objects') return Response.json({ data: [peopleObject] }) + requests.push(url.pathname) + if (url.pathname === '/v2/objects/people') return fixtureResponse(url) + if (url.pathname !== '/v2/objects/object-people/records/query') throw new Error(`Unexpected object request ${url.pathname}`) const { offset } = JSON.parse(String(init.body)) offsets.push(offset) if (offset === 0) return Response.json({ data: page }) @@ -224,36 +219,38 @@ describe('Attio template', () => { }) const journal = createMemoryJournal() const destination = createMemoryDestination() - const first = await runIngestion({ selected: selectStreams([fixturePipeline(deps, ['records'])], []), backfill: undefined }, { journal, destination }) + const first = await runIngestion({ selected: selectStreams([fixturePipeline(deps)], ['resource:records', 'object:people']), backfill: undefined }, { journal, destination }) expect(first.ok).toBe(false) - expect((await journal.readCheckpoint('attio.primary.records')).envelope?.state).toMatchObject({ phase: 'scanning', completedParents: [] }) + expect((await journal.readCheckpoint('attio.primary.records.people')).envelope).toBeUndefined() fail = false const restored = await restoredJournal(journal) - const next = await runIngestion({ selected: selectStreams([fixturePipeline(deps, ['records'])], []), backfill: undefined }, { - journal: restored, destination, - }) - expect(next.ok).toBe(true) + const resumed = await runIngestion({ selected: selectStreams([fixturePipeline(deps)], ['resource:records', 'object:people']), backfill: undefined }, { journal: restored, destination }) + expect(resumed.ok).toBe(true) expect(offsets).toEqual([0, 500, 0, 500]) - expect((await restored.readCheckpoint('attio.primary.records')).envelope?.state).toMatchObject({ phase: 'complete', cycle: 1 }) + expect(requests).not.toContain('/v2/objects') expect(destination.tables.get('default.attio_records_raw')).toHaveLength(500) }) - test('empty collections still checkpoint completion and start a new cycle on the next run', async () => { - const deps = fixtureDeps(() => Response.json({ data: [] })) + test('reruns publish changed observations with the same stable raw ID', async () => { + let changed = false + const deps = fixtureDeps((url) => url.pathname === '/v2/objects/object-people/records/query' + ? Response.json({ data: [{ ...person, values: { ...person.values, name: [{ full_name: changed ? 'Ada Updated' : 'Ada Example' }] } }] }) : fixtureResponse(url)) const journal = createMemoryJournal() const destination = createMemoryDestination() - for (const cycle of [1, 2]) { - const result = await runIngestion({ selected: selectStreams([fixturePipeline(deps, ['records', 'notes'])], []), backfill: undefined }, { journal, destination }) - expect(result.ok).toBe(true) - for (const resource of ['records', 'notes']) { - expect((await journal.readCheckpoint(`attio.primary.${resource}`)).envelope?.state).toMatchObject({ phase: 'complete', cycle }) - } - } - expect(destination.tables.size).toBe(0) + const request = { selected: selectStreams([fixturePipeline(deps)], ['resource:records', 'object:people']), backfill: undefined } + expect((await runIngestion(request, { journal, destination })).ok).toBe(true) + changed = true + expect((await runIngestion(request, { journal, destination })).ok).toBe(true) + const rows = destination.tables.get('default.attio_records_raw') ?? [] + expect(rows).toHaveLength(2) + expect(rows[0]?.id).toBe(rows[1]?.id) + expect(rows[1]?.raw).toMatchObject({ data: { values: { name: [{ full_name: 'Ada Updated' }] } } }) }) - test('rows accepted before a lost journal write cannot mark their parent complete', async () => { - const deps = fixtureDeps() + test('a fresh runtime after a lost journal acknowledgement replays changed rows safely', async () => { + let changed = false + const deps = fixtureDeps((url) => url.pathname === '/v2/objects/object-people/records/query' + ? Response.json({ data: [{ ...person, values: { ...person.values, revision: [{ value: changed ? 'new' : 'old' }] } }] }) : fixtureResponse(url)) const journal = createMemoryJournal() const destination = createMemoryDestination() const failingJournal = { @@ -265,90 +262,32 @@ describe('Attio template', () => { await journal.append(events) }, } - const failed = await runIngestion({ selected: selectStreams([fixturePipeline(deps, ['records'])], []), backfill: undefined }, { + const failed = await runIngestion({ selected: selectStreams([fixturePipeline(deps)], ['resource:records', 'object:people']), backfill: undefined }, { journal: failingJournal, destination, }) expect(failed.ok).toBe(false) - expect((await journal.readCheckpoint('attio.primary.records')).envelope?.state).toMatchObject({ phase: 'scanning', completedParents: [] }) expect(destination.tables.get('default.attio_records_raw')).toHaveLength(1) - const restored = await restoredJournal(journal) - const resumed = await runIngestion({ selected: selectStreams([fixturePipeline(deps, ['records'])], []), backfill: undefined }, { - journal: restored, destination, + changed = true + const resumed = await runIngestion({ selected: selectStreams([fixturePipeline(deps)], ['resource:records', 'object:people']), backfill: undefined }, { + journal: await restoredJournal(journal), destination, }) expect(resumed.ok).toBe(true) - expect(destination.tables.get('default.attio_records_raw')).toHaveLength(2) - expect((await restored.readCheckpoint('attio.primary.records')).envelope?.state).toMatchObject({ phase: 'complete' }) + const rows = destination.tables.get('default.attio_records_raw') ?? [] + expect(rows).toHaveLength(2) + expect(rows[0]?.id).toBe(rows[1]?.id) + expect(rows[1]?.raw).toMatchObject({ data: { values: { revision: [{ value: 'new' }] } } }) }, 10_000) - - test('workspace note scans restart offset zero after an execution budget stops a page', async () => { - const offsets: string[] = [] - const notes = Array.from({ length: 50 }, (_, index) => ({ id: { workspace_id: 'workspace-1', note_id: `note-${index}` } })) - const deps = fixtureDeps((url) => { - const offset = url.searchParams.get('offset') ?? '' - offsets.push(offset) - return Response.json({ data: offset === '0' ? notes : [] }) - }) - const pipeline = fixturePipeline(deps, ['notes']) - const bounded = definePipeline({ ...pipeline, streams: pipeline.streams.map((stream) => ({ ...stream, budget: { maxChunks: 2 } })) }) - const journal = createMemoryJournal() - const destination = createMemoryDestination() - const first = await runIngestion({ selected: selectStreams([bounded], []), backfill: undefined }, { journal, destination }) - expect(first.streams[0]?.outcome).toBe('budget_exhausted') - expect((await journal.readCheckpoint('attio.primary.notes')).envelope?.state).toMatchObject({ phase: 'scanning' }) - const restored = await restoredJournal(journal) - const resumed = await runIngestion({ selected: selectStreams([fixturePipeline(deps, ['notes'])], []), backfill: undefined }, { - journal: restored, destination, - }) - expect(resumed.ok).toBe(true) - expect(offsets).toEqual(['0', '0', '50']) - expect((await restored.readCheckpoint('attio.primary.notes')).envelope?.state).toMatchObject({ phase: 'complete', cycle: 1 }) - }) - - test('changed scope and malformed saved frontiers fail before another source request', async () => { - const strategy = scanCheckpoint('records') - const initial = { scope: 'changed-selection', cycle: 1, phase: 'scanning', parents: [], completedParents: [] } - for (const invalid of [{ ...initial, cycle: -1 }, { ...initial, completedParents: ['unknown'] }, { ...initial, parents: [{}] }]) { - expect(() => strategy.parseState(invalid)).toThrow('Invalid Attio scan checkpoint') - } - let requests = 0 - const deps = fixtureDeps(() => { requests += 1; return Response.json({ data: [] }) }) - await expect(collect(readRecords({ ...context(), state: strategy.parseState(initial) }, deps))).rejects.toThrow('checkpoint scope changed') - expect(requests).toBe(0) - }) - - test.each(['workspace_id', 'object_id'])('resumed parents reject a response with a different %s', async (field) => { - const pipeline = fixturePipeline(fixtureDeps(), ['records']) - const bounded = definePipeline({ ...pipeline, streams: pipeline.streams.map((stream) => ({ ...stream, budget: { maxChunks: 1 } })) }) - const journal = createMemoryJournal() - const destination = createMemoryDestination() - const first = await runIngestion({ selected: selectStreams([bounded], []), backfill: undefined }, { journal, destination }) - expect(first.streams[0]?.outcome).toBe('budget_exhausted') - const foreign = fixtureDeps(() => Response.json({ data: [{ ...person, id: { ...person.id, [field]: 'another-account-or-parent' } }] })) - const restored = await restoredJournal(journal) - const result = await runIngestion({ selected: selectStreams([fixturePipeline(foreign, ['records'])], []), backfill: undefined }, { - journal: restored, destination, - }) - expect(result.ok).toBe(false) - expect(result.streams[0]?.error).toContain('different workspace or parent') - expect(destination.tables.size).toBe(0) - expect((await restored.readCheckpoint('attio.primary.records')).envelope?.state).toMatchObject({ phase: 'scanning', completedParents: [] }) - }) }) -function fixturePipeline(deps: AttioClientDeps, selectedResources?: readonly string[]) { +function fixturePipeline(deps: AttioClientDeps) { return definePipeline({ - ...attio, + ...createAttioPipeline({ ...attioConfig, objects: ['people', 'subscriptions'], lists: ['sales'] }, deps), retry: { retries: 1, minTimeout: 0, maxTimeout: 0, randomize: false }, - streams: attio.streams.filter((stream) => selectedResources === undefined || selectedResources.includes(stream.id.split('.').at(-1) ?? '')).map((stream) => { - const read = readers[stream.id.split('.').at(-1) ?? ''] - if (!read) throw new Error(`No fixture reader for ${stream.id}`) - return { ...stream, read: (context: AttioReadContext) => read(context, deps) } - }), }) } -function context(signal = new AbortController().signal): AttioReadContext { - return { signal, attempt: async (operation) => operation(signal), streamId: 'fixture', state: undefined, selection: undefined, cutoff: new Date('2026-01-01T00:00:00Z') } +function context(signal = new AbortController().signal): FetchContext { + return { signal, attempt: async (operation) => operation(signal) } } async function collect(items: AsyncIterable): Promise { diff --git a/registry/attio/tests/fixtures.ts b/registry/attio/tests/fixtures.ts index 984ee044..4d4819f1 100644 --- a/registry/attio/tests/fixtures.ts +++ b/registry/attio/tests/fixtures.ts @@ -1,8 +1,21 @@ import type { AttioClientDeps } from '../client.js' +import { attioConfig } from '../config.js' export const peopleObject = { id: { workspace_id: 'workspace-1', object_id: 'object-people' }, api_slug: 'people' } +export const companiesObject = { id: { workspace_id: 'workspace-1', object_id: 'object-companies' }, api_slug: 'companies' } export const customObject = { id: { workspace_id: 'workspace-1', object_id: 'object-custom' }, api_slug: 'subscriptions' } -const salesList = { id: { workspace_id: 'workspace-1', list_id: 'list-sales' }, api_slug: 'sales' } +export const salesList = { id: { workspace_id: 'workspace-1', list_id: 'list-sales' }, api_slug: 'sales' } + +const fixtureParents: Record = { + '/v2/objects/people': peopleObject, + '/v2/objects/object-people': peopleObject, + '/v2/objects/companies': companiesObject, + '/v2/objects/object-companies': companiesObject, + '/v2/objects/subscriptions': customObject, + '/v2/objects/object-custom': customObject, + '/v2/lists/sales': salesList, + '/v2/lists/list-sales': salesList, +} export const person = { id: { workspace_id: 'workspace-1', object_id: 'object-people', record_id: 'record-1' }, @@ -19,8 +32,10 @@ export const fixtureCollections: Record = { '/v2/objects': [peopleObject, customObject], '/v2/objects/object-people/records/query': [person], '/v2/objects/object-custom/records/query': [{ id: { ...person.id, object_id: 'object-custom' }, values: { custom_number: [{ value: 7 }] } }], + '/v2/objects/object-companies/records/query': [], '/v2/objects/object-people/attributes': [{ id: { ...peopleObject.id, attribute_id: 'attribute-1' }, api_slug: 'name', type: 'personal-name' }], '/v2/objects/object-custom/attributes': [{ id: { ...customObject.id, attribute_id: 'attribute-1' }, api_slug: 'custom_number', type: 'number' }], + '/v2/objects/object-companies/attributes': [], '/v2/lists': [salesList], '/v2/lists/list-sales/attributes': [{ id: { workspace_id: 'workspace-1', object_id: 'list-sales', attribute_id: 'attribute-1' }, api_slug: 'stage', type: 'status' }], '/v2/lists/list-sales/entries/query': [{ id: { ...salesList.id, entry_id: 'entry-1' }, parent_object: 'people', parent_record_id: 'record-1', entry_values: { stage: [{ status: { title: 'Prospect' } }] } }], @@ -30,15 +45,18 @@ export const fixtureCollections: Record = { } export function fixtureDeps( - respond: (url: URL, init: RequestInit) => Response | Promise = (url) => { - const data = fixtureCollections[url.pathname] - if (data === undefined) throw new Error(`Unexpected fixture URL: ${url.pathname}`) - return Response.json({ data }) - }, + respond: (url: URL, init: RequestInit) => Response | Promise = fixtureResponse, ): AttioClientDeps { return { + config: attioConfig, token: () => 'fixture-token', wait: async () => {}, fetch: (url, init) => Promise.resolve(respond(new URL(url), init)), } } + +export function fixtureResponse(url: URL): Response { + const data = fixtureParents[url.pathname] ?? fixtureCollections[url.pathname] + if (data === undefined) throw new Error(`Unexpected fixture URL: ${url.pathname}`) + return Response.json({ data }) +} diff --git a/registry/attio/tests/install.e2e.test.ts b/registry/attio/tests/install.e2e.test.ts index 6749b9d5..35082b74 100644 --- a/registry/attio/tests/install.e2e.test.ts +++ b/registry/attio/tests/install.e2e.test.ts @@ -70,11 +70,16 @@ describe.serial('built Attio template installed into a consumer', () => { expect(await Bun.file(join(project, 'src/integrations/attio/tests/install.e2e.test.ts')).exists()).toBe(false) const configPath = join(project, 'src/integrations/attio/config.ts') const config = await readFile(configPath, 'utf8') - await writeFile(configPath, config.replace("database: 'default'", `database: ${JSON.stringify(env.clickhouseDatabase)}`).replace("tablePrefix: 'attio'", `tablePrefix: '${prefix}'`)) + await writeFile(configPath, config.replace("database: 'default'", `database: ${JSON.stringify(env.clickhouseDatabase)}`) + .replace("tablePrefix: 'attio'", `tablePrefix: '${prefix}'`) + .replace("objects: ['people', 'companies']", "objects: ['people', 'subscriptions']")) const list = cli(['ingest', 'list', '--tag', 'provider:attio', '--json']) expect(list.exitCode, formatTestDiagnostic('list without token', list)).toBe(0) expect(JSON.parse(list.stdout).streams).toHaveLength(9) + const peopleOnly = cli(['ingest', 'list', '--tag', 'provider:attio', '--tag', 'resource:records', '--tag', 'object:people', '--json']) + expect(peopleOnly.exitCode, formatTestDiagnostic('one object stream', peopleOnly)).toBe(0) + expect(JSON.parse(peopleOnly.stdout).streams.map((stream: { streamId: string }) => stream.streamId)).toEqual(['attio.primary.records.people']) const generate = cli(['generate', '--name', 'attio_fixture', '--json']) expect(generate.exitCode, formatTestDiagnostic('generate', generate)).toBe(0) const migrate = cli(['migrate', '--apply', '--json']) @@ -83,16 +88,18 @@ describe.serial('built Attio template installed into a consumer', () => { await waitForView(executor, env.clickhouseDatabase, `${prefix}_people`) const installed: typeof import('../index.js') = await import(pathToFileURL(join(project, 'src/integrations/attio/index.ts')).href) - const readers: typeof import('../sources/records.js') = await import(pathToFileURL(join(project, 'src/integrations/attio/sources/records.ts')).href) - const recordStream = installed.attio.streams.find((stream) => stream.id.endsWith('.records')) - expect(recordStream).toBeDefined() - if (!recordStream) throw new Error('Installed records stream is missing') + const factory: typeof import('../pipeline.js') = await import(pathToFileURL(join(project, 'src/integrations/attio/pipeline.ts')).href) + const installedConfig: typeof import('../config.js') = await import(pathToFileURL(join(project, 'src/integrations/attio/config.ts')).href) + expect(installed.attio.streams.filter((stream) => stream.tags.includes('resource:records')).map((stream) => stream.id)) + .toEqual(['attio.primary.records.people', 'attio.primary.records.subscriptions']) let revision = 1 const deps = fixtureDeps((url) => { - if (url.pathname === '/v2/objects') return Response.json({ data: [ + if (url.pathname === '/v2/objects/people') return Response.json({ data: { id: { workspace_id: 'workspace-1', object_id: 'people' }, api_slug: 'people' }, + }) + if (url.pathname === '/v2/objects/subscriptions') return Response.json({ data: { id: { workspace_id: 'workspace-1', object_id: 'custom' }, api_slug: 'subscriptions' }, - ] }) + }) if (url.pathname === '/v2/objects/people/records/query') return Response.json({ data: [ { ...person, id: { ...person.id, object_id: 'people' }, values: { ...person.values, name: [{ full_name: revision === 1 ? 'Ada Example' : 'Ada Updated' }], email_addresses: [{ email_address: 'one@example.test' }, { email_address: 'two@example.test' }] } }, { id: { ...person.id, object_id: 'people', record_id: 'empty-attributes' }, values: {} }, @@ -100,7 +107,8 @@ describe.serial('built Attio template installed into a consumer', () => { if (url.pathname === '/v2/objects/custom/records/query') return Response.json({ data: [{ id: { workspace_id: 'workspace-1', object_id: 'custom', record_id: 'custom-1' }, values: { unmodeled: [{ value: 'retained' }] } }] }) throw new Error(`Unexpected provider fixture request: ${url.pathname}`) }) - const pipeline = definePipeline({ id: 'fixture', streams: [{ ...recordStream, read: (context) => readers.readRecords(context, deps) }], retry: { retries: 0 } }) + const configured = factory.createAttioPipeline(installedConfig.attioConfig, deps) + const pipeline = definePipeline({ id: 'fixture', streams: configured.streams.filter((stream) => stream.tags.includes('resource:records')), retry: { retries: 0 } }) const journal = createMemoryJournal() for (const next of [1, 2]) { revision = next diff --git a/skills/chkit-ingestion/SKILL.md b/skills/chkit-ingestion/SKILL.md index a79533f5..cdf4775c 100644 --- a/skills/chkit-ingestion/SKILL.md +++ b/skills/chkit-ingestion/SKILL.md @@ -13,7 +13,9 @@ Establish auth, response shape, pagination, record IDs, change filters/cursors, Treat stored shape and mapping together: prefer raw objects plus a SQL view when the shape may evolve and storage is affordable; map inside `read` for a known schema. Model warehouse entities separately; embed bounded child collections when consumers need complete documents. A unified document table is a specialized retrieval model. -Choose the strongest appropriate sync mechanism the provider actually supports: a change feed or sync token, reliable modification windows, or checkpointed full reconciliation. A simple full read remains appropriate for small bounded catalogs or explicitly requested prototypes. Use the default loader unless another publication contract is needed. Explain consequential choices and coverage limits. +Choose the strongest appropriate sync mechanism the provider actually supports: a change feed or sync token, reliable modification windows, or full reads with safe replay. A simple full read is appropriate when the API lacks reliable change selection and its cost is acceptable, as well as for small catalogs or explicitly requested prototypes. Use the default loader unless another publication contract is needed. Explain consequential choices and coverage limits. + +Choose stream boundaries that match the entities users sync independently. Several streams can write to one raw table; table grouping does not require one reader to traverse every entity type. Prefer existing `paginate()` and incremental strategies. Add custom state only when a provider's recovery contract needs progress beyond the executor's journal; ordinary `fullSync()` already records successful work, including empty reads. Put repeated generic mechanics in the library when the existing API cannot express them concisely. ## Registry quality bar @@ -21,7 +23,7 @@ Before creating or materially changing an official registry integration, read [S For each resource, establish its provider contract and choose a safe completion boundary before coding. Keep provider payloads raw; application-specific projections are optional and separate. Compare existing registry providers and established implementations such as Brain when accessible, then verify the API contract rather than copying their assumptions. Private repository access is not a dependency of the installed skill. -The release gate is evidence that interruptions resume or safely replay, progress follows destination acknowledgement, empty reads commit completion, and late changes have a reconciliation plan. Full scans are valid when justified by the API; completion checkpoints do not make them change feeds. Match manifest strategy labels to actual behavior, document limitations and identity/state migrations, and add portable failure/restart fixtures. +The release gate is evidence that interruptions resume or safely replay, progress follows destination acknowledgement, empty reads commit completion, and late changes have a reconciliation plan. Full scans are valid when justified by the API; completion checkpoints do not make them change feeds. Reliable full-sync replay does not require a provider-owned scan state machine. Match manifest strategy labels to actual behavior, document limitations and identity/state migrations, and add portable failure/restart fixtures. ## Read only the relevant docs @@ -40,7 +42,7 @@ Use local `apps/docs/src/content/docs/api-sync/` or these URLs. Match the instal - TypeScript and direct ClickHouse config are required. Preserve existing schema discovery: add the provider entry to `schema` paths, or re-export its schema and active pipelines from the project `entry`. Migrations create destinations. - Stream IDs own checkpoints: keep them stable and account-scoped. Pipelines group streams without dependency ordering. - Use `context.attempt`/`paginate`, forward cancellation, and throw `HttpError.fromResponse` for HTTP failures. Yield bounded chunks. -- Support or explicitly reject historical bounds. When supported, enforce both bounds and freeze unfinished selections across restarts. Persist complete validated resume state; the executor commits it after writes. +- Support historical bounds or state explicitly that a full-sync reader ignores them and does not provide date-range backfills. When supported, enforce both bounds and freeze unfinished selections across restarts. Persist complete validated resume state; the executor commits it after writes. - Separate the completed watermark from unfinished traversal state. Newest-first pages cannot alone advance a completed timestamp watermark; documented fixed-window timestamp frontiers or scoped page positions can still resume unfinished reads. Preserve timestamp precision and equal-timestamp coverage. - Choose overlap from indexing/arrival behavior. For children that appear or change without updating their parent, retain bounded pending parents or reconcile old roots; a recent overlap alone is insufficient. - Custom tables need `ingestionColumns`. Prefer `simpleLoader`; custom loaders must preserve write identity and acknowledge publication before returning receipts. diff --git a/skills/chkit-ingestion/references/stable-sync.md b/skills/chkit-ingestion/references/stable-sync.md index 10ccdef5..654d1c81 100644 --- a/skills/chkit-ingestion/references/stable-sync.md +++ b/skills/chkit-ingestion/references/stable-sync.md @@ -13,24 +13,26 @@ Choose the strongest appropriate mechanism, independently for each resource: | Change feed or sync token | Bootstrap completely, retain the input token throughout pagination, and promote a terminal replacement token only after all corresponding writes | | Reliable modification filter | Fixed upper cutoff and overlapping lower bound; checkpoint a fully consumed window or independently complete bounded interval | | Creation-time filter | Use for proven immutable events or parent discovery; distinguish discovery progress from freshness, and retain delayed-child work/reconcile later edits separately | -| Full mutable listing | Record scan completion; for expensive reads, freeze parent selection and checkpoint completed parents, replaying unfinished mutable collections from their beginning | +| Full mutable listing | Use `fullSync()` with ordinary pagination and journaled execution completion; replay an interrupted stream from the beginning when offsets are mutable. Add finer progress only when the collection's cost and provider contract justify it | -A small catalog can remain a simple full read if its complete cost is acceptable. For a registry resource, explain that choice. A creation timestamp is not a modification cursor; a pagination cursor is not automatically a change token. Runtime `cursorState()` can implement a full scan, so the helper alone does not determine the manifest's strategy label. +A full read is appropriate when the API lacks a reliable change mechanism and its complete cost is acceptable. For a registry resource, explain that choice. The executor records successful work, including empty `fullSync()` executions; do not add cycle counters or state-only markers merely to duplicate that completion. A creation timestamp is not a modification cursor; a pagination cursor is not automatically a change token. Runtime `cursorState()` can implement a full scan, so the helper alone does not determine the manifest's strategy label. -In a chkit checkout, inspect relevant `registry//` implementations and tests. Calendar demonstrates terminal sync-token promotion; Slack fixed timestamp frontiers and retained threads; Meet late artifacts and nested cursors; Attio/Circleback completed-parent scans; GitHub/Linear complete discussion windows; Lemlist completed intervals. Reuse the invariant, not provider-specific dates, budgets, or account assumptions. Consult Brain's provider implementations when available while keeping its application projections out of raw registry schemas. +Choose meaningful stream boundaries before designing custom recovery. Attio demonstrates independent People/Companies readers sharing a raw records table and using ordinary full-sync pagination. This is appropriate for independently selected entity types; it is not a universal requirement to split every parent record into a stream. Pipelines are built from configured selections, and stream order is not a discovery dependency. Prefer the existing pagination and checkpoint primitives; extract genuinely repeated mechanics into the library rather than copying a generic state machine into each provider. + +In a chkit checkout, inspect relevant `registry//` implementations and tests. Calendar demonstrates terminal sync-token promotion; Slack fixed timestamp frontiers and retained threads; Meet late artifacts and nested cursors; Circleback completed-parent scans; GitHub/Linear complete discussion windows; Lemlist completed intervals. Reuse the invariant, not provider-specific dates, budgets, or account assumptions. Consult Brain's provider implementations when available while keeping its application projections out of raw registry schemas. ## Keep progress recoverable - Bind stream IDs and saved state to stable non-secret source, account, and query identities. Check authenticated identity when the provider exposes it; otherwise require new stream/destination identities when switching accounts. Credentials themselves must not become checkpoint scope. Validate saved types, bounds, parent positions, and scope before reuse. Version custom strategies and document migrations. - Separate completed coverage from unfinished work. Persist fixed window bounds and enough scoped page/parent/child position to resume them; recomputing a cutoff must not move an unfinished selection. Preserve exact provider timestamp precision and handle equal timestamps without skipping records. -- Yield fresh, complete candidate state. The executor owns its durable commit after destination evidence. Never advance a watermark past rows or required work that is neither loaded nor durably retained. Empty terminal pages and successful empty scans must also checkpoint completion. Use state-only markers or batching that flushes safe boundaries. -- Persist offsets or opaque page tokens only under a documented resumption contract. Expiring page tokens can be saved while valid if state also retains the original input and a safe recovery path. For mutable offset lists, restart the unfinished parent/interval at zero; skip acknowledged parents only within the saved scan cycle. Frozen selection means unchanged predicates and retained discovered parent IDs, not a frozen source dataset. Coverage still depends on provider ordering, snapshots, or eventual stability plus reconciliation. A completed cycle resets parent progress and discovers current parents again. +- When custom progress is necessary, yield fresh, complete candidate state. The executor owns its durable commit after destination evidence. Never advance a watermark past rows or required work that is neither loaded nor durably retained. Empty terminal pages must also commit required token/watermark progress; use state-only markers or batching that flushes those safe boundaries. Ordinary empty full-sync success needs no custom marker. +- Persist offsets or opaque page tokens only under a documented resumption contract. Expiring page tokens can be saved while valid if state also retains the original input and a safe recovery path. For mutable offset lists, restart the interrupted stream's collection at zero. If justified custom state tracks multiple work units within a stream, skip acknowledged units only within the saved selection and restart its unfinished mutable unit. Frozen selection means unchanged predicates and retained discovered parent IDs, not a frozen source dataset. Coverage still depends on provider ordering, snapshots, or eventual stability plus reconciliation. A completed cycle resets any such progress and selects current work again. - On an expired token, replay the same scoped collection from its committed input or rebuild a baseline as the provider requires. Bound protocol resets across the unfinished selection, not just one process: retain reset debt or detect repeated recovery without useful durable progress. Repeated rejection/stalled recovery fails visibly. Keep existing raw observations during baseline recovery. Absence from a rebuilt baseline is not automatically deletion. -- Treat explicit backfills as separate checkpoint namespaces. Honor both bounds or reject unsupported bounds explicitly. Retain the original unfinished bounds, reject incompatible reuse of a backfill ID, and prevent interval watermarks from regressing. A bundled helper ignoring `range` does not establish a source's historical-range behavior. +- Treat explicit backfills as separate checkpoint namespaces. Honor both bounds for historical selection. Reject unsupported bounds when the reader can enforce that contract, or document that a simple full-sync reader ignores bounds and provides no date-range backfill. Retain the original unfinished bounds for supported backfills, reject incompatible reuse of a backfill ID, and prevent interval watermarks from regressing. A bundled helper ignoring `range` does not establish historical-range coverage. ## Preserve raw data and delayed children -Store complete returned objects as raw JSON with stable source/parent/provider keys; preserve sparse cancellation payloads. Keep application projections separate. For large child collections, prefer independently paged raw child rows over unbounded nested buffering. Each stream must discover its own parents; stream order is not a dependency contract. +Store complete returned objects as raw JSON with stable source/parent/provider keys; preserve sparse cancellation payloads. Keep application projections separate. For large child collections, prefer independently paged raw child rows over unbounded nested buffering. A stream that needs a parent must resolve it independently from its configured identity or its own discovery; stream order is not a dependency contract. A child failure must not publish an incomplete embedded document as complete. Either finish the required children first, or emit raw parent/child observations with durable pending work and explicit completion semantics. Retain relevant parents for delayed recordings/transcripts, and periodically revisit old roots when edits or replies do not update the discovery timestamp. Choose overlap and reconciliation cadence from provider retention, delay, volume, and freshness requirements. From 6d8711158c9dd305e73a8104bc9febd89cfcc7e0 Mon Sep 17 00:00:00 2001 From: "replicas-connector[bot]" Date: Sun, 4 Oct 2026 19:53:27 +0000 Subject: [PATCH 03/18] Fix unused Attio fixture exports flagged by fallow MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Keep Companies and Sales fixture constants private and publish the corrected bundle as Attio 0.2.1, preserving the immutable 0.2.0 artifact. Co-Authored-By: Marc Höffl Co-Authored-By: replicas-connector[bot] --- registry/attio/manifest.json | 2 +- registry/attio/releases/0.2.1.json | 382 +++++++++++++++++++++++++++++ registry/attio/tests/fixtures.ts | 4 +- 3 files changed, 385 insertions(+), 3 deletions(-) create mode 100644 registry/attio/releases/0.2.1.json diff --git a/registry/attio/manifest.json b/registry/attio/manifest.json index 5c72271f..4e57b60a 100644 --- a/registry/attio/manifest.json +++ b/registry/attio/manifest.json @@ -95,7 +95,7 @@ "meta": { "chkit": { "formatVersion": 1, - "version": "0.2.0", + "version": "0.2.1", "documentation": "https://chkit.obsessiondb.com/integrations/attio/", "logo": "https://chkit.obsessiondb.com/logos/attio.svg", "language": "typescript", diff --git a/registry/attio/releases/0.2.1.json b/registry/attio/releases/0.2.1.json new file mode 100644 index 00000000..40f1f3a3 --- /dev/null +++ b/registry/attio/releases/0.2.1.json @@ -0,0 +1,382 @@ +{ + "$schema": "https://ui.shadcn.com/schema/registry-item.json", + "name": "attio", + "type": "registry:item", + "title": "Attio", + "description": "Editable Attio CRM schemas, raw data readers, typed views, and independent streams for People, Companies, configured custom types, and workspace resources.", + "dependencies": [ + "@chkit/core@^0.2.0-beta.10", + "@chkit/plugin-ingest@^0.2.0-beta.10" + ], + "devDependencies": [ + "@types/bun@^1.2.0" + ], + "files": [ + { + "path": "attio/README.md", + "type": "registry:file", + "target": "src/integrations/attio/README.md", + "content": "# Attio CRM ingestion\n\nThis is your source code: edit the API readers, schemas, views, and `pipeline.ts` directly. It uses the [chkit CLI](https://www.npmjs.com/package/chkit), `@chkit/core`, and `@chkit/plugin-ingest`; there is no runtime connection to the template registry.\n\n## First run\n\nSet `ATTIO_API_TOKEN` in your execution environment and configure the project's direct ClickHouse connection. A single-workspace API token works; an OAuth access token also works, but this template does not implement OAuth issuance or refresh. See [Attio authentication](https://docs.attio.com/rest-api/guides/authentication).\n\n### Create an Attio API key\n\n1. As a workspace admin, open the workspace-name dropdown in Attio and select **Workspace settings**.\n2. Open **Developers**, select **+ New access token**, and name the token, for example **chkit ClickHouse**.\n3. Grant the read scopes in the resource table below for the streams to run.\n4. Click the token on the Developers page to copy it. Set `ATTIO_API_TOKEN` in the project's `.env` file or the scheduler's secret environment.\n\nSee Attio's [API key instructions](https://attio.com/help/reference/apps/generating-an-api-key). Use a single-workspace API token for this setup; OAuth token creation and refresh require a separate application flow.\n\nBun loads a project `.env` automatically. With Node or a scheduler, explicitly supply environment variables using that runtime's environment-loading mechanism. `.env.example` only lists required variable names; it does not load values. Schema imports, `check`, and `ingest list` do not read the token or make API requests.\n\nReview `config.ts` before the first migration. The default target database is `default`; raw table and view names begin with `attio_`. Use a stable, non-secret `sourceId` for this source instance. For another installation, give it a distinct source ID and table prefix or database before exporting its pipeline. Changing these after ingestion changes stream/storage identities and requires a deliberate migration.\n\n```sh\nbunx chkit check\nbunx chkit ingest list --tag provider:attio\nbunx chkit generate --name add_attio\n# Review the generated SQL.\nbunx chkit migrate --apply\nbunx chkit ingest run --tag provider:attio\n```\n\nThe schema uses native JSON (ClickHouse 25.3 or newer). A direct ClickHouse connection is required; workbench-only execution is not supported. Use an external scheduler to repeat the run command, with one ingestion process per project/target at a time.\n\n## Included resources\n\nEach configured object type has its own records and object-attributes streams. The defaults create separate People and Companies readers; tasks, notes, members, and the object/list catalogs have workspace-wide streams. Each configured list similarly gets independent entries and list-attributes streams. The number of streams depends on `objects` and `lists`; the default configuration has nine streams writing to seven resource tables. All nine raw tables are defined, including entries and list attributes, whose readers are enabled by configuring `lists`.\n\nEvery stream uses `fullSync()` and the existing pagination helper. A records reader fetches one configured object type, such as People, rather than looping over all object types. An interrupted stream restarts its collection at offset zero because Attio's mutable offsets are not durable cursors. Each stream has its own execution journal; no parent-completion ledger or custom scan state is needed.\n\n| Stream tag | Raw table | Required Attio scopes |\n| --- | --- | --- |\n| `resource:objects` | `attio_objects_raw` | `object_configuration:read` |\n| `resource:object_attributes` | `attio_object_attributes_raw` | `object_configuration:read` |\n| `resource:records` | `attio_records_raw` | `object_configuration:read`, `record_permission:read` |\n| `resource:lists` | `attio_lists_raw` | `list_configuration:read` |\n| `resource:list_attributes` | `attio_list_attributes_raw` | `list_configuration:read` |\n| `resource:entries` | `attio_entries_raw` | `list_configuration:read`, `list_entry:read` |\n| `resource:notes` | `attio_notes_raw` | `note:read`, `object_configuration:read`, `record_permission:read` |\n| `resource:tasks` | `attio_tasks_raw` | `task:read`, `object_configuration:read`, `record_permission:read`, `user_management:read` |\n| `resource:members` | `attio_members_raw` | `user_management:read` |\n\nThe objects and lists catalog streams retain all accessible metadata. Records and attributes are fetched only for the object types configured in `objects`; entries and list attributes require configured `lists`. Standard `users` records, when explicitly configured, are distinct from [workspace members](https://docs.attio.com/rest-api/endpoint-reference/workspace-members/list-workspace-members). Attribute streams include archived attribute definitions; separate select-option/status catalogs and historical attribute values are not fetched. Active record/entry payloads retain their returned select/status values.\n\nThe template does not collect meetings, call recordings, transcripts, emails, files, comments, or historical attribute values. It does not receive webhooks or detect source deletions/merges. Coverage is limited to data the token can access. A permission error fails that stream with a diagnostic; it is never treated as an empty dataset.\n\nAPI references: [attributes](https://docs.attio.com/rest-api/endpoint-reference/attributes/list-attributes), [records](https://docs.attio.com/rest-api/endpoint-reference/records/list-records), [lists](https://docs.attio.com/rest-api/endpoint-reference/lists/list-all-lists), [entries](https://docs.attio.com/rest-api/endpoint-reference/entries/list-entries), [notes](https://docs.attio.com/rest-api/endpoint-reference/notes/list-notes), [tasks](https://docs.attio.com/rest-api/endpoint-reference/tasks/list-tasks).\n\n## Customize the source\n\nEach file in `sources/` contains one resource's raw table and reader, such as `sources/notes.ts`, `sources/tasks.ts`, or `sources/members.ts`. `sources/records.ts` also defines the people, company, and deal views. `client.ts` shares HTTP, validation, and pagination logic; `pipeline.ts` creates streams from the explicit object/list configuration.\n\nRemove an entry from `pipeline.ts` to stop collecting that resource. Keep its schema export in `index.ts` to preserve already collected data. Removing schema exports can generate destructive migration operations, which should be reviewed. After removing a resource's stream and schema export, its local files can also be deleted.\n\nSet `objects` and `lists` in `config.ts` to explicit arrays of API slugs or IDs. Defaults are `objects: ['people', 'companies']` and `lists: []`. Add `deals` or a custom object slug to collect that type; add list slugs or IDs to collect entries. `[]` disables the corresponding object/list child streams. The stream graph is built from these arrays without API discovery, so schema imports and `ingest list` remain offline. A reader resolves its one configured slug or ID through Attio and fails visibly if it is unknown or inaccessible. The objects/lists catalogs still collect all accessible metadata; notes, tasks, and members remain workspace-wide.\n\nObject streams use `.records.` and `.object_attributes.`. List streams use `.entries.` and `.list_attributes.`. Each has independent journal progress even though streams of the same resource write to the same raw table. Keep configured references stable: changing from a slug to its equivalent ID changes the stream ID.\n\nTo run all configured records streams:\n\n```sh\nbunx chkit ingest run --tag provider:attio --tag resource:records\n```\n\nTo run only People records:\n\n```sh\nbunx chkit ingest run --tag provider:attio --tag resource:records --tag object:people\n```\n\nRecord streams carry `resource:records` and `object:`. Object-attribute streams carry `resource:object_attributes` and `object:`; entries/list-attribute streams carry their resource tag and `list:`.\n\n`pageSize` defaults to 500. Notes have a separate `notesPageSize` of 50, the documented maximum. Requests go through the ingestion executor's retry/cancellation context, with a 30-second request timeout. The serial pipeline leaves a 25ms gap before each request and a 125ms gap for notes; raising concurrency requires revisiting rate control. `429` responses retain `Retry-After` for executor backoff. Attio also limits query complexity; a permanently expensive query may need simpler filters/sorts rather than repeated retries. See [rate limits](https://docs.attio.com/rest-api/guides/rate-limiting).\n\n## Stored data and editable views\n\nEach raw table contains a stable `id`, native JSON `raw`, and chkit batch/run/ingestion-time metadata. The key includes source identity, resource kind, workspace ID, and every relevant object/list/entity ID. Names, emails, and domains are never keys.\n\nThe complete returned entity is retained under `raw.data`. The envelope adds `source_id`; record and object-attribute rows also include `object_slug`, and list-entry/list-attribute rows include `list_slug`. No custom attributes or multivalued arrays are flattened away. For example:\n\n```sql\nSELECT id, raw.data, _chkit_ingested_at\nFROM default.attio_records_raw FINAL;\n```\n\n`sources/records.ts` defines three ordinary `FINAL` views:\n\n| View | Editable projection |\n| --- | --- |\n| `attio_people` | name, email address array, company record ID array |\n| `attio_companies` | name, domain array, description |\n| `attio_deals` | name, stage, nullable numeric value, currency code, company/people record ID arrays |\n\nAll include the stable row `id`, `source_id`, `workspace_id`, `object_id`, `record_id`, nullable `created_at`, `web_url`, and `_chkit_ingested_at`. Missing optional strings yield empty strings, absent arrays yield empty arrays, and absent/unparseable dates or deal amounts yield null. The deals view remains empty until `deals` is configured and observed; previously loaded deal rows remain when it is later deselected. Edit these SQL projections to expose your custom fields, then generate and review a migration. The original payload remains available for further projections.\n\nThe executor journal records per-stream execution and load completion, including successful empty reads. Every scheduled run reads each selected stream from the beginning. An interruption in Companies affects that stream's completion independently of People; the next Companies attempt starts at offset zero. The reader validates returned child workspace and parent IDs against its resolved object/list before publishing rows. The journal stores no Attio-specific cycle, parent inventory, or offset checkpoint. These full-sync readers ignore date bounds; date-range backfills are not supported.\n\nVersion `0.2.0` changes records, object-attributes, entries, and list-attributes stream IDs to the per-type/per-list forms above. Those new stream IDs begin with fresh journal progress. Raw table schemas, row identities for the same `sourceId`, and all three views remain unchanged. The previous `0.1.3` artifact remains available. If upgrading from its checkpointed-scan preview, the unchanged workspace stream IDs would have saved a different strategy; use a new `sourceId` and review the existing rows because source identity is also part of every raw row ID. Installed source is editable and is not automatically replaced by a registry update.\n\nThese views show the latest **observed** version per ID, not a guaranteed snapshot of Attio now. Full reads update returned IDs without clearing tables. Deleted, merged, inaccessible, or deselected entities remain until you implement reconciliation. Data is published batch by batch, so a failed run may have loaded some updates. Offset pagination over changing source data can miss or duplicate entities within a read. There is no universal modification cursor, atomic snapshot replacement, source-version ordering, or complete change history in this template. No data-bearing chunk uses a page offset as its identity.\n\n## Fixture tests\n\nInstall the optional test suite with `bunx chkit add attio --with-tests`, then run:\n\n```sh\nbun test src/integrations/attio/tests/attio.test.ts\n```\n\nFor a custom install path, point the command at that folder's `tests/attio.test.ts`. Tests mock Attio HTTP responses and use in-memory ingestion destinations and journals. They require Bun but no Attio credentials or ClickHouse server. The fixture suite checks per-type stream selection, independent pagination, preserved custom fields, repeat runs, cancellation, errors, safe offset-zero restart, successful empty reads, and rows loaded before a journal failure. Adjust the fixtures and expectations when customizing the source.\n\n## Smoke check\n\nAfter a migration and one successful run, inspect `chkit ingest status --tag provider:attio` and query `SELECT count() FROM default.attio_records_raw FINAL`. Compare a known record's retained `raw.data` with its Attio API response. Run again after changing that record and verify its view reflects the new observation. This smoke check needs a real test workspace and database; the registry's sanitized fixture tests do not assert complete live API coverage.\n" + }, + { + "path": "attio/client.ts", + "type": "registry:file", + "target": "src/integrations/attio/client.ts", + "content": "import { setTimeout as sleep } from 'node:timers/promises'\n\nimport { HttpError, IngestConfigError, paginate, rawRows, type ErrorClassifier, type FetchContext } from '@chkit/plugin-ingest'\n\nimport { attioConfig, type AttioConfig } from './config.js'\n\nexport type AttioEntity = Record & { id: Record }\n\nexport interface AttioClientDeps {\n config: AttioConfig\n fetch: (url: string, init: RequestInit) => Promise\n token: () => string | undefined\n wait: (milliseconds: number, signal: AbortSignal) => Promise\n}\n\ninterface CollectionRequest {\n path: string\n idFields: readonly string[]\n method?: 'GET' | 'POST'\n body?: Record\n query?: Record\n /** Omit for the unpaginated objects, lists, and workspace_members endpoints. */\n pageSize?: number\n valuesField?: 'values' | 'entry_values'\n parent?: { field: 'object_id' | 'list_id'; id: string; workspaceId: string }\n}\n\nexport const defaultAttioClientDeps: AttioClientDeps = {\n config: attioConfig,\n fetch: (url, init) => fetch(url, init),\n token: () => process.env.ATTIO_API_TOKEN,\n wait: async (milliseconds, signal) => { await sleep(milliseconds, undefined, { signal }) },\n}\n\n/** Resolve one configured object/list; each stream owns one collection. */\nexport async function* readParentCollection(\n context: FetchContext,\n input: {\n resource: string\n parent: { kind: 'objects' | 'lists'; ref: string }\n request: (parent: AttioEntity) => CollectionRequest\n },\n deps: AttioClientDeps = defaultAttioClientDeps,\n) {\n const idField = input.parent.kind === 'objects' ? 'object_id' : 'list_id'\n const path = `/${input.parent.kind}/${encodeURIComponent(input.parent.ref)}`\n const parent = await context.attempt(async (signal) => {\n const entity = parseEntity(await requestData({ path, idFields: ['workspace_id', idField] }, 0, signal, deps), {\n path, idFields: ['workspace_id', idField],\n })\n entitySlug(entity)\n if (input.parent.ref !== entityId(entity, idField) && input.parent.ref !== entitySlug(entity)) {\n throw new IngestConfigError(`Attio ${path} returned a different configured parent.`)\n }\n return entity\n }, { label: `GET /v2${path}` })\n const request = { ...input.request(parent), parent: {\n field: input.parent.kind === 'objects' || input.resource === 'list_attributes' ? 'object_id' as const : 'list_id' as const,\n id: entityId(parent, idField), workspaceId: entityId(parent, 'workspace_id'),\n } }\n yield* readRows(context, {\n resource: input.resource, request,\n metadata: input.parent.kind === 'objects' ? { object_slug: entitySlug(parent) } : { list_slug: entitySlug(parent) },\n }, deps)\n}\n\nexport async function* readRows(\n context: FetchContext,\n input: { resource: string; request: CollectionRequest; metadata?: { object_slug?: string; list_slug?: string } },\n deps: AttioClientDeps = defaultAttioClientDeps,\n) {\n for await (const page of readCollection(context, input.request, deps)) {\n yield { rows: toAttioRows(page, input.resource, input.request.idFields, input.metadata, deps.config.sourceId) }\n }\n}\n\n/** Each page is validated inside attempt, so executor retry/cancellation owns every request. */\nexport function readCollection(\n context: FetchContext,\n request: CollectionRequest,\n deps: AttioClientDeps = defaultAttioClientDeps,\n) {\n if (request.pageSize !== undefined && (!Number.isSafeInteger(request.pageSize) || request.pageSize <= 0)) {\n throw new IngestConfigError('Attio pageSize must be a positive safe integer.')\n }\n return paginate({\n context, initial: 0, label: `${request.method ?? 'GET'} /v2${request.path}`,\n fetchPage: async (offset = 0, signal) => {\n const data = await requestData(request, offset, signal, deps)\n if (!Array.isArray(data)) throw new IngestConfigError(`Attio ${request.path} returned no data array.`)\n const items = data.map((value: unknown) => parseEntity(value, request))\n if (request.pageSize !== undefined && items.length > request.pageSize) {\n throw new IngestConfigError(`Attio ${request.path} returned more rows than the requested page size.`)\n }\n const next = request.pageSize !== undefined && items.length === request.pageSize ? offset + items.length : undefined\n if (next !== undefined && !Number.isSafeInteger(next)) {\n throw new IngestConfigError('Attio pagination offset exceeded the safe integer range.')\n }\n return { items, next }\n },\n })\n}\n\n/** Original provider fields stay under data; metadata does not overwrite custom attributes. */\nexport function toAttioRows(\n items: readonly AttioEntity[],\n resource: string,\n idFields: readonly string[],\n metadata: { object_slug?: string; list_slug?: string } = {},\n sourceId = attioConfig.sourceId,\n) {\n return rawRows(\n items.map((data) => ({ source_id: sourceId, ...metadata, data })),\n ({ data }) => JSON.stringify([sourceId, resource, ...idFields.map((field) => entityId(data, field))]),\n )\n}\n\nexport function entityId(entity: AttioEntity, field: string): string {\n const value = entity.id[field]\n if (!value) throw new IngestConfigError(`Attio entity is missing id.${field}.`)\n return value\n}\n\nexport function entitySlug(entity: AttioEntity): string {\n if (typeof entity.api_slug !== 'string' || entity.api_slug.length === 0) {\n throw new IngestConfigError('Attio object/list is missing api_slug.')\n }\n return entity.api_slug\n}\n\n/** Invalid successful responses/configurations cannot be fixed by retrying. */\nexport const classifyAttioError: ErrorClassifier = (cause) => {\n if (cause instanceof IngestConfigError || cause instanceof SyntaxError) return { kind: 'permanent' }\n // HttpError preserves 429 Retry-After dates, 5xx retries, and permanent 4xx semantics.\n return undefined\n}\n\nasync function requestData(\n request: CollectionRequest,\n offset: number,\n signal: AbortSignal,\n deps: AttioClientDeps,\n): Promise {\n const token = deps.token()?.trim()\n if (!token) throw new IngestConfigError('Set ATTIO_API_TOKEN before running Attio ingestion. Schema imports do not need credentials.')\n const url = new URL(`https://api.attio.com/v2${request.path}`)\n for (const [key, value] of Object.entries(request.query ?? {})) url.searchParams.set(key, value)\n const method = request.method ?? 'GET'\n const pagination = request.pageSize === undefined ? {} : { limit: request.pageSize, offset }\n if (method === 'GET') {\n for (const [key, value] of Object.entries(pagination)) url.searchParams.set(key, String(value))\n }\n // The default pipeline runs one request at a time: <=40 reads/s and <=8 notes/s.\n // Attio may lower limits and applies query-complexity limits; 429s still use executor backoff.\n await deps.wait(request.path === '/notes' ? 125 : 25, signal)\n signal.throwIfAborted()\n const response = await deps.fetch(url.toString(), {\n method,\n headers: { Authorization: `Bearer ${token}`, Accept: 'application/json', 'Content-Type': 'application/json' },\n ...(method === 'POST' ? { body: JSON.stringify({ ...request.body, ...pagination }) } : {}),\n signal: AbortSignal.any([signal, AbortSignal.timeout(30_000)]),\n redirect: 'error',\n })\n if (!response.ok) {\n const error = await HttpError.fromResponse(response)\n if (response.status === 401 || response.status === 403) {\n error.message += ' Check ATTIO_API_TOKEN and the resource scopes in the Attio README, or disable this stream.'\n }\n throw error\n }\n const payload: unknown = await response.json()\n if (!isObject(payload) || !('data' in payload)) {\n throw new IngestConfigError(`Attio ${request.path} returned no data field.`)\n }\n return payload.data\n}\n\nfunction parseEntity(value: unknown, request: CollectionRequest): AttioEntity {\n if (!isEntity(value)) throw new IngestConfigError(`Attio ${request.path} returned an entity without a valid id object.`)\n for (const field of request.idFields) entityId(value, field)\n if (request.parent && (entityId(value, 'workspace_id') !== request.parent.workspaceId\n || entityId(value, request.parent.field) !== request.parent.id)) {\n throw new IngestConfigError(`Attio ${request.path} returned an entity from a different workspace or parent than the resolved collection.`)\n }\n if (request.valuesField !== undefined) {\n const values = value[request.valuesField]\n if (!isObject(values) || !Object.values(values).every(Array.isArray)) {\n throw new IngestConfigError(`Attio ${request.path} returned invalid ${request.valuesField}; expected arrays of attribute values.`)\n }\n }\n return value\n}\n\nfunction isEntity(value: unknown): value is AttioEntity {\n return isObject(value) && isObject(value.id) && Object.values(value.id).every((part) => typeof part === 'string')\n}\n\nfunction isObject(value: unknown): value is Record {\n return typeof value === 'object' && value !== null && !Array.isArray(value)\n}\n" + }, + { + "path": "attio/config.ts", + "type": "registry:file", + "target": "src/integrations/attio/config.ts", + "content": "export interface AttioConfig {\n /** Stable, non-secret name for this installation. Changing it creates new stream identities. */\n sourceId: string\n database: string\n tablePrefix: string\n /** Each UUID or API slug creates its own records and attributes streams. */\n objects: readonly string[]\n /** Each UUID or API slug creates its own entries and attributes streams. */\n lists: readonly string[]\n pageSize: number\n notesPageSize: number\n}\n\n// Keep these values stable across runs. Credentials are read only in client.ts at request time.\nexport const attioConfig: AttioConfig = {\n sourceId: 'attio.primary',\n database: 'default',\n tablePrefix: 'attio',\n objects: ['people', 'companies'],\n lists: [],\n pageSize: 500,\n notesPageSize: 50,\n}\n" + }, + { + "path": "attio/index.ts", + "type": "registry:file", + "target": "src/integrations/attio/index.ts", + "content": "export { attio } from './pipeline.js'\nexport { attioObjectsRaw } from './sources/objects.js'\nexport { attioObjectAttributesRaw } from './sources/object-attributes.js'\nexport { attioRecordsRaw, attioPeople, attioCompanies, attioDeals } from './sources/records.js'\nexport { attioListsRaw } from './sources/lists.js'\nexport { attioListAttributesRaw } from './sources/list-attributes.js'\nexport { attioEntriesRaw } from './sources/entries.js'\nexport { attioNotesRaw } from './sources/notes.js'\nexport { attioTasksRaw } from './sources/tasks.js'\nexport { attioMembersRaw } from './sources/members.js'\n" + }, + { + "path": "attio/pipeline.ts", + "type": "registry:file", + "target": "src/integrations/attio/pipeline.ts", + "content": "import { definePipeline, defineStream } from '@chkit/plugin-ingest'\n\nimport { classifyAttioError, defaultAttioClientDeps, type AttioClientDeps } from './client.js'\nimport { attioConfig, type AttioConfig } from './config.js'\nimport { attioObjectsRaw, readObjects } from './sources/objects.js'\nimport { attioObjectAttributesRaw, readObjectAttributes } from './sources/object-attributes.js'\nimport { attioRecordsRaw, readRecords } from './sources/records.js'\nimport { attioListsRaw, readLists } from './sources/lists.js'\nimport { attioListAttributesRaw, readListAttributes } from './sources/list-attributes.js'\nimport { attioEntriesRaw, readEntries } from './sources/entries.js'\nimport { attioNotesRaw, readNotes } from './sources/notes.js'\nimport { attioTasksRaw, readTasks } from './sources/tasks.js'\nimport { attioMembersRaw, readMembers } from './sources/members.js'\n\nconst streamOptions = { classifyError: classifyAttioError, batchSize: 500 }\n\n// One records stream per object type; no reader traverses other objects.\n// Keep schema exports when removing streams to retain existing stored data.\nexport function createAttioPipeline(config: AttioConfig = attioConfig, deps: AttioClientDeps = defaultAttioClientDeps) {\n const client = { ...deps, config }\n return definePipeline({\n id: config.sourceId,\n tags: ['provider:attio'],\n maxStreams: 1,\n maxFetches: 1,\n maxLoads: 1,\n retry: { retries: 5, minTimeout: 1_000, maxTimeout: 30_000, randomize: true },\n streams: [\n defineStream({ ...streamOptions, id: `${config.sourceId}.objects`, tags: ['resource:objects'], destination: attioObjectsRaw, read: (context) => readObjects(context, client) }),\n ...config.objects.flatMap((object) => [\n defineStream({ ...streamOptions, id: `${config.sourceId}.records.${object}`, tags: ['resource:records', `object:${object}`], destination: attioRecordsRaw, read: (context) => readRecords(context, object, client) }),\n defineStream({ ...streamOptions, id: `${config.sourceId}.object_attributes.${object}`, tags: ['resource:object_attributes', `object:${object}`], destination: attioObjectAttributesRaw, read: (context) => readObjectAttributes(context, object, client) }),\n ]),\n defineStream({ ...streamOptions, id: `${config.sourceId}.lists`, tags: ['resource:lists'], destination: attioListsRaw, read: (context) => readLists(context, client) }),\n ...config.lists.flatMap((list) => [\n defineStream({ ...streamOptions, id: `${config.sourceId}.entries.${list}`, tags: ['resource:entries', `list:${list}`], destination: attioEntriesRaw, read: (context) => readEntries(context, list, client) }),\n defineStream({ ...streamOptions, id: `${config.sourceId}.list_attributes.${list}`, tags: ['resource:list_attributes', `list:${list}`], destination: attioListAttributesRaw, read: (context) => readListAttributes(context, list, client) }),\n ]),\n defineStream({ ...streamOptions, id: `${config.sourceId}.notes`, tags: ['resource:notes'], destination: attioNotesRaw, read: (context) => readNotes(context, client) }),\n defineStream({ ...streamOptions, id: `${config.sourceId}.tasks`, tags: ['resource:tasks'], destination: attioTasksRaw, read: (context) => readTasks(context, client) }),\n defineStream({ ...streamOptions, id: `${config.sourceId}.members`, tags: ['resource:members'], destination: attioMembersRaw, read: (context) => readMembers(context, client) }),\n ],\n })\n}\n\nexport const attio = createAttioPipeline()\n" + }, + { + "path": "attio/sources/objects.ts", + "type": "registry:file", + "target": "src/integrations/attio/sources/objects.ts", + "content": "import { rawTable, type FetchContext } from '@chkit/plugin-ingest'\n\nimport { readRows, type AttioClientDeps } from '../client.js'\nimport { attioConfig } from '../config.js'\n\nexport const attioObjectsRaw = rawTable({ database: attioConfig.database, name: `${attioConfig.tablePrefix}_objects_raw` })\n\nexport function readObjects(context: FetchContext, deps?: AttioClientDeps) {\n return readRows(context, {\n resource: 'objects', request: { path: '/objects', idFields: ['workspace_id', 'object_id'] },\n }, deps)\n}\n" + }, + { + "path": "attio/sources/object-attributes.ts", + "type": "registry:file", + "target": "src/integrations/attio/sources/object-attributes.ts", + "content": "import { rawTable, type FetchContext } from '@chkit/plugin-ingest'\n\nimport { defaultAttioClientDeps, entityId, readParentCollection, type AttioClientDeps } from '../client.js'\nimport { attioConfig } from '../config.js'\n\nexport const attioObjectAttributesRaw = rawTable({ database: attioConfig.database, name: `${attioConfig.tablePrefix}_object_attributes_raw` })\n\nexport function readObjectAttributes(context: FetchContext, object: string, deps: AttioClientDeps = defaultAttioClientDeps) {\n return readParentCollection(context, {\n resource: 'object_attributes', parent: { kind: 'objects', ref: object },\n request: (parent) => ({\n path: `/objects/${encodeURIComponent(entityId(parent, 'object_id'))}/attributes`,\n idFields: ['workspace_id', 'object_id', 'attribute_id'],\n query: { show_archived: 'true' },\n pageSize: deps.config.pageSize,\n }),\n }, deps)\n}\n" + }, + { + "path": "attio/sources/records.ts", + "type": "registry:file", + "target": "src/integrations/attio/sources/records.ts", + "content": "import { view } from '@chkit/core'\nimport { rawTable, type FetchContext } from '@chkit/plugin-ingest'\n\nimport { defaultAttioClientDeps, entityId, readParentCollection, type AttioClientDeps } from '../client.js'\nimport { attioConfig } from '../config.js'\n\nexport const attioRecordsRaw = rawTable({ database: attioConfig.database, name: `${attioConfig.tablePrefix}_records_raw` })\n\n// Each view depends only on records. JSON extraction tolerates absent optional attributes.\nexport const attioPeople = recordView('people', `\n JSONExtractString(values, 'name', 1, 'full_name') AS name,\n arrayMap(value -> JSONExtractString(value, 'email_address'), JSONExtractArrayRaw(values, 'email_addresses')) AS email_addresses,\n arrayMap(value -> JSONExtractString(value, 'target_record_id'), JSONExtractArrayRaw(values, 'company')) AS company_record_ids`)\n\nexport const attioCompanies = recordView('companies', `\n JSONExtractString(values, 'name', 1, 'value') AS name,\n arrayMap(value -> JSONExtractString(value, 'domain'), JSONExtractArrayRaw(values, 'domains')) AS domains,\n JSONExtractString(values, 'description', 1, 'value') AS description`)\n\nexport const attioDeals = recordView('deals', `\n JSONExtractString(values, 'name', 1, 'value') AS name,\n JSONExtractString(values, 'stage', 1, 'status', 'title') AS stage,\n JSONExtract(values, 'value', 1, 'currency_value', 'Nullable(Float64)') AS value,\n JSONExtractString(values, 'value', 1, 'currency_code') AS currency_code,\n arrayMap(value -> JSONExtractString(value, 'target_record_id'), JSONExtractArrayRaw(values, 'associated_company')) AS company_record_ids,\n arrayMap(value -> JSONExtractString(value, 'target_record_id'), JSONExtractArrayRaw(values, 'associated_people')) AS people_record_ids`)\n\nexport function readRecords(context: FetchContext, object: string, deps: AttioClientDeps = defaultAttioClientDeps) {\n return readParentCollection(context, {\n resource: 'records', parent: { kind: 'objects', ref: object },\n request: (parent) => ({\n path: `/objects/${encodeURIComponent(entityId(parent, 'object_id'))}/records/query`,\n idFields: ['workspace_id', 'object_id', 'record_id'],\n method: 'POST',\n valuesField: 'values',\n pageSize: deps.config.pageSize,\n }),\n }, deps)\n}\n\nfunction recordView(objectSlug: string, columns: string) {\n return view({\n database: attioConfig.database,\n name: `${attioConfig.tablePrefix}_${objectSlug}`,\n as: `WITH toJSONString(raw) AS payload,\n JSONExtractRaw(payload, 'data') AS record,\n JSONExtractRaw(record, 'values') AS values\nSELECT id,\n JSONExtractString(payload, 'source_id') AS source_id,\n JSONExtractString(record, 'id', 'workspace_id') AS workspace_id,\n JSONExtractString(record, 'id', 'object_id') AS object_id,\n JSONExtractString(record, 'id', 'record_id') AS record_id,\n parseDateTime64BestEffortOrNull(JSONExtractString(record, 'created_at'), 6, 'UTC') AS created_at,\n JSONExtractString(record, 'web_url') AS web_url,${columns},\n _chkit_ingested_at\nFROM ${quoteIdentifier(attioRecordsRaw.database)}.${quoteIdentifier(attioRecordsRaw.name)} FINAL\nWHERE JSONExtractString(payload, 'object_slug') = '${objectSlug}'`,\n })\n}\n\nfunction quoteIdentifier(value: string): string {\n return `\\`${value.replaceAll('\\\\', '\\\\\\\\').replaceAll('`', '\\\\`')}\\``\n}\n" + }, + { + "path": "attio/sources/lists.ts", + "type": "registry:file", + "target": "src/integrations/attio/sources/lists.ts", + "content": "import { rawTable, type FetchContext } from '@chkit/plugin-ingest'\n\nimport { readRows, type AttioClientDeps } from '../client.js'\nimport { attioConfig } from '../config.js'\n\nexport const attioListsRaw = rawTable({ database: attioConfig.database, name: `${attioConfig.tablePrefix}_lists_raw` })\n\nexport function readLists(context: FetchContext, deps?: AttioClientDeps) {\n return readRows(context, {\n resource: 'lists', request: { path: '/lists', idFields: ['workspace_id', 'list_id'] },\n }, deps)\n}\n" + }, + { + "path": "attio/sources/list-attributes.ts", + "type": "registry:file", + "target": "src/integrations/attio/sources/list-attributes.ts", + "content": "import { rawTable, type FetchContext } from '@chkit/plugin-ingest'\n\nimport { defaultAttioClientDeps, entityId, readParentCollection, type AttioClientDeps } from '../client.js'\nimport { attioConfig } from '../config.js'\n\nexport const attioListAttributesRaw = rawTable({ database: attioConfig.database, name: `${attioConfig.tablePrefix}_list_attributes_raw` })\n\nexport function readListAttributes(context: FetchContext, list: string, deps: AttioClientDeps = defaultAttioClientDeps) {\n return readParentCollection(context, {\n resource: 'list_attributes', parent: { kind: 'lists', ref: list },\n request: (parent) => ({\n path: `/lists/${encodeURIComponent(entityId(parent, 'list_id'))}/attributes`,\n idFields: ['workspace_id', 'object_id', 'attribute_id'],\n // Attio's shared attribute schema calls the parent list ID object_id too.\n query: { show_archived: 'true' },\n pageSize: deps.config.pageSize,\n }),\n }, deps)\n}\n" + }, + { + "path": "attio/sources/entries.ts", + "type": "registry:file", + "target": "src/integrations/attio/sources/entries.ts", + "content": "import { rawTable, type FetchContext } from '@chkit/plugin-ingest'\n\nimport { defaultAttioClientDeps, entityId, readParentCollection, type AttioClientDeps } from '../client.js'\nimport { attioConfig } from '../config.js'\n\nexport const attioEntriesRaw = rawTable({ database: attioConfig.database, name: `${attioConfig.tablePrefix}_entries_raw` })\n\nexport function readEntries(context: FetchContext, list: string, deps: AttioClientDeps = defaultAttioClientDeps) {\n return readParentCollection(context, {\n resource: 'entries', parent: { kind: 'lists', ref: list },\n request: (parent) => ({\n path: `/lists/${encodeURIComponent(entityId(parent, 'list_id'))}/entries/query`,\n idFields: ['workspace_id', 'list_id', 'entry_id'],\n method: 'POST',\n valuesField: 'entry_values',\n pageSize: deps.config.pageSize,\n }),\n }, deps)\n}\n" + }, + { + "path": "attio/sources/notes.ts", + "type": "registry:file", + "target": "src/integrations/attio/sources/notes.ts", + "content": "import { IngestConfigError, rawTable, type FetchContext } from '@chkit/plugin-ingest'\n\nimport { defaultAttioClientDeps, readRows, type AttioClientDeps } from '../client.js'\nimport { attioConfig } from '../config.js'\n\nexport const attioNotesRaw = rawTable({ database: attioConfig.database, name: `${attioConfig.tablePrefix}_notes_raw` })\n\nexport function readNotes(context: FetchContext, deps: AttioClientDeps = defaultAttioClientDeps) {\n if (deps.config.notesPageSize > 50) throw new IngestConfigError('Attio notesPageSize cannot exceed 50.')\n return readRows(context, {\n resource: 'notes', request: {\n path: '/notes', idFields: ['workspace_id', 'note_id'], pageSize: deps.config.notesPageSize,\n },\n }, deps)\n}\n" + }, + { + "path": "attio/sources/tasks.ts", + "type": "registry:file", + "target": "src/integrations/attio/sources/tasks.ts", + "content": "import { rawTable, type FetchContext } from '@chkit/plugin-ingest'\n\nimport { defaultAttioClientDeps, readRows, type AttioClientDeps } from '../client.js'\nimport { attioConfig } from '../config.js'\n\nexport const attioTasksRaw = rawTable({ database: attioConfig.database, name: `${attioConfig.tablePrefix}_tasks_raw` })\n\nexport function readTasks(context: FetchContext, deps: AttioClientDeps = defaultAttioClientDeps) {\n return readRows(context, {\n resource: 'tasks', request: {\n path: '/tasks', idFields: ['workspace_id', 'task_id'], pageSize: deps.config.pageSize,\n query: { sort: 'created_at:asc' },\n },\n }, deps)\n}\n" + }, + { + "path": "attio/sources/members.ts", + "type": "registry:file", + "target": "src/integrations/attio/sources/members.ts", + "content": "import { rawTable, type FetchContext } from '@chkit/plugin-ingest'\n\nimport { readRows, type AttioClientDeps } from '../client.js'\nimport { attioConfig } from '../config.js'\n\nexport const attioMembersRaw = rawTable({ database: attioConfig.database, name: `${attioConfig.tablePrefix}_members_raw` })\n\nexport function readMembers(context: FetchContext, deps?: AttioClientDeps) {\n return readRows(context, {\n resource: 'members', request: {\n path: '/workspace_members', idFields: ['workspace_id', 'workspace_member_id'],\n },\n }, deps)\n}\n" + }, + { + "path": "attio/tests/attio.test.ts", + "type": "registry:file", + "target": "src/integrations/attio/tests/attio.test.ts", + "role": "test", + "content": "import { describe, expect, test } from 'bun:test'\n\nimport { HttpError, definePipeline, runIngestion, selectStreams, type FetchContext } from '@chkit/plugin-ingest'\nimport { createMemoryDestination, createMemoryJournal } from '@chkit/plugin-ingest/testing'\n\nimport { classifyAttioError, readCollection, type AttioClientDeps } from '../client.js'\nimport { attioConfig } from '../config.js'\nimport * as definitions from '../index.js'\nimport { attio, createAttioPipeline } from '../pipeline.js'\nimport { readEntries } from '../sources/entries.js'\nimport { readMembers } from '../sources/members.js'\nimport { readNotes } from '../sources/notes.js'\nimport { readRecords } from '../sources/records.js'\nimport { readTasks } from '../sources/tasks.js'\nimport { fixtureDeps, fixtureResponse, peopleObject, person } from './fixtures.js'\n\ndescribe('Attio template', () => {\n test('exports independently selectable object streams and raw destinations without credentials', () => {\n const exported = Object.values(definitions)\n expect(exported).toContain(attio)\n expect(attio.streams).toHaveLength(9)\n expect(attio.streams.filter((stream) => stream.tags.includes('resource:records')).map((stream) => stream.id))\n .toEqual(['attio.primary.records.people', 'attio.primary.records.companies'])\n expect(selectStreams([attio], ['resource:records', 'object:people']).map(({ stream }) => stream.id)).toEqual(['attio.primary.records.people'])\n expect(selectStreams([attio], ['resource:records', 'object:companies']).map(({ stream }) => stream.id))\n .toEqual(['attio.primary.records.companies'])\n for (const stream of attio.streams) {\n expect(exported).toContain(stream.destination)\n expect(stream.incremental.id).toBe('chkit.full_sync')\n }\n expect([definitions.attioPeople, definitions.attioCompanies, definitions.attioDeals].map((view) => view.name))\n .toEqual(['attio_people', 'attio_companies', 'attio_deals'])\n })\n\n test('an object named tasks does not overlap the workspace tasks resource tag', () => {\n const pipeline = createAttioPipeline({ ...attioConfig, objects: ['tasks'], lists: [] }, fixtureDeps())\n expect(selectStreams([pipeline], ['resource:tasks']).map(({ stream }) => stream.id)).toEqual(['attio.primary.tasks'])\n expect(selectStreams([pipeline], ['resource:records', 'object:tasks']).map(({ stream }) => stream.id))\n .toEqual(['attio.primary.records.tasks'])\n })\n\n test('configured custom objects and lists have their own streams sharing raw tables', async () => {\n const pipeline = fixturePipeline(fixtureDeps())\n const journal = createMemoryJournal()\n const destination = createMemoryDestination()\n const result = await runIngestion({ selected: selectStreams([pipeline], []), backfill: undefined }, { journal, destination })\n expect(result.ok, JSON.stringify(result)).toBe(true)\n expect(pipeline.streams).toHaveLength(11)\n expect(result.streams.filter((stream) => stream.streamId.includes('.records.')).map((stream) => stream.rows)).toEqual([1, 1])\n expect(destination.tables.size).toBe(9)\n expect(destination.tables.get('default.attio_records_raw')?.map((row) => row.raw)).toContainEqual({\n source_id: 'attio.primary', object_slug: 'people', data: person,\n })\n expect(destination.tables.get('default.attio_entries_raw')?.[0]?.raw).toMatchObject({ list_slug: 'sales' })\n expect(selectStreams([pipeline], ['resource:records', 'object:subscriptions']).map(({ stream }) => stream.id))\n .toEqual(['attio.primary.records.subscriptions'])\n expect(selectStreams([pipeline], ['resource:entries']).map(({ stream }) => stream.id))\n .toEqual(['attio.primary.entries.sales'])\n })\n\n test('one object reader paginates only that object and preserves raw custom values and IDs', async () => {\n const requests: Array<{ path: string; body: unknown }> = []\n const firstPage = Array.from({ length: 500 }, (_, index) => ({ ...person, id: { ...person.id, record_id: `record-${index}` } }))\n const deps = fixtureDeps((url, init) => {\n const body = typeof init.body === 'string' ? JSON.parse(init.body) : undefined\n requests.push({ path: url.pathname, body })\n if (url.pathname === '/v2/objects/people') return Response.json({ data: peopleObject })\n if (url.pathname === '/v2/objects/object-people/records/query') {\n return Response.json({ data: body.offset === 0 ? firstPage : [{ ...person, id: { ...person.id, record_id: 'last' } }] })\n }\n throw new Error(`Unexpected URL ${url.pathname}`)\n })\n const chunks = await collect(readRecords(context(), 'people', deps))\n expect(chunks.map((chunk) => chunk.rows.length)).toEqual([500, 1])\n expect(requests).toEqual([\n { path: '/v2/objects/people', body: undefined },\n { path: '/v2/objects/object-people/records/query', body: { limit: 500, offset: 0 } },\n { path: '/v2/objects/object-people/records/query', body: { limit: 500, offset: 500 } },\n ])\n expect(chunks.every((chunk) => !('state' in chunk) && !('id' in chunk))).toBe(true)\n expect(chunks[0]?.rows[0]?.id).toBe(JSON.stringify(['attio.primary', 'records', 'workspace-1', 'object-people', 'record-0']))\n expect(chunks[0]?.rows[0]?.raw).toMatchObject({ data: { values: { custom_multi: [{ value: 'one' }, { value: 'two' }] } } })\n const custom = await collect(readRecords(context(), 'subscriptions', fixtureDeps()))\n expect(custom[0]?.rows[0]?.raw).toMatchObject({ object_slug: 'subscriptions' })\n expect(custom[0]?.rows[0]?.id).not.toBe(chunks[0]?.rows[0]?.id)\n })\n\n test('a source identity supplied to the pipeline owns stream IDs and raw row IDs', async () => {\n const pipeline = createAttioPipeline({ ...attioConfig, sourceId: 'attio.secondary', objects: ['people'], lists: [] }, fixtureDeps())\n const destination = createMemoryDestination()\n const result = await runIngestion({ selected: selectStreams([pipeline], ['resource:records', 'object:people']), backfill: undefined }, {\n journal: createMemoryJournal(), destination,\n })\n expect(result.ok).toBe(true)\n expect(result.streams[0]?.streamId).toBe('attio.secondary.records.people')\n const row = destination.tables.get('default.attio_records_raw')?.[0]\n expect(row?.id).toBe(JSON.stringify(['attio.secondary', 'records', 'workspace-1', 'object-people', 'record-1']))\n expect(row?.raw).toMatchObject({ source_id: 'attio.secondary' })\n })\n\n test('notes use their 50-item limit and stop fetching after cancellation', async () => {\n const controller = new AbortController()\n const waits: number[] = []\n const urls: string[] = []\n const page = Array.from({ length: 50 }, (_, index) => ({ id: { workspace_id: 'workspace-1', note_id: `note-${index}` } }))\n const deps = fixtureDeps((url) => { urls.push(url.toString()); return Response.json({ data: page }) })\n deps.wait = async (milliseconds, signal) => { waits.push(milliseconds); signal.throwIfAborted() }\n const iterator = readNotes(context(controller.signal), deps)\n expect((await iterator.next()).value?.rows).toHaveLength(50)\n controller.abort(new Error('fixture cancelled'))\n await expect(iterator.next()).rejects.toThrow('fixture cancelled')\n expect(urls).toHaveLength(1)\n expect(new URL(urls[0] ?? '').searchParams.get('limit')).toBe('50')\n expect(waits).toEqual([125])\n })\n\n test('GET offset pagination requests the terminating empty page for an exact multiple', async () => {\n const offsets: string[] = []\n const deps = fixtureDeps((url) => {\n const offset = url.searchParams.get('offset') ?? ''\n offsets.push(offset)\n return Response.json({ data: offset === '0' ? [{ id: { task_id: 'a' } }, { id: { task_id: 'b' } }] : [] })\n })\n const pages = await collect(readCollection(context(), { path: '/tasks', idFields: ['task_id'], pageSize: 2 }, deps))\n expect(pages.map((page) => page.length)).toEqual([2])\n expect(offsets).toEqual(['0', '2'])\n })\n\n test('malformed successful responses and missing identities fail visibly', async () => {\n for (const payload of [{}, { data: null }, { data: [{ id: { task_id: 'task-1' } }] }]) {\n await expect(collect(readTasks(context(), fixtureDeps(() => Response.json(payload))))).rejects.toThrow('Attio')\n }\n const malformed = fixtureDeps((url) => url.pathname === '/v2/objects/people'\n ? fixtureResponse(url) : Response.json({ data: [{ ...person, values: { name: 'not-an-array' } }] }))\n await expect(collect(readRecords(context(), 'people', malformed))).rejects.toThrow('invalid values')\n await expect(collect(readRecords(context(), 'people', fixtureDeps(() => Response.json({ data: [peopleObject] }))))).rejects.toThrow('Attio')\n })\n\n test.each(['workspace_id', 'object_id'])('object readers reject children with a different %s', async (field) => {\n const deps = fixtureDeps((url) => url.pathname === '/v2/objects/people'\n ? fixtureResponse(url) : Response.json({ data: [{ ...person, id: { ...person.id, [field]: 'foreign' } }] }))\n await expect(collect(readRecords(context(), 'people', deps))).rejects.toThrow('different workspace or parent')\n })\n\n test('list readers resolve one list and reject entries belonging to another list', async () => {\n const requests: string[] = []\n const deps = fixtureDeps((url) => {\n requests.push(url.pathname)\n return fixtureResponse(url)\n })\n const chunks = await collect(readEntries(context(), 'sales', deps))\n expect(requests).toEqual(['/v2/lists/sales', '/v2/lists/list-sales/entries/query'])\n expect(chunks[0]?.rows[0]?.raw).toMatchObject({ list_slug: 'sales' })\n const invalid = fixtureDeps((url) => url.pathname === '/v2/lists/sales'\n ? fixtureResponse(url) : Response.json({ data: [{ id: { workspace_id: 'workspace-1', list_id: 'foreign', entry_id: 'one' }, entry_values: {} }] }))\n await expect(collect(readEntries(context(), 'sales', invalid))).rejects.toThrow('different workspace or parent')\n })\n\n test('permission failures fail one object stream while another remains independent', async () => {\n const requests: string[] = []\n const deps = fixtureDeps((url) => {\n requests.push(url.pathname)\n return url.pathname === '/v2/objects/object-custom/records/query'\n ? Response.json({ code: 'insufficient_scope' }, { status: 403 }) : fixtureResponse(url)\n })\n const pipeline = fixturePipeline(deps)\n const destination = createMemoryDestination()\n const journal = createMemoryJournal()\n const result = await runIngestion({ selected: selectStreams([pipeline], ['resource:records']), backfill: undefined }, { journal, destination })\n expect(result.ok).toBe(false)\n expect(result.streams.map((stream) => [stream.streamId, stream.outcome, stream.rows])).toEqual([\n ['attio.primary.records.people', 'succeeded', 1], ['attio.primary.records.subscriptions', 'failed', 0],\n ])\n expect(result.streams[1]?.error).toContain('resource scopes')\n expect(requests.filter((path) => path === '/v2/objects/object-custom/records/query')).toHaveLength(1)\n expect(destination.tables.get('default.attio_records_raw')).toHaveLength(1)\n expect((await journal.readCheckpoint('attio.primary.records.people')).lastSuccessSeq).toBeGreaterThan(0)\n expect((await journal.readCheckpoint('attio.primary.records.subscriptions')).lastSuccessSeq).toBe(0)\n })\n\n test('429 retries remain under executor authority and retain Retry-After', async () => {\n let calls = 0\n const deps = fixtureDeps((url) => {\n calls += 1\n return calls === 1 ? Response.json({ code: 'rate_limit_exceeded' }, { status: 429, headers: { 'Retry-After': '0' } }) : fixtureResponse(url)\n })\n const journal = createMemoryJournal()\n const result = await runIngestion({ selected: selectStreams([fixturePipeline(deps)], ['resource:notes']), backfill: undefined }, {\n journal, destination: createMemoryDestination(),\n })\n expect(result.ok).toBe(true)\n expect(calls).toBe(2)\n expect(journal.events.some((event) => event.eventKind === 'retry_scheduled' && event.errorClass === 'rate_limited')).toBe(true)\n const http = await HttpError.fromResponse(new Response(null, { status: 429, headers: { 'Retry-After': '2' } }))\n expect(http.retryAfterMs).toBe(2_000)\n expect(classifyAttioError(http, { kind: 'rate_limited', retryAfterMs: 2_000 })).toBeUndefined()\n })\n\n test('a missing token fails only when a reader runs', async () => {\n const deps = fixtureDeps(() => { throw new Error('must not fetch') })\n deps.token = () => undefined\n expect(attio.streams).toHaveLength(9)\n await expect(collect(readMembers(context(), deps))).rejects.toThrow('Set ATTIO_API_TOKEN')\n })\n\n test('an interrupted object restarts offset zero without requesting other objects', async () => {\n let fail = true\n const requests: string[] = []\n const offsets: number[] = []\n const page = Array.from({ length: 500 }, (_, index) => ({ ...person, id: { ...person.id, record_id: `record-${index}` } }))\n const deps = fixtureDeps((url, init) => {\n requests.push(url.pathname)\n if (url.pathname === '/v2/objects/people') return fixtureResponse(url)\n if (url.pathname !== '/v2/objects/object-people/records/query') throw new Error(`Unexpected object request ${url.pathname}`)\n const { offset } = JSON.parse(String(init.body))\n offsets.push(offset)\n if (offset === 0) return Response.json({ data: page })\n return fail ? Response.json({ code: 'insufficient_scope' }, { status: 403 }) : Response.json({ data: [] })\n })\n const journal = createMemoryJournal()\n const destination = createMemoryDestination()\n const first = await runIngestion({ selected: selectStreams([fixturePipeline(deps)], ['resource:records', 'object:people']), backfill: undefined }, { journal, destination })\n expect(first.ok).toBe(false)\n expect((await journal.readCheckpoint('attio.primary.records.people')).envelope).toBeUndefined()\n fail = false\n const restored = await restoredJournal(journal)\n const resumed = await runIngestion({ selected: selectStreams([fixturePipeline(deps)], ['resource:records', 'object:people']), backfill: undefined }, { journal: restored, destination })\n expect(resumed.ok).toBe(true)\n expect(offsets).toEqual([0, 500, 0, 500])\n expect(requests).not.toContain('/v2/objects')\n expect(destination.tables.get('default.attio_records_raw')).toHaveLength(500)\n })\n\n test('reruns publish changed observations with the same stable raw ID', async () => {\n let changed = false\n const deps = fixtureDeps((url) => url.pathname === '/v2/objects/object-people/records/query'\n ? Response.json({ data: [{ ...person, values: { ...person.values, name: [{ full_name: changed ? 'Ada Updated' : 'Ada Example' }] } }] }) : fixtureResponse(url))\n const journal = createMemoryJournal()\n const destination = createMemoryDestination()\n const request = { selected: selectStreams([fixturePipeline(deps)], ['resource:records', 'object:people']), backfill: undefined }\n expect((await runIngestion(request, { journal, destination })).ok).toBe(true)\n changed = true\n expect((await runIngestion(request, { journal, destination })).ok).toBe(true)\n const rows = destination.tables.get('default.attio_records_raw') ?? []\n expect(rows).toHaveLength(2)\n expect(rows[0]?.id).toBe(rows[1]?.id)\n expect(rows[1]?.raw).toMatchObject({ data: { values: { name: [{ full_name: 'Ada Updated' }] } } })\n })\n\n test('a fresh runtime after a lost journal acknowledgement replays changed rows safely', async () => {\n let changed = false\n const deps = fixtureDeps((url) => url.pathname === '/v2/objects/object-people/records/query'\n ? Response.json({ data: [{ ...person, values: { ...person.values, revision: [{ value: changed ? 'new' : 'old' }] } }] }) : fixtureResponse(url))\n const journal = createMemoryJournal()\n const destination = createMemoryDestination()\n const failingJournal = {\n ...journal,\n async append(events: Parameters[0]) {\n if (events.some((event) => event.eventKind === 'batch_committed' && Number(event.detail.rows) > 0)) {\n throw new Error('fixture journal unavailable after rows loaded')\n }\n await journal.append(events)\n },\n }\n const failed = await runIngestion({ selected: selectStreams([fixturePipeline(deps)], ['resource:records', 'object:people']), backfill: undefined }, {\n journal: failingJournal, destination,\n })\n expect(failed.ok).toBe(false)\n expect(destination.tables.get('default.attio_records_raw')).toHaveLength(1)\n changed = true\n const resumed = await runIngestion({ selected: selectStreams([fixturePipeline(deps)], ['resource:records', 'object:people']), backfill: undefined }, {\n journal: await restoredJournal(journal), destination,\n })\n expect(resumed.ok).toBe(true)\n const rows = destination.tables.get('default.attio_records_raw') ?? []\n expect(rows).toHaveLength(2)\n expect(rows[0]?.id).toBe(rows[1]?.id)\n expect(rows[1]?.raw).toMatchObject({ data: { values: { revision: [{ value: 'new' }] } } })\n }, 10_000)\n})\n\nfunction fixturePipeline(deps: AttioClientDeps) {\n return definePipeline({\n ...createAttioPipeline({ ...attioConfig, objects: ['people', 'subscriptions'], lists: ['sales'] }, deps),\n retry: { retries: 1, minTimeout: 0, maxTimeout: 0, randomize: false },\n })\n}\n\nfunction context(signal = new AbortController().signal): FetchContext {\n return { signal, attempt: async (operation) => operation(signal) }\n}\n\nasync function collect(items: AsyncIterable): Promise {\n const result: T[] = []\n for await (const item of items) result.push(item)\n return result\n}\n\nasync function restoredJournal(previous: ReturnType) {\n const journal = createMemoryJournal()\n await journal.append(previous.events.map((event) => ({\n ...event, checkpoint: event.checkpoint === undefined ? undefined : JSON.parse(JSON.stringify(event.checkpoint)),\n })))\n return journal\n}\n" + }, + { + "path": "attio/tests/fixtures.ts", + "type": "registry:file", + "target": "src/integrations/attio/tests/fixtures.ts", + "role": "test", + "content": "import type { AttioClientDeps } from '../client.js'\nimport { attioConfig } from '../config.js'\n\nexport const peopleObject = { id: { workspace_id: 'workspace-1', object_id: 'object-people' }, api_slug: 'people' }\nconst companiesObject = { id: { workspace_id: 'workspace-1', object_id: 'object-companies' }, api_slug: 'companies' }\nexport const customObject = { id: { workspace_id: 'workspace-1', object_id: 'object-custom' }, api_slug: 'subscriptions' }\nconst salesList = { id: { workspace_id: 'workspace-1', list_id: 'list-sales' }, api_slug: 'sales' }\n\nconst fixtureParents: Record = {\n '/v2/objects/people': peopleObject,\n '/v2/objects/object-people': peopleObject,\n '/v2/objects/companies': companiesObject,\n '/v2/objects/object-companies': companiesObject,\n '/v2/objects/subscriptions': customObject,\n '/v2/objects/object-custom': customObject,\n '/v2/lists/sales': salesList,\n '/v2/lists/list-sales': salesList,\n}\n\nexport const person = {\n id: { workspace_id: 'workspace-1', object_id: 'object-people', record_id: 'record-1' },\n created_at: '2026-01-01T00:00:00Z',\n web_url: 'https://app.attio.com/example/person/record-1',\n values: {\n name: [{ full_name: 'Ada Example', first_name: 'Ada', last_name: 'Example', attribute_type: 'personal-name', active_until: null }],\n email_addresses: [{ email_address: 'ada@example.test', attribute_type: 'email-address' }],\n custom_multi: [{ value: 'one' }, { value: 'two' }],\n },\n}\n\nexport const fixtureCollections: Record = {\n '/v2/objects': [peopleObject, customObject],\n '/v2/objects/object-people/records/query': [person],\n '/v2/objects/object-custom/records/query': [{ id: { ...person.id, object_id: 'object-custom' }, values: { custom_number: [{ value: 7 }] } }],\n '/v2/objects/object-companies/records/query': [],\n '/v2/objects/object-people/attributes': [{ id: { ...peopleObject.id, attribute_id: 'attribute-1' }, api_slug: 'name', type: 'personal-name' }],\n '/v2/objects/object-custom/attributes': [{ id: { ...customObject.id, attribute_id: 'attribute-1' }, api_slug: 'custom_number', type: 'number' }],\n '/v2/objects/object-companies/attributes': [],\n '/v2/lists': [salesList],\n '/v2/lists/list-sales/attributes': [{ id: { workspace_id: 'workspace-1', object_id: 'list-sales', attribute_id: 'attribute-1' }, api_slug: 'stage', type: 'status' }],\n '/v2/lists/list-sales/entries/query': [{ id: { ...salesList.id, entry_id: 'entry-1' }, parent_object: 'people', parent_record_id: 'record-1', entry_values: { stage: [{ status: { title: 'Prospect' } }] } }],\n '/v2/notes': [{ id: { workspace_id: 'workspace-1', note_id: 'note-1' }, parent_object: 'people', parent_record_id: 'record-1', content_markdown: '# First call' }],\n '/v2/tasks': [{ id: { workspace_id: 'workspace-1', task_id: 'task-1' }, content_plaintext: 'Follow up', is_completed: false }],\n '/v2/workspace_members': [{ id: { workspace_id: 'workspace-1', workspace_member_id: 'member-1' }, first_name: 'Grace' }],\n}\n\nexport function fixtureDeps(\n respond: (url: URL, init: RequestInit) => Response | Promise = fixtureResponse,\n): AttioClientDeps {\n return {\n config: attioConfig,\n token: () => 'fixture-token',\n wait: async () => {},\n fetch: (url, init) => Promise.resolve(respond(new URL(url), init)),\n }\n}\n\nexport function fixtureResponse(url: URL): Response {\n const data = fixtureParents[url.pathname] ?? fixtureCollections[url.pathname]\n if (data === undefined) throw new Error(`Unexpected fixture URL: ${url.pathname}`)\n return Response.json({ data })\n}\n" + } + ], + "meta": { + "chkit": { + "formatVersion": 1, + "version": "0.2.1", + "language": "typescript", + "license": "MIT", + "documentation": "https://chkit.obsessiondb.com/integrations/attio/", + "logo": "https://chkit.obsessiondb.com/logos/attio.svg", + "authentication": { + "method": "Bearer token", + "env": [ + "ATTIO_API_TOKEN" + ], + "setup": [ + "As a workspace admin, open the workspace-name dropdown in Attio and select Workspace settings.", + "Open Developers, select + New access token, and name the token (for example, chkit ClickHouse).", + "Grant the read scopes listed for the resources to sync.", + "Click the token on the Developers page to copy it. Set ATTIO_API_TOKEN in the project .env file or the scheduler secret environment." + ], + "documentation": "https://attio.com/help/reference/apps/generating-an-api-key" + }, + "chkit": "^0.2.0-beta.10", + "ingest": "^0.2.0-beta.10", + "clickhouse": ">=25.3.0", + "root": "src/integrations/attio", + "entry": "index.ts", + "exports": [ + "attio", + "attioObjectsRaw", + "attioObjectAttributesRaw", + "attioRecordsRaw", + "attioPeople", + "attioCompanies", + "attioDeals", + "attioListsRaw", + "attioListAttributesRaw", + "attioEntriesRaw", + "attioNotesRaw", + "attioTasksRaw", + "attioMembersRaw" + ], + "resources": [ + { + "name": "objects", + "title": "Objects", + "table": "attio_objects_raw", + "description": "Accessible standard and custom object definitions, IDs, API slug, singular and plural names, creation time", + "scopes": [ + "object_configuration:read" + ], + "strategy": "full", + "endpoints": [ + { + "method": "GET", + "path": "/objects", + "documentation": "https://docs.attio.com/rest-api/endpoint-reference/objects/list-objects" + } + ] + }, + { + "name": "object_attributes", + "title": "Object attributes", + "table": "attio_object_attributes_raw", + "description": "Attribute IDs, slug, title, description, type, flags, defaults, relationship definitions, configuration, creation time; includes archived definitions", + "scopes": [ + "object_configuration:read" + ], + "strategy": "full", + "endpoints": [ + { + "method": "GET", + "path": "/objects/{object_id}/attributes", + "documentation": "https://docs.attio.com/rest-api/endpoint-reference/attributes/list-attributes" + }, + { + "method": "GET", + "path": "/objects/{object_id}", + "documentation": "https://docs.attio.com/rest-api/endpoint-reference/objects/get-an-object" + } + ] + }, + { + "name": "records", + "title": "Records", + "table": "attio_records_raw", + "description": "Independent records streams for configured object types (People and Companies by default), IDs, creation time, Attio URL, and the complete values object with all returned attribute arrays", + "scopes": [ + "object_configuration:read", + "record_permission:read" + ], + "strategy": "full", + "endpoints": [ + { + "method": "POST", + "path": "/objects/{object_id}/records/query", + "documentation": "https://docs.attio.com/rest-api/endpoint-reference/records/list-records" + }, + { + "method": "GET", + "path": "/objects/{object_id}", + "documentation": "https://docs.attio.com/rest-api/endpoint-reference/objects/get-an-object" + } + ] + }, + { + "name": "lists", + "title": "Lists", + "table": "attio_lists_raw", + "description": "List IDs, slug, name, parent objects, workspace and member access configuration, creator, creation time", + "scopes": [ + "list_configuration:read" + ], + "strategy": "full", + "endpoints": [ + { + "method": "GET", + "path": "/lists", + "documentation": "https://docs.attio.com/rest-api/endpoint-reference/lists/list-all-lists" + } + ] + }, + { + "name": "list_attributes", + "title": "List attributes", + "table": "attio_list_attributes_raw", + "description": "Attribute definitions for selected lists, including types, configuration, flags, defaults, relationships, and archived definitions", + "scopes": [ + "list_configuration:read" + ], + "strategy": "full", + "endpoints": [ + { + "method": "GET", + "path": "/lists/{list_id}/attributes", + "documentation": "https://docs.attio.com/rest-api/endpoint-reference/attributes/list-attributes" + }, + { + "method": "GET", + "path": "/lists/{list_id}", + "documentation": "https://docs.attio.com/rest-api/endpoint-reference/lists/get-a-list" + } + ] + }, + { + "name": "entries", + "title": "List entries", + "table": "attio_entries_raw", + "description": "Entry IDs, parent object and record ID, creation time, and the complete entry_values object", + "scopes": [ + "list_configuration:read", + "list_entry:read" + ], + "strategy": "full", + "endpoints": [ + { + "method": "POST", + "path": "/lists/{list_id}/entries/query", + "documentation": "https://docs.attio.com/rest-api/endpoint-reference/entries/list-entries" + }, + { + "method": "GET", + "path": "/lists/{list_id}", + "documentation": "https://docs.attio.com/rest-api/endpoint-reference/lists/get-a-list" + } + ] + }, + { + "name": "notes", + "title": "Notes", + "table": "attio_notes_raw", + "description": "Note IDs, parent record links, title, plaintext and Markdown content, tags, creator, creation time, and meeting reference when returned", + "scopes": [ + "note:read", + "object_configuration:read", + "record_permission:read" + ], + "strategy": "full", + "endpoints": [ + { + "method": "GET", + "path": "/notes", + "documentation": "https://docs.attio.com/rest-api/endpoint-reference/notes/list-notes" + } + ] + }, + { + "name": "tasks", + "title": "Tasks", + "table": "attio_tasks_raw", + "description": "Task IDs, plaintext content, deadline, completion state and time, linked records, assignees, creator, creation time", + "scopes": [ + "task:read", + "object_configuration:read", + "record_permission:read", + "user_management:read" + ], + "strategy": "full", + "endpoints": [ + { + "method": "GET", + "path": "/tasks", + "documentation": "https://docs.attio.com/rest-api/endpoint-reference/tasks/list-tasks" + } + ] + }, + { + "name": "members", + "title": "Workspace members", + "table": "attio_members_raw", + "description": "Member IDs, first and last name, email address, avatar URL, access level, creation time", + "scopes": [ + "user_management:read" + ], + "strategy": "full", + "endpoints": [ + { + "method": "GET", + "path": "/workspace_members", + "documentation": "https://docs.attio.com/rest-api/endpoint-reference/workspace-members/list-workspace-members" + } + ] + } + ], + "views": [ + { + "name": "attio_people", + "source": "records", + "description": "Names, email addresses, and linked companies for people records." + }, + { + "name": "attio_companies", + "source": "records", + "description": "Names, domains, and descriptions for company records." + }, + { + "name": "attio_deals", + "source": "records", + "description": "Deal names, stages, values, currencies, and linked people and companies." + } + ], + "sync": { + "description": "People, Companies, each configured custom object/list, and workspace resources have independent full-sync streams. Each reader paginates one collection through chkit-ingest; interrupted mutable offset reads replay that collection from zero. Completed and empty reads are recorded by the executor journal without a custom scan checkpoint. Custom types and lists must be configured explicitly.", + "schedule": "Run chkit ingest run through cron, CI, or an existing scheduler; the template does not start a background worker.", + "deletions": "Records absent from a later API response remain in ClickHouse. The template does not propagate deletions or consume webhooks." + }, + "env": { + "ATTIO_API_TOKEN": "" + }, + "fileHashes": { + "src/integrations/attio/README.md": "0d4472f92632ed1b12eea5f05622deccf027ae9d488998ec193f01f542397514", + "src/integrations/attio/client.ts": "ae90b76cfa969ea871e589e48a1f22efbf8488a4000ea25af22ac271609cc5bf", + "src/integrations/attio/config.ts": "08688e8b38ceab1c30573cb8ab32e620b19219d080c7e2aa56546d6e4395aed3", + "src/integrations/attio/index.ts": "74be5d3684988a95cef41a53ad44071b3d06000b8adb49bebbf30c3b2698a663", + "src/integrations/attio/pipeline.ts": "338be6e5b14f155d4d67c53daa47a61d844caf98016287f7e66cfd82acd5d436", + "src/integrations/attio/sources/objects.ts": "3262f82c51f3367b169548af28e6e98ca859fb85dc60cfe20816a0352365293d", + "src/integrations/attio/sources/object-attributes.ts": "4c6e543f1f13a1ac9afb10e10be8524efbc817df23f5dab6c3ba47742f3b7178", + "src/integrations/attio/sources/records.ts": "2dd116f2ea2e3476c278153e762adbeecbf6d61952c2e5f69332ab3674fe75b7", + "src/integrations/attio/sources/lists.ts": "89cd338931fb8959fb16c88b872b02229be9737cbf9987ee9d0aa025ea647a88", + "src/integrations/attio/sources/list-attributes.ts": "95f4e2aaac531e05115916c26d8b97973daa358e7679970aec6ac1db16060d93", + "src/integrations/attio/sources/entries.ts": "9329630f494d8cdc0ea2711a49845f09cfc9ab1775a7e39cb381a85e4c5960aa", + "src/integrations/attio/sources/notes.ts": "ca5b71bf24d432ff1967eefa9a1b5feef938ee6b102a06c6a2cc8058d23fb743", + "src/integrations/attio/sources/tasks.ts": "a2d64a19efe7bdeedf0925d7883feb6e931b1b62264f83532c1932c893625e97", + "src/integrations/attio/sources/members.ts": "7ed50b334465e4060ab3ecc4d47c6863c1c03b9940026f8e7089603504b72462", + "src/integrations/attio/tests/attio.test.ts": "9200421e0876f99147b8e7d7a8f7d0e0b2bc4784460d3ba0f67211ca7a38119e", + "src/integrations/attio/tests/fixtures.ts": "5e193c211cf40e8eee5331c5b4649a772294005e5bc878b1f79b7ccdbf390912" + } + } + } +} diff --git a/registry/attio/tests/fixtures.ts b/registry/attio/tests/fixtures.ts index 4d4819f1..19716a93 100644 --- a/registry/attio/tests/fixtures.ts +++ b/registry/attio/tests/fixtures.ts @@ -2,9 +2,9 @@ import type { AttioClientDeps } from '../client.js' import { attioConfig } from '../config.js' export const peopleObject = { id: { workspace_id: 'workspace-1', object_id: 'object-people' }, api_slug: 'people' } -export const companiesObject = { id: { workspace_id: 'workspace-1', object_id: 'object-companies' }, api_slug: 'companies' } +const companiesObject = { id: { workspace_id: 'workspace-1', object_id: 'object-companies' }, api_slug: 'companies' } export const customObject = { id: { workspace_id: 'workspace-1', object_id: 'object-custom' }, api_slug: 'subscriptions' } -export const salesList = { id: { workspace_id: 'workspace-1', list_id: 'list-sales' }, api_slug: 'sales' } +const salesList = { id: { workspace_id: 'workspace-1', list_id: 'list-sales' }, api_slug: 'sales' } const fixtureParents: Record = { '/v2/objects/people': peopleObject, From 5027f8e413e2f7071483ed622c8e85d7e89f20b9 Mon Sep 17 00:00:00 2001 From: "replicas-connector[bot]" Date: Sun, 4 Oct 2026 23:52:38 +0000 Subject: [PATCH 04/18] Keep one integration release per PR and add changelogs MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Consolidate Attio's unmerged releases into one 0.2.0 draft. Add cumulative integration changelog metadata and render it in CLI inspection and guides. Refresh drafts against the merged base and validate one new artifact per integration, immutable published history, and current source agreement. Document the lifecycle in authoring guidance and verify recovery and release protection through fixture and installed-consumer tests. Co-Authored-By: Marc Höffl Co-Authored-By: replicas-connector[bot] --- .changeset/checkpointed-registry-ingestion.md | 2 + .github/workflows/ci.yml | 2 + CLAUDE.md | 6 + .../src/components/RegistryReference.astro | 6 + .../docs/api-sync/registry-authoring.md | 47 ++- apps/docs/src/content/docs/cli/registry.md | 4 +- .../src/content/docs/integrations/attio.mdx | 6 +- .../content/docs/integrations/circleback.mdx | 4 + .../src/content/docs/integrations/github.mdx | 4 + .../docs/integrations/google-calendar.mdx | 4 + .../content/docs/integrations/google-meet.mdx | 4 + .../src/content/docs/integrations/lemlist.mdx | 4 + .../src/content/docs/integrations/linear.mdx | 4 + .../src/content/docs/integrations/slack.mdx | 4 + apps/docs/src/data/registry.ts | 9 +- apps/docs/src/integrations/raw-markdown.ts | 2 +- package.json | 4 +- packages/cli/src/commands/registry.ts | 9 +- packages/cli/src/registry/model.ts | 16 +- .../cli/src/test/registry/artifact.test.ts | 53 +++ packages/cli/src/test/registry/cli.test.ts | 26 ++ registry/attio/README.md | 2 +- registry/attio/manifest.json | 30 +- registry/attio/releases/0.1.3.json | 369 ----------------- registry/attio/releases/0.2.0.json | 36 +- registry/attio/releases/0.2.1.json | 382 ------------------ registry/attio/tests/install.e2e.test.ts | 4 +- registry/circleback/manifest.json | 16 + registry/circleback/releases/0.2.0.json | 16 + registry/github/manifest.json | 16 + registry/github/releases/0.2.0.json | 16 + registry/google-calendar/manifest.json | 16 + registry/google-calendar/releases/0.2.0.json | 16 + registry/google-meet/manifest.json | 16 + registry/google-meet/releases/0.2.0.json | 16 + registry/lemlist/manifest.json | 16 + registry/lemlist/releases/0.2.0.json | 16 + registry/linear/manifest.json | 16 + registry/linear/releases/0.2.0.json | 16 + registry/package.json | 2 +- registry/slack/manifest.json | 22 + registry/slack/releases/0.2.0.json | 22 + registry/slack/tests/install.e2e.test.ts | 4 +- scripts/check-registry-docs.test.ts | 27 +- scripts/check-registry-docs.ts | 7 +- scripts/registry-releases.test.ts | 225 +++++++++++ scripts/registry-releases.ts | 146 +++++++ skills/chkit-ingestion/SKILL.md | 2 + .../chkit-ingestion/references/stable-sync.md | 10 +- 49 files changed, 917 insertions(+), 785 deletions(-) delete mode 100644 registry/attio/releases/0.1.3.json delete mode 100644 registry/attio/releases/0.2.1.json create mode 100644 scripts/registry-releases.test.ts create mode 100644 scripts/registry-releases.ts diff --git a/.changeset/checkpointed-registry-ingestion.md b/.changeset/checkpointed-registry-ingestion.md index 21035a00..57d442ff 100644 --- a/.changeset/checkpointed-registry-ingestion.md +++ b/.changeset/checkpointed-registry-ingestion.md @@ -5,3 +5,5 @@ Allow registry resources to declare cursor checkpoints and timestamp windows alongside full scans. Upgrade the eight raw API integrations with provider-specific sync tokens, overlapping incremental ranges, and safe restart recovery, while preserving historical registry releases. Attio now creates independent configured object-type and list streams using existing full-sync pagination, with People and Companies enabled by default. Update the ingestion authoring skill with the registry sync quality bar, provider capability selection, meaningful stream boundaries, existing pagination/checkpoint primitives, durable recovery rules, and required failure/restart verification. + +Add per-integration changelog metadata, shown in registry inspection and integration guides. Keep one draft release per integration in each PR, refresh it in place until merge, and validate release artifacts against the PR base to protect published versions and changelog history. diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 26104484..8911f652 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -15,6 +15,8 @@ jobs: steps: - name: Checkout uses: actions/checkout@v4 + with: + fetch-depth: 0 - name: Setup Bun uses: ./.github/actions/setup diff --git a/CLAUDE.md b/CLAUDE.md index 7c275d5d..61aec71a 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -101,6 +101,12 @@ A test may skip itself on `CHKIT_E2E_TARGET=obsessiondb` only for a known Obsess Uses changesets. Run `bun run changeset` to create a changeset, then `bun run version-packages` to bump versions. +### Registry integration releases + +Keep at most one new release artifact per integration in a PR. Versions already on the merged base are immutable; artifacts added by the unmerged PR are drafts. Further edits refresh that same draft version and its current `meta.chkit.changelog` entry, without adding intermediate versions. Preserve earlier released changelog entries. + +Regenerate with `bun run registry:release -- --base origin/main ` and validate with `bun run check:registry-releases -- --base origin/main`; use the actual PR base if different. Omit provider names to refresh all drafts. Do not hand-edit generated artifacts. `bun run verify` checks release history and source/artifact agreement. + ### Releasing (GitHub Actions) Versions are semver; all 10 published packages share one version (a changesets `fixed` group). A release takes two steps, so the version reaches git before npm and any failure can be re-run: diff --git a/apps/docs/src/components/RegistryReference.astro b/apps/docs/src/components/RegistryReference.astro index 611c8e0d..afff6585 100644 --- a/apps/docs/src/components/RegistryReference.astro +++ b/apps/docs/src/components/RegistryReference.astro @@ -67,3 +67,9 @@ if (!app) throw new Error(`Unknown registry app: ${name}`); )} +{section === 'changelog' && app.changelog?.map((release) => ( + <> +

    Version {release.version}

    +
      {release.changes.map((change) =>
    • {change}
      • )}
    + +))} diff --git a/apps/docs/src/content/docs/api-sync/registry-authoring.md b/apps/docs/src/content/docs/api-sync/registry-authoring.md index a8451b3b..cca2be22 100644 --- a/apps/docs/src/content/docs/api-sync/registry-authoring.md +++ b/apps/docs/src/content/docs/api-sync/registry-authoring.md @@ -96,7 +96,8 @@ The item name `registry` is reserved because `registry.json` contains the catalo | Field under `meta.chkit` | Meaning | |---|---| | `formatVersion` | Registry metadata format; currently `1` | -| `version` | Immutable semantic version of this template | +| `version` | Semantic version of this template; immutable after publication | +| `changelog` | Newest-first version history with a semantic `version` and nonempty `changes` array per entry; optional for custom and legacy items, required for official current manifests | | `language` | Currently `typescript` | | `license` | License for the copied source | | `documentation` | Optional HTTP(S) URL of the app's integration guide; shown by CLI list and inspect | @@ -112,6 +113,26 @@ The item name `registry` is reserved because `registry.json` contains the catalo Resource strategies describe selection: `full` performs complete scan cycles, including scans with completion checkpoints; `timestamp` selects time windows; `cursor` follows checkpointed provider state, such as a change token or a resumable snapshot traversal. Describe restart behavior, cursor lifetime, and deletion handling in `sync`. Metadata does not configure the runtime strategy. Older strict registry clients accept only `full`; templates advertising new labels must require a CLI version that accepts them. +Keep a cumulative integration changelog in `meta.chkit.changelog`: + +```json +{ + "version": "0.2.0", + "changelog": [ + { + "version": "0.2.0", + "changes": ["Give People and Companies independent ingestion streams."] + }, + { + "version": "0.1.2", + "changes": ["Include fixture tests and expanded authentication and resource metadata."] + } + ] +} +``` + +The first entry matches the template version, and subsequent versions descend without duplicates. Retain notes for earlier releases, including migration details users need when upgrading. CLI inspect, integration guides, and agent-readable Markdown display this same history. Historical artifacts without the optional field remain valid. Older strict clients reject unknown metadata fields; current templates using `changelog` require a supporting CLI version. + Declare npm dependencies as `package@semver-range`. Every item must include `@chkit/core` and `@chkit/plugin-ingest`. Git URLs, local dependencies, and package-manager aliases are outside this format. Use blank values for secrets in `env`. Never include credentials in metadata or source artifacts. State resource limitations in the item's description and copied README, including deletion behavior and inaccessible API families. @@ -207,17 +228,28 @@ Each `sources/` module keeps a resource's reader and schema together. Shared req `manifest.json` contains one registry item. Its source paths are relative to the provider directory (for example, `sources/notes.ts`); target paths still use the full consumer path such as `src/integrations/attio/sources/notes.ts`. The repository's catalog loader discovers these provider-local manifests and aggregates them for the build, CLI artifacts, and documentation. `bun scripts/build-registry.ts` builds the official catalog; it does not require a handwritten catalog at the registry root. -Released artifacts are committed under `registry//releases/.json`. The official CLI reads provider directories and manifests from GitHub and installs those committed release artifacts directly. The documentation build also copies that history into `apps/docs/public/r//` before building the current catalog and latest aliases. History and the source are colocated without installing release files into consumer projects. +Released artifacts are committed under `registry//releases/.json`. The official CLI reads provider directories and manifests from GitHub's `main` branch and installs those release artifacts directly. The documentation build also copies that history into `apps/docs/public/r//` before building the current catalog and latest aliases. History and the source are colocated without installing release files into consumer projects. + +An artifact already present in the PR's merged base is immutable: preserve its exact bytes and path. Each integration contributes at most one new version artifact per PR. Choose its version above the latest merged release when first changing distributed source or metadata; subsequent edits in the same unmerged PR update that draft's source and current changelog entry without another version bump. Intermediate PR revisions do not become separate release artifacts or changelog entries. + +Regenerate the draft at the same version with the repository's release command: + +```sh +bun run registry:release -- --base origin/main attio +bun run check:registry-releases -- --base origin/main +``` + +The base defaults to `origin/main`; use the actual PR base when it differs. Pass provider names to refresh selected integrations, or omit names to refresh all current drafts. The release command regenerates artifacts from source and refuses to replace versions already in the base. The check rejects changed or deleted published artifacts, multiple new versions for one integration, and a current artifact that does not match its source and manifest. The same draft can be regenerated after every edit, including after it has been committed or pushed to the PR branch. Once merged, further template changes require a new version and changelog entry in another PR. -After validating a new release artifact, copy its immutable file into the provider's `releases/` directory and commit it with the corresponding source and version change. Do not hand-edit the artifact or commit generated latest aliases. Adding a provider or template version does not require an npm package release; changes to CLI behavior do. The docs build also publishes a static catalog at `https://chkit.obsessiondb.com/r/registry.json` for web and custom-registry use. +Commit the generated draft with its source, manifest, and changelog updates. Do not hand-edit artifact content or commit generated latest aliases. Adding a provider or template version does not require an npm package release; changes to CLI behavior do. The docs build also publishes a static catalog at `https://chkit.obsessiondb.com/r/registry.json` for web and custom-registry use. ## Add an app to the official documentation -Each provider declared in `registry//manifest.json` has a guide at `apps/docs/src/content/docs/integrations/.md` or `.mdx`. Set its title to `Integrating ClickHouse with ` and write a specific one-sentence description. Give the sidebar a short app label. +Each provider declared in `registry//manifest.json` has an MDX guide at `apps/docs/src/content/docs/integrations/.mdx` so the shared resource and changelog components render. Set its title to `Integrating ClickHouse with ` and write a specific one-sentence description. Give the sidebar a short app label. Set `meta.chkit.documentation` to `https://chkit.obsessiondb.com/integrations//`. Every official app requires a provider logo: store the official asset in `apps/docs/public/logos/`, record its source in that directory's `README.md`, and set `meta.chkit.logo` to its full HTTPS URL on the docs site. Preserve the asset's proportions and brand colors. -Every official manifest includes authentication setup steps, resource titles and destination tables, provider endpoint references, derived views, and sync/deletion metadata. Credential setup must explain where an administrator creates a token in the source system, which permissions it requires, and how the execution environment receives it. Link to the provider's current instructions and verify the UI path before publishing. +Every official manifest includes authentication setup steps, resource titles and destination tables, provider endpoint references, derived views, sync/deletion metadata, and an integration changelog. Credential setup must explain where an administrator creates a token in the source system, which permissions it requires, and how the execution environment receives it. Link to the provider's current instructions and verify the UI path before publishing. Every guide also explains installation, migrations, raw and projected fields, pagination, repeat runs, failure recovery, unsupported data, and scheduling. Verify the claims against the installed readers and schema. A name-swapped introduction alone is not a complete integration guide. @@ -229,9 +261,10 @@ import RegistryReference from '../../../components/RegistryReference.astro'; + ``` -The other sections are `overview`, `views`, and `sync`. The raw-Markdown build expands the same components for agents. The `resources` section documents every declared resource; handwritten guides must include each exact resource name in backticks. Keep the explanation of provider-specific behavior as prose alongside the generated reference tables. +The other sections are `overview`, `views`, and `sync`. The raw-Markdown build expands the same components for agents. The `resources` section documents every declared resource; handwritten guides must include each exact resource name in backticks. Include `changelog` under a Changelog heading in each official integration guide. Keep the explanation of provider-specific behavior as prose alongside the generated reference tables. The [integration list](/integrations/), its agent-readable Markdown, CLI discovery, and search structured data read the same manifest. The docs build checks that every official item has its guide, description, resource coverage, and required local logo asset. New guides also enter site search, the sitemap, and `llms.txt` automatically. @@ -240,7 +273,7 @@ bun run scripts/check-registry-docs.ts bun run --cwd apps/docs build ``` -Preview the app listing and guide in both themes, check the rendered resource tables, and verify the guide appears in `apps/docs/dist/_raw/index.md` and `apps/docs/dist/llms.txt`. Metadata changes require a new immutable registry version just like source changes. +Preview the app listing and guide in both themes, check the rendered resource tables and changelog, and verify the guide appears in `apps/docs/dist/_raw/index.md` and `apps/docs/dist/llms.txt`. Changes to released template metadata require a new version just like source changes; edits to an unmerged draft update its one version artifact. ## Related pages diff --git a/apps/docs/src/content/docs/cli/registry.md b/apps/docs/src/content/docs/cli/registry.md index 48dbe5ba..3a8f6305 100644 --- a/apps/docs/src/content/docs/cli/registry.md +++ b/apps/docs/src/content/docs/cli/registry.md @@ -37,6 +37,8 @@ Loads and validates a built item. In the official GitHub registry, a bare name r Inspection exposes the version, copied files, optional test set, dependency requirements, environment examples, resource coverage, scopes, and compatibility metadata. Rich provider metadata also includes credential setup steps, destination tables, API references, derived views, schedules, and deletion behavior. When declared, documentation and logo URLs are included. Human output labels environment values as `.env.example` defaults, shows each resource's scopes and strategy, and prints an install command preserving the selected registry. It does not install files or execute the provider. +When declared, the changelog lists release versions and their changes, newest first. The CLI and integration guide use the same `meta.chkit.changelog` history. Inspecting a pinned version shows the history bundled with that release; older or custom items without changelog metadata remain readable. + ### Build Reads the source catalog and each declared source file relative to the manifest directory. Validation covers the supported item format, safe paths, unique targets, required dependencies, and explicit entry exports. The builder embeds source content, computes file hashes, and emits: @@ -114,7 +116,7 @@ chkit registry list --json | jq '{command, schemaVersion, ok, action, items: [.i } ``` -The `resources` array preserves the manifest's resource descriptions, scopes, strategies, and optional titles, destination tables, and API endpoints. `inspect --json` includes provider authentication, derived views, and sync behavior under `item.meta.chkit` when declared. `documentation` and `logo` are omitted for older or custom items without those optional fields. +The `resources` array preserves the manifest's resource descriptions, scopes, strategies, and optional titles, destination tables, and API endpoints. `inspect --json` includes provider authentication, derived views, sync behavior, and versioned `changelog` entries under `item.meta.chkit` when declared. `documentation`, `logo`, and `changelog` are omitted for older or custom items without those optional fields. **Inspect:** returns `action: "inspect"`, the resolved `origin`, and the full built `item`, including source contents and `meta.chkit`. To extract only the item identity: diff --git a/apps/docs/src/content/docs/integrations/attio.mdx b/apps/docs/src/content/docs/integrations/attio.mdx index 9a4406aa..ee79e44a 100644 --- a/apps/docs/src/content/docs/integrations/attio.mdx +++ b/apps/docs/src/content/docs/integrations/attio.mdx @@ -188,7 +188,7 @@ Every stream uses `fullSync()` and begins its next execution at offset zero. Att Child response workspace and parent IDs are checked against the reader's resolved object/list before rows are published. Adding a configured type creates a new stream; removing it stops future reads without removing existing rows. Every scheduled run rereads the selected collections, so edits remain observable. -Version `0.2.0` changes the former combined records, object-attributes, entries, and list-attributes stream IDs to per-type/per-list IDs. Those streams begin with fresh journal progress; the raw table schemas, row IDs for the same `sourceId`, and views are unchanged. The `0.1.3` release artifact remains available. When upgrading from its checkpointed-scan preview, use a new `sourceId` to avoid reusing the old custom strategy for unchanged workspace stream IDs. A new source ID also changes raw row IDs, so review retained observations as part of that migration. Installed files are not automatically overwritten by a newer registry release. +Upgrading from `0.1.2` to `0.2.0` changes the former combined records, object-attributes, entries, and list-attributes stream IDs to per-type/per-list IDs. Those streams begin with fresh journal progress; the raw table schemas, row IDs for the same `sourceId`, and views are unchanged. Installed files are not automatically overwritten by a newer registry release. The destination stores the latest **observed** entity. New observations replace earlier observations for the same row identity when ClickHouse reconciles versions. Freshness depends on the external schedule, scan duration, and successful API responses. There are no webhooks or continuous change capture in this integration. @@ -435,6 +435,10 @@ The tests use fixture responses, a mocked HTTP client, and an in-memory destinat Repository-only packaging and live-database tests remain outside the installed test set. The manifest selects the portable tests explicitly. +## Changelog + + + ## Related pages - [Integration list](/integrations/): available apps with dedicated ClickHouse integration guides. diff --git a/apps/docs/src/content/docs/integrations/circleback.mdx b/apps/docs/src/content/docs/integrations/circleback.mdx index 0655591d..484db163 100644 --- a/apps/docs/src/content/docs/integrations/circleback.mdx +++ b/apps/docs/src/content/docs/integrations/circleback.mdx @@ -42,6 +42,10 @@ bunx chkit add circleback --with-tests bun test src/integrations/circleback/tests/basic.test.ts ``` +## Changelog + + + ## Related pages - [App registry](/integrations/) diff --git a/apps/docs/src/content/docs/integrations/github.mdx b/apps/docs/src/content/docs/integrations/github.mdx index 062e4983..8d29c9e7 100644 --- a/apps/docs/src/content/docs/integrations/github.mdx +++ b/apps/docs/src/content/docs/integrations/github.mdx @@ -41,6 +41,10 @@ bunx chkit add github --with-tests bun test src/integrations/github/tests/basic.test.ts ``` +## Changelog + + + ## Related pages - [App registry](/integrations/) diff --git a/apps/docs/src/content/docs/integrations/google-calendar.mdx b/apps/docs/src/content/docs/integrations/google-calendar.mdx index bcbe1dd3..e9e14c61 100644 --- a/apps/docs/src/content/docs/integrations/google-calendar.mdx +++ b/apps/docs/src/content/docs/integrations/google-calendar.mdx @@ -46,6 +46,10 @@ bun test src/integrations/google-calendar/tests/basic.test.ts Fixtures cover page resumption, empty deltas, sink failures before token promotion, cancellation payloads, expired tokens, and calendar-account changes. Schedule repeat runs externally with one ingestion process per destination at a time. +## Changelog + + + ## Related pages - [App registry](/integrations/) diff --git a/apps/docs/src/content/docs/integrations/google-meet.mdx b/apps/docs/src/content/docs/integrations/google-meet.mdx index 4993d5f1..35a11659 100644 --- a/apps/docs/src/content/docs/integrations/google-meet.mdx +++ b/apps/docs/src/content/docs/integrations/google-meet.mdx @@ -58,6 +58,10 @@ bun test src/integrations/google-meet/tests/basic.test.ts Fixtures cover nested entry resumption, late transcripts from retained parents, fixed end-time and ongoing discovery, rejected tokens, sink failures, isolated backfill bounds, and expiry gaps. +## Changelog + + + ## Related pages - [App registry](/integrations/) diff --git a/apps/docs/src/content/docs/integrations/lemlist.mdx b/apps/docs/src/content/docs/integrations/lemlist.mdx index b6501c0d..b5dcf5b0 100644 --- a/apps/docs/src/content/docs/integrations/lemlist.mdx +++ b/apps/docs/src/content/docs/integrations/lemlist.mdx @@ -43,6 +43,10 @@ bunx chkit add lemlist --with-tests bun test src/integrations/lemlist/tests/basic.test.ts ``` +## Changelog + + + ## Related pages - [App registry](/integrations/) diff --git a/apps/docs/src/content/docs/integrations/linear.mdx b/apps/docs/src/content/docs/integrations/linear.mdx index e22ff9d2..95f1e2c6 100644 --- a/apps/docs/src/content/docs/integrations/linear.mdx +++ b/apps/docs/src/content/docs/integrations/linear.mdx @@ -43,6 +43,10 @@ bunx chkit add linear --with-tests bun test src/integrations/linear/tests/basic.test.ts ``` +## Changelog + + + ## Related pages - [App registry](/integrations/) diff --git a/apps/docs/src/content/docs/integrations/slack.mdx b/apps/docs/src/content/docs/integrations/slack.mdx index 8ba97edb..fd181ee5 100644 --- a/apps/docs/src/content/docs/integrations/slack.mdx +++ b/apps/docs/src/content/docs/integrations/slack.mdx @@ -294,6 +294,10 @@ bun test src/integrations/slack/tests/slack.test.ts src/integrations/slack/tests The optional tests mock Slack responses and use an in-memory destination. They need Bun, with no Slack token or ClickHouse server. Adjust the test path for a custom `--path`. Installed files belong to the project; reinstalling does not overwrite local edits or merge newer integration versions automatically. +## Changelog + + + ## Related pages - [Integration list](/integrations/): available apps and dedicated sync guides. diff --git a/apps/docs/src/data/registry.ts b/apps/docs/src/data/registry.ts index 7c25d35c..2d876a15 100644 --- a/apps/docs/src/data/registry.ts +++ b/apps/docs/src/data/registry.ts @@ -6,7 +6,7 @@ import type { RegistryCatalog } from '../../../../packages/cli/src/registry/mode // Read it as build-time data so source analysis does not require generated files. const registry: RegistryCatalog = JSON.parse(readFileSync(resolve('public/r/registry.json'), 'utf8')); -export type RegistryReferenceSection = 'overview' | 'authentication' | 'scopes' | 'resources' | 'views' | 'sync'; +export type RegistryReferenceSection = 'overview' | 'authentication' | 'scopes' | 'resources' | 'views' | 'sync' | 'changelog'; export function registryStrategyLabel(strategy: 'full' | 'timestamp' | 'cursor'): string { return { full: 'Full scans', timestamp: 'Timestamp windows', cursor: 'Cursor checkpoints' }[strategy]; @@ -27,6 +27,7 @@ export const registryApps = registry.items.map((item) => ({ authentication: item.meta.chkit.authentication, views: item.meta.chkit.views, sync: item.meta.chkit.sync, + changelog: item.meta.chkit.changelog, clickhouse: item.meta.chkit.clickhouse, language: item.meta.chkit.language, strategies: [...new Set(item.meta.chkit.resources.map((resource) => resource.strategy))], @@ -90,6 +91,12 @@ export function registryReferenceMarkdown(name: string, section: RegistryReferen ['Schedule', app.sync.schedule], ['Deletions', app.sync.deletions], ]) : ''; + case 'changelog': + return (app.changelog ?? []).map((release) => [ + `### Version ${release.version}`, + '', + ...release.changes.map((change) => `- ${change}`), + ].join('\n')).join('\n\n'); } } diff --git a/apps/docs/src/integrations/raw-markdown.ts b/apps/docs/src/integrations/raw-markdown.ts index 2a30a6a8..9870f84a 100644 --- a/apps/docs/src/integrations/raw-markdown.ts +++ b/apps/docs/src/integrations/raw-markdown.ts @@ -74,7 +74,7 @@ function expandRegistryReferences(source: string): string { .map((part, index) => index % 2 === 1 ? part : part .replace(/^import Registry(?:Apps|Reference) from .+;\n/gm, '') .replace('', () => registryMarkdown()) - .replace(//g, (_match, name: string, section: RegistryReferenceSection) => registryReferenceMarkdown(name, section))) + .replace(//g, (_match, name: string, section: RegistryReferenceSection) => registryReferenceMarkdown(name, section))) .join(''); } diff --git a/package.json b/package.json index 41438d80..81e5c44e 100644 --- a/package.json +++ b/package.json @@ -24,6 +24,7 @@ "docs:dev": "bun run --cwd apps/docs dev", "docs:build": "bun run --cwd apps/docs build", "registry:build": "bun run scripts/build-registry.ts", + "registry:release": "bun run scripts/registry-releases.ts refresh", "typecheck": "turbo run typecheck", "lint": "turbo run lint", "test": "turbo run test", @@ -42,11 +43,12 @@ "fallow:dead-code": "bunx fallow --only dead-code", "fallow:pr": "bunx fallow --changed-since main --summary || true", "fallow:ci": "bunx fallow --fail-on-issues", - "verify": "bun run check:workspace-deps && turbo run typecheck lint test build && bun run check:packed-deps", + "verify": "bun run check:workspace-deps && bun run check:registry-releases && turbo run typecheck lint test build && bun run check:packed-deps", "changeset": "changeset", "version-packages": "changeset version", "check:examples": "bun run scripts/check-examples-manifest.ts", "check:workspace-deps": "bun run scripts/check-workspace-deps.ts", + "check:registry-releases": "bun run scripts/registry-releases.ts check", "check:packed-deps": "bun run scripts/check-packed-deps.ts", "clean": "turbo run clean", "chkit": "bun run ./packages/cli/src/bin/chkit.ts", diff --git a/packages/cli/src/commands/registry.ts b/packages/cli/src/commands/registry.ts index 5fc1e779..d93150d4 100644 --- a/packages/cli/src/commands/registry.ts +++ b/packages/cli/src/commands/registry.ts @@ -128,6 +128,13 @@ export async function cmdRegistry(command: 'add' | 'registry', argv: string[]): console.log(` Schedule: ${sync.schedule}`) console.log(` Deletions: ${sync.deletions}`) } + if (item.meta.chkit.changelog?.length) { + console.log('\nChangelog:') + for (const release of item.meta.chkit.changelog) { + console.log(` ${release.version}`) + for (const change of release.changes) console.log(` - ${change}`) + } + } console.log('\nFiles:') for (const file of item.files) console.log(` ${file.target}${file.role === 'test' ? ' (optional; --with-tests)' : ''}`) } @@ -163,7 +170,7 @@ function registryHelp(command: 'add' | 'registry'): string { if (command === 'registry') return `chkit registry [options] list Browse apps, sync resources, and integration guides - inspect