diff --git a/lib/linkedin/sync-accepted.ts b/lib/linkedin/sync-accepted.ts index 7c9dd6d1..7959a722 100644 --- a/lib/linkedin/sync-accepted.ts +++ b/lib/linkedin/sync-accepted.ts @@ -2,44 +2,11 @@ import type { Page } from "playwright"; import { getDb } from "@/lib/db"; import { getSessionPage, saveSessionState, markNeedsReauth } from "@/lib/linkedin/session"; -/** - * Accepted-connection sync via the authoritative Voyager connections API. - * - * Replaces the old absence-inference (scroll the SENT list, treat a vanished - * invite as "accepted"). That was measurably wrong: of 1,600 contacts it had - * marked degree=1, only 325 were genuinely connected — 1,275 were phantoms - * (invites that expired/withdrew/were ignored, not accepted). See docs §19. - * - * Sources — all proved against prod (Jul 2026), NO scrolling: - * - DATA: GET /voyager/api/relationships/dash/connections - * ?decorationId=…ConnectionListWithProfile-16&q=search - * &sortType=RECENTLY_ADDED&start=N&count=100 - * → data["*elements"] (newest-first) + included[]: Connection { createdAt(ms), - * connectedMember } and Profile { publicIdentifier (vanity), entityUrn }. - * The API paginates to the end (30 pages for ~2947) in ~60s. It does NOT - * expose a grand total. - * - TOTAL (checksum): the connections PAGE header " connections"

. Read - * once per full pass via a single navigation (NO scroll). A full pass is - * "verified complete" only when unique-pulled == declared total (matched ±0 - * in prod). Presence in the list = 100% proof of a 1st-degree connection. - * - * Behaviour: - * - First run (boundary NULL) → FULL pass: page to the end, stamp degree=1 + - * connected_at from createdAt for every matched contact. If the pass is - * checksum-verified complete, ALSO un-mark phantom degree=1 contacts whose - * vanity is absent from the authoritative list (resets degree/connected_at). - * - Later runs → incremental: page from start=0 and STOP at the first - * connection older than the stored boundary (minus a 24h overlap margin). - * Incremental/incomplete passes are ADD-ONLY — they never un-mark (a partial - * pull must not wipe real accepts). Un-marking happens ONLY on a verified - * full pass. - * - Reuses the runner's shared browser (getSessionPage) — never a 2nd browser. - */ - +/** Shared manual/scheduled detector. Only positive connections-list evidence marks + * acceptance; absence never implies acceptance or removes a connection. */ const ACCEPTED_SYNC_INTERVAL_MS = 8 * 60 * 60 * 1000; // 8h — 3x per day const PAGE_SIZE = 100; const MAX_PAGES = 60; // safety cap (60 * 100 = 6000) -const OVERLAP_MARGIN_MS = 24 * 60 * 60 * 1000; // re-check a day of overlap (idempotent) const DECORATION = "com.linkedin.voyager.dash.deco.web.mynetwork.ConnectionListWithProfile-16"; export function shouldSyncAccepted(accountId: string): boolean { @@ -62,15 +29,8 @@ export async function syncAcceptedConnections(accountId: string): Promise { const m = document.body.innerText.match(/([\d.,]+)\s+connections?/i); return m ? parseInt(m[1].replace(/[.,]/g, ""), 10) : null; }); - const findByVanity = db.prepare( - `SELECT id, full_name, connected_at, degree FROM targets - WHERE linkedin_url LIKE ? AND connection_requested_at IS NOT NULL` - ); + const waiting = db.prepare(`SELECT DISTINCT t.id, t.linkedin_url, t.connected_at, t.degree + FROM targets t JOIN run_profiles rp ON rp.target_id=t.id + JOIN runs r ON r.id=rp.run_id WHERE r.account_id=? + AND t.connection_requested_at IS NOT NULL`).all(accountId) as Array<{ + id: string; + linkedin_url: string; + connected_at: string | null; + degree: number | null; + }>; const stampAccepted = db.prepare( "UPDATE targets SET degree = 1, connected_at = COALESCE(connected_at, ?) WHERE id = ?" ); - const seenVanities = new Set(); // full-pass phantom check - let uniquePulled = 0; + const seenVanities = new Set(); let newestSeen: number | null = null; - let reachedBoundary = false; + let reachedEnd = false; for (let pageIdx = 0; pageIdx < MAX_PAGES; pageIdx++) { const conns = await fetchConnectionsPage(page, pageIdx * PAGE_SIZE, PAGE_SIZE); if (conns === null) { - console.warn(`[sync-accepted] connections API failed at start=${pageIdx * PAGE_SIZE} — stopping`); - break; + throw new Error(`LinkedIn connections API unavailable at offset ${pageIdx * PAGE_SIZE}; verification incomplete`); } - if (conns.length === 0) break; // end of list + if (conns.length === 0) { + reachedEnd = true; + break; + } // end of list for (const c of conns) { - uniquePulled++; - if (c.vanity) seenVanities.add(c.vanity); + if (c.vanity) seenVanities.add(c.vanity.toLowerCase()); if (newestSeen === null || c.createdAt > newestSeen) newestSeen = c.createdAt; - - // Incremental early-exit (list is newest-first). - if (stopBefore !== null && c.createdAt < stopBefore) { - reachedBoundary = true; - break; - } if (!c.vanity) continue; - for (const m of findByVanity.all(`%/in/${c.vanity}/%`) as Array<{ - id: string; full_name: string | null; connected_at: string | null; degree: number | null; - }>) { + for (const m of waiting.filter(t => profileVanity(t.linkedin_url) === c.vanity?.toLowerCase())) { if (m.degree === 1 && m.connected_at) continue; // already correct stampAccepted.run(msToSqlite(c.createdAt), m.id); - console.log(`[sync-accepted] Accepted: ${m.full_name ?? c.vanity}`); + console.log(`[sync-accepted] Accepted: ${c.vanity}`); stamped++; + m.degree = 1; + m.connected_at = msToSqlite(c.createdAt); } } - if (reachedBoundary) break; + // Do not assume upstream pages are globally sorted by connection date. + if (waiting.every(t => t.degree === 1 && t.connected_at || seenVanities.has(profileVanity(t.linkedin_url) || ''))) { + reachedEnd = true; + break; + } await page.waitForTimeout(900 + Math.random() * 700); // gentle, API-only } - // Completeness checksum: a full pass is trustworthy only if what we pulled - // matches LinkedIn's own declared total (±5 slack for live churn). - const verifiedComplete = - isFullPass && declaredTotal !== null && Math.abs(uniquePulled - declaredTotal) <= 5; - - // Correction: ONLY on a verified-complete full pass, un-mark phantom - // degree=1 contacts (present nowhere in the authoritative list). Never on an - // incremental/incomplete pass — a partial pull must not wipe real accepts. - let unmarked = 0; - if (verifiedComplete) { - const deg1 = db.prepare( - "SELECT id, full_name, linkedin_url FROM targets WHERE degree = 1 AND linkedin_url LIKE '%/in/%'" - ).all() as Array<{ id: string; full_name: string | null; linkedin_url: string }>; - const unmark = db.prepare("UPDATE targets SET degree = NULL, connected_at = NULL WHERE id = ?"); - const tx = db.transaction((rows: typeof deg1) => { - for (const t of rows) { - const mm = t.linkedin_url.match(/\/in\/([^/?#]+)/); - const v = mm ? decodeURIComponent(mm[1]).toLowerCase() : null; - if (!v || !seenVanities.has(v)) { - unmark.run(t.id); - unmarked++; - } - } - }); - tx(deg1); - console.log(`[sync-accepted] Verified full pass (pulled ${uniquePulled} == declared ${declaredTotal}). Un-marked ${unmarked} phantom degree=1.`); - } else if (isFullPass) { - console.warn(`[sync-accepted] Full pass NOT verified complete (pulled ${uniquePulled}, declared ${declaredTotal}) — add-only, no un-marking.`); - } - + if (!reachedEnd) + throw new Error("Connection verification reached the page limit; verification incomplete"); // Advance the boundary to the newest connection seen this run. if (newestSeen !== null) { db.prepare("UPDATE accounts SET connections_synced_through_ms = ? WHERE id = ?").run(newestSeen, accountId); @@ -170,7 +105,8 @@ export async function syncAcceptedConnections(accountId: string): Promise { - for (const target of waiting) { - const match = target.linkedin_url?.match(/\/in\/([^/?#]+)/); - if (!match) continue; // no /in/ URL — can't check - const vanity = match[1].toLowerCase(); - - if (!stillPending.has(vanity)) { - // Not in the pending list anymore → accepted (or expired, but treat as accepted) - markAccepted.run(now, target.id); - accepted.push(target.full_name ?? vanity); - } else { - skipped.push(target.full_name ?? vanity); - } - } - })(); - - return res.json({ - pending_on_linkedin: stillPending.size, - waiting_in_db: waiting.length, - newly_accepted: accepted.length, - still_pending: skipped.length, - accepted_names: accepted, - }); - - } catch (err) { - console.error("[sync-accepted]", err); - return res.status(500).json({ error: err instanceof Error ? err.message : "Sync failed" }); - } finally { - await page?.close().catch(() => {}); + const count = await syncAcceptedConnections(id); + return res.json({ newly_accepted: count, source: "connections-list" }); + } catch { + return res.status(502).json({ error: "Connection verification failed or was incomplete. Check account authentication and retry verification; no outreach was started." }); } } diff --git a/tests/ACCEPTANCE-SYNC.md b/tests/ACCEPTANCE-SYNC.md new file mode 100644 index 00000000..509cf0d1 --- /dev/null +++ b/tests/ACCEPTANCE-SYNC.md @@ -0,0 +1,16 @@ +# Acceptance synchronization + +After `npm ci`, run `node tests/accepted-sync.cjs`. +The fixture uses real in-memory SQLite and a stubbed LinkedIn session. No account +cookies, network, invitations, or messages are used. + +Coverage: exact canonical profile matching; account-scoped enrollment; an older +connection on a later page despite a newer stored boundary; absent profiles left +unchanged; historical connection date preservation; idempotent rechecks; and API +failure without a successful-sync timestamp. + +The manual endpoint and scheduled runner share the same detector. Only positive +connections-list evidence updates a target. Positive results can remain if a +later page fails; incomplete scans do not advance accepted_sync_at. The existing +8-hour polling interval and runner scheduling conditions are unchanged. A stored +historical connection date is not proof that the current campaign caused it. diff --git a/tests/accepted-sync.cjs b/tests/accepted-sync.cjs new file mode 100644 index 00000000..2cf73126 --- /dev/null +++ b/tests/accepted-sync.cjs @@ -0,0 +1,35 @@ +const fs = require('fs'), ts = require('typescript'), Module = require('module'), assert = require('node:assert/strict'), DB = require('better-sqlite3'); +(async () => { + let passed = 0; + for (const failure of [false, true]) { + const db = new DB(':memory:'); + db.exec(`CREATE TABLE accounts(id TEXT,connections_synced_through_ms INTEGER,accepted_sync_at TEXT,li_connections INTEGER);INSERT INTO accounts(id,connections_synced_through_ms) VALUES('a',1900000000000);CREATE TABLE targets(id TEXT,linkedin_url TEXT,connection_requested_at TEXT,connected_at TEXT,degree INTEGER);CREATE TABLE runs(id TEXT,account_id TEXT);CREATE TABLE run_profiles(target_id TEXT,run_id TEXT);INSERT INTO runs VALUES('r','a'),('other','b');INSERT INTO targets VALUES('match','https://www.linkedin.com/in/synthetic-person', 'sent',NULL,NULL),('foreign','https://www.linkedin.com/in/synthetic-person/','sent',NULL,NULL),('absent','https://www.linkedin.com/in/absent/','sent',NULL,NULL);INSERT INTO run_profiles VALUES('match','r'),('foreign','other'),('absent','r');`); + let pages = 0; + const page = { goto: async () => { }, waitForTimeout: async () => { }, url: () => 'https://www.linkedin.com/mynetwork/invite-connect/connections/', close: async () => { }, evaluate: async (fn, args) => !args ? null : failure ? null : pages++ === 0 ? [{ vanity: 'unrelated', createdAt: 1789751000000 }] : pages === 2 ? [{ vanity: 'synthetic-person', createdAt: 1658235047000 }] : [] }; + const m = new Module('/app/sync-fixture.cjs'); + m.require = n => n === '@/lib/db' ? { getDb: () => db } : n === '@/lib/linkedin/session' ? { getSessionPage: async () => page, saveSessionState: async () => { }, markNeedsReauth: async () => { } } : require(n); + m._compile(ts.transpileModule(fs.readFileSync(require('path').join(__dirname, '../lib/linkedin/sync-accepted.ts'), 'utf8'), { compilerOptions: { module: ts.ModuleKind.CommonJS, target: ts.ScriptTarget.ES2022 } }).outputText, '/app/sync-fixture.cjs'); + assert.equal(m.exports.profileVanity('https://www.linkedin.com/in/Synthetic-person/?x=y'), 'synthetic-person'); + assert.equal(m.exports.profileVanity('https://evil.test/in/synthetic-person'), null); + passed += 2; + if (failure) { + await assert.rejects(m.exports.syncAcceptedConnections('a'), /unavailable/); + assert.equal(db.prepare('SELECT accepted_sync_at FROM accounts').get().accepted_sync_at, null); + passed += 2; + } + else { + assert.equal(await m.exports.syncAcceptedConnections('a'), 1); + assert.equal(db.prepare("SELECT degree FROM targets WHERE id='match'").get().degree, 1); + assert.equal(db.prepare("SELECT degree FROM targets WHERE id='foreign'").get().degree, null); + assert.equal(db.prepare("SELECT degree FROM targets WHERE id='absent'").get().degree, null); + passed += 4; + assert.equal(db.prepare("SELECT connected_at FROM targets WHERE id='match'").get().connected_at, '2022-07-19 12:50:47'); + assert.ok(db.prepare('SELECT accepted_sync_at FROM accounts').get().accepted_sync_at); + pages = 0; + assert.equal(await m.exports.syncAcceptedConnections('a'), 0); + passed += 3; + } + db.close(); + } + console.log(JSON.stringify({ passed, network: 'none', synthetic: true })); +})().catch(e => { console.error(e); process.exitCode = 1; });