From c5d0cd85b099a770797eb8e9193bfdf038299e37 Mon Sep 17 00:00:00 2001 From: TheGuyWithoutH Date: Sun, 20 Sep 2026 16:31:27 +0200 Subject: [PATCH 1/2] Share positive-evidence connection acceptance synchronization --- lib/linkedin/sync-accepted.ts | 408 ++++++++++------------- pages/api/accounts/[id]/sync-accepted.ts | 86 +---- tests/ACCEPTANCE-SYNC.md | 16 + tests/accepted-sync.cjs | 35 ++ 4 files changed, 246 insertions(+), 299 deletions(-) create mode 100644 tests/ACCEPTANCE-SYNC.md create mode 100644 tests/accepted-sync.cjs diff --git a/lib/linkedin/sync-accepted.ts b/lib/linkedin/sync-accepted.ts index 7c9dd6d1..1dfd5471 100644 --- a/lib/linkedin/sync-accepted.ts +++ b/lib/linkedin/sync-accepted.ts @@ -1,256 +1,202 @@ import type { Page } from "playwright"; import { getDb } from "@/lib/db"; import { getSessionPage, saveSessionState, markNeedsReauth } from "@/lib/linkedin/session"; - -/** - * Accepted-connection sync via the authoritative Voyager connections API. - * - * Replaces the old absence-inference (scroll the SENT list, treat a vanished - * invite as "accepted"). That was measurably wrong: of 1,600 contacts it had - * marked degree=1, only 325 were genuinely connected — 1,275 were phantoms - * (invites that expired/withdrew/were ignored, not accepted). See docs §19. - * - * Sources — all proved against prod (Jul 2026), NO scrolling: - * - DATA: GET /voyager/api/relationships/dash/connections - * ?decorationId=…ConnectionListWithProfile-16&q=search - * &sortType=RECENTLY_ADDED&start=N&count=100 - * → data["*elements"] (newest-first) + included[]: Connection { createdAt(ms), - * connectedMember } and Profile { publicIdentifier (vanity), entityUrn }. - * The API paginates to the end (30 pages for ~2947) in ~60s. It does NOT - * expose a grand total. - * - TOTAL (checksum): the connections PAGE header " connections"

. Read - * once per full pass via a single navigation (NO scroll). A full pass is - * "verified complete" only when unique-pulled == declared total (matched ±0 - * in prod). Presence in the list = 100% proof of a 1st-degree connection. - * - * Behaviour: - * - First run (boundary NULL) → FULL pass: page to the end, stamp degree=1 + - * connected_at from createdAt for every matched contact. If the pass is - * checksum-verified complete, ALSO un-mark phantom degree=1 contacts whose - * vanity is absent from the authoritative list (resets degree/connected_at). - * - Later runs → incremental: page from start=0 and STOP at the first - * connection older than the stored boundary (minus a 24h overlap margin). - * Incremental/incomplete passes are ADD-ONLY — they never un-mark (a partial - * pull must not wipe real accepts). Un-marking happens ONLY on a verified - * full pass. - * - Reuses the runner's shared browser (getSessionPage) — never a 2nd browser. - */ - +/** Shared manual/scheduled detector. Only positive connections-list evidence marks + * acceptance; absence never implies acceptance or removes a connection. */ const ACCEPTED_SYNC_INTERVAL_MS = 8 * 60 * 60 * 1000; // 8h — 3x per day const PAGE_SIZE = 100; const MAX_PAGES = 60; // safety cap (60 * 100 = 6000) -const OVERLAP_MARGIN_MS = 24 * 60 * 60 * 1000; // re-check a day of overlap (idempotent) const DECORATION = "com.linkedin.voyager.dash.deco.web.mynetwork.ConnectionListWithProfile-16"; - export function shouldSyncAccepted(accountId: string): boolean { - const db = getDb(); - const row = db.prepare("SELECT accepted_sync_at FROM accounts WHERE id = ?").get(accountId) as - | { accepted_sync_at: string | null } - | undefined; - if (!row?.accepted_sync_at) return true; - return Date.now() - new Date(row.accepted_sync_at).getTime() >= ACCEPTED_SYNC_INTERVAL_MS; + const db = getDb(); + const row = db.prepare("SELECT accepted_sync_at FROM accounts WHERE id = ?").get(accountId) as { + accepted_sync_at: string | null; + } | undefined; + if (!row?.accepted_sync_at) + return true; + return Date.now() - new Date(row.accepted_sync_at).getTime() >= ACCEPTED_SYNC_INTERVAL_MS; } - interface ApiConnection { - vanity: string | null; - createdAt: number; // epoch ms + vanity: string | null; + createdAt: number; // epoch ms } - export async function syncAcceptedConnections(accountId: string): Promise { - const db = getDb(); - const page = await getSessionPage(accountId); - let stamped = 0; - - try { - const boundaryRow = db.prepare("SELECT connections_synced_through_ms FROM accounts WHERE id = ?").get(accountId) as - | { connections_synced_through_ms: number | null } - | undefined; - const boundary = boundaryRow?.connections_synced_through_ms ?? null; - const isFullPass = boundary === null; - const stopBefore = boundary === null ? null : boundary - OVERLAP_MARGIN_MS; - - // Read the declared total from the connections PAGE header (single nav, NO - // scroll). This is also our login-wall check and the completeness checksum. - await page.goto("https://www.linkedin.com/mynetwork/invite-connect/connections/", { - waitUntil: "domcontentloaded", - timeout: 35000, - }); - await page.waitForTimeout(3500 + Math.random() * 1500); - if (/\/login|\/authwall|\/checkpoint|\/uas\//.test(page.url())) { - console.warn(`[sync-accepted] Session looks logged out (${page.url()}) — skipping`); - return 0; - } - const declaredTotal = await page.evaluate(() => { - const m = document.body.innerText.match(/([\d.,]+)\s+connections?/i); - return m ? parseInt(m[1].replace(/[.,]/g, ""), 10) : null; - }); - - const findByVanity = db.prepare( - `SELECT id, full_name, connected_at, degree FROM targets - WHERE linkedin_url LIKE ? AND connection_requested_at IS NOT NULL` - ); - const stampAccepted = db.prepare( - "UPDATE targets SET degree = 1, connected_at = COALESCE(connected_at, ?) WHERE id = ?" - ); - - const seenVanities = new Set(); // full-pass phantom check - let uniquePulled = 0; - let newestSeen: number | null = null; - let reachedBoundary = false; - - for (let pageIdx = 0; pageIdx < MAX_PAGES; pageIdx++) { - const conns = await fetchConnectionsPage(page, pageIdx * PAGE_SIZE, PAGE_SIZE); - if (conns === null) { - console.warn(`[sync-accepted] connections API failed at start=${pageIdx * PAGE_SIZE} — stopping`); - break; - } - if (conns.length === 0) break; // end of list - - for (const c of conns) { - uniquePulled++; - if (c.vanity) seenVanities.add(c.vanity); - if (newestSeen === null || c.createdAt > newestSeen) newestSeen = c.createdAt; - - // Incremental early-exit (list is newest-first). - if (stopBefore !== null && c.createdAt < stopBefore) { - reachedBoundary = true; - break; + const db = getDb(); + const page = await getSessionPage(accountId); + let stamped = 0; + try { + // Read the declared total from the connections PAGE header (single nav, NO + // scroll). This is also our login-wall check and an informational account count. + await page.goto("https://www.linkedin.com/mynetwork/invite-connect/connections/", { + waitUntil: "domcontentloaded", + timeout: 35000, + }); + await page.waitForTimeout(3500 + Math.random() * 1500); + if (/\/login|\/authwall|\/checkpoint|\/uas\//.test(page.url())) { + console.warn(`[sync-accepted] Session looks logged out (${page.url()}) — skipping`); + throw new Error("LinkedIn login or verification required"); } - if (!c.vanity) continue; - - for (const m of findByVanity.all(`%/in/${c.vanity}/%`) as Array<{ - id: string; full_name: string | null; connected_at: string | null; degree: number | null; - }>) { - if (m.degree === 1 && m.connected_at) continue; // already correct - stampAccepted.run(msToSqlite(c.createdAt), m.id); - console.log(`[sync-accepted] Accepted: ${m.full_name ?? c.vanity}`); - stamped++; + const declaredTotal = await page.evaluate(() => { + const m = document.body.innerText.match(/([\d.,]+)\s+connections?/i); + return m ? parseInt(m[1].replace(/[.,]/g, ""), 10) : null; + }); + const waiting = db.prepare(`SELECT DISTINCT t.id, t.linkedin_url, t.connected_at, t.degree + FROM targets t JOIN run_profiles rp ON rp.target_id=t.id + JOIN runs r ON r.id=rp.run_id WHERE r.account_id=? + AND t.connection_requested_at IS NOT NULL`).all(accountId) as Array<{ + id: string; + linkedin_url: string; + connected_at: string | null; + degree: number | null; + }>; + const stampAccepted = db.prepare("UPDATE targets SET degree = 1, connected_at = COALESCE(connected_at, ?) WHERE id = ?"); + const seenVanities = new Set(); + let newestSeen: number | null = null; + let reachedEnd = false; + for (let pageIdx = 0; pageIdx < MAX_PAGES; pageIdx++) { + const conns = await fetchConnectionsPage(page, pageIdx * PAGE_SIZE, PAGE_SIZE); + if (conns === null) { + throw new Error(`LinkedIn connections API unavailable at offset ${pageIdx * PAGE_SIZE}; verification incomplete`); + } + if (conns.length === 0) { + reachedEnd = true; + break; + } // end of list + for (const c of conns) { + if (c.vanity) + seenVanities.add(c.vanity.toLowerCase()); + if (newestSeen === null || c.createdAt > newestSeen) + newestSeen = c.createdAt; + if (!c.vanity) + continue; + for (const m of waiting.filter(t => profileVanity(t.linkedin_url) === c.vanity?.toLowerCase())) { + if (m.degree === 1 && m.connected_at) + continue; // already correct + stampAccepted.run(msToSqlite(c.createdAt), m.id); + console.log(`[sync-accepted] Accepted: ${c.vanity}`); + stamped++; + m.degree = 1; + m.connected_at = msToSqlite(c.createdAt); + } + } + // Do not assume upstream pages are globally sorted by connection date. + if (waiting.every(t => t.degree === 1 && t.connected_at || seenVanities.has(profileVanity(t.linkedin_url) || ''))) { + reachedEnd = true; + break; + } + await page.waitForTimeout(900 + Math.random() * 700); // gentle, API-only } - } - - if (reachedBoundary) break; - await page.waitForTimeout(900 + Math.random() * 700); // gentle, API-only - } - - // Completeness checksum: a full pass is trustworthy only if what we pulled - // matches LinkedIn's own declared total (±5 slack for live churn). - const verifiedComplete = - isFullPass && declaredTotal !== null && Math.abs(uniquePulled - declaredTotal) <= 5; - - // Correction: ONLY on a verified-complete full pass, un-mark phantom - // degree=1 contacts (present nowhere in the authoritative list). Never on an - // incremental/incomplete pass — a partial pull must not wipe real accepts. - let unmarked = 0; - if (verifiedComplete) { - const deg1 = db.prepare( - "SELECT id, full_name, linkedin_url FROM targets WHERE degree = 1 AND linkedin_url LIKE '%/in/%'" - ).all() as Array<{ id: string; full_name: string | null; linkedin_url: string }>; - const unmark = db.prepare("UPDATE targets SET degree = NULL, connected_at = NULL WHERE id = ?"); - const tx = db.transaction((rows: typeof deg1) => { - for (const t of rows) { - const mm = t.linkedin_url.match(/\/in\/([^/?#]+)/); - const v = mm ? decodeURIComponent(mm[1]).toLowerCase() : null; - if (!v || !seenVanities.has(v)) { - unmark.run(t.id); - unmarked++; - } + if (!reachedEnd) + throw new Error("Connection verification reached the page limit; verification incomplete"); + // Advance the boundary to the newest connection seen this run. + if (newestSeen !== null) { + db.prepare("UPDATE accounts SET connections_synced_through_ms = ? WHERE id = ?").run(newestSeen, accountId); } - }); - tx(deg1); - console.log(`[sync-accepted] Verified full pass (pulled ${uniquePulled} == declared ${declaredTotal}). Un-marked ${unmarked} phantom degree=1.`); - } else if (isFullPass) { - console.warn(`[sync-accepted] Full pass NOT verified complete (pulled ${uniquePulled}, declared ${declaredTotal}) — add-only, no un-marking.`); - } - - // Advance the boundary to the newest connection seen this run. - if (newestSeen !== null) { - db.prepare("UPDATE accounts SET connections_synced_through_ms = ? WHERE id = ?").run(newestSeen, accountId); - } - // Store the declared total for visibility (Settings shows LinkedIn's count). - if (declaredTotal !== null) { - db.prepare("UPDATE accounts SET li_connections = ? WHERE id = ?").run(declaredTotal, accountId); + // Store the declared total for visibility (Settings shows LinkedIn's count). + if (declaredTotal !== null) { + db.prepare("UPDATE accounts SET li_connections = ? WHERE id = ?").run(declaredTotal, accountId); + } + console.log(`[sync-accepted] Stamped ${stamped} accepted (boundary=${newestSeen}).`); + db.prepare("UPDATE accounts SET accepted_sync_at = datetime('now') WHERE id = ?").run(accountId); } - console.log(`[sync-accepted] Stamped ${stamped} accepted, un-marked ${unmarked} phantom (boundary=${newestSeen}).`); - } finally { - // B5 safety: only persist the session if still on a valid page. - let url = ""; - try { url = page.url(); } catch { /* gone */ } - try { await page.close(); } catch { /* ignore */ } - if (/\/login|\/authwall|\/checkpoint|\/uas\//.test(url)) { - console.warn(`[sync-accepted] Ended on a wall (${url}) — not persisting; flagging re-auth`); - try { await markNeedsReauth(accountId); } catch { /* ignore */ } - } else { - try { await saveSessionState(accountId); } catch { /* ignore */ } + finally { + // B5 safety: only persist the session if still on a valid page. + let url = ""; + try { + url = page.url(); + } + catch { /* gone */ } + try { + await page.close(); + } + catch { /* ignore */ } + if (/\/login|\/authwall|\/checkpoint|\/uas\//.test(url)) { + console.warn(`[sync-accepted] Ended on a wall (${url}) — not persisting; flagging re-auth`); + try { + await markNeedsReauth(accountId); + } + catch { /* ignore */ } + } + else { + try { + await saveSessionState(accountId); + } + catch { /* ignore */ } + } } - db.prepare("UPDATE accounts SET accepted_sync_at = datetime('now') WHERE id = ?").run(accountId); - } - - return stamped; + return stamped; } - // ── helpers ────────────────────────────────────────────────────────────────── - function msToSqlite(ms: number): string { - return new Date(ms).toISOString().replace("T", " ").slice(0, 19); + return new Date(ms).toISOString().replace("T", " ").slice(0, 19); } - async function fetchConnectionsPage(page: Page, start: number, count: number): Promise { - return page.evaluate( - async ({ start, count, decoration }): Promise => { - const cookies = document.cookie.split("; ").reduce((a: Record, c) => { - const i = c.indexOf("="); - if (i > 0) a[c.slice(0, i)] = c.slice(i + 1); - return a; - }, {}); - const csrf = (cookies["JSESSIONID"] || "").replace(/"/g, ""); - const url = - `https://www.linkedin.com/voyager/api/relationships/dash/connections` + - `?decorationId=${decoration}&count=${count}&q=search&sortType=RECENTLY_ADDED&start=${start}`; - let json: { - included?: Array<{ - $type?: string; - entityUrn?: string; - createdAt?: number; - connectedMember?: string; - publicIdentifier?: string; - }>; - }; - try { - const r = await fetch(url, { - headers: { - "csrf-token": csrf, - "accept": "application/vnd.linkedin.normalized+json+2.1", - "x-restli-protocol-version": "2.0.0", - "x-li-lang": "en_US", - }, - credentials: "include", - }); - if (!r.ok) return null; - json = await r.json(); - } catch { - return null; - } - - const included = json.included || []; - const vanityByUrn: Record = {}; - for (const x of included) { - if ((x.$type || "").includes("identity.profile.Profile") && x.entityUrn && x.publicIdentifier) { - vanityByUrn[x.entityUrn] = x.publicIdentifier; + return page.evaluate(async ({ start, count, decoration }): Promise => { + const cookies = document.cookie.split("; ").reduce((a: Record, c) => { + const i = c.indexOf("="); + if (i > 0) + a[c.slice(0, i)] = c.slice(i + 1); + return a; + }, {}); + const csrf = (cookies["JSESSIONID"] || "").replace(/"/g, ""); + const url = `https://www.linkedin.com/voyager/api/relationships/dash/connections` + + `?decorationId=${decoration}&count=${count}&q=search&sortType=RECENTLY_ADDED&start=${start}`; + let json: { + included?: Array<{ + $type?: string; + entityUrn?: string; + createdAt?: number; + connectedMember?: string; + publicIdentifier?: string; + }>; + }; + try { + const r = await fetch(url, { + headers: { + "csrf-token": csrf, + "accept": "application/vnd.linkedin.normalized+json+2.1", + "x-restli-protocol-version": "2.0.0", + "x-li-lang": "en_US", + }, + credentials: "include", + }); + if (!r.ok) + return null; + json = await r.json(); + } + catch { + return null; } - } - const out: ApiConnection[] = []; - for (const x of included) { - if ((x.$type || "").includes("relationships.Connection") && typeof x.createdAt === "number") { - const memberUrn = x.connectedMember || null; - out.push({ - createdAt: x.createdAt, - vanity: memberUrn ? vanityByUrn[memberUrn] ?? null : null, - }); + const included = json.included || []; + const vanityByUrn: Record = {}; + for (const x of included) { + if ((x.$type || "").includes("identity.profile.Profile") && x.entityUrn && x.publicIdentifier) { + vanityByUrn[x.entityUrn] = x.publicIdentifier; + } } - } - out.sort((a, b) => b.createdAt - a.createdAt); - return out; - }, - { start, count, decoration: DECORATION } - ); + const out: ApiConnection[] = []; + for (const x of included) { + if ((x.$type || "").includes("relationships.Connection") && typeof x.createdAt === "number") { + const memberUrn = x.connectedMember || null; + out.push({ + createdAt: x.createdAt, + vanity: memberUrn ? vanityByUrn[memberUrn] ?? null : null, + }); + } + } + out.sort((a, b) => b.createdAt - a.createdAt); + return out; + }, { start, count, decoration: DECORATION }); +} +export function profileVanity(value: string): string | null { + try { + const u = new URL(value); + if (!['linkedin.com', 'www.linkedin.com'].includes(u.hostname)) + return null; + const m = u.pathname.match(/^\/in\/([^/]+)\/?$/); + return m ? decodeURIComponent(m[1]).toLowerCase() : null; + } + catch { + return null; + } } diff --git a/pages/api/accounts/[id]/sync-accepted.ts b/pages/api/accounts/[id]/sync-accepted.ts index 490d42a7..a16eadb1 100644 --- a/pages/api/accounts/[id]/sync-accepted.ts +++ b/pages/api/accounts/[id]/sync-accepted.ts @@ -1,72 +1,22 @@ import type { NextApiRequest, NextApiResponse } from "next"; import { getDb } from "@/lib/db"; -import { getSessionPage, saveSessionState } from "@/lib/linkedin/session"; -import { scrapePendingInvitationVanityNames } from "@/lib/linkedin/pending-invitations"; - +import { syncAcceptedConnections } from "@/lib/linkedin/sync-accepted"; export default async function handler(req: NextApiRequest, res: NextApiResponse) { - if (req.method !== "POST") return res.status(405).end(); - - const accountId = req.query.id as string; - const db = getDb(); - - const account = db.prepare("SELECT id, is_authenticated FROM accounts WHERE id = ?").get(accountId) as - | { id: string; is_authenticated: number } - | undefined; - - if (!account) return res.status(404).json({ error: "Account not found" }); - if (!account.is_authenticated) return res.status(400).json({ error: "Account not authenticated" }); - - let page; - try { - page = await getSessionPage(accountId); - const stillPending = await scrapePendingInvitationVanityNames(page); - await saveSessionState(accountId); - - // Find all targets that we sent a request to but haven't marked as connected - const waiting = db.prepare(` - SELECT id, linkedin_url, full_name - FROM targets - WHERE connection_requested_at IS NOT NULL - AND (degree IS NULL OR degree != 1) - AND connected_at IS NULL - `).all() as { id: string; linkedin_url: string; full_name: string | null }[]; - - const now = new Date().toISOString(); - const accepted: string[] = []; - const skipped: string[] = []; - - const markAccepted = db.prepare( - "UPDATE targets SET degree = 1, connected_at = ? WHERE id = ?" - ); - - db.transaction(() => { - for (const target of waiting) { - const match = target.linkedin_url?.match(/\/in\/([^/?#]+)/); - if (!match) continue; // no /in/ URL — can't check - const vanity = match[1].toLowerCase(); - - if (!stillPending.has(vanity)) { - // Not in the pending list anymore → accepted (or expired, but treat as accepted) - markAccepted.run(now, target.id); - accepted.push(target.full_name ?? vanity); - } else { - skipped.push(target.full_name ?? vanity); - } - } - })(); - - return res.json({ - pending_on_linkedin: stillPending.size, - waiting_in_db: waiting.length, - newly_accepted: accepted.length, - still_pending: skipped.length, - accepted_names: accepted, - }); - - } catch (err) { - console.error("[sync-accepted]", err); - return res.status(500).json({ error: err instanceof Error ? err.message : "Sync failed" }); - } finally { - await page?.close().catch(() => {}); - } + if (req.method !== "POST") + return res.status(405).end(); + const id = req.query.id as string; + const account = getDb().prepare("SELECT is_authenticated FROM accounts WHERE id=?").get(id) as { + is_authenticated: number; + } | undefined; + if (!account) + return res.status(404).json({ error: "Account not found" }); + if (!account.is_authenticated) + return res.status(400).json({ error: "Account not authenticated" }); + try { + const count = await syncAcceptedConnections(id); + return res.json({ newly_accepted: count, source: "connections-list" }); + } + catch { + return res.status(502).json({ error: "Connection verification failed or was incomplete. Check account authentication and retry verification; no outreach was started." }); + } } diff --git a/tests/ACCEPTANCE-SYNC.md b/tests/ACCEPTANCE-SYNC.md new file mode 100644 index 00000000..509cf0d1 --- /dev/null +++ b/tests/ACCEPTANCE-SYNC.md @@ -0,0 +1,16 @@ +# Acceptance synchronization + +After `npm ci`, run `node tests/accepted-sync.cjs`. +The fixture uses real in-memory SQLite and a stubbed LinkedIn session. No account +cookies, network, invitations, or messages are used. + +Coverage: exact canonical profile matching; account-scoped enrollment; an older +connection on a later page despite a newer stored boundary; absent profiles left +unchanged; historical connection date preservation; idempotent rechecks; and API +failure without a successful-sync timestamp. + +The manual endpoint and scheduled runner share the same detector. Only positive +connections-list evidence updates a target. Positive results can remain if a +later page fails; incomplete scans do not advance accepted_sync_at. The existing +8-hour polling interval and runner scheduling conditions are unchanged. A stored +historical connection date is not proof that the current campaign caused it. diff --git a/tests/accepted-sync.cjs b/tests/accepted-sync.cjs new file mode 100644 index 00000000..2cf73126 --- /dev/null +++ b/tests/accepted-sync.cjs @@ -0,0 +1,35 @@ +const fs = require('fs'), ts = require('typescript'), Module = require('module'), assert = require('node:assert/strict'), DB = require('better-sqlite3'); +(async () => { + let passed = 0; + for (const failure of [false, true]) { + const db = new DB(':memory:'); + db.exec(`CREATE TABLE accounts(id TEXT,connections_synced_through_ms INTEGER,accepted_sync_at TEXT,li_connections INTEGER);INSERT INTO accounts(id,connections_synced_through_ms) VALUES('a',1900000000000);CREATE TABLE targets(id TEXT,linkedin_url TEXT,connection_requested_at TEXT,connected_at TEXT,degree INTEGER);CREATE TABLE runs(id TEXT,account_id TEXT);CREATE TABLE run_profiles(target_id TEXT,run_id TEXT);INSERT INTO runs VALUES('r','a'),('other','b');INSERT INTO targets VALUES('match','https://www.linkedin.com/in/synthetic-person', 'sent',NULL,NULL),('foreign','https://www.linkedin.com/in/synthetic-person/','sent',NULL,NULL),('absent','https://www.linkedin.com/in/absent/','sent',NULL,NULL);INSERT INTO run_profiles VALUES('match','r'),('foreign','other'),('absent','r');`); + let pages = 0; + const page = { goto: async () => { }, waitForTimeout: async () => { }, url: () => 'https://www.linkedin.com/mynetwork/invite-connect/connections/', close: async () => { }, evaluate: async (fn, args) => !args ? null : failure ? null : pages++ === 0 ? [{ vanity: 'unrelated', createdAt: 1789751000000 }] : pages === 2 ? [{ vanity: 'synthetic-person', createdAt: 1658235047000 }] : [] }; + const m = new Module('/app/sync-fixture.cjs'); + m.require = n => n === '@/lib/db' ? { getDb: () => db } : n === '@/lib/linkedin/session' ? { getSessionPage: async () => page, saveSessionState: async () => { }, markNeedsReauth: async () => { } } : require(n); + m._compile(ts.transpileModule(fs.readFileSync(require('path').join(__dirname, '../lib/linkedin/sync-accepted.ts'), 'utf8'), { compilerOptions: { module: ts.ModuleKind.CommonJS, target: ts.ScriptTarget.ES2022 } }).outputText, '/app/sync-fixture.cjs'); + assert.equal(m.exports.profileVanity('https://www.linkedin.com/in/Synthetic-person/?x=y'), 'synthetic-person'); + assert.equal(m.exports.profileVanity('https://evil.test/in/synthetic-person'), null); + passed += 2; + if (failure) { + await assert.rejects(m.exports.syncAcceptedConnections('a'), /unavailable/); + assert.equal(db.prepare('SELECT accepted_sync_at FROM accounts').get().accepted_sync_at, null); + passed += 2; + } + else { + assert.equal(await m.exports.syncAcceptedConnections('a'), 1); + assert.equal(db.prepare("SELECT degree FROM targets WHERE id='match'").get().degree, 1); + assert.equal(db.prepare("SELECT degree FROM targets WHERE id='foreign'").get().degree, null); + assert.equal(db.prepare("SELECT degree FROM targets WHERE id='absent'").get().degree, null); + passed += 4; + assert.equal(db.prepare("SELECT connected_at FROM targets WHERE id='match'").get().connected_at, '2022-07-19 12:50:47'); + assert.ok(db.prepare('SELECT accepted_sync_at FROM accounts').get().accepted_sync_at); + pages = 0; + assert.equal(await m.exports.syncAcceptedConnections('a'), 0); + passed += 3; + } + db.close(); + } + console.log(JSON.stringify({ passed, network: 'none', synthetic: true })); +})().catch(e => { console.error(e); process.exitCode = 1; }); From af128b15a42a04176b7aa5a4c552e2cdc4827117 Mon Sep 17 00:00:00 2001 From: TheGuyWithoutH Date: Mon, 21 Sep 2026 00:58:28 +0200 Subject: [PATCH 2/2] Restore existing formatting in acceptance sync changes --- lib/linkedin/sync-accepted.ts | 344 +++++++++++------------ pages/api/accounts/[id]/sync-accepted.ts | 34 +-- 2 files changed, 189 insertions(+), 189 deletions(-) diff --git a/lib/linkedin/sync-accepted.ts b/lib/linkedin/sync-accepted.ts index 1dfd5471..7959a722 100644 --- a/lib/linkedin/sync-accepted.ts +++ b/lib/linkedin/sync-accepted.ts @@ -1,202 +1,202 @@ import type { Page } from "playwright"; import { getDb } from "@/lib/db"; import { getSessionPage, saveSessionState, markNeedsReauth } from "@/lib/linkedin/session"; + /** Shared manual/scheduled detector. Only positive connections-list evidence marks * acceptance; absence never implies acceptance or removes a connection. */ const ACCEPTED_SYNC_INTERVAL_MS = 8 * 60 * 60 * 1000; // 8h — 3x per day const PAGE_SIZE = 100; const MAX_PAGES = 60; // safety cap (60 * 100 = 6000) const DECORATION = "com.linkedin.voyager.dash.deco.web.mynetwork.ConnectionListWithProfile-16"; + export function shouldSyncAccepted(accountId: string): boolean { - const db = getDb(); - const row = db.prepare("SELECT accepted_sync_at FROM accounts WHERE id = ?").get(accountId) as { - accepted_sync_at: string | null; - } | undefined; - if (!row?.accepted_sync_at) - return true; - return Date.now() - new Date(row.accepted_sync_at).getTime() >= ACCEPTED_SYNC_INTERVAL_MS; + const db = getDb(); + const row = db.prepare("SELECT accepted_sync_at FROM accounts WHERE id = ?").get(accountId) as + | { accepted_sync_at: string | null } + | undefined; + if (!row?.accepted_sync_at) return true; + return Date.now() - new Date(row.accepted_sync_at).getTime() >= ACCEPTED_SYNC_INTERVAL_MS; } + interface ApiConnection { - vanity: string | null; - createdAt: number; // epoch ms + vanity: string | null; + createdAt: number; // epoch ms } + export async function syncAcceptedConnections(accountId: string): Promise { - const db = getDb(); - const page = await getSessionPage(accountId); - let stamped = 0; - try { - // Read the declared total from the connections PAGE header (single nav, NO - // scroll). This is also our login-wall check and an informational account count. - await page.goto("https://www.linkedin.com/mynetwork/invite-connect/connections/", { - waitUntil: "domcontentloaded", - timeout: 35000, - }); - await page.waitForTimeout(3500 + Math.random() * 1500); - if (/\/login|\/authwall|\/checkpoint|\/uas\//.test(page.url())) { - console.warn(`[sync-accepted] Session looks logged out (${page.url()}) — skipping`); - throw new Error("LinkedIn login or verification required"); - } - const declaredTotal = await page.evaluate(() => { - const m = document.body.innerText.match(/([\d.,]+)\s+connections?/i); - return m ? parseInt(m[1].replace(/[.,]/g, ""), 10) : null; - }); - const waiting = db.prepare(`SELECT DISTINCT t.id, t.linkedin_url, t.connected_at, t.degree + const db = getDb(); + const page = await getSessionPage(accountId); + let stamped = 0; + + try { + // Read the declared total from the connections PAGE header (single nav, NO + // scroll). This is also our login-wall check and an informational account count. + await page.goto("https://www.linkedin.com/mynetwork/invite-connect/connections/", { + waitUntil: "domcontentloaded", + timeout: 35000, + }); + await page.waitForTimeout(3500 + Math.random() * 1500); + if (/\/login|\/authwall|\/checkpoint|\/uas\//.test(page.url())) { + console.warn(`[sync-accepted] Session looks logged out (${page.url()}) — skipping`); + throw new Error("LinkedIn login or verification required"); + } + const declaredTotal = await page.evaluate(() => { + const m = document.body.innerText.match(/([\d.,]+)\s+connections?/i); + return m ? parseInt(m[1].replace(/[.,]/g, ""), 10) : null; + }); + + const waiting = db.prepare(`SELECT DISTINCT t.id, t.linkedin_url, t.connected_at, t.degree FROM targets t JOIN run_profiles rp ON rp.target_id=t.id JOIN runs r ON r.id=rp.run_id WHERE r.account_id=? AND t.connection_requested_at IS NOT NULL`).all(accountId) as Array<{ - id: string; - linkedin_url: string; - connected_at: string | null; - degree: number | null; - }>; - const stampAccepted = db.prepare("UPDATE targets SET degree = 1, connected_at = COALESCE(connected_at, ?) WHERE id = ?"); - const seenVanities = new Set(); - let newestSeen: number | null = null; - let reachedEnd = false; - for (let pageIdx = 0; pageIdx < MAX_PAGES; pageIdx++) { - const conns = await fetchConnectionsPage(page, pageIdx * PAGE_SIZE, PAGE_SIZE); - if (conns === null) { - throw new Error(`LinkedIn connections API unavailable at offset ${pageIdx * PAGE_SIZE}; verification incomplete`); - } - if (conns.length === 0) { - reachedEnd = true; - break; - } // end of list - for (const c of conns) { - if (c.vanity) - seenVanities.add(c.vanity.toLowerCase()); - if (newestSeen === null || c.createdAt > newestSeen) - newestSeen = c.createdAt; - if (!c.vanity) - continue; - for (const m of waiting.filter(t => profileVanity(t.linkedin_url) === c.vanity?.toLowerCase())) { - if (m.degree === 1 && m.connected_at) - continue; // already correct - stampAccepted.run(msToSqlite(c.createdAt), m.id); - console.log(`[sync-accepted] Accepted: ${c.vanity}`); - stamped++; - m.degree = 1; - m.connected_at = msToSqlite(c.createdAt); - } - } - // Do not assume upstream pages are globally sorted by connection date. - if (waiting.every(t => t.degree === 1 && t.connected_at || seenVanities.has(profileVanity(t.linkedin_url) || ''))) { - reachedEnd = true; - break; - } - await page.waitForTimeout(900 + Math.random() * 700); // gentle, API-only - } - if (!reachedEnd) - throw new Error("Connection verification reached the page limit; verification incomplete"); - // Advance the boundary to the newest connection seen this run. - if (newestSeen !== null) { - db.prepare("UPDATE accounts SET connections_synced_through_ms = ? WHERE id = ?").run(newestSeen, accountId); + id: string; + linkedin_url: string; + connected_at: string | null; + degree: number | null; + }>; + const stampAccepted = db.prepare( + "UPDATE targets SET degree = 1, connected_at = COALESCE(connected_at, ?) WHERE id = ?" + ); + + const seenVanities = new Set(); + let newestSeen: number | null = null; + let reachedEnd = false; + + for (let pageIdx = 0; pageIdx < MAX_PAGES; pageIdx++) { + const conns = await fetchConnectionsPage(page, pageIdx * PAGE_SIZE, PAGE_SIZE); + if (conns === null) { + throw new Error(`LinkedIn connections API unavailable at offset ${pageIdx * PAGE_SIZE}; verification incomplete`); + } + if (conns.length === 0) { + reachedEnd = true; + break; + } // end of list + + for (const c of conns) { + if (c.vanity) seenVanities.add(c.vanity.toLowerCase()); + if (newestSeen === null || c.createdAt > newestSeen) newestSeen = c.createdAt; + if (!c.vanity) continue; + + for (const m of waiting.filter(t => profileVanity(t.linkedin_url) === c.vanity?.toLowerCase())) { + if (m.degree === 1 && m.connected_at) continue; // already correct + stampAccepted.run(msToSqlite(c.createdAt), m.id); + console.log(`[sync-accepted] Accepted: ${c.vanity}`); + stamped++; + m.degree = 1; + m.connected_at = msToSqlite(c.createdAt); } - // Store the declared total for visibility (Settings shows LinkedIn's count). - if (declaredTotal !== null) { - db.prepare("UPDATE accounts SET li_connections = ? WHERE id = ?").run(declaredTotal, accountId); - } - console.log(`[sync-accepted] Stamped ${stamped} accepted (boundary=${newestSeen}).`); - db.prepare("UPDATE accounts SET accepted_sync_at = datetime('now') WHERE id = ?").run(accountId); + } + + // Do not assume upstream pages are globally sorted by connection date. + if (waiting.every(t => t.degree === 1 && t.connected_at || seenVanities.has(profileVanity(t.linkedin_url) || ''))) { + reachedEnd = true; + break; + } + await page.waitForTimeout(900 + Math.random() * 700); // gentle, API-only } - finally { - // B5 safety: only persist the session if still on a valid page. - let url = ""; - try { - url = page.url(); - } - catch { /* gone */ } - try { - await page.close(); - } - catch { /* ignore */ } - if (/\/login|\/authwall|\/checkpoint|\/uas\//.test(url)) { - console.warn(`[sync-accepted] Ended on a wall (${url}) — not persisting; flagging re-auth`); - try { - await markNeedsReauth(accountId); - } - catch { /* ignore */ } - } - else { - try { - await saveSessionState(accountId); - } - catch { /* ignore */ } - } + + if (!reachedEnd) + throw new Error("Connection verification reached the page limit; verification incomplete"); + // Advance the boundary to the newest connection seen this run. + if (newestSeen !== null) { + db.prepare("UPDATE accounts SET connections_synced_through_ms = ? WHERE id = ?").run(newestSeen, accountId); } - return stamped; + // Store the declared total for visibility (Settings shows LinkedIn's count). + if (declaredTotal !== null) { + db.prepare("UPDATE accounts SET li_connections = ? WHERE id = ?").run(declaredTotal, accountId); + } + console.log(`[sync-accepted] Stamped ${stamped} accepted (boundary=${newestSeen}).`); + db.prepare("UPDATE accounts SET accepted_sync_at = datetime('now') WHERE id = ?").run(accountId); + } finally { + // B5 safety: only persist the session if still on a valid page. + let url = ""; + try { url = page.url(); } catch { /* gone */ } + try { await page.close(); } catch { /* ignore */ } + if (/\/login|\/authwall|\/checkpoint|\/uas\//.test(url)) { + console.warn(`[sync-accepted] Ended on a wall (${url}) — not persisting; flagging re-auth`); + try { await markNeedsReauth(accountId); } catch { /* ignore */ } + } else { + try { await saveSessionState(accountId); } catch { /* ignore */ } + } + } + + return stamped; } + // ── helpers ────────────────────────────────────────────────────────────────── + function msToSqlite(ms: number): string { - return new Date(ms).toISOString().replace("T", " ").slice(0, 19); + return new Date(ms).toISOString().replace("T", " ").slice(0, 19); } + async function fetchConnectionsPage(page: Page, start: number, count: number): Promise { - return page.evaluate(async ({ start, count, decoration }): Promise => { - const cookies = document.cookie.split("; ").reduce((a: Record, c) => { - const i = c.indexOf("="); - if (i > 0) - a[c.slice(0, i)] = c.slice(i + 1); - return a; - }, {}); - const csrf = (cookies["JSESSIONID"] || "").replace(/"/g, ""); - const url = `https://www.linkedin.com/voyager/api/relationships/dash/connections` + - `?decorationId=${decoration}&count=${count}&q=search&sortType=RECENTLY_ADDED&start=${start}`; - let json: { - included?: Array<{ - $type?: string; - entityUrn?: string; - createdAt?: number; - connectedMember?: string; - publicIdentifier?: string; - }>; - }; - try { - const r = await fetch(url, { - headers: { - "csrf-token": csrf, - "accept": "application/vnd.linkedin.normalized+json+2.1", - "x-restli-protocol-version": "2.0.0", - "x-li-lang": "en_US", - }, - credentials: "include", - }); - if (!r.ok) - return null; - json = await r.json(); - } - catch { - return null; - } - const included = json.included || []; - const vanityByUrn: Record = {}; - for (const x of included) { - if ((x.$type || "").includes("identity.profile.Profile") && x.entityUrn && x.publicIdentifier) { - vanityByUrn[x.entityUrn] = x.publicIdentifier; - } + return page.evaluate( + async ({ start, count, decoration }): Promise => { + const cookies = document.cookie.split("; ").reduce((a: Record, c) => { + const i = c.indexOf("="); + if (i > 0) a[c.slice(0, i)] = c.slice(i + 1); + return a; + }, {}); + const csrf = (cookies["JSESSIONID"] || "").replace(/"/g, ""); + const url = + `https://www.linkedin.com/voyager/api/relationships/dash/connections` + + `?decorationId=${decoration}&count=${count}&q=search&sortType=RECENTLY_ADDED&start=${start}`; + let json: { + included?: Array<{ + $type?: string; + entityUrn?: string; + createdAt?: number; + connectedMember?: string; + publicIdentifier?: string; + }>; + }; + try { + const r = await fetch(url, { + headers: { + "csrf-token": csrf, + "accept": "application/vnd.linkedin.normalized+json+2.1", + "x-restli-protocol-version": "2.0.0", + "x-li-lang": "en_US", + }, + credentials: "include", + }); + if (!r.ok) return null; + json = await r.json(); + } catch { + return null; + } + + const included = json.included || []; + const vanityByUrn: Record = {}; + for (const x of included) { + if ((x.$type || "").includes("identity.profile.Profile") && x.entityUrn && x.publicIdentifier) { + vanityByUrn[x.entityUrn] = x.publicIdentifier; } - const out: ApiConnection[] = []; - for (const x of included) { - if ((x.$type || "").includes("relationships.Connection") && typeof x.createdAt === "number") { - const memberUrn = x.connectedMember || null; - out.push({ - createdAt: x.createdAt, - vanity: memberUrn ? vanityByUrn[memberUrn] ?? null : null, - }); - } + } + const out: ApiConnection[] = []; + for (const x of included) { + if ((x.$type || "").includes("relationships.Connection") && typeof x.createdAt === "number") { + const memberUrn = x.connectedMember || null; + out.push({ + createdAt: x.createdAt, + vanity: memberUrn ? vanityByUrn[memberUrn] ?? null : null, + }); } - out.sort((a, b) => b.createdAt - a.createdAt); - return out; - }, { start, count, decoration: DECORATION }); + } + out.sort((a, b) => b.createdAt - a.createdAt); + return out; + }, + { start, count, decoration: DECORATION } + ); } + export function profileVanity(value: string): string | null { - try { - const u = new URL(value); - if (!['linkedin.com', 'www.linkedin.com'].includes(u.hostname)) - return null; - const m = u.pathname.match(/^\/in\/([^/]+)\/?$/); - return m ? decodeURIComponent(m[1]).toLowerCase() : null; - } - catch { - return null; - } + try { + const u = new URL(value); + if (!['linkedin.com', 'www.linkedin.com'].includes(u.hostname)) return null; + const m = u.pathname.match(/^\/in\/([^/]+)\/?$/); + return m ? decodeURIComponent(m[1]).toLowerCase() : null; + } catch { + return null; + } } diff --git a/pages/api/accounts/[id]/sync-accepted.ts b/pages/api/accounts/[id]/sync-accepted.ts index a16eadb1..7da2c69b 100644 --- a/pages/api/accounts/[id]/sync-accepted.ts +++ b/pages/api/accounts/[id]/sync-accepted.ts @@ -1,22 +1,22 @@ import type { NextApiRequest, NextApiResponse } from "next"; import { getDb } from "@/lib/db"; import { syncAcceptedConnections } from "@/lib/linkedin/sync-accepted"; + export default async function handler(req: NextApiRequest, res: NextApiResponse) { - if (req.method !== "POST") - return res.status(405).end(); - const id = req.query.id as string; - const account = getDb().prepare("SELECT is_authenticated FROM accounts WHERE id=?").get(id) as { - is_authenticated: number; - } | undefined; - if (!account) - return res.status(404).json({ error: "Account not found" }); - if (!account.is_authenticated) - return res.status(400).json({ error: "Account not authenticated" }); - try { - const count = await syncAcceptedConnections(id); - return res.json({ newly_accepted: count, source: "connections-list" }); - } - catch { - return res.status(502).json({ error: "Connection verification failed or was incomplete. Check account authentication and retry verification; no outreach was started." }); - } + if (req.method !== "POST") return res.status(405).end(); + + const id = req.query.id as string; + const account = getDb().prepare("SELECT is_authenticated FROM accounts WHERE id=?").get(id) as { + is_authenticated: number; + } | undefined; + + if (!account) return res.status(404).json({ error: "Account not found" }); + if (!account.is_authenticated) return res.status(400).json({ error: "Account not authenticated" }); + + try { + const count = await syncAcceptedConnections(id); + return res.json({ newly_accepted: count, source: "connections-list" }); + } catch { + return res.status(502).json({ error: "Connection verification failed or was incomplete. Check account authentication and retry verification; no outreach was started." }); + } }