Skip to content

Unify connection acceptance synchronization #3

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
TheGuyWithoutH wants to merge 2 commits into from
Closed

Unify connection acceptance synchronization #3

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
c5d0cd8
Share positive-evidence connection acceptance synchronization
TheGuyWithoutH Sep 20, 2026
af128b1
Restore existing formatting in acceptance sync changes
TheGuyWithoutH Sep 20, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
144 changes: 45 additions & 99 deletions lib/linkedin/sync-accepted.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,44 +2,11 @@ import type { Page } from "playwright";
import { getDb } from "@/lib/db";
import { getSessionPage, saveSessionState, markNeedsReauth } from "@/lib/linkedin/session";

/**
* Accepted-connection sync via the authoritative Voyager connections API.
*
* Replaces the old absence-inference (scroll the SENT list, treat a vanished
* invite as "accepted"). That was measurably wrong: of 1,600 contacts it had
* marked degree=1, only 325 were genuinely connected — 1,275 were phantoms
* (invites that expired/withdrew/were ignored, not accepted). See docs §19.
*
* Sources — all proved against prod (Jul 2026), NO scrolling:
* - DATA: GET /voyager/api/relationships/dash/connections
* ?decorationId=…ConnectionListWithProfile-16&q=search
* &sortType=RECENTLY_ADDED&start=N&count=100
* → data["*elements"] (newest-first) + included[]: Connection { createdAt(ms),
* connectedMember } and Profile { publicIdentifier (vanity), entityUrn }.
* The API paginates to the end (30 pages for ~2947) in ~60s. It does NOT
* expose a grand total.
* - TOTAL (checksum): the connections PAGE header "<N> connections" <p>. Read
* once per full pass via a single navigation (NO scroll). A full pass is
* "verified complete" only when unique-pulled == declared total (matched ±0
* in prod). Presence in the list = 100% proof of a 1st-degree connection.
*
* Behaviour:
* - First run (boundary NULL) → FULL pass: page to the end, stamp degree=1 +
* connected_at from createdAt for every matched contact. If the pass is
* checksum-verified complete, ALSO un-mark phantom degree=1 contacts whose
* vanity is absent from the authoritative list (resets degree/connected_at).
* - Later runs → incremental: page from start=0 and STOP at the first
* connection older than the stored boundary (minus a 24h overlap margin).
* Incremental/incomplete passes are ADD-ONLY — they never un-mark (a partial
* pull must not wipe real accepts). Un-marking happens ONLY on a verified
* full pass.
* - Reuses the runner's shared browser (getSessionPage) — never a 2nd browser.
*/

/** Shared manual/scheduled detector. Only positive connections-list evidence marks
* acceptance; absence never implies acceptance or removes a connection. */
const ACCEPTED_SYNC_INTERVAL_MS = 8 * 60 * 60 * 1000; // 8h — 3x per day
const PAGE_SIZE = 100;
const MAX_PAGES = 60; // safety cap (60 * 100 = 6000)
const OVERLAP_MARGIN_MS = 24 * 60 * 60 * 1000; // re-check a day of overlap (idempotent)
const DECORATION = "com.linkedin.voyager.dash.deco.web.mynetwork.ConnectionListWithProfile-16";

export function shouldSyncAccepted(accountId: string): boolean {
Expand All @@ -62,106 +29,74 @@ export async function syncAcceptedConnections(accountId: string): Promise<number
let stamped = 0;

try {
const boundaryRow = db.prepare("SELECT connections_synced_through_ms FROM accounts WHERE id = ?").get(accountId) as
| { connections_synced_through_ms: number | null }
| undefined;
const boundary = boundaryRow?.connections_synced_through_ms ?? null;
const isFullPass = boundary === null;
const stopBefore = boundary === null ? null : boundary - OVERLAP_MARGIN_MS;

// Read the declared total from the connections PAGE header (single nav, NO
// scroll). This is also our login-wall check and the completeness checksum.
// scroll). This is also our login-wall check and an informational account count.
await page.goto("https://www.linkedin.com/mynetwork/invite-connect/connections/", {
waitUntil: "domcontentloaded",
timeout: 35000,
});
await page.waitForTimeout(3500 + Math.random() * 1500);
if (/\/login|\/authwall|\/checkpoint|\/uas\//.test(page.url())) {
console.warn(`[sync-accepted] Session looks logged out (${page.url()}) — skipping`);
return 0;
throw new Error("LinkedIn login or verification required");
}
const declaredTotal = await page.evaluate(() => {
const m = document.body.innerText.match(/([\d.,]+)\s+connections?/i);
return m ? parseInt(m[1].replace(/[.,]/g, ""), 10) : null;
});

const findByVanity = db.prepare(
`SELECT id, full_name, connected_at, degree FROM targets
WHERE linkedin_url LIKE ? AND connection_requested_at IS NOT NULL`
);
const waiting = db.prepare(`SELECT DISTINCT t.id, t.linkedin_url, t.connected_at, t.degree
FROM targets t JOIN run_profiles rp ON rp.target_id=t.id
JOIN runs r ON r.id=rp.run_id WHERE r.account_id=?
AND t.connection_requested_at IS NOT NULL`).all(accountId) as Array<{
id: string;
linkedin_url: string;
connected_at: string | null;
degree: number | null;
}>;
const stampAccepted = db.prepare(
"UPDATE targets SET degree = 1, connected_at = COALESCE(connected_at, ?) WHERE id = ?"
);

const seenVanities = new Set<string>(); // full-pass phantom check
let uniquePulled = 0;
const seenVanities = new Set<string>();
let newestSeen: number | null = null;
let reachedBoundary = false;
let reachedEnd = false;

for (let pageIdx = 0; pageIdx < MAX_PAGES; pageIdx++) {
const conns = await fetchConnectionsPage(page, pageIdx * PAGE_SIZE, PAGE_SIZE);
if (conns === null) {
console.warn(`[sync-accepted] connections API failed at start=${pageIdx * PAGE_SIZE} — stopping`);
break;
throw new Error(`LinkedIn connections API unavailable at offset ${pageIdx * PAGE_SIZE}; verification incomplete`);
}
if (conns.length === 0) break; // end of list
if (conns.length === 0) {
reachedEnd = true;
break;
} // end of list

for (const c of conns) {
uniquePulled++;
if (c.vanity) seenVanities.add(c.vanity);
if (c.vanity) seenVanities.add(c.vanity.toLowerCase());
if (newestSeen === null || c.createdAt > newestSeen) newestSeen = c.createdAt;

// Incremental early-exit (list is newest-first).
if (stopBefore !== null && c.createdAt < stopBefore) {
reachedBoundary = true;
break;
}
if (!c.vanity) continue;

for (const m of findByVanity.all(`%/in/${c.vanity}/%`) as Array<{
id: string; full_name: string | null; connected_at: string | null; degree: number | null;
}>) {
for (const m of waiting.filter(t => profileVanity(t.linkedin_url) === c.vanity?.toLowerCase())) {
if (m.degree === 1 && m.connected_at) continue; // already correct
stampAccepted.run(msToSqlite(c.createdAt), m.id);
console.log(`[sync-accepted] Accepted: ${m.full_name ?? c.vanity}`);
console.log(`[sync-accepted] Accepted: ${c.vanity}`);
stamped++;
m.degree = 1;
m.connected_at = msToSqlite(c.createdAt);
}
}

if (reachedBoundary) break;
// Do not assume upstream pages are globally sorted by connection date.
if (waiting.every(t => t.degree === 1 && t.connected_at || seenVanities.has(profileVanity(t.linkedin_url) || ''))) {
reachedEnd = true;
break;
}
await page.waitForTimeout(900 + Math.random() * 700); // gentle, API-only
}

// Completeness checksum: a full pass is trustworthy only if what we pulled
// matches LinkedIn's own declared total (±5 slack for live churn).
const verifiedComplete =
isFullPass && declaredTotal !== null && Math.abs(uniquePulled - declaredTotal) <= 5;

// Correction: ONLY on a verified-complete full pass, un-mark phantom
// degree=1 contacts (present nowhere in the authoritative list). Never on an
// incremental/incomplete pass — a partial pull must not wipe real accepts.
let unmarked = 0;
if (verifiedComplete) {
const deg1 = db.prepare(
"SELECT id, full_name, linkedin_url FROM targets WHERE degree = 1 AND linkedin_url LIKE '%/in/%'"
).all() as Array<{ id: string; full_name: string | null; linkedin_url: string }>;
const unmark = db.prepare("UPDATE targets SET degree = NULL, connected_at = NULL WHERE id = ?");
const tx = db.transaction((rows: typeof deg1) => {
for (const t of rows) {
const mm = t.linkedin_url.match(/\/in\/([^/?#]+)/);
const v = mm ? decodeURIComponent(mm[1]).toLowerCase() : null;
if (!v || !seenVanities.has(v)) {
unmark.run(t.id);
unmarked++;
}
}
});
tx(deg1);
console.log(`[sync-accepted] Verified full pass (pulled ${uniquePulled} == declared ${declaredTotal}). Un-marked ${unmarked} phantom degree=1.`);
} else if (isFullPass) {
console.warn(`[sync-accepted] Full pass NOT verified complete (pulled ${uniquePulled}, declared ${declaredTotal}) — add-only, no un-marking.`);
}

if (!reachedEnd)
throw new Error("Connection verification reached the page limit; verification incomplete");
// Advance the boundary to the newest connection seen this run.
if (newestSeen !== null) {
db.prepare("UPDATE accounts SET connections_synced_through_ms = ? WHERE id = ?").run(newestSeen, accountId);
Expand All @@ -170,7 +105,8 @@ export async function syncAcceptedConnections(accountId: string): Promise<number
if (declaredTotal !== null) {
db.prepare("UPDATE accounts SET li_connections = ? WHERE id = ?").run(declaredTotal, accountId);
}
console.log(`[sync-accepted] Stamped ${stamped} accepted, un-marked ${unmarked} phantom (boundary=${newestSeen}).`);
console.log(`[sync-accepted] Stamped ${stamped} accepted (boundary=${newestSeen}).`);
db.prepare("UPDATE accounts SET accepted_sync_at = datetime('now') WHERE id = ?").run(accountId);
} finally {
// B5 safety: only persist the session if still on a valid page.
let url = "";
Expand All @@ -182,7 +118,6 @@ export async function syncAcceptedConnections(accountId: string): Promise<number
} else {
try { await saveSessionState(accountId); } catch { /* ignore */ }
}
db.prepare("UPDATE accounts SET accepted_sync_at = datetime('now') WHERE id = ?").run(accountId);
}

return stamped;
Expand Down Expand Up @@ -254,3 +189,14 @@ async function fetchConnectionsPage(page: Page, start: number, count: number): P
{ start, count, decoration: DECORATION }
);
}

export function profileVanity(value: string): string | null {
try {
const u = new URL(value);
if (!['linkedin.com', 'www.linkedin.com'].includes(u.hostname)) return null;
const m = u.pathname.match(/^\/in\/([^/]+)\/?$/);
return m ? decodeURIComponent(m[1]).toLowerCase() : null;
} catch {
return null;
}
}
68 changes: 9 additions & 59 deletions pages/api/accounts/[id]/sync-accepted.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,72 +1,22 @@
import type { NextApiRequest, NextApiResponse } from "next";
import { getDb } from "@/lib/db";
import { getSessionPage, saveSessionState } from "@/lib/linkedin/session";
import { scrapePendingInvitationVanityNames } from "@/lib/linkedin/pending-invitations";
import { syncAcceptedConnections } from "@/lib/linkedin/sync-accepted";

export default async function handler(req: NextApiRequest, res: NextApiResponse) {
if (req.method !== "POST") return res.status(405).end();

const accountId = req.query.id as string;
const db = getDb();

const account = db.prepare("SELECT id, is_authenticated FROM accounts WHERE id = ?").get(accountId) as
| { id: string; is_authenticated: number }
| undefined;
const id = req.query.id as string;
const account = getDb().prepare("SELECT is_authenticated FROM accounts WHERE id=?").get(id) as {
is_authenticated: number;
} | undefined;

if (!account) return res.status(404).json({ error: "Account not found" });
if (!account.is_authenticated) return res.status(400).json({ error: "Account not authenticated" });

let page;
try {
page = await getSessionPage(accountId);
const stillPending = await scrapePendingInvitationVanityNames(page);
await saveSessionState(accountId);

// Find all targets that we sent a request to but haven't marked as connected
const waiting = db.prepare(`
SELECT id, linkedin_url, full_name
FROM targets
WHERE connection_requested_at IS NOT NULL
AND (degree IS NULL OR degree != 1)
AND connected_at IS NULL
`).all() as { id: string; linkedin_url: string; full_name: string | null }[];

const now = new Date().toISOString();
const accepted: string[] = [];
const skipped: string[] = [];

const markAccepted = db.prepare(
"UPDATE targets SET degree = 1, connected_at = ? WHERE id = ?"
);

db.transaction(() => {
for (const target of waiting) {
const match = target.linkedin_url?.match(/\/in\/([^/?#]+)/);
if (!match) continue; // no /in/ URL — can't check
const vanity = match[1].toLowerCase();

if (!stillPending.has(vanity)) {
// Not in the pending list anymore → accepted (or expired, but treat as accepted)
markAccepted.run(now, target.id);
accepted.push(target.full_name ?? vanity);
} else {
skipped.push(target.full_name ?? vanity);
}
}
})();

return res.json({
pending_on_linkedin: stillPending.size,
waiting_in_db: waiting.length,
newly_accepted: accepted.length,
still_pending: skipped.length,
accepted_names: accepted,
});

} catch (err) {
console.error("[sync-accepted]", err);
return res.status(500).json({ error: err instanceof Error ? err.message : "Sync failed" });
} finally {
await page?.close().catch(() => {});
const count = await syncAcceptedConnections(id);
return res.json({ newly_accepted: count, source: "connections-list" });
} catch {
return res.status(502).json({ error: "Connection verification failed or was incomplete. Check account authentication and retry verification; no outreach was started." });
}
}
16 changes: 16 additions & 0 deletions tests/ACCEPTANCE-SYNC.md
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,16 @@
# Acceptance synchronization

After `npm ci`, run `node tests/accepted-sync.cjs`.
The fixture uses real in-memory SQLite and a stubbed LinkedIn session. No account
cookies, network, invitations, or messages are used.

Coverage: exact canonical profile matching; account-scoped enrollment; an older
connection on a later page despite a newer stored boundary; absent profiles left
unchanged; historical connection date preservation; idempotent rechecks; and API
failure without a successful-sync timestamp.

The manual endpoint and scheduled runner share the same detector. Only positive
connections-list evidence updates a target. Positive results can remain if a
later page fails; incomplete scans do not advance accepted_sync_at. The existing
8-hour polling interval and runner scheduling conditions are unchanged. A stored
historical connection date is not proof that the current campaign caused it.
35 changes: 35 additions & 0 deletions tests/accepted-sync.cjs
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,35 @@
const fs = require('fs'), ts = require('typescript'), Module = require('module'), assert = require('node:assert/strict'), DB = require('better-sqlite3');
(async () => {
let passed = 0;
for (const failure of [false, true]) {
const db = new DB(':memory:');
db.exec(`CREATE TABLE accounts(id TEXT,connections_synced_through_ms INTEGER,accepted_sync_at TEXT,li_connections INTEGER);INSERT INTO accounts(id,connections_synced_through_ms) VALUES('a',1900000000000);CREATE TABLE targets(id TEXT,linkedin_url TEXT,connection_requested_at TEXT,connected_at TEXT,degree INTEGER);CREATE TABLE runs(id TEXT,account_id TEXT);CREATE TABLE run_profiles(target_id TEXT,run_id TEXT);INSERT INTO runs VALUES('r','a'),('other','b');INSERT INTO targets VALUES('match','https://www.linkedin.com/in/synthetic-person', 'sent',NULL,NULL),('foreign','https://www.linkedin.com/in/synthetic-person/','sent',NULL,NULL),('absent','https://www.linkedin.com/in/absent/','sent',NULL,NULL);INSERT INTO run_profiles VALUES('match','r'),('foreign','other'),('absent','r');`);
let pages = 0;
const page = { goto: async () => { }, waitForTimeout: async () => { }, url: () => 'https://www.linkedin.com/mynetwork/invite-connect/connections/', close: async () => { }, evaluate: async (fn, args) => !args ? null : failure ? null : pages++ === 0 ? [{ vanity: 'unrelated', createdAt: 1789751000000 }] : pages === 2 ? [{ vanity: 'synthetic-person', createdAt: 1658235047000 }] : [] };
const m = new Module('/app/sync-fixture.cjs');
m.require = n => n === '@/lib/db' ? { getDb: () => db } : n === '@/lib/linkedin/session' ? { getSessionPage: async () => page, saveSessionState: async () => { }, markNeedsReauth: async () => { } } : require(n);
m._compile(ts.transpileModule(fs.readFileSync(require('path').join(__dirname, '../lib/linkedin/sync-accepted.ts'), 'utf8'), { compilerOptions: { module: ts.ModuleKind.CommonJS, target: ts.ScriptTarget.ES2022 } }).outputText, '/app/sync-fixture.cjs');
assert.equal(m.exports.profileVanity('https://www.linkedin.com/in/Synthetic-person/?x=y'), 'synthetic-person');
assert.equal(m.exports.profileVanity('https://evil.test/in/synthetic-person'), null);
passed += 2;
if (failure) {
await assert.rejects(m.exports.syncAcceptedConnections('a'), /unavailable/);
assert.equal(db.prepare('SELECT accepted_sync_at FROM accounts').get().accepted_sync_at, null);
passed += 2;
}
else {
assert.equal(await m.exports.syncAcceptedConnections('a'), 1);
assert.equal(db.prepare("SELECT degree FROM targets WHERE id='match'").get().degree, 1);
assert.equal(db.prepare("SELECT degree FROM targets WHERE id='foreign'").get().degree, null);
assert.equal(db.prepare("SELECT degree FROM targets WHERE id='absent'").get().degree, null);
passed += 4;
assert.equal(db.prepare("SELECT connected_at FROM targets WHERE id='match'").get().connected_at, '2022-07-19 12:50:47');
assert.ok(db.prepare('SELECT accepted_sync_at FROM accounts').get().accepted_sync_at);
pages = 0;
assert.equal(await m.exports.syncAcceptedConnections('a'), 0);
passed += 3;
}
db.close();
}
console.log(JSON.stringify({ passed, network: 'none', synthetic: true }));
})().catch(e => { console.error(e); process.exitCode = 1; });