diff --git a/CURRENT_PLAN.md b/CURRENT_PLAN.md index 115496f..4887db5 100644 --- a/CURRENT_PLAN.md +++ b/CURRENT_PLAN.md @@ -52,3 +52,12 @@ Complete the latest GitHub API ranking in all five languages, including the unre - [x] Reuse the existing project title image and keep the facts table readable on narrow screens. - [x] Check generated site consistency in the repository verifier. - [ ] Publish an independent draft PR for review; do not merge automatically. + +## Star-farm filter (2026-11-14) + +- [x] Find how the `/ai--manus` flood reached the AI and Web new-repo boards (API samples, history replay). +- [x] Drop clone clusters, fake-watcher and zero-fork farms before ranking; flag borderline signals; blocklist on every board (`data/anti_abuse.json`). +- [x] Log dropped repos in `stats.json`, never in a README; tests in `tests/test_antiabuse.py`. +- [x] Replay the rules on every committed board and on a live discovery run. +- [ ] Publish through a pull request; merge only on the owner's word. The next daily run regenerates the boards. + diff --git a/TODO_BACKLOG.md b/TODO_BACKLOG.md index a9327ee..be84852 100644 --- a/TODO_BACKLOG.md +++ b/TODO_BACKLOG.md @@ -3,7 +3,8 @@ ## Now - [x] Verified delayed automatic update and same-day duplicate skip on 2026-11-08 (runs 42487291739 and 42490659376); another real update succeeded on 2026-11-09 (42659639717) -- [ ] Developer Tools new-repo board: 15 of 20 entries were flagged ⚠ in the last run because star-farmed repos use `cli` / `developer-tools`; decide whether to add a quality signal (owner call) +- [x] Star-farm filter for new-repo boards (`tools/antiabuse.py`, 2026-11-14): clone floods / fake watchers / zero-fork farms are dropped +- [ ] Owner call: also drop (not just flag) single cracked / pirated repos - [ ] Design and Data new-repo boards are thin (0-2 entries): widen with specific topics only (no generic ones; `tests/test_spam.py` guards that) ## Next diff --git a/data/anti_abuse.json b/data/anti_abuse.json new file mode 100644 index 0000000..b84bb76 --- /dev/null +++ b/data/anti_abuse.json @@ -0,0 +1,45 @@ +{ + "_comment": "Anti star-farming rules for the new-repo boards (tools/antiabuse.py, docs/ANTI_ABUSE.md). Thresholds were checked against every new-repo board in rankings/ (history replay). 'blocklist' applies to every board.", + "clone_clusters": { + "enabled": true, + "_doc": "Repos from different owners with the same normalized repo name, or the same normalized description, form a cluster. Same description (>= min_desc_chars) across >= desc_min_owners owners, or same name across >= name_min_owners owners, is a clone cluster: every member is dropped, except the earliest member whose owner is credible (when one exists).", + "name_min_owners": 3, + "desc_min_owners": 2, + "name_and_desc_min_owners": 2, + "min_desc_chars": 24, + "keep_check_earliest": 3 + }, + "credible_owner": { + "_doc": "Only used to keep the original of a clone cluster.", + "min_account_days": 365, + "min_followers": 50 + }, + "new_owner": { + "_doc": "A user account younger than max_account_days when the repo was created, with fewer than max_followers followers. Alone: flagged and ranked last. Together with low_fork: dropped.", + "max_account_days": 60, + "max_followers": 5 + }, + "low_fork": { + "_doc": "stars / (forks + 1). >= flag_ratio (and stars >= min_stars): flagged and ranked last. >= drop_ratio: dropped.", + "min_stars": 150, + "flag_ratio": 100, + "drop_ratio": 500 + }, + "watch_farm": { + "_doc": "subscribers_count (watchers) >= share x stars: dropped. Real new repos stay under ~10 %.", + "min_stars": 100, + "share": 0.4 + }, + "fork_farm": { + "_doc": "forks >= share x stars: flagged and ranked last.", + "min_stars": 100, + "share": 0.9 + }, + "owner_lookup_budget": 150, + "blocklist": { + "_doc": "Case-insensitive. repos: owner/name. owners: account logins. names: repo-name regexes (full match). Applies to every board.", + "repos": [], + "owners": [], + "names": ["ai--manus"] + } +} diff --git a/docs/ANTI_ABUSE.md b/docs/ANTI_ABUSE.md new file mode 100644 index 0000000..7f7dd07 --- /dev/null +++ b/docs/ANTI_ABUSE.md @@ -0,0 +1,57 @@ +# Star-farm filter + +The "hottest new repos (7 days)" boards are found by GitHub search, so they are the boards a star farm can +reach. On 2026-11-14 the AI and Web boards were flooded by one farm: hundreds of throwaway accounts each +published the same repo, `/ai--manus` ("AI Manus is a general-purpose AI Agent system ..."), all +created within minutes on 2026-11-13, ~270 stars each, 0 forks, owner accounts one month old with 0 followers. +GitHub search returned 547 copies. + +Code: `tools/antiabuse.py`. Thresholds: `data/anti_abuse.json` (edit there, not in code). +Tests: `tests/test_antiabuse.py`. Only fields the GitHub API already returns are used +(search, `GET /repos/{r}`, `GET /users/{u}`); nothing is estimated and nothing is sent anywhere. + +## Rules + +Screened **before** the `GET /repos` refresh budget is spent, so a flood cannot push real repos out: + +| Rule | Condition (defaults) | Result | +|---|---|---| +| Blocklist | `blocklist.repos` / `owners` / `names` (regex, full match, case-insensitive) | dropped, on **every** board | +| Clone cluster | same normalized description (>= 24 letters) under >= 2 owners, or same name **and** description under >= 2 owners, or same normalized name under >= 3 owners | every member dropped, except the earliest member whose owner is credible (account >= 365 days and >= 50 followers); none credible -> all dropped | +| Stars nobody forks (extreme) | stars >= 150 and stars / (forks + 1) >= 500 | dropped | + +After the refresh (needs `subscribers_count` and the owner account): + +| Rule | Condition (defaults) | Result | +|---|---|---| +| Fake watchers | watchers (`subscribers_count`) >= 40 % of stars, stars >= 100 | dropped | +| Fresh throwaway owner + low forks | user account < 60 days old at repo creation, < 5 followers, and stars / (forks + 1) >= 100 | dropped | +| Low forks | stars >= 150 and stars / (forks + 1) >= 100 | ⚠ kept, ranked last, never a leader | +| Fresh throwaway owner | user account < 60 days old at repo creation, < 5 followers | ⚠ kept, ranked last, never a leader | +| Fork farm | forks >= 90 % of stars, stars >= 100 | ⚠ kept, ranked last, never a leader | + +- A cluster or a dropped repo seen in one category is dropped from the later categories too. +- Owner lookups are cached and capped (`owner_lookup_budget`, default 150 per run). An owner that cannot be + read never triggers an owner rule (fail open), and a missing `forks_count` never triggers a fork rule. +- Tracked projects (week / month / year / total boards) are hand-picked in `data/projects.json`; only the + blocklist applies to them (`blocked` in `stats.json`, never ranked, still fetched). + +## Report + +Dropped repos never reach a README, `latest/` or a ranking page. Each run records them in +`data/stats.json` (and the history snapshot) under `newcomers.categories..anti_abuse`: +`dropped_count`, `by_reason`, `clusters` (key, size, owners, kept), `bulk_sample` (20 names from clusters and +the blocklist) and `dropped` (every other dropped repo with its reason, up to 200). `newcomers.anti_abuse` has the run totals and the number of owner lookups. The update log prints the +same summary. + +## Calibration + +The thresholds were replayed against every new-repo board committed under `rankings/` (408 repos, re-read +from the API on 2026-11-14): 74 repos would have been dropped (38 `ai--manus` copies, 26 repos with ~1,100 or +~300 stars and 0-1 forks, 4 fake-watcher repos, ...), none of them a repo with normal fork / watcher numbers. +8 repos would only be flagged ⚠, see the pull request for the list. Real hot newcomers (for example +`FrigadeHQ/yogurt`, `opencoredev/islandcn`, `alexeyw/lot-map`, `HolyArrs/HolyArrs`) have forks at 5-15 % of +stars and watchers under 10 %, far from every threshold. + +To silence a farm by hand, add it to `blocklist` in `data/anti_abuse.json`. To relax a rule, change its +threshold there and run `python3 tools/verify.py`. diff --git a/docs/CATEGORIES.md b/docs/CATEGORIES.md index eadafa3..c29dd08 100644 --- a/docs/CATEGORIES.md +++ b/docs/CATEGORIES.md @@ -49,7 +49,9 @@ Rules (`tools/spam.py`, `suspicious_reason`), matched on repo name, description 4. A commercial software name in the repo name (Adobe, Photoshop, Canva, Autodesk, AutoCAD, Cinema 4D, Microsoft Office, Wondershare, ...) unless the owner is the vendor. 5. A `-Pro` / `-Premium` / year-suffixed name with no description and no language. -This is a heuristic, not a verdict: a flagged repo is only moved down, never hidden. `tests/test_spam.py` pins the rules, the ordering, the leader exclusion and that the scheduled entry points use the same code. +This is a heuristic, not a verdict: a flagged repo is only moved down, never hidden. +Clear star farms (clone floods such as hundreds of `/ai--manus`, fake watchers, stars nobody forks) are +a different case: they are **dropped** by the star-farm filter, see [ANTI_ABUSE.md](ANTI_ABUSE.md). `tests/test_spam.py` pins the rules, the ordering, the leader exclusion and that the scheduled entry points use the same code. ## Per-category boards diff --git a/docs/NEW_ARRIVALS.md b/docs/NEW_ARRIVALS.md index c45ae6a..c47db82 100644 --- a/docs/NEW_ARRIVALS.md +++ b/docs/NEW_ARRIVALS.md @@ -25,9 +25,12 @@ Suspected cracked / pirated / activator repos are listed last and never lead a c 1. Created within the last 7 days (exact timestamp check on `created_at`). 2. At least 100 stars, not a fork, not archived, not private. -3. The best 30 candidates by search star count (per category) are re-read with `GET /repos/{owner}/{repo}`. -4. **Hotness = stars ÷ max(age in days, 1).** -5. Ranked by hotness (ties: more stars, then name), top 20 shown per category, with suspected cracked / pirated repos after all normal ones. +3. Star farms are dropped before and after the refresh: clone clusters, blocklist, stars nobody forks, fake + watchers, fresh throwaway owners ([ANTI_ABUSE.md](ANTI_ABUSE.md)). Dropped repos are logged under + `newcomers.categories..anti_abuse` and never shown. +4. The best 30 remaining candidates by search star count (per category) are re-read with `GET /repos/{owner}/{repo}`. +5. **Hotness = stars ÷ max(age in days, 1).** +6. Ranked by hotness (ties: more stars, then name), top 20 shown per category, with suspected cracked / pirated repos after all normal ones. If every search query for a category fails, that category's new-repo board is shown as missing. If only some queries fail, the README says so. Search is index-based and can lag by minutes. diff --git a/docs/project-governance/CHANGELOG.md b/docs/project-governance/CHANGELOG.md index a09526a..79c8c10 100644 --- a/docs/project-governance/CHANGELOG.md +++ b/docs/project-governance/CHANGELOG.md @@ -1,5 +1,10 @@ # CHANGELOG.md +## 2026-11-14 (star-farm filter) + +- New-repo boards drop star farms: clone floods (the same repo under many throwaway accounts, e.g. `/ai--manus`), fake watchers, stars nobody forks, fresh throwaway owners. Borderline cases are kept with ⚠ and ranked last. Rules: `docs/ANTI_ABUSE.md`, thresholds `data/anti_abuse.json`. +- A blocklist in `data/anti_abuse.json` applies to every board. Dropped repos are logged in `stats.json`, never shown. + ## 2026-11-07 (public first screen) - README first screen now states the value, the live-board URL, and three quick-use steps before the category leaders. diff --git a/docs/project-governance/DECISIONS.md b/docs/project-governance/DECISIONS.md index 1a9cfbe..8d3cfc5 100644 --- a/docs/project-governance/DECISIONS.md +++ b/docs/project-governance/DECISIONS.md @@ -121,3 +121,12 @@ - The raw-data refresh stays fail-safe and independent of translation availability. The local translation job is complete only when both its cache-aware description and hook audits report zero gaps across current categories. - Valid partial output is retained and published, but remaining gaps return status 4 and are retried on the next existing scheduled invocation. Audit failures cannot be interpreted as zero work. One invocation makes one bounded generation pass; a push race rechecks the new ranking without opening an unbounded model loop. - Existing detailed descriptions, translations and valid hooks are preserved. This completion also fills the previous 98 missing hook fields for reuse without rewriting past numeric snapshots. No model/backend, credentials, installed schedule or ranking policy changes are part of this fix. + +## 2026-11-14 + +### Star farms are dropped, not just flagged + +- Owner request: the boards must not be gamed. Clear farms (clone clusters, fake watchers, >= 500 stars per fork, fresh throwaway owner + low forks, blocklist) are removed from the new-repo boards and logged in `stats.json`; borderline signals (low forks alone, fresh owner alone, fork farm) keep the existing ⚠ / rank-last treatment so a real repo is never hidden by one weak signal. +- Single cracked / pirated repos keep the 2026-10-30 decision (flag, rank last) unless a farm rule drops them. +- Thresholds live in `data/anti_abuse.json`, calibrated by replaying every committed board (`docs/ANTI_ABUSE.md`). + diff --git a/docs/project-governance/WORKLOG.md b/docs/project-governance/WORKLOG.md index 1f90692..04556b2 100644 --- a/docs/project-governance/WORKLOG.md +++ b/docs/project-governance/WORKLOG.md @@ -1,5 +1,12 @@ # WORKLOG.md +## 2026-11-14 + +- The 2026-11-14 AI board had 19 of 20 entries and the Web board 19 of 20 from one farm: `/ai--manus`, same description, created minutes apart on 2026-11-13, ~270 stars, 0 forks, owners ~30 days old with 0 followers (search finds 547 copies). Root cause: discovery ranked any non-fork repo with >= 100 stars by stars/day and had no rule for identical repos under different owners; the flood also used the whole refresh budget. +- Added `tools/antiabuse.py` + `data/anti_abuse.json` (clone clusters, blocklist, low-fork, fake watchers, fresh owners), wired into `newcomers.discover_category` before and after the refresh, report under `anti_abuse`; `risers_core.rank` skips `blocked` tracked projects. +- Replay over every committed new-repo board (408 repos re-read from the API): 74 dropped, all farm-shaped; 8 flagged only. +- Verification: `python3 tools/verify.py` green; `tests/test_antiabuse.py` added. + ## 2026-11-06 - PLAN-005: production cadence is once daily at America/Los_Angeles 00:00. Dropped the Beijing cron (`0 16 * * *`) and the Beijing launchd agent. Scheduled Actions runs always use slot `pt`. diff --git a/tests/test_antiabuse.py b/tests/test_antiabuse.py new file mode 100644 index 0000000..68e19fa --- /dev/null +++ b/tests/test_antiabuse.py @@ -0,0 +1,280 @@ +"""Star-farm filter (tools/antiabuse.py + data/anti_abuse.json). + +Pins: the 2026-11-13 `/ai--manus` clone flood is removed from every new-repo board (and does not +use up the refresh budget), the original of a cloned repo survives when its owner is credible, the per-repo +signals (stars nobody forks, fake watchers, fresh throwaway owners), the blocklist on every board, and that +normal hot repos measured from real history snapshots are untouched. +""" +import datetime as dt +import json +import sys +import unittest +from pathlib import Path +from urllib.parse import parse_qs, urlparse + +ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(ROOT / "tools")) +import antiabuse as aa # noqa: E402 +import newcomers as nc # noqa: E402 +import risers_core as core # noqa: E402 + +NOW = dt.datetime(2026, 11, 14, 8, 0, tzinfo=dt.timezone.utc) +NOSLEEP = lambda s: None # noqa: E731 +MANUS_DESC = ("AI Manus is a general-purpose AI Agent system that supports running various tools and " + "operations in a sandbox environment.") +MANUS_OWNERS = ["AdmiralEmporium", "ApothecaryDesign", "ArcherAmplify", "Archiveeutrough", "ArcticPrawnEmporium", + "armsworminterface", "armysystemsun", "Aspectnisafeguard", "BarrierCenter", "BladesmithPanel", + "Boarddrunail", "Boardglacode", "ClampOverlord46", "Feathershosmash", "GryphonEmpress", + "HurricaneMonarchDart", "TealAgentRevive", "TorusBarbarianMap9", "WinnerBrokerBarrow", + "AdvisorShake", "AgentLobster", "AmplitudeBoss", "AtomEmissary", "StudentSeparate"] + + +def ts(days_ago: float) -> str: + return (NOW - dt.timedelta(days=days_ago)).strftime("%Y-%m-%dT%H:%M:%SZ") + + +def repo(name, stars, days_old=1.0, desc="a normal repo", topics=("ai", "web"), forks=None, subs=None, **kw): + r = {"full_name": name, "id": abs(hash(name)) % 10**9, "html_url": f"https://github.com/{name}", + "description": desc, "language": "TypeScript", "stargazers_count": stars, "created_at": ts(days_old), + "fork": False, "archived": False, "private": False, "topics": list(topics), + "forks_count": max(stars // 10, 1) if forks is None else forks} + if subs is not None: + r["subscribers_count"] = subs + r.update(kw) + return r + + +def cfg_no_blocklist(): + """Real thresholds, empty blocklist: proves the rules catch the flood without a hand-written entry.""" + return {**aa.load_config(), "blocklist": {"repos": [], "owners": [], "names": []}} + + +def user(login, account_days, followers, kind="User"): + return {"login": login, "type": kind, "created_at": ts(account_days), "followers": followers} + + +def manus_pool(): + # exactly what the 2026-11-14 board showed: same name, same description, 0 forks, fresh 0-follower accounts + return [repo(f"{o}/ai--manus", 266 + i % 3, 0.6, MANUS_DESC, forks=0, subs=1) for i, o in enumerate(MANUS_OWNERS)] + + +def fake_api(pool, users): + def api(path): + if path.startswith("/search/repositories"): + q = parse_qs(urlparse(path).query)["q"][0] + topic = q.split()[0].removeprefix("topic:") + hits = sorted((r for r in pool if topic in r["topics"]), key=lambda r: -r["stargazers_count"]) + return {"total_count": len(hits), "items": hits} + if path.startswith("/users/"): + login = path.removeprefix("/users/") + if login.lower() in users: + return users[login.lower()] + raise RuntimeError("404") + return next(r for r in pool if r["full_name"] == path.removeprefix("/repos/")) + return api + + +def default_users(pool): + users = {} + for r in pool: + o = r["full_name"].split("/")[0] + users[o.lower()] = user(o, 31, 0) if r["full_name"].endswith("ai--manus") else user(o, 2000, 120) + return users + + +NORMAL = [ # shaped after real boards: yogurt, islandcn, lot-map, HolyArrs, serveros panel + repo("FrigadeHQ/yogurt", 1368, 3.7, "Private AI meeting notes that stay on your computer", forks=71, subs=18), + repo("opencoredev/islandcn", 587, 1.1, "The open-source Dynamic Island for React", forks=55, subs=0), + repo("alexeyw/lot-map", 780, 4.8, "A live map beside the chat that shows what Claude is doing", forks=88, subs=6), + repo("HolyArrs/HolyArrs", 1141, 4.6, "One self-hosted app for your shows, movies and music", forks=59, subs=3), + repo("serveroshq/panel", 771, 5.5, "The open source ServerOS panel", forks=62, subs=2), +] + + +class CloneClusterTests(unittest.TestCase): + def test_ai_manus_flood_is_one_cluster_and_fully_dropped(self): + g = aa.Guard(fake_api([], default_users(manus_pool())), NOW, cfg_no_blocklist()) + kept, dropped, clusters = g.screen_candidates(manus_pool() + NORMAL) + self.assertEqual({r["full_name"] for r in kept}, {r["full_name"] for r in NORMAL}) + self.assertEqual(len(dropped), len(MANUS_OWNERS)) + self.assertTrue(all(d["reason"].startswith("clone-cluster:ai--manus") for d in dropped)) + self.assertEqual(len(clusters), 1) + self.assertIsNone(clusters[0]["kept"]) + self.assertEqual(clusters[0]["owners"], len(MANUS_OWNERS)) + + def test_same_description_under_new_names_is_still_a_cluster(self): + pool = [repo(f"farm{i}/agent-{i}", 300, 1, "Agent OS 2026: run every tool in a sandbox!!", forks=3) + for i in range(2)] + clusters = aa.find_clone_clusters(pool, aa.load_config()) + self.assertEqual(len(clusters), 1) + + def test_original_survives_when_its_owner_is_credible(self): + orig = repo("realorg/cool-agent", 900, 5, "A cool agent framework with a long description", forks=80) + copies = [repo(f"copy{i}/cool-agent", 300, 1, "A cool agent framework with a long description", forks=0) + for i in range(3)] + users = {"realorg": user("realorg", 3000, 500, "Organization")} + users.update({f"copy{i}": user(f"copy{i}", 10, 0) for i in range(3)}) + g = aa.Guard(fake_api([], users), NOW) + kept, dropped, clusters = g.screen_candidates([orig] + copies) + self.assertEqual([r["full_name"] for r in kept], ["realorg/cool-agent"]) + self.assertEqual(clusters[0]["kept"], "realorg/cool-agent") + self.assertEqual(len(dropped), 3) + + def test_two_unrelated_repos_sharing_a_generic_name_are_kept(self): + pool = [repo("alice/agent-skills", 400, 2, "Skills for Claude Code by Alice"), + repo("bob/agent-skills", 300, 3, "My curated list of Codex prompts")] + self.assertEqual(aa.find_clone_clusters(pool, aa.load_config()), []) + + def test_same_owner_duplicates_are_not_a_cluster(self): + pool = [repo("me/dsh-go", 900, 2, "DeepSeek Harness in your pocket, runs on the phone"), + repo("me/dsh-Go", 900, 2, "DeepSeek Harness in your pocket, runs on the phone")] + self.assertEqual(aa.find_clone_clusters(pool, aa.load_config()), []) + + def test_name_normalization(self): + self.assertEqual(aa.norm_name("AI--Manus"), aa.norm_name("ai_manus")) + self.assertEqual(aa.norm_desc("Tool 2026: setup!"), aa.norm_desc("tool 2023 setup")) + + +class SignalTests(unittest.TestCase): + cfg = aa.load_config() + + def test_stars_nobody_forks_are_dropped(self): + drop, _ = aa.repo_signals(repo("x/rufus-usb-formatter", 1103, forks=0), user("x", 2000, 0), self.cfg) + self.assertTrue(drop.startswith("low-fork")) + + def test_fake_watchers_are_dropped(self): + drop, _ = aa.repo_signals(repo("x/Blender-AI-MCP-Studio", 272, forks=4, subs=195), None, self.cfg) + self.assertTrue(drop.startswith("watch-farm")) + + def test_fresh_owner_plus_low_forks_is_dropped_alone_only_flagged(self): + r = repo("x/cubase-studio-desktop", 300, forks=0) + self.assertTrue(aa.repo_signals(r, user("x", 20, 0), self.cfg)[0].startswith("new-owner+low-fork")) + drop, flag = aa.repo_signals(repo("x/thing", 300, forks=40), user("x", 20, 0), self.cfg) + self.assertIsNone(drop) + self.assertEqual(flag, "new-owner") + + def test_low_fork_alone_is_only_flagged(self): + drop, flag = aa.repo_signals(repo("o/ChaosIntelligence", 428, forks=0), user("o", 3000, 3), self.cfg) + self.assertIsNone(drop) + self.assertTrue(flag.startswith("low-fork")) + + def test_normal_repos_have_no_signal(self): + for r in NORMAL: + with self.subTest(r["full_name"]): + self.assertEqual(aa.repo_signals(r, user("o", 1500, 100), self.cfg), (None, None)) + + def test_unknown_owner_and_missing_forks_never_trigger(self): + r = repo("o/x", 5000) + del r["forks_count"] + self.assertEqual(aa.repo_signals(r, None, self.cfg), (None, None)) + + def test_owner_lookup_budget_fails_open(self): + calls = [] + g = aa.Guard(lambda p: calls.append(p) or user("x", 1, 0), NOW, {**aa.load_config(), "owner_lookup_budget": 1}) + g.owner("a") + self.assertIsNone(g.owner("b")) + self.assertEqual(len(calls), 1) + + +class BlocklistTests(unittest.TestCase): + def test_blocklist_matches_repo_owner_and_name_pattern(self): + cfg = {**aa.load_config(), "blocklist": {"repos": ["A/B"], "owners": ["evil"], "names": ["ai--manus"]}} + self.assertEqual(aa.blocked_reason("a/b", cfg), "blocklist:repo") + self.assertEqual(aa.blocked_reason("EVIL/x", cfg), "blocklist:owner") + self.assertTrue(aa.blocked_reason("someone/AI--Manus", cfg).startswith("blocklist:name")) + self.assertIsNone(aa.blocked_reason("Simpleyyt/ai-manus", cfg)) + + def test_blocked_tracked_project_is_never_ranked_on_any_board(self): + g = {"per_day": 1.0} + projs = [{"name": "a/a", "ok": True, "stars": 10, "growth": {"week": g, "month": g, "year": g}}, + {"name": "b/b", "ok": True, "stars": 99, "blocked": "blocklist:repo", + "growth": {"week": {"per_day": 9}, "month": {"per_day": 9}, "year": {"per_day": 9}}}] + for key in ("week", "month", "year", "total"): + self.assertEqual([p["name"] for p in core.rank(projs, key)], ["a/a"]) + + def test_tracked_projects_are_not_blocklisted(self): + cfg = aa.load_config() + for p in json.loads((ROOT / "data/projects.json").read_text("utf-8")): + self.assertIsNone(aa.blocked_reason(p["name"], cfg), p["name"]) + + def test_config_file_is_valid(self): + cfg = aa.load_config() + for k in ("clone_clusters", "low_fork", "watch_farm", "new_owner", "blocklist"): + self.assertIn(k, cfg) + + +class DiscoveryTests(unittest.TestCase): + def test_board_backfills_with_normal_repos_and_reports_the_drop(self): + pool = manus_pool() + NORMAL + [repo("dev/real-small", 150, 5, "A real small tool", forks=12, subs=2)] + orig = nc.REFRESH_TOP + nc.REFRESH_TOP = 6 # the flood must not eat the refresh budget + try: + api = fake_api(pool, default_users(pool)) + res = nc.discover_category(api, NOW, {"id": "ai", "topics": ["ai"]}, set(), NOSLEEP, + aa.Guard(api, NOW, cfg_no_blocklist())) + finally: + nc.REFRESH_TOP = orig + names = [i["name"] for i in res["items"]] + self.assertFalse([n for n in names if n.endswith("ai--manus")]) + self.assertEqual(len(names), 6) + self.assertEqual(names[0], "opencoredev/islandcn") + self.assertEqual(res["anti_abuse"]["dropped_count"], len(MANUS_OWNERS)) + self.assertEqual(res["anti_abuse"]["by_reason"], {"clone-cluster": len(MANUS_OWNERS)}) + + def test_every_category_board_is_clean(self): + pool = manus_pool() + NORMAL + cats = [{"id": "ai", "primary": True, "order": 0, "topics": ["ai"]}, + {"id": "web", "order": 1, "topics": ["web"]}] + res = nc.discover_all(fake_api(pool, default_users(pool)), NOW, cats, [], [], core.rank, NOSLEEP) + for cid, block in res["categories"].items(): + with self.subTest(cid): + self.assertFalse([i for i in block["items"] if i["name"].endswith("ai--manus")]) + self.assertEqual(res["anti_abuse"]["dropped_total"], 2 * len(MANUS_OWNERS)) + + def test_flagged_repo_is_ranked_last_with_reason(self): + pool = NORMAL[:2] + [repo("o/zero-forks", 300, 1, "Something", forks=0, subs=0)] + res = nc.discover_category(fake_api(pool, {"o": user("o", 3000, 3)} | {r["full_name"].split("/")[0].lower(): user("x", 3000, 90) for r in NORMAL}), + NOW, {"id": "ai", "topics": ["ai"]}, set(), NOSLEEP) + self.assertEqual(res["items"][-1]["name"], "o/zero-forks") + self.assertTrue(res["items"][-1]["suspicious"].startswith("low-fork")) + + +class RenderTests(unittest.TestCase): + def test_farm_flag_gets_its_own_note_in_every_language(self): + import archive + from i18n import S + item = {"name": "o/x", "url": "u", "description": "d", "created_at": "2026-11-13T00:00:00Z", "stars": 300, + "per_day": 300.0, "age_days": 1.0, "suspicious": "low-fork:300 stars/0 forks"} + stats = {"generated_at": "2026-11-14T08:00:00Z", "today_utc": "2026-11-14", "projects": [], + "category_ids": ["ai"], "newcomers": {"ok": True, "categories": {"ai": {"ok": True, "items": [item]}}}} + cats = [{"id": "ai", "primary": True, "order": 0, "name": {}, "topics": ["ai"]}] + for lang in S: + with self.subTest(lang): + md = archive.render_rankings_md(stats, lang, cats) + self.assertIn(S[lang]["farm_note"], md) + self.assertNotIn(S[lang]["susp_note"], md) + + +class HistoryReplayTests(unittest.TestCase): + """Committed boards: the clone rule never fires on a board without a real clone flood.""" + + def test_history_boards_only_cluster_known_farms(self): + cfg = aa.load_config() + for f in sorted((ROOT / "rankings").glob("*/*/stats.json")): + nc_root = json.loads(f.read_text("utf-8")).get("newcomers") or {} + for cid, block in (nc_root.get("categories") or {"ai": nc_root}).items(): + repos = [{"full_name": i["name"], "description": i.get("description"), "created_at": i.get("created_at"), + "stargazers_count": i.get("stars")} for i in block.get("items") or []] + for cl in aa.find_clone_clusters(repos, cfg): + for m in cl["members"]: + with self.subTest(f=str(f.relative_to(ROOT)), cid=cid, repo=m["full_name"]): + # every clustered member is either the ai--manus flood or already a ⚠ crack repo + item = next(i for i in block["items"] if i["name"] == m["full_name"]) + self.assertTrue(m["full_name"].lower().endswith("/ai--manus") or item.get("suspicious") + or "full working installer" in (item.get("description") or "").lower() + or "standalone" in (item.get("description") or "").lower(), + m["full_name"]) + + +if __name__ == "__main__": + unittest.main() diff --git a/tools/antiabuse.py b/tools/antiabuse.py new file mode 100644 index 0000000..948b038 --- /dev/null +++ b/tools/antiabuse.py @@ -0,0 +1,338 @@ +"""Anti star-farming filters for the "hottest new repos" boards, plus a blocklist for every board. + +Star farms create many throwaway accounts that each publish the same repo (same name and/or the same +description, e.g. hundreds of `/ai--manus`) and pump them with bought stars, or push one repo +with stars nobody forks or with fake watchers. The rules below only read fields the GitHub API already +returns (search / GET /repos/{r} / GET /users/{u}); thresholds live in data/anti_abuse.json and are +documented in docs/ANTI_ABUSE.md. tests/test_antiabuse.py pins them. + +Two outcomes: + drop -> removed from the board, recorded under newcomers.categories..anti_abuse (never in a README) + flag -> kept with `suspicious: `, shown with ⚠, ranked after every normal repo, never a leader +""" +from __future__ import annotations + +import datetime as dt +import json +import re +from pathlib import Path +from typing import Any, Callable + +ROOT = Path(__file__).resolve().parents[1] +CONFIG_PATH = ROOT / "data/anti_abuse.json" + +DEFAULTS: dict[str, Any] = { + "clone_clusters": {"enabled": True, "name_min_owners": 3, "desc_min_owners": 2, + "name_and_desc_min_owners": 2, "min_desc_chars": 24, "keep_check_earliest": 3}, + "credible_owner": {"min_account_days": 365, "min_followers": 50}, + "new_owner": {"max_account_days": 60, "max_followers": 5}, + "low_fork": {"min_stars": 150, "flag_ratio": 100, "drop_ratio": 500}, + "watch_farm": {"min_stars": 100, "share": 0.4}, + "fork_farm": {"min_stars": 100, "share": 0.9}, + "owner_lookup_budget": 150, + "blocklist": {"repos": [], "owners": [], "names": []}, +} +REPORT_CAP = 200 # single drops listed per category (counts are always exact) +SAMPLE_CAP = 20 # names listed for clone-cluster / blocklist drops + +Api = Callable[[str], Any] + + +def load_config(path: Path | None = None) -> dict[str, Any]: + """data/anti_abuse.json merged over DEFAULTS (missing file -> defaults).""" + cfg = json.loads(json.dumps(DEFAULTS)) + p = path or CONFIG_PATH + if p.exists(): + raw = json.loads(p.read_text("utf-8")) + for k, v in raw.items(): + if k.startswith("_"): + continue + if isinstance(v, dict) and isinstance(cfg.get(k), dict): + cfg[k].update({kk: vv for kk, vv in v.items() if not kk.startswith("_")}) + else: + cfg[k] = v + return cfg + + +def _parse_ts(value: str | None) -> dt.datetime | None: + if not value: + return None + try: + return dt.datetime.fromisoformat(str(value).replace("Z", "+00:00")) + except ValueError: + return None + + +def split_name(full_name: str) -> tuple[str, str]: + owner, _, name = (full_name or "").partition("/") + return owner, name + + +def norm_name(name: str) -> str: + """Repo name without case and separators: 'AI--Manus', 'ai_manus' -> 'aimanus'.""" + return re.sub(r"[^0-9a-z]+", "", (name or "").lower()) + + +def norm_desc(desc: str | None) -> str: + """Description without case, digits, punctuation or spacing, so '... 2026 ...' variants still match.""" + return re.sub(r"[\W\d_]+", "", (desc or "").lower()) + + +# ---------------------------------------------------------------- blocklist (every board) + +def blocked_reason(full_name: str, cfg: dict[str, Any]) -> str | None: + bl = cfg.get("blocklist") or {} + owner, name = split_name(full_name) + if full_name.lower() in {r.lower() for r in bl.get("repos") or []}: + return "blocklist:repo" + if owner.lower() in {o.lower() for o in bl.get("owners") or []}: + return "blocklist:owner" + for pat in bl.get("names") or []: + if re.fullmatch(pat, name, flags=re.I): + return f"blocklist:name:{pat}" + return None + + +def filter_blocked_projects(projects: list[dict], cfg: dict[str, Any]) -> tuple[list[dict], list[dict]]: + """Tracked projects (week / month / year / total boards) minus blocklisted ones.""" + kept, dropped = [], [] + for p in projects: + reason = blocked_reason(p.get("name", ""), cfg) + if reason: + dropped.append({"name": p.get("name"), "reason": reason}) + else: + kept.append(p) + return kept, dropped + + +# ---------------------------------------------------------------- clone clusters + +def find_clone_clusters(repos: list[dict], cfg: dict[str, Any]) -> list[dict[str, Any]]: + """Group repos from different owners that share a normalized name or description. + + Returns [{"key", "members": [repo, ...] (oldest first), "owners": n}] for groups that pass the + configured owner thresholds. A repo is in at most one cluster (union of name and description links). + """ + cc = cfg["clone_clusters"] + if not cc.get("enabled", True) or not repos: + return [] + parent = list(range(len(repos))) + + def find(i: int) -> int: + while parent[i] != i: + parent[i] = parent[parent[i]] + i = parent[i] + return i + + def union(a: int, b: int) -> None: + ra, rb = find(a), find(b) + if ra != rb: + parent[rb] = ra + + by_name: dict[str, list[int]] = {} + by_desc: dict[str, list[int]] = {} + for i, r in enumerate(repos): + _, name = split_name(r.get("full_name", "")) + if norm_name(name): + by_name.setdefault(norm_name(name), []).append(i) + d = norm_desc(r.get("description")) + if len(d) >= int(cc["min_desc_chars"]): + by_desc.setdefault(d, []).append(i) + for idx in list(by_name.values()) + list(by_desc.values()): + for j in idx[1:]: + union(idx[0], j) + + groups: dict[int, list[int]] = {} + for i in range(len(repos)): + groups.setdefault(find(i), []).append(i) + + out = [] + for members in groups.values(): + if len(members) < 2: + continue + owners = {split_name(repos[i].get("full_name", ""))[0].lower() for i in members} + if len(owners) < 2: + continue + names = {norm_name(split_name(repos[i].get("full_name", ""))[1]) for i in members} + descs = {norm_desc(repos[i].get("description")) for i in members + if len(norm_desc(repos[i].get("description"))) >= int(cc["min_desc_chars"])} + # owners per shared name / shared description + name_owners = max(len({split_name(repos[i]["full_name"])[0].lower() for i in members + if norm_name(split_name(repos[i]["full_name"])[1]) == n}) for n in names) + desc_owners = max((len({split_name(repos[i]["full_name"])[0].lower() for i in members + if norm_desc(repos[i].get("description")) == d}) for d in descs), default=0) + both = 0 + for n in names: + for d in descs: + both = max(both, len({split_name(repos[i]["full_name"])[0].lower() for i in members + if norm_name(split_name(repos[i]["full_name"])[1]) == n + and norm_desc(repos[i].get("description")) == d})) + if (name_owners >= int(cc["name_min_owners"]) or desc_owners >= int(cc["desc_min_owners"]) + or both >= int(cc["name_and_desc_min_owners"])): + ms = sorted((repos[i] for i in members), key=lambda r: (r.get("created_at") or "", r["full_name"].lower())) + key = split_name(ms[0]["full_name"])[1].lower() + out.append({"key": key, "members": ms, "owners": len(owners)}) + return out + + +# ---------------------------------------------------------------- per-repo signals + +def owner_age_days(repo: dict, owner: dict | None) -> float | None: + created, ocreated = _parse_ts(repo.get("created_at")), _parse_ts((owner or {}).get("created_at")) + if not created or not ocreated: + return None + return (created - ocreated).total_seconds() / 86400 + + +def is_new_owner(repo: dict, owner: dict | None, cfg: dict[str, Any]) -> bool: + if not owner or owner.get("type") != "User": + return False + age = owner_age_days(repo, owner) + no = cfg["new_owner"] + return (age is not None and age < float(no["max_account_days"]) + and int(owner.get("followers") or 0) < int(no["max_followers"])) + + +def is_credible_owner(owner: dict | None, now: dt.datetime, cfg: dict[str, Any]) -> bool: + if not owner: + return False + created = _parse_ts(owner.get("created_at")) + co = cfg["credible_owner"] + return bool(created and (now - created).days >= int(co["min_account_days"]) + and int(owner.get("followers") or 0) >= int(co["min_followers"])) + + +def repo_signals(repo: dict, owner: dict | None, cfg: dict[str, Any]) -> tuple[str | None, str | None]: + """(drop_reason, flag_reason) for one repo. subscribers_count is only on GET /repos/{r}.""" + stars = int(repo.get("stargazers_count") or 0) + has_forks = repo.get("forks_count") is not None # always present in real API payloads; unknown -> no fork rule + forks = int(repo.get("forks_count") or 0) + lf, wf, ff = cfg["low_fork"], cfg["watch_farm"], cfg["fork_farm"] + ratio = stars / (forks + 1) + low_fork = has_forks and stars >= int(lf["min_stars"]) and ratio >= float(lf["flag_ratio"]) + new_owner = is_new_owner(repo, owner, cfg) + subs = repo.get("subscribers_count") + if subs is not None and stars >= int(wf["min_stars"]) and int(subs) >= float(wf["share"]) * stars: + return f"watch-farm:{int(subs)} watchers/{stars} stars", None + if has_forks and stars >= int(lf["min_stars"]) and ratio >= float(lf["drop_ratio"]): + return f"low-fork:{stars} stars/{forks} forks", None + if low_fork and new_owner: + return f"new-owner+low-fork:{stars} stars/{forks} forks", None + if low_fork: + return None, f"low-fork:{stars} stars/{forks} forks" + if new_owner: + return None, "new-owner" + if has_forks and stars >= int(ff["min_stars"]) and forks >= float(ff["share"]) * stars: + return None, f"fork-farm:{forks} forks/{stars} stars" + return None, None + + +# ---------------------------------------------------------------- run state + +class Guard: + """One per daily run: config, a cached / budgeted owner lookup, cross-category memory and the report.""" + + def __init__(self, api: Api | None, now: dt.datetime, cfg: dict[str, Any] | None = None): + self.api = api + self.now = now + self.cfg = cfg or load_config() + self.owners: dict[str, dict | None] = {} + self.lookups = 0 + self.cluster_keys: set[str] = set() # norm names / descs of clusters seen in earlier categories + self.dropped_names: set[str] = set() + + def owner(self, login: str) -> dict | None: + key = login.lower() + if key in self.owners: + return self.owners[key] + if self.api is None or self.lookups >= int(self.cfg.get("owner_lookup_budget", 0)): + return None # fail open: unknown owner never triggers an owner rule + self.lookups += 1 + try: + data = self.api(f"/users/{login}") + self.owners[key] = data if isinstance(data, dict) else None + except Exception: + self.owners[key] = None + return self.owners[key] + + def _remembered(self, repo: dict) -> bool: + _, name = split_name(repo.get("full_name", "")) + d = norm_desc(repo.get("description")) + return (repo.get("full_name", "").lower() in self.dropped_names + or ("n:" + norm_name(name)) in self.cluster_keys + or (len(d) >= int(self.cfg["clone_clusters"]["min_desc_chars"]) and ("d:" + d) in self.cluster_keys)) + + def screen_candidates(self, cands: list[dict]) -> tuple[list[dict], list[dict], list[dict]]: + """Before the GET /repos refresh: blocklist, clone clusters, extreme low-fork (search fields only). + + Returns (kept, dropped [{name, reason}], clusters [{key, owners, size, kept}]). + """ + dropped: list[dict] = [] + kept: list[dict] = [] + for r in cands: + reason = blocked_reason(r.get("full_name", ""), self.cfg) + if not reason and self._remembered(r): + reason = "clone-cluster:seen in an earlier category" + if reason: + dropped.append({"name": r["full_name"], "reason": reason}) + else: + kept.append(r) + clusters_out = [] + drop_ids: set[str] = set() + for cl in find_clone_clusters(kept, self.cfg): + keep = None + for m in cl["members"][: int(self.cfg["clone_clusters"]["keep_check_earliest"])]: + if is_credible_owner(self.owner(split_name(m["full_name"])[0]), self.now, self.cfg): + keep = m + break + for m in cl["members"]: + if m is keep: + continue + drop_ids.add(m["full_name"].lower()) + dropped.append({"name": m["full_name"], + "reason": f"clone-cluster:{cl['key']} ({cl['owners']} owners)"}) + _, name = split_name(m["full_name"]) + self.cluster_keys.add("n:" + norm_name(name)) + d = norm_desc(m.get("description")) + if len(d) >= int(self.cfg["clone_clusters"]["min_desc_chars"]): + self.cluster_keys.add("d:" + d) + clusters_out.append({"key": cl["key"], "owners": cl["owners"], "size": len(cl["members"]), + "kept": keep["full_name"] if keep else None}) + kept = [r for r in kept if r["full_name"].lower() not in drop_ids] + final = [] + for r in kept: + drop, _ = repo_signals({k: v for k, v in r.items() if k != "subscribers_count"}, None, self.cfg) + if drop: + dropped.append({"name": r["full_name"], "reason": drop}) + else: + final.append(r) + self.dropped_names |= {d["name"].lower() for d in dropped} + return final, dropped, clusters_out + + def screen_refreshed(self, repo: dict) -> tuple[str | None, str | None]: + """After GET /repos/{r}: watcher / fork / owner signals (one cached owner lookup).""" + reason = blocked_reason(repo.get("full_name", ""), self.cfg) + if reason: + return reason, None + owner = self.owner(split_name(repo.get("full_name", ""))[0]) + drop, flag = repo_signals(repo, owner, self.cfg) + if drop: + self.dropped_names.add(repo["full_name"].lower()) + return drop, flag + + +def report(dropped: list[dict], clusters: list[dict]) -> dict[str, Any]: + """Compact log for stats.json: counts by reason, clusters (with a sample), and every other drop by name.""" + reasons: dict[str, int] = {} + for d in dropped: + k = d["reason"].split(":")[0] + reasons[k] = reasons.get(k, 0) + 1 + bulk = [d for d in dropped if d["reason"].startswith(("clone-cluster", "blocklist"))] + single = [d for d in dropped if not d["reason"].startswith(("clone-cluster", "blocklist"))] + return { + "dropped_count": len(dropped), + "by_reason": dict(sorted(reasons.items())), + "clusters": sorted(clusters, key=lambda c: -c["size"]), + "bulk_sample": sorted({d["name"] for d in bulk}, key=str.lower)[:SAMPLE_CAP], + "dropped": sorted(single, key=lambda d: d["name"].lower())[:REPORT_CAP], + } diff --git a/tools/archive.py b/tools/archive.py index 157ddab..d670c0f 100644 --- a/tools/archive.py +++ b/tools/archive.py @@ -91,6 +91,13 @@ def _rows(ranked: list[dict], key: str, lang: str = "en") -> list[str]: return out +FARM_FLAGS = ("low-fork", "new-owner", "fork-farm") + + +def _farm_flag(item: dict) -> bool: + return str(item.get("suspicious") or "").startswith(FARM_FLAGS) + + def _new_rows(items: list[dict]) -> list[str]: return [f"| {i} | [{it['name']}]({it['url']}){' ⚠' if it.get('suspicious') else ''} | {it['created_at'][:10]} | {it['stars']:,} | " f"{it['per_day']:,.1f} |" for i, it in enumerate(items, 1)] @@ -156,8 +163,11 @@ def render_rankings_md(stats: dict[str, Any], lang: str = "en", lines += [f"| {s['h_rank']} | {s['h_repo']} | {s['h_created']} | {s['h_stars_short']} | {s['h_perday']} |", "|---|---|---|---:|---:|"] lines += _new_rows(new_block.get("items", [])[:TOP_N]) + [""] - if any(i.get("suspicious") for i in new_block.get("items", [])[:TOP_N]): + shown = new_block.get("items", [])[:TOP_N] + if any(i.get("suspicious") and not _farm_flag(i) for i in shown): lines += [s["susp_note"], ""] + if any(_farm_flag(i) for i in shown): # star-farm signals (tools/antiabuse.py) + lines += [s["farm_note"], ""] bad = [q for q in new_block.get("queries", []) if not q.get("ok")] if bad: lines += [s["new_partial"].format(failed=len(bad), total=len(new_block["queries"])), ""] diff --git a/tools/i18n.py b/tools/i18n.py index 0010ca0..9fb6e8d 100644 --- a/tools/i18n.py +++ b/tools/i18n.py @@ -19,6 +19,7 @@ "new_failed": "The new-repo search failed in the latest run, so this ranking is missing (nothing is estimated): {error}", "new_partial": "Note: {failed} of {total} search queries failed in this run, so the candidate pool may be incomplete.", "susp_note": "⚠ = suspected cracked / pirated / activator repo (matched by name or description rules in [docs/CATEGORIES.md](docs/CATEGORIES.md)). These are still listed, but always ranked after every normal repo and never shown as a category leader.", + "farm_note": "⚠ (low-fork / new-owner / fork-farm) = signs of star farming (stars nobody forks, a brand-new owner account, forks close to stars), see [docs/ANTI_ABUSE.md](docs/ANTI_ABUSE.md). Listed after every normal repo, never a category leader. Clear star farms (clone floods, fake watchers) are removed entirely.", "history": "These tables are the latest ranking. Every update is also saved per time slot: [ranking history]({index}) · [{latest}]({latest}).", "shown": "Each board in the README shows the top {m}. The full top {n} of each board (from {total} tracked projects) is in the latest/ files.", "title": "GitHub Risers: Ranked by Stars Gained per Day", @@ -123,6 +124,7 @@ "new_failed": "最近一次运行中新仓库搜索失败,因此缺少这个榜单(未做任何估算):{error}", "new_partial": "注意:本次运行中 {total} 条搜索查询有 {failed} 条失败,候选池可能不完整。", "susp_note": "⚠ = 疑似破解 / 盗版 / 激活工具类仓库(按名称或描述规则识别,规则见 [docs/CATEGORIES.md](docs/CATEGORIES.md))。这类仓库照常收录,但一律排在所有正常项目之后,也不会出现在各类榜首里。", + "farm_note": "⚠(刷星迹象)= 星多却几乎没人 fork、账号刚注册、或 fork 数接近星数,规则见 [docs/ANTI_ABUSE.md](docs/ANTI_ABUSE.md)。照常收录,但排在所有正常项目之后,也不会当榜首。明显刷榜的(批量克隆、假关注等)直接剔除。", "history": "下方表格为最新榜单。每次更新还会按时段另存一份:[榜单历史]({index}) · [{latest}]({latest})。", "shown": "README 里每张榜只列前 {m} 名;每张榜完整的前 {n} 名(来自 {total} 个已收录项目)在 latest/ 文件里。", "title": "GitHub 涨星榜 / GitHub Risers", @@ -231,6 +233,7 @@ "new_failed": "最近一次執行中新儲存庫搜尋失敗,因此缺少這個榜單(未做任何估算):{error}", "new_partial": "注意:本次執行中 {total} 筆搜尋查詢有 {failed} 筆失敗,候選名單可能不完整。", "susp_note": "⚠ = 疑似破解 / 盜版 / 啟用工具類儲存庫(依名稱或描述規則辨識,規則見 [docs/CATEGORIES.md](docs/CATEGORIES.md))。這類儲存庫照常收錄,但一律排在所有正常專案之後,也不會出現在各類榜首裡。", + "farm_note": "⚠(刷星跡象)= 星多卻幾乎沒人 fork、帳號剛註冊、或 fork 數接近星數,規則見 [docs/ANTI_ABUSE.md](docs/ANTI_ABUSE.md)。照常收錄,但排在所有正常專案之後,也不會當榜首。明顯刷榜的(大量複製、假關注等)直接剔除。", "history": "下方表格為最新榜單。每次更新還會依時段另存一份:[榜單歷史]({index}) · [{latest}]({latest})。", "shown": "README 裡每張榜只列前 {m} 名;每張榜完整的前 {n} 名(來自 {total} 個已收錄專案)在 latest/ 檔案裡。", "title": "GitHub 漲星榜 / GitHub Risers", @@ -339,6 +342,7 @@ "new_failed": "最新の実行で新着リポジトリの検索に失敗したため、このランキングは欠損しています(推定はしていません):{error}", "new_partial": "注意:今回は検索 {total} 件のうち {failed} 件が失敗したため、候補が不完全な可能性があります。", "susp_note": "⚠ = クラック / 海賊版 / アクティベーションツールの疑いがあるリポジトリ(名前・説明のルールで判定。ルールは [docs/CATEGORIES.md](docs/CATEGORIES.md))。掲載はしますが、常に通常のリポジトリより後ろに並べ、カテゴリ 1 位には表示しません。", + "farm_note": "⚠(スター水増しの兆候)= スターが多いのにほぼフォークされない、作成直後のアカウント、フォーク数がスター数に近い、のいずれか。ルールは [docs/ANTI_ABUSE.md](docs/ANTI_ABUSE.md)。掲載はしますが通常のリポジトリより後ろに並べ、カテゴリ 1 位にはしません。明らかな水増し(大量コピー、偽ウォッチャーなど)は除外します。", "history": "以下が最新のランキングです。更新のたびに時間帯ごとにも保存しています:[ランキング履歴]({index}) · [{latest}]({latest})。", "shown": "README では各ランキングの上位 {m} 件を表示します。各ランキングの上位 {n} 件の全体(収録 {total} プロジェクトから)は latest/ のファイルにあります。", "title": "GitHub スター急上昇ランキング / GitHub Risers", @@ -443,6 +447,7 @@ "new_failed": "최근 실행에서 신규 저장소 검색이 실패해 이 랭킹이 누락되었습니다(추정하지 않음): {error}", "new_partial": "참고: 이번 실행에서 검색 {total}건 중 {failed}건이 실패해 후보 목록이 불완전할 수 있습니다.", "susp_note": "⚠ = 크랙 / 불법 복제 / 정품 인증 우회 도구로 의심되는 저장소(이름·설명 규칙으로 판별, 규칙은 [docs/CATEGORIES.md](docs/CATEGORIES.md)). 목록에는 포함하지만 항상 정상 저장소 뒤에 두며 카테고리 1위로는 표시하지 않습니다.", + "farm_note": "⚠(스타 조작 징후) = 스타는 많은데 포크가 거의 없음, 갓 만든 계정, 포크 수가 스타 수에 가까움 중 하나. 규칙은 [docs/ANTI_ABUSE.md](docs/ANTI_ABUSE.md). 목록에는 포함하지만 정상 저장소 뒤에 두며 카테고리 1위로는 표시하지 않습니다. 명백한 조작(대량 복제, 가짜 워처 등)은 제외합니다.", "history": "아래가 최신 순위입니다. 업데이트할 때마다 시간대별로도 저장합니다: [순위 기록]({index}) · [{latest}]({latest}).", "shown": "README에는 랭킹마다 상위 {m}개를 싣습니다. 각 랭킹의 상위 {n}개 전체({total}개 수록 프로젝트 기준)는 latest/ 파일에 있습니다.", "title": "GitHub 스타 급상승 랭킹 / GitHub Risers", diff --git a/tools/newcomers.py b/tools/newcomers.py index f5bc066..2bf7658 100644 --- a/tools/newcomers.py +++ b/tools/newcomers.py @@ -7,6 +7,9 @@ Everything comes from the API (search + GET /repos/{r} for top candidates). Pure functions take an `api` callable so they are unit-testable offline. +Star farms (clone clusters such as hundreds of `/ai--manus`, stars nobody forks, fake watchers, +blocklist) are dropped before ranking and logged under `anti_abuse` (tools/antiabuse.py, docs/ANTI_ABUSE.md). + Category rules (topics / keywords / optional languages) live in data/categories.json and are documented in docs/CATEGORIES.md and docs/NEW_ARRIVALS.md. """ @@ -20,8 +23,9 @@ from typing import Any, Callable try: # works both as `tools.newcomers` and when tools/ is on sys.path - from . import spam + from . import antiabuse, spam except ImportError: # pragma: no cover + import antiabuse # type: ignore import spam # type: ignore WINDOW_DAYS = 7 @@ -75,7 +79,7 @@ def search_all(api: Api, q: str, sleep: Callable[[float], None] = time.sleep) -> return items, total -def to_item(repo: dict, now: dt.datetime) -> dict[str, Any]: +def to_item(repo: dict, now: dt.datetime, flag: str | None = None) -> dict[str, Any]: created = parse_ts(repo["created_at"]) age = max((now - created).total_seconds() / 86400, 1.0) stars = int(repo["stargazers_count"]) @@ -91,7 +95,7 @@ def to_item(repo: dict, now: dt.datetime) -> dict[str, Any]: "per_day": round(stars / age, 1), "topics": list(repo.get("topics") or []), } - reason = spam.suspicious_reason(repo) + reason = spam.suspicious_reason(repo) or flag if reason: item["suspicious"] = reason return item @@ -128,8 +132,10 @@ def rank_items(items: list[dict]) -> list[dict]: def discover_category(api: Api, now: dt.datetime, category: dict[str, Any], excluded_names: set[str], - sleep: Callable[[float], None] = time.sleep) -> dict[str, Any]: - """Run searches for one category, refresh candidates, dedup, rank top N.""" + sleep: Callable[[float], None] = time.sleep, + guard: "antiabuse.Guard | None" = None) -> dict[str, Any]: + """Run searches for one category, screen star farms, refresh candidates, dedup, rank top N.""" + guard = guard or antiabuse.Guard(api, now) queries, found = [], {} for label, q in build_queries(now, category): entry: dict[str, Any] = {"label": label, "q": q} @@ -156,6 +162,8 @@ def discover_category(api: Api, now: dt.datetime, category: dict[str, Any], result.update(ok=False, error="every search query failed: " + queries[0].get("error", "?"), items=[]) return result cands = [r for r in found.values() if qualifies(r, now) and has_category_topic(r, category)] + # star farms (clone clusters, blocklist, stars nobody forks) go before the refresh budget is spent + cands, farm_dropped, clusters = guard.screen_candidates(cands) # normal repos first so a flood of star-farmed spam cannot crowd them out of the refresh budget cands.sort(key=lambda r: (bool(spam.suspicious_reason(r)), -int(r["stargazers_count"]))) refreshed = [] @@ -165,24 +173,31 @@ def discover_category(api: Api, now: dt.datetime, category: dict[str, Any], except Exception: fresh = r if qualifies(fresh, now) and has_category_topic(fresh, category): - refreshed.append(fresh) - items, dropped = exclude_duplicates([to_item(r, now) for r in refreshed], excluded_names) + drop, flag = guard.screen_refreshed(fresh) + if drop: + farm_dropped.append({"name": fresh["full_name"], "reason": drop}) + continue + refreshed.append((fresh, flag)) + items, dropped = exclude_duplicates([to_item(r, now, flag) for r, flag in refreshed], excluded_names) ranked = rank_items(items)[:TOP_N] result.update(ok=True, excluded_duplicates=dropped, items=ranked, - suspicious_in_board=sum(1 for i in ranked if i.get("suspicious"))) + suspicious_in_board=sum(1 for i in ranked if i.get("suspicious")), + anti_abuse=antiabuse.report(farm_dropped, clusters)) return result def discover_all(api: Api, now: dt.datetime, categories: list[dict[str, Any]], projects_meta: list[dict[str, Any]], stats_projects: list[dict], rank: Callable[[list[dict], str], list[dict]], - sleep: Callable[[float], None] = time.sleep) -> dict[str, Any]: + sleep: Callable[[float], None] = time.sleep, + guard: "antiabuse.Guard | None" = None) -> dict[str, Any]: """Discover newcomers for every category (AI first). Exclusive across categories by order. Within a category, newcomers never repeat that category's tracked projects or its week/month/year/total top N. A repo already shown in an earlier category's newcomers is also excluded (AI is primary, so it claims first). """ + guard = guard or antiabuse.Guard(api, now) # shared: a farm seen in one category is dropped in the others by_cat: dict[str, dict[str, Any]] = {} claimed: set[str] = set() # lower-cased names already shown in an earlier category ordered = sorted(categories, key=lambda c: (0 if c.get("primary") else 1, c.get("order", 99), c["id"])) @@ -191,7 +206,7 @@ def discover_all(api: Api, now: dt.datetime, categories: list[dict[str, Any]], tracked = {p["name"] for p in projects_meta if p.get("category") == cid} cat_stats = [p for p in stats_projects if p.get("name") in tracked] excluded = tracked | displayed_names(cat_stats, TOP_N, rank) | claimed - result = discover_category(api, now, cat, excluded, sleep) + result = discover_category(api, now, cat, excluded, sleep, guard) by_cat[cid] = result if result.get("ok"): claimed |= {i["name"].lower() for i in result.get("items", [])} @@ -203,6 +218,8 @@ def discover_all(api: Api, now: dt.datetime, categories: list[dict[str, Any]], "min_stars": MIN_STARS, "top_n": TOP_N, "categories": by_cat, + "anti_abuse": {"owner_lookups": guard.lookups, + "dropped_total": sum((r.get("anti_abuse") or {}).get("dropped_count", 0) for r in by_cat.values())}, "error": None if ok_any else "every category search failed", } diff --git a/tools/risers_core.py b/tools/risers_core.py index 4312ced..31de6ac 100644 --- a/tools/risers_core.py +++ b/tools/risers_core.py @@ -107,8 +107,11 @@ def snapshot_delta( def rank(projects: list[dict[str, Any]], key: str) -> list[dict[str, Any]]: - """Rank ok projects by growth window (per_day desc), or by total stars for key='total'.""" - ok = [p for p in projects if p.get("ok")] + """Rank ok projects by growth window (per_day desc), or by total stars for key='total'. + + A project marked `blocked` (data/anti_abuse.json blocklist, set by tools/update_stars.py) is never ranked. + """ + ok = [p for p in projects if p.get("ok") and not p.get("blocked")] if key == "total": return sorted(ok, key=lambda p: (-p["stars"], p["name"])) return sorted(ok, key=lambda p: (-p["growth"][key]["per_day"], -p["stars"], p["name"])) diff --git a/tools/update_stars.py b/tools/update_stars.py index 3d785ea..61376f9 100755 --- a/tools/update_stars.py +++ b/tools/update_stars.py @@ -22,6 +22,7 @@ from typing import Any sys.path.insert(0, str(Path(__file__).resolve().parent)) +import antiabuse # noqa: E402 import archive # noqa: E402 import i18n_pipeline # noqa: E402 import risers_core as core # noqa: E402 @@ -192,18 +193,28 @@ def main(argv: list[str] | None = None) -> int: # Attach category from projects.json onto each result for per-category rankings. meta_by_name = {p["name"]: p for p in projects} + abuse_cfg = antiabuse.load_config() for r in results: meta = meta_by_name.get(r["name"], {}) r["category"] = meta.get("category", "ai") + blocked = antiabuse.blocked_reason(r["name"], abuse_cfg) # every board honours the blocklist + if blocked: + r["blocked"] = blocked + print(f"BLOCK {r['name']:36s} {blocked} (kept in data, never ranked)") categories = json.loads((ROOT / "data/categories.json").read_text("utf-8")) print("discovering new repos per category (GitHub search API, several minutes) ...", flush=True) + guard = antiabuse.Guard(lambda path: api_get(path, token), now, abuse_cfg) new = newcomers.discover_all( - lambda path: api_get(path, token), now, categories, projects, results, core.rank) + lambda path: api_get(path, token), now, categories, projects, results, core.rank, guard=guard) if new["ok"]: for cid, block in new["categories"].items(): if block.get("ok"): - print(f"new[{cid}]: {len(block['items'])} ranked from {block['candidates_seen']} candidates") + ab = block.get("anti_abuse") or {} + print(f"new[{cid}]: {len(block['items'])} ranked from {block['candidates_seen']} candidates; " + f"star-farm filter dropped {ab.get('dropped_count', 0)} {ab.get('by_reason', {})}") + for c in (ab.get("clusters") or [])[:5]: + print(f" clone cluster {c['key']}: {c['size']} repos / {c['owners']} owners, kept={c['kept']}") else: print(f"MISS new[{cid}]: {block.get('error')}", file=sys.stderr) else: