From c07b618350c1477f009f6354eafa863d24cafe92 Mon Sep 17 00:00:00 2001 From: Dmitry Ivanov Date: Sat, 26 Sep 2026 11:34:34 +0300 Subject: [PATCH 1/6] some re-org of dirs/modules --- .github/workflows/collect_commits.yml | 18 ------------------ src/{ => api/forgejo}/ForgejoRepoAPI.py | 0 src/{ => api/github}/GitHubRepoAPI.py | 0 .../github}/graphql/pull_request_parser.py | 0 src/git_logger.py | 2 +- src/interface_wrapper.py | 8 ++++---- 6 files changed, 5 insertions(+), 23 deletions(-) delete mode 100644 .github/workflows/collect_commits.yml rename src/{ => api/forgejo}/ForgejoRepoAPI.py (100%) rename src/{ => api/github}/GitHubRepoAPI.py (100%) rename src/{ => api/github}/graphql/pull_request_parser.py (100%) diff --git a/.github/workflows/collect_commits.yml b/.github/workflows/collect_commits.yml deleted file mode 100644 index cc50997a..00000000 --- a/.github/workflows/collect_commits.yml +++ /dev/null @@ -1,18 +0,0 @@ -name: Collect Commits -on: - push: - -jobs: - collect-commits: - runs-on: [self-hosted, mse] - steps: - - name: Collect info - run: | - cd /data/ - mkdir -p '${{github.repository}}' - - cat > ./run.json < str: class RepositoryFactory: @staticmethod def create_api(token: str, base_url: str | None = None) -> IRepositoryAPI: - from src.ForgejoRepoAPI import ForgejoRepoAPI - from src.GitHubRepoAPI import GitHubRepoAPI + from src.api.forgejo.ForgejoRepoAPI import ForgejoRepoAPI + from src.api.github.GitHubRepoAPI import GitHubRepoAPI errors = [] From f91b1e76bd876f845232e1a21c538050ee68afde Mon Sep 17 00:00:00 2001 From: Dmitry Ivanov Date: Sat, 26 Sep 2026 12:05:12 +0300 Subject: [PATCH 2/6] re-org api wrapper modules (and upd imports) --- src/{interface_wrapper.py => api/baseAPI.py} | 152 +------------------ src/api/forgejo/ForgejoRepoAPI.py | 24 +-- src/api/github/GitHubRepoAPI.py | 40 ++--- src/api/models.py | 127 ++++++++++++++++ src/commits_parser.py | 3 +- src/contributors_parser.py | 3 +- src/git_logger.py | 6 +- src/invites_parser.py | 3 +- src/issues_parser.py | 3 +- src/pull_requests_parser.py | 3 +- src/workflow_runs_parser.py | 3 +- 11 files changed, 167 insertions(+), 200 deletions(-) rename src/{interface_wrapper.py => api/baseAPI.py} (53%) create mode 100644 src/api/models.py diff --git a/src/interface_wrapper.py b/src/api/baseAPI.py similarity index 53% rename from src/interface_wrapper.py rename to src/api/baseAPI.py index 58e0af61..0b8145c8 100644 --- a/src/interface_wrapper.py +++ b/src/api/baseAPI.py @@ -1,139 +1,15 @@ -import logging from abc import ABC, abstractmethod -from dataclasses import dataclass -from datetime import datetime from github import Auth, Github from pyforgejo import PyforgejoApi -# Настройка логирования -logging.basicConfig( - level=logging.INFO, format="%(asctime)s - %(levelname)s - %(message)s" -) - - -# Модельные классы -@dataclass -class Contributor: - username: str - email: str - - -@dataclass -class User: - _id: int - login: str - username: str - email: str - html_url: str - node_id: str - type: str - bio: str - site_admin: bool - - -@dataclass -class Commit: - _id: str - message: str - author: User - date: datetime - files: list[str] - additions: int - deletions: int - - -@dataclass -class Branch: - name: str - last_commit: Commit | None - - -@dataclass -class Repository: - _id: str - name: str - url: str - default_branch: Branch - owner: User - - -@dataclass -class Issue: - _id: int - number: int - title: str - state: str - created_at: datetime - closed_at: datetime - body: str - user: User - closed_by: User - labels: list[str] - milestone: str - - -@dataclass -class PullRequest: - _id: int - title: str - author: User - state: str - created_at: datetime - head_label: str - base_label: str - head_ref: str - base_ref: str - merged_by: User - merged: bool - files: list[str] - issue_url: str - labels: list[str] - milestone: str - comments: int = 0 - review_comments: int = 0 - - -@dataclass -class Invite: - _id: int - invitee: User - created_at: datetime | None - html_url: str - - -@dataclass -class Comment: - body: str - created_at: datetime - author: User - - -@dataclass -class WikiPage: - title: str - content: str - - -@dataclass -class WorkflowRun: - display_title: str - event: str - head_branch: str - head_sha: str - name: str - path: str - created_at: datetime - run_started_at: datetime - updated_at: datetime - conclusion: str - status: str - url: str +from src.api.models import (Branch, Comment, Commit, Contributor, Invite, + Issue, PullRequest, Repository, User, WikiPage, + WorkflowRun) # Интерфейс API class IRepositoryAPI(ABC): - @abstractmethod def get_user_data(self, user) -> User: pass @@ -224,25 +100,3 @@ def create_api(token: str, base_url: str | None = None) -> IRepositoryAPI: errors.append(f"Forgejo login failed: {e}") raise Exception(" / ".join(errors)) - - -# Сервис для расчёта метрик -class CommitmentCalculator: - def __init__(self, api: IRepositoryAPI): - self.api = api - - def calculate(self, repo: Repository) -> dict[str, int]: - if not repo: - return {} - try: - commits = self.api.get_commits(repo) - if not commits: - return {} - result = {} - for commit in commits: - author = commit.author.username - result[author] = result.get(author, 0) + 1 - return result - except Exception as e: - logging.error(f"Failed to calculate commitment for repo {repo.name}: {e}") - return {} diff --git a/src/api/forgejo/ForgejoRepoAPI.py b/src/api/forgejo/ForgejoRepoAPI.py index d76fb937..9186187d 100644 --- a/src/api/forgejo/ForgejoRepoAPI.py +++ b/src/api/forgejo/ForgejoRepoAPI.py @@ -1,27 +1,15 @@ import base64 -import sys import logging +import sys import isodate from pyforgejo import PyforgejoApi -from src.utils import ( - log_exceptions, -) -from src.interface_wrapper import ( - Branch, - Comment, - Commit, - Contributor, - Invite, - IRepositoryAPI, - Issue, - PullRequest, - Repository, - User, - WikiPage, - WorkflowRun -) +from src.api.baseAPI import IRepositoryAPI +from src.api.models import (Branch, Comment, Commit, Contributor, Invite, + Issue, PullRequest, Repository, User, WikiPage, + WorkflowRun) +from src.utils import log_exceptions class ForgejoRepoAPI(IRepositoryAPI): diff --git a/src/api/github/GitHubRepoAPI.py b/src/api/github/GitHubRepoAPI.py index ca87a169..d9be9f11 100644 --- a/src/api/github/GitHubRepoAPI.py +++ b/src/api/github/GitHubRepoAPI.py @@ -1,22 +1,10 @@ from github import Github -from src.utils import ( - log_exceptions, -) -from src.interface_wrapper import ( - Branch, - Comment, - Commit, - Contributor, - Invite, - IRepositoryAPI, - Issue, - PullRequest, - Repository, - User, - WikiPage, - WorkflowRun, -) +from src.api.baseAPI import IRepositoryAPI +from src.api.models import (Branch, Comment, Commit, Contributor, Invite, + Issue, PullRequest, Repository, User, WikiPage, + WorkflowRun) +from src.utils import log_exceptions class GitHubRepoAPI(IRepositoryAPI): @@ -83,7 +71,7 @@ def get_contributors(self, repo: Repository) -> list[Contributor]: @log_exceptions(default_return=[], message="Failed to get issues from GitHub") def get_issues(self, repo: Repository) -> list[Issue]: - issues = self.client.get_repo(repo._id).get_issues(state='all') + issues = self.client.get_repo(repo._id).get_issues(state="all") return [ Issue( _id=i.number, @@ -101,9 +89,11 @@ def get_issues(self, repo: Repository) -> list[Issue]: for i in issues ] - @log_exceptions(default_return=[], message="Failed to get pull requests from GitHub") + @log_exceptions( + default_return=[], message="Failed to get pull requests from GitHub" + ) def get_pull_requests(self, repo: Repository) -> list[PullRequest]: - pulls = self.client.get_repo(repo._id).get_pulls(state='all') + pulls = self.client.get_repo(repo._id).get_pulls(state="all") return [ PullRequest( _id=p.number, @@ -139,7 +129,7 @@ def get_branches(self, repo: Repository) -> list[Branch]: return result def get_wiki_pages(self, repo: Repository) -> list[WikiPage]: - raise Exception('not implemented') + raise Exception("not implemented") @log_exceptions(default_return=[], message="Failed to get forks from GitHub") def get_forks(self, repo: Repository) -> list[Repository]: @@ -200,7 +190,9 @@ def get_invites(self, repo: Repository) -> list[Invite]: def get_rate_limiting(self) -> tuple[int, int]: return self.client.rate_limiting - @log_exceptions(default_return=[], message="Failed to get workflow runs from GitHub") + @log_exceptions( + default_return=[], message="Failed to get workflow runs from GitHub" + ) def get_workflow_runs(self, repo) -> list[WorkflowRun]: runs = self.client.get_repo(repo._id).get_workflow_runs() return [ @@ -222,14 +214,14 @@ def get_workflow_runs(self, repo) -> list[WorkflowRun]: ] def get_base_url(self) -> str: - return 'https://api.github.com' + return "https://api.github.com" # Точка входа для тестирования if __name__ == "__main__": # Создайте клиент GitHub (используйте ваш токен) # client = Github("") - api = GitHubRepoAPI('client') + api = GitHubRepoAPI("client") # Укажите ваш репозиторий repo_name = "" diff --git a/src/api/models.py b/src/api/models.py new file mode 100644 index 00000000..5d442a29 --- /dev/null +++ b/src/api/models.py @@ -0,0 +1,127 @@ +import logging +from dataclasses import dataclass +from datetime import datetime + +# Настройка логирования +logging.basicConfig( + level=logging.INFO, format="%(asctime)s - %(levelname)s - %(message)s" +) + + +# Модельные классы +@dataclass +class Contributor: + username: str + email: str + + +@dataclass +class User: + _id: int + login: str + username: str + email: str + html_url: str + node_id: str + type: str + bio: str + site_admin: bool + + +@dataclass +class Commit: + _id: str + message: str + author: User + date: datetime + files: list[str] + additions: int + deletions: int + + +@dataclass +class Branch: + name: str + last_commit: Commit | None + + +@dataclass +class Repository: + _id: str + name: str + url: str + default_branch: Branch + owner: User + + +@dataclass +class Issue: + _id: int + number: int + title: str + state: str + created_at: datetime + closed_at: datetime + body: str + user: User + closed_by: User + labels: list[str] + milestone: str + + +@dataclass +class PullRequest: + _id: int + title: str + author: User + state: str + created_at: datetime + head_label: str + base_label: str + head_ref: str + base_ref: str + merged_by: User + merged: bool + files: list[str] + issue_url: str + labels: list[str] + milestone: str + comments: int = 0 + review_comments: int = 0 + + +@dataclass +class Invite: + _id: int + invitee: User + created_at: datetime | None + html_url: str + + +@dataclass +class Comment: + body: str + created_at: datetime + author: User + + +@dataclass +class WikiPage: + title: str + content: str + + +@dataclass +class WorkflowRun: + display_title: str + event: str + head_branch: str + head_sha: str + name: str + path: str + created_at: datetime + run_started_at: datetime + updated_at: datetime + conclusion: str + status: str + url: str diff --git a/src/commits_parser.py b/src/commits_parser.py index 74a2d8c4..20843215 100644 --- a/src/commits_parser.py +++ b/src/commits_parser.py @@ -5,8 +5,9 @@ import pytz +from src.api.baseAPI import IRepositoryAPI from src.constants import EMPTY_FIELD, GOOGLE_MAX_CELL_LEN, TIMEDELTA, TIMEZONE -from src.interface_wrapper import IRepositoryAPI, Repository +from src.api.models import Repository from src.utils import logger diff --git a/src/contributors_parser.py b/src/contributors_parser.py index e87fc824..e04c9214 100644 --- a/src/contributors_parser.py +++ b/src/contributors_parser.py @@ -2,8 +2,9 @@ from time import sleep from typing import Generator +from src.api.baseAPI import IRepositoryAPI from src.constants import EMPTY_FIELD, TIMEDELTA -from src.interface_wrapper import IRepositoryAPI, Repository +from src.api.models import Repository from src.utils import logger diff --git a/src/git_logger.py b/src/git_logger.py index 6c3a5b5f..98d6888c 100644 --- a/src/git_logger.py +++ b/src/git_logger.py @@ -5,10 +5,10 @@ import requests +from src.api.baseAPI import IRepositoryAPI from src.api.github.GitHubRepoAPI import GitHubRepoAPI -from src.interface_wrapper import ( - RepositoryFactory, - IRepositoryAPI +from src.api.baseAPI import ( + RepositoryFactory ) from src.constants import ( EMPTY_FIELD, diff --git a/src/invites_parser.py b/src/invites_parser.py index f56ad778..503975d4 100644 --- a/src/invites_parser.py +++ b/src/invites_parser.py @@ -2,8 +2,9 @@ from time import sleep from typing import Generator +from src.api.baseAPI import IRepositoryAPI from src.constants import TIMEDELTA -from src.interface_wrapper import IRepositoryAPI, Repository +from src.api.models import Repository from src.utils import logger diff --git a/src/issues_parser.py b/src/issues_parser.py index 1d5d673f..dc199db8 100644 --- a/src/issues_parser.py +++ b/src/issues_parser.py @@ -6,9 +6,10 @@ import pytz import requests +from src.api.baseAPI import IRepositoryAPI from src.constants import EMPTY_FIELD, TIMEDELTA, TIMEZONE from src.git_logger import get_assignee_story -from src.interface_wrapper import IRepositoryAPI, Repository +from src.api.models import Repository from src.utils import logger diff --git a/src/pull_requests_parser.py b/src/pull_requests_parser.py index ac177d87..b90400d4 100644 --- a/src/pull_requests_parser.py +++ b/src/pull_requests_parser.py @@ -7,9 +7,10 @@ import pytz import requests +from src.api.baseAPI import IRepositoryAPI from src.constants import EMPTY_FIELD, TIMEDELTA, TIMEZONE from src.git_logger import get_assignee_story -from src.interface_wrapper import IRepositoryAPI, Repository +from src.api.models import Repository from src.utils import logger from src.repo_dataclasses import PullRequestData diff --git a/src/workflow_runs_parser.py b/src/workflow_runs_parser.py index be69bbf8..25cedae5 100644 --- a/src/workflow_runs_parser.py +++ b/src/workflow_runs_parser.py @@ -2,8 +2,9 @@ from time import sleep from typing import Generator +from src.api.baseAPI import IRepositoryAPI from src.constants import TIMEDELTA -from src.interface_wrapper import IRepositoryAPI, Repository +from src.api.models import Repository from src.utils import logger From ef2f9bff5fdb918f5d4e3e6cb1f75d9d3fccb58d Mon Sep 17 00:00:00 2001 From: Dmitry Ivanov Date: Sat, 26 Sep 2026 13:55:17 +0300 Subject: [PATCH 3/6] add PoC for Forgejo DB --- main.py | 11 +++- src/api/baseAPI.py | 67 ++++++++++++++++------ src/api/forgejo/ForgejoRepoDB.py | 96 ++++++++++++++++++++++++++++++++ src/api/forgejo/sql_queries.py | 57 +++++++++++++++++++ src/api/utils.py | 7 +++ src/git_logger.py | 29 ++++------ 6 files changed, 229 insertions(+), 38 deletions(-) create mode 100644 src/api/forgejo/ForgejoRepoDB.py create mode 100644 src/api/forgejo/sql_queries.py create mode 100644 src/api/utils.py diff --git a/main.py b/main.py index 1d703264..b2d8b74f 100644 --- a/main.py +++ b/main.py @@ -12,10 +12,18 @@ from src import wikipars from src import workflow_runs_parser from src.utils import parse_time, validate_and_normalize_cell +from src.api.utils import APITypes def parse_args(): parser = argparse.ArgumentParser() + parser.add_argument( + '--source', + choices=list(APITypes), + default=APITypes.github, + required=False, + help='Data source: github API (by default), forgejo API, forgejo db (direct access)' + ) parser.add_argument("--invites", help="print pending invites", action="store_true") parser.add_argument("-c", "--commits", help="log commits", action="store_true") parser.add_argument( @@ -51,6 +59,7 @@ def parse_args(): '--base_url', type=str, required=False, + default="https://git.moevm.pro/api/v1", help='Base URL for Forgejo instance (if using Forgejo)', ) @@ -212,7 +221,7 @@ def main(): repositories = git_logger.get_repos_from_file(args.list) try: - clients = git_logger.Clients(tokens, args.base_url) + clients = git_logger.Clients(api_type=args.source, auth_data=dict(tokens=tokens, base_url=args.base_url)) binded_repos = git_logger.get_next_binded_repo(clients, repositories) except Exception as e: print(f"Failed to initialize any clients: {e}") diff --git a/src/api/baseAPI.py b/src/api/baseAPI.py index 0b8145c8..78657a0c 100644 --- a/src/api/baseAPI.py +++ b/src/api/baseAPI.py @@ -3,9 +3,24 @@ from github import Auth, Github from pyforgejo import PyforgejoApi -from src.api.models import (Branch, Comment, Commit, Contributor, Invite, - Issue, PullRequest, Repository, User, WikiPage, - WorkflowRun) +from src.api.models import ( + Branch, + Comment, + Commit, + Contributor, + Invite, + Issue, + PullRequest, + Repository, + User, + WikiPage, + WorkflowRun, +) + +import logging +import traceback + +from src.api.utils import APITypes # Интерфейс API @@ -80,23 +95,39 @@ def get_base_url(self) -> str: class RepositoryFactory: @staticmethod - def create_api(token: str, base_url: str | None = None) -> IRepositoryAPI: + def create_api(api_type: APITypes, auth_data: dict) -> IRepositoryAPI: from src.api.forgejo.ForgejoRepoAPI import ForgejoRepoAPI from src.api.github.GitHubRepoAPI import GitHubRepoAPI errors = [] - try: - client = GitHubRepoAPI(Github(auth=Auth.Token(token))) - if client.client: - return client - except Exception as e: - errors.append(f"GitHub login failed: {e}") - - if base_url: - try: - return ForgejoRepoAPI(PyforgejoApi(api_key=token, base_url=base_url)) - except Exception as e: - errors.append(f"Forgejo login failed: {e}") - - raise Exception(" / ".join(errors)) + match api_type: + case APITypes.github: + try: + token = auth_data.get("token") + client = GitHubRepoAPI(Github(auth=Auth.Token(token))) + if client.client: + return client + except Exception as e: + errors.append( + f"GitHub login failed: {e}. Token: {str(token)[:4]}..." + ) + case APITypes.forgejo | auth_data.get("base_url"): # legacy code for backward compability + try: + base_url = auth_data.get("base_url") + token = auth_data.get("token") + return ForgejoRepoAPI( + PyforgejoApi(api_key=token, base_url=base_url) + ) + except Exception as e: + errors.append( + f"Forgejo login failed: {e}. Base url: {base_url}. Token: {str(token)[:4]}..." + ) + case APITypes.forgejo_db: + ... + + if errors: + logging.error(" / ".join(errors)) + logging.error("\n".join(traceback.format_stack())) + + return None diff --git a/src/api/forgejo/ForgejoRepoDB.py b/src/api/forgejo/ForgejoRepoDB.py new file mode 100644 index 00000000..b4d1b8f2 --- /dev/null +++ b/src/api/forgejo/ForgejoRepoDB.py @@ -0,0 +1,96 @@ +import base64 +import logging + +import psycopg2 + + +from src.api.baseAPI import IRepositoryAPI +from src.api.models import ( + Branch, + Comment, + Commit, + Contributor, + Issue, + PullRequest, + Repository, + User, + WikiPage, +) +from src.utils import log_exceptions +from src.api.forgejo.sql_queries import GET_FULL_PR_INFO + + +class ForgejoDBClient: + def __init__(self, connect_info: dict): + self.connection_params = dict( + host=connect_info.get("FORGEJO_DB_HOST"), + port=int(connect_info.get("FORGEJO_DB_PORT", 5432)), + dbname=connect_info.get("FORGEJO_DB_NAME"), + user=connect_info.get("FORGEJO_DB_USER"), + password=connect_info.get("FORGEJO_DB_PASSWORD"), + ) + self.connection = psycopg2.connect(**self.connection_params) + + def __enter__(self): + self.connection = psycopg2.connect(**self.connection_params) + return self + + def __exit__(self, exc_type, exc_val, exc_tb): + if self.connection and not self.connection.closed: + self.connection.close() + return False + + def get_all_pull_requests(self): + with self.connection.cursor() as cur: + cur.execute(GET_FULL_PR_INFO, ([], )) + columns = [d[0] for d in cur.description] + rows = [dict(zip(columns, r)) for r in cur.fetchall()] + + return [self._row_to_pr(row) for row in rows] + + @staticmethod + def _row_to_pr(row: dict) -> PullRequest: + author = User( + name=row["creator_name"], + login=row["creator_login"], + email=row["creator_email"], + ) + + merged_by = None + if row["merged"] and row["merger_login"]: + merged_by = User( + name=row["merger_name"], + login=row["merger_login"], + email=row["merger_email"], + ) + + return PullRequest( + _id=row["number"], # per-repo номер PR (GitHub API: p.number) + title=row["title"], + author=author, + state=row["state"], # 'open' / 'closed' + created_at=row["created_at"], + head_label=row["source_branch"], + base_label=row["target_branch"], + head_ref=row["source_branch"], + base_ref=row["target_branch"], + merged_by=merged_by, + merged=bool(row["merged"]), + files=[], + issue_url=None, + labels=row["labels"].split(", ") if row["labels"] else [], + milestone=row["milestone"], + comments=row["num_comments"] or 0, + review_comments=0, + ) + +class ForgejoRepoDB(IRepositoryAPI): + def __init__(self, client: ForgejoDBClient): + self.client = client + + @log_exceptions( + default_return=[], message="Failed to get all pull requests from Forgejo DB" + ) + def get_pull_requests(self, repo: Repository) -> list[PullRequest]: + logging.info(f"{self} doesn't support per-repo - it returns all PR in DB") + return self.client.get_all_pull_requests() diff --git a/src/api/forgejo/sql_queries.py b/src/api/forgejo/sql_queries.py new file mode 100644 index 00000000..a3807c2b --- /dev/null +++ b/src/api/forgejo/sql_queries.py @@ -0,0 +1,57 @@ +GET_FULL_PR_INFO = """ +SELECT + r.owner_name || '/' || r.name AS repository_name, + i.name AS title, + i.index AS number, + i.num_comments AS num_comments, + CASE WHEN i.is_closed THEN 'closed' ELSE 'open' END AS state, + pr.base_branch AS commit_into, + pr.head_branch AS commit_from, + to_timestamp(i.created_unix) AS created_at, + creator.name AS creator_name, + creator.lower_name AS creator_login, + creator.email AS creator_email, + NULL::text AS changed_files, + (SELECT string_agg(c.content, E'\\n---\\n' ORDER BY c.created_unix) + FROM comment c WHERE c.issue_id = i.id) AS comment_body, + (SELECT string_agg(to_timestamp(c.created_unix)::text, ', ' ORDER BY c.created_unix) + FROM comment c WHERE c.issue_id = i.id) AS comment_created_at, + (SELECT string_agg(cu.name, ', ' ORDER BY c.created_unix) + FROM comment c JOIN "user" cu ON c.poster_id = cu.id + WHERE c.issue_id = i.id) AS comment_author_name, + (SELECT string_agg(cu.lower_name, ', ' ORDER BY c.created_unix) + FROM comment c JOIN "user" cu ON c.poster_id = cu.id + WHERE c.issue_id = i.id) AS comment_author_login, + (SELECT string_agg(cu.email, ', ' ORDER BY c.created_unix) + FROM comment c JOIN "user" cu ON c.poster_id = cu.id + WHERE c.issue_id = i.id) AS comment_author_email, + merger.name AS merger_name, + merger.lower_name AS merger_login, + merger.email AS merger_email, + pr.head_branch AS source_branch, + pr.base_branch AS target_branch, + (SELECT string_agg(a.name, ', ' ORDER BY a.name) + FROM issue_assignees ia JOIN "user" a ON ia.assignee_id = a.id + WHERE ia.issue_id = i.id) AS assignee_story, + (SELECT string_agg(related_id::text, ', ' ORDER BY related_id) + FROM ( + SELECT dependency_id AS related_id + FROM issue_dependency WHERE issue_id = i.id + UNION + SELECT issue_id AS related_id + FROM issue_dependency WHERE dependency_id = i.id + ) AS rel) AS related_issues, + (SELECT string_agg(l.name, ', ' ORDER BY l.name) + FROM issue_label il JOIN label l ON il.label_id = l.id + WHERE il.issue_id = i.id) AS labels, + m.name AS milestone, + pr.has_merged AS merged +FROM pull_request pr +INNER JOIN issue i ON pr.issue_id = i.id +INNER JOIN repository r ON pr.base_repo_id = r.id +INNER JOIN "user" creator ON i.poster_id = creator.id +LEFT JOIN "user" merger ON pr.merger_id = merger.id +LEFT JOIN milestone m ON i.milestone_id = m.id +WHERE r.owner_name || '/' || r.name = ANY(%s) +ORDER BY i.created_unix DESC; +""" \ No newline at end of file diff --git a/src/api/utils.py b/src/api/utils.py new file mode 100644 index 00000000..3277c48e --- /dev/null +++ b/src/api/utils.py @@ -0,0 +1,7 @@ +from enum import StrEnum + + +class APITypes(StrEnum): + forgejo = "forgejo" + forgejo_db = "forgejo_db" + github = "github" diff --git a/src/git_logger.py b/src/git_logger.py index 98d6888c..acbf4cfe 100644 --- a/src/git_logger.py +++ b/src/git_logger.py @@ -10,22 +10,13 @@ from src.api.baseAPI import ( RepositoryFactory ) +from src.api.utils import APITypes from src.constants import ( EMPTY_FIELD, TIMEDELTA, ) -def login(token, base_url): - try: - client = RepositoryFactory.create_api(token, base_url) - return client - except Exception as e: - logging.error(e) - logging.error(traceback.format_exc()) - return None - - def get_tokens_from_file(tokens_path: str) -> list[str]: with open(tokens_path, 'r') as file: tokens = [token for token in file.read().split('\n') if token] @@ -46,20 +37,20 @@ def get_repos_from_file(repos_path: str) -> list[str]: class Clients: - def __init__(self, tokens: list[str], base_url: str | None = None): + def __init__(self, api_type: APITypes, auth_data: list[dict]): self.clients = [] self.token_map = {} - for token in tokens: - client = login(token, base_url) - if client: - self.clients.append(client) - self.token_map[client] = token + if api_type != APITypes.forgejo_db: + + for auth in auth_data: + client, token = RepositoryFactory.create_api(api_type, auth) + if client: + self.clients.append(client) + self.token_map[client] = token if not self.clients: - if base_url: - raise Exception("No valid tokens for either GitHub or Forgejo") - raise Exception("Make sure that base_url is provided") + raise Exception(f"No clients after login: type={api_type}") def _get_next_client(self) -> tuple[IRepositoryAPI, str]: client = None From 97fc39fb32d27e3f6ab6080ee8ad3b8475784aed Mon Sep 17 00:00:00 2001 From: Dmitry Ivanov Date: Sat, 26 Sep 2026 15:03:34 +0300 Subject: [PATCH 4/6] add forgejodb exporter for PR --- .dockerignore | 3 +- main.py | 95 ++++++++++++---------- requirements.txt | 9 ++- src/api/baseAPI.py | 25 ++++-- src/api/forgejo/ForgejoRepoDB.py | 133 +++++++++++++++++++++++++------ src/api/forgejo/sql_queries.py | 109 +++++++++++++++++++------ src/git_logger.py | 12 ++- src/pull_requests_parser.py | 2 +- 8 files changed, 272 insertions(+), 116 deletions(-) diff --git a/.dockerignore b/.dockerignore index fd9abfc9..1d219f00 100644 --- a/.dockerignore +++ b/.dockerignore @@ -2,4 +2,5 @@ .git forgejo Industrial_Nikolaev_AA_9381 -reports \ No newline at end of file +reports +output \ No newline at end of file diff --git a/main.py b/main.py index b2d8b74f..70f1b447 100644 --- a/main.py +++ b/main.py @@ -18,11 +18,11 @@ def parse_args(): parser = argparse.ArgumentParser() parser.add_argument( - '--source', + "--source", choices=list(APITypes), default=APITypes.github, required=False, - help='Data source: github API (by default), forgejo API, forgejo db (direct access)' + help="Data source: github API (by default), forgejo API, forgejo db (direct access)", ) parser.add_argument("--invites", help="print pending invites", action="store_true") parser.add_argument("-c", "--commits", help="log commits", action="store_true") @@ -30,7 +30,9 @@ def parse_args(): "-p", "--pull_requests", help="log pull requests", action="store_true" ) parser.add_argument( - "--graphql", help="use graphql for requesting data (work only with --pull_requests) ", action="store_true" + "--graphql", + help="use graphql for requesting data (work only with --pull_requests) ", + action="store_true", ) parser.add_argument("-i", "--issues", help="log issues", action="store_true") parser.add_argument("-w", "--wikis", help="log wikis", action="store_true") @@ -48,91 +50,91 @@ def parse_args(): action="store_true", ) parser.add_argument( - '--start_cell', + "--start_cell", type=str, required=False, help='Starting cell for Google Sheets export (e.g., "A1", "B3")', - default="A1" + default="A1", ) parser.add_argument( - '--base_url', + "--base_url", type=str, required=False, default="https://git.moevm.pro/api/v1", - help='Base URL for Forgejo instance (if using Forgejo)', + help="Base URL for Forgejo instance (if using Forgejo)", ) token = parser.add_mutually_exclusive_group(required=True) - token.add_argument('-t', '--token', type=str, help='account access token') - token.add_argument('--tokens', type=str, help='path to your tokens') + token.add_argument("-t", "--token", type=str, help="account access token") + token.add_argument("--tokens", type=str, help="path to your tokens") parser.add_argument( - '-l', - '--list', + "-l", + "--list", type=str, required=True, help=( - 'Path to the file containing the list of repositories. ' - 'Repositories should be separated by a line break. ' - 'Names must be in the format / ' + "Path to the file containing the list of repositories. " + "Repositories should be separated by a line break. " + "Names must be in the format / " ), ) parser.add_argument( "--download_repos", type=str, help="path to downloaded repositories", - default='./', + default="./", ) - parser.add_argument('-o', '--out', type=str, required=True, help='output filename') + parser.add_argument("-o", "--out", type=str, required=True, help="output filename") parser.add_argument( "--pr_comments", help="log comments for PR", action="store_true" ) parser.add_argument( - '-s', - '--start', + "-s", + "--start", type=str, required=False, - help='start time', - default='2000/01/01-00:00:00', + help="start time", + default="2000/01/01-00:00:00", ) parser.add_argument( - '-f', - '--finish', + "-f", + "--finish", type=str, required=False, - help='finish time', - default='2400/01/01-00:00:00', + help="finish time", + default="2400/01/01-00:00:00", ) parser.add_argument( - '-b', - '--branch', + "-b", + "--branch", type=str, required=False, help=( - 'branch to select commits, ' + "branch to select commits, " 'by default use "default" repository branch, ' 'use "all" to get all commits from all branches', ), default=None, ) parser.add_argument( - '--google_token', + "--google_token", type=str, required=False, - help='Specify path to google token file', + help="Specify path to google token file", ) parser.add_argument( - '--table_id', + "--table_id", type=str, required=False, - help='Specify Google sheet document id (can find in url)', + help="Specify Google sheet document id (can find in url)", ) parser.add_argument( - '--sheet_name', + "--sheet_name", type=str, required=False, - help='Specify title for a sheet in a document in which data will be printed', + help="Specify title for a sheet in a document in which data will be printed", ) parser.add_argument( "--clear_sheet", @@ -144,18 +146,18 @@ def parse_args(): if args.export_google_sheets: for action in parser._actions: - if action.dest == 'google_token': + if action.dest == "google_token": action.required = True - if action.dest == 'table_id': + if action.dest == "table_id": action.required = True - if action.dest == 'sheet_name': + if action.dest == "sheet_name": action.required = True return parser.parse_args() def run(args, binded_repos, repos_for_wiki=None): - start = parse_time(args.start.split('-')) - finish = parse_time(args.finish.split('-')) + start = parse_time(args.start.split("-")) + finish = parse_time(args.finish.split("-")) if args.commits: commits_parser.log_commits( @@ -164,8 +166,7 @@ def run(args, binded_repos, repos_for_wiki=None): if args.pull_requests: if args.graphql: pull_requests_parser.log_pull_requests_by_graphql( - binded_repos=binded_repos, - csv_name=args.out + binded_repos=binded_repos, csv_name=args.out ) else: pull_requests_parser.log_pull_requests( @@ -178,7 +179,12 @@ def run(args, binded_repos, repos_for_wiki=None): ) if args.issues: issues_parser.log_issues( - binded_repos, args.out, start, finish, args.forks_include, args.base_url, + binded_repos, + args.out, + start, + finish, + args.forks_include, + args.base_url, ) if args.invites: invites_parser.log_invitations( @@ -221,7 +227,10 @@ def main(): repositories = git_logger.get_repos_from_file(args.list) try: - clients = git_logger.Clients(api_type=args.source, auth_data=dict(tokens=tokens, base_url=args.base_url)) + clients = git_logger.Clients( + api_type=args.source, + auth_data=[dict(token=token, base_url=args.base_url) for token in tokens], + ) binded_repos = git_logger.get_next_binded_repo(clients, repositories) except Exception as e: print(f"Failed to initialize any clients: {e}") @@ -231,5 +240,5 @@ def main(): run(args, binded_repos, repositories) -if __name__ == '__main__': +if __name__ == "__main__": main() diff --git a/requirements.txt b/requirements.txt index 800f5b1f..a2257550 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,9 +1,10 @@ GitPython==3.1.41 +isodate==0.7.2 +pandas==2.2.3 +psycopg2-binary==2.9.13 +pyforgejo==2.0.7 PyGithub==2.4.0 pygsheets==2.0.6 -pandas==2.2.3 pytz==2024.2 requests==2.32.3 -pyforgejo==2.0.7 -isodate==0.7.2 -unittest-parametrize==1.6.0 +unittest-parametrize==1.6.0 \ No newline at end of file diff --git a/src/api/baseAPI.py b/src/api/baseAPI.py index 78657a0c..24904e12 100644 --- a/src/api/baseAPI.py +++ b/src/api/baseAPI.py @@ -97,37 +97,46 @@ class RepositoryFactory: @staticmethod def create_api(api_type: APITypes, auth_data: dict) -> IRepositoryAPI: from src.api.forgejo.ForgejoRepoAPI import ForgejoRepoAPI + from src.api.forgejo.ForgejoRepoDB import ForgejoDBClient, ForgejoRepoDB from src.api.github.GitHubRepoAPI import GitHubRepoAPI errors = [] + base_url = auth_data.get("base_url") + token = auth_data.get("token") + if auth_data.get("base_url") and api_type != APITypes.forgejo_db: + api_type = APITypes.forgejo # legacy code for backward compability + match api_type: case APITypes.github: try: - token = auth_data.get("token") client = GitHubRepoAPI(Github(auth=Auth.Token(token))) if client.client: - return client + return client, token except Exception as e: errors.append( f"GitHub login failed: {e}. Token: {str(token)[:4]}..." ) - case APITypes.forgejo | auth_data.get("base_url"): # legacy code for backward compability + case APITypes.forgejo: try: - base_url = auth_data.get("base_url") - token = auth_data.get("token") return ForgejoRepoAPI( PyforgejoApi(api_key=token, base_url=base_url) - ) + ), token except Exception as e: errors.append( f"Forgejo login failed: {e}. Base url: {base_url}. Token: {str(token)[:4]}..." ) case APITypes.forgejo_db: - ... + try: + token = auth_data.get("token") + return ForgejoRepoDB(ForgejoDBClient({'base_url': base_url, 'FORGEJO_DB_PASSWORD': token})), None + except Exception as e: + errors.append( + f"Forgejo DB login failed: {e}. Base url: {base_url}. Token: {str(token)[:4]}..." + ) if errors: logging.error(" / ".join(errors)) logging.error("\n".join(traceback.format_stack())) - return None + return None, None diff --git a/src/api/forgejo/ForgejoRepoDB.py b/src/api/forgejo/ForgejoRepoDB.py index b4d1b8f2..3cd607df 100644 --- a/src/api/forgejo/ForgejoRepoDB.py +++ b/src/api/forgejo/ForgejoRepoDB.py @@ -17,19 +17,20 @@ WikiPage, ) from src.utils import log_exceptions -from src.api.forgejo.sql_queries import GET_FULL_PR_INFO +from src.api.forgejo.sql_queries import GET_FULL_PR_INFO, GET_USER_BY_ID, GET_REPO class ForgejoDBClient: def __init__(self, connect_info: dict): self.connection_params = dict( - host=connect_info.get("FORGEJO_DB_HOST"), + host=connect_info.get("FORGEJO_DB_HOST", 'db'), port=int(connect_info.get("FORGEJO_DB_PORT", 5432)), - dbname=connect_info.get("FORGEJO_DB_NAME"), - user=connect_info.get("FORGEJO_DB_USER"), - password=connect_info.get("FORGEJO_DB_PASSWORD"), + dbname=connect_info.get("FORGEJO_DB_NAME", 'forgejo'), + user=connect_info.get("FORGEJO_DB_USER", "forgejo"), + password=connect_info.get("FORGEJO_DB_PASSWORD", "moevmforgejopassword"), ) - self.connection = psycopg2.connect(**self.connection_params) + self.connection = None + self.base_url = connect_info.get("base_url") def __enter__(self): self.connection = psycopg2.connect(**self.connection_params) @@ -41,34 +42,45 @@ def __exit__(self, exc_type, exc_val, exc_tb): return False def get_all_pull_requests(self): - with self.connection.cursor() as cur: - cur.execute(GET_FULL_PR_INFO, ([], )) - columns = [d[0] for d in cur.description] - rows = [dict(zip(columns, r)) for r in cur.fetchall()] - + with self: + with self.connection.cursor() as cur: + cur.execute(GET_FULL_PR_INFO) + columns = [d[0] for d in cur.description] + rows = [dict(zip(columns, r)) for r in cur.fetchall()] + logging.error(f"{rows}") return [self._row_to_pr(row) for row in rows] @staticmethod - def _row_to_pr(row: dict) -> PullRequest: - author = User( - name=row["creator_name"], - login=row["creator_login"], - email=row["creator_email"], + def _user_from_row(row: dict, prefix: str, base_url: str = "") -> User | None: + uid = row.get(f"{prefix}_id") + if not uid: + return None + + login = row.get(f"{prefix}_login") or "" + return User( + _id=uid, + login=login, + username=row.get(f"{prefix}_full_name") or row.get(f"{prefix}_name") or "No name", + email=row.get(f"{prefix}_email") or "", + html_url=f"{base_url.rstrip('/')}/{login}" if base_url and login else "", + node_id=str(uid), + type={0: "User", 1: "Organization"}.get(row.get(f"{prefix}_type") or 0, ""), + bio=row.get(f"{prefix}_bio") or "", + site_admin=bool(row.get(f"{prefix}_is_admin")), ) - merged_by = None - if row["merged"] and row["merger_login"]: - merged_by = User( - name=row["merger_name"], - login=row["merger_login"], - email=row["merger_email"], - ) + def _row_to_pr(self, row: dict) -> PullRequest: + author = self._user_from_row(row, "creator", self.base_url) + merged_by = ( + self._user_from_row(row, "merger", self.base_url) + if row["merged"] else None + ) return PullRequest( - _id=row["number"], # per-repo номер PR (GitHub API: p.number) + _id=row["number"], title=row["title"], author=author, - state=row["state"], # 'open' / 'closed' + state=row["state"], created_at=row["created_at"], head_label=row["source_branch"], base_label=row["target_branch"], @@ -84,13 +96,82 @@ def _row_to_pr(row: dict) -> PullRequest: review_comments=0, ) -class ForgejoRepoDB(IRepositoryAPI): +class ForgejoRepoDB(): + _USER_TYPES = {0: "User", 1: "Organization", 2: "Bot"} + def __init__(self, client: ForgejoDBClient): self.client = client + def get_user_data(self, row: dict) -> User: + """ + row — dict w/keys: id, name, lower_name, full_name, + email, description, is_admin, type. + """ + login = row.get("lower_name") or row.get("name") or "" + return User( + login=login, + username=row.get("full_name") or row.get("name") or "No name", + email=row.get("email") or "", + html_url=f"{self.client.base_url}/{login}" if self.client.base_url else None, + node_id=row.get("id"), + type=self._USER_TYPES.get(row.get("type") or 0, ""), + bio=row.get("description") or "", + site_admin=bool(row.get("is_admin")), + _id=row.get("id"), + ) + + def get_user_by_id(self, user_id: int) -> User | None: + with self.client: + with self.client.connection.cursor() as cur: + cur.execute(GET_USER_BY_ID, (user_id,)) + row = cur.fetchone() + if not row: + return None + cols = [d[0] for d in cur.description] + return self.get_user_data(dict(zip(cols, row))) + + @log_exceptions(default_return=None, message="Failed to get repository from Forgejo DB") + def get_repository(self, id: str) -> Repository | None: + logging.info(f"{self} doesn't support get_repository (it parses all PR in DB), so returned Repo is Mock") + with self.client: + with self.client.connection.cursor() as cur: + cur.execute(GET_REPO, (id,)) + row = cur.fetchone() + if not row: + logging.error(f"Repository {id} not found in Forgejo DB.") + return None + cols = [d[0] for d in cur.description] + row = dict(zip(cols, row)) + + owner = self.get_user_data({ + "id": row["owner_id"], + "name": row["owner_name_user"], + "lower_name": row["owner_login"], + "full_name": row["owner_full_name"], + "email": row["owner_email"], + "description": row["owner_description"], + "is_admin": row["owner_is_admin"], + "type": row["owner_type"], + }) + + return Repository( + _id=f"{row['owner_name']}/{row['name']}", + name=row["name"], + url=f"{self.client.base_url}/{row['owner_name']}/{row['name']}" if self.client.base_url else None, + default_branch=Branch(name=row["default_branch"], last_commit=None), + owner=owner, + ) + @log_exceptions( default_return=[], message="Failed to get all pull requests from Forgejo DB" ) def get_pull_requests(self, repo: Repository) -> list[PullRequest]: logging.info(f"{self} doesn't support per-repo - it returns all PR in DB") return self.client.get_all_pull_requests() + + def get_rate_limiting(self) -> tuple[int, int]: + import sys + return sys.maxsize, sys.maxsize + + def get_base_url(self): + return self.client.base_url \ No newline at end of file diff --git a/src/api/forgejo/sql_queries.py b/src/api/forgejo/sql_queries.py index a3807c2b..d2f4574e 100644 --- a/src/api/forgejo/sql_queries.py +++ b/src/api/forgejo/sql_queries.py @@ -1,38 +1,53 @@ GET_FULL_PR_INFO = """ SELECT + -- Репозиторий r.owner_name || '/' || r.name AS repository_name, - i.name AS title, + + -- Основные поля PR (для PullRequest) i.index AS number, - i.num_comments AS num_comments, + i.name AS title, CASE WHEN i.is_closed THEN 'closed' ELSE 'open' END AS state, - pr.base_branch AS commit_into, - pr.head_branch AS commit_from, to_timestamp(i.created_unix) AS created_at, + pr.head_branch AS source_branch, + pr.base_branch AS target_branch, + pr.head_branch AS head_ref, + pr.base_branch AS base_ref, + pr.has_merged AS merged, + i.num_comments AS num_comments, + 0 AS review_comments, + m.name AS milestone, + + -- Автор PR (для User) + creator.id AS creator_id, creator.name AS creator_name, creator.lower_name AS creator_login, + creator.full_name AS creator_full_name, creator.email AS creator_email, - NULL::text AS changed_files, - (SELECT string_agg(c.content, E'\\n---\\n' ORDER BY c.created_unix) - FROM comment c WHERE c.issue_id = i.id) AS comment_body, - (SELECT string_agg(to_timestamp(c.created_unix)::text, ', ' ORDER BY c.created_unix) - FROM comment c WHERE c.issue_id = i.id) AS comment_created_at, - (SELECT string_agg(cu.name, ', ' ORDER BY c.created_unix) - FROM comment c JOIN "user" cu ON c.poster_id = cu.id - WHERE c.issue_id = i.id) AS comment_author_name, - (SELECT string_agg(cu.lower_name, ', ' ORDER BY c.created_unix) - FROM comment c JOIN "user" cu ON c.poster_id = cu.id - WHERE c.issue_id = i.id) AS comment_author_login, - (SELECT string_agg(cu.email, ', ' ORDER BY c.created_unix) - FROM comment c JOIN "user" cu ON c.poster_id = cu.id - WHERE c.issue_id = i.id) AS comment_author_email, + creator.description AS creator_bio, + creator.is_admin AS creator_is_admin, + creator.type AS creator_type, + + -- Мержер (для User) + merger.id AS merger_id, merger.name AS merger_name, merger.lower_name AS merger_login, + merger.full_name AS merger_full_name, merger.email AS merger_email, - pr.head_branch AS source_branch, - pr.base_branch AS target_branch, + merger.description AS merger_bio, + merger.is_admin AS merger_is_admin, + merger.type AS merger_type, + + -- Лейблы + (SELECT string_agg(l.name, ', ' ORDER BY l.name) + FROM issue_label il JOIN label l ON il.label_id = l.id + WHERE il.issue_id = i.id) AS labels, + + -- Назначенные исполнители (SELECT string_agg(a.name, ', ' ORDER BY a.name) FROM issue_assignees ia JOIN "user" a ON ia.assignee_id = a.id WHERE ia.issue_id = i.id) AS assignee_story, + + -- Связанные issues (SELECT string_agg(related_id::text, ', ' ORDER BY related_id) FROM ( SELECT dependency_id AS related_id @@ -41,17 +56,59 @@ SELECT issue_id AS related_id FROM issue_dependency WHERE dependency_id = i.id ) AS rel) AS related_issues, - (SELECT string_agg(l.name, ', ' ORDER BY l.name) - FROM issue_label il JOIN label l ON il.label_id = l.id - WHERE il.issue_id = i.id) AS labels, - m.name AS milestone, - pr.has_merged AS merged + + -- Комментарии (только пользовательские, type = 0) + (SELECT string_agg(c.content, E'\n---\n' ORDER BY c.created_unix) + FROM comment c + WHERE c.issue_id = i.id AND c.type = 0) AS comment_body, + (SELECT string_agg(to_timestamp(c.created_unix)::text, ', ' ORDER BY c.created_unix) + FROM comment c + WHERE c.issue_id = i.id AND c.type = 0) AS comment_created_at, + (SELECT string_agg(cu.name, ', ' ORDER BY c.created_unix) + FROM comment c JOIN "user" cu ON c.poster_id = cu.id + WHERE c.issue_id = i.id AND c.type = 0) AS comment_author_name, + (SELECT string_agg(cu.lower_name, ', ' ORDER BY c.created_unix) + FROM comment c JOIN "user" cu ON c.poster_id = cu.id + WHERE c.issue_id = i.id AND c.type = 0) AS comment_author_login, + (SELECT string_agg(cu.email, ', ' ORDER BY c.created_unix) + FROM comment c JOIN "user" cu ON c.poster_id = cu.id + WHERE c.issue_id = i.id AND c.type = 0) AS comment_author_email, + + -- Изменённые файлы (через БД недоступны) + NULL::text AS changed_files + FROM pull_request pr INNER JOIN issue i ON pr.issue_id = i.id INNER JOIN repository r ON pr.base_repo_id = r.id INNER JOIN "user" creator ON i.poster_id = creator.id LEFT JOIN "user" merger ON pr.merger_id = merger.id LEFT JOIN milestone m ON i.milestone_id = m.id -WHERE r.owner_name || '/' || r.name = ANY(%s) ORDER BY i.created_unix DESC; +""" + +GET_USER_BY_ID = """ +SELECT id, name, lower_name, full_name, email, description, is_admin, type + FROM "user" WHERE id = %s; +""" + +GET_REPO = """ +SELECT + r.id, + r.owner_name, + r.name, + r.lower_name, + r.description, + r.default_branch, + r.owner_id, + u.id AS owner_id, + u.name AS owner_name_user, + u.lower_name AS owner_login, + u.full_name AS owner_full_name, + u.email AS owner_email, + u.description AS owner_description, + u.is_admin AS owner_is_admin, + u.type AS owner_type +FROM repository r +LEFT JOIN "user" u ON r.owner_id = u.id +WHERE r.owner_name || '/' || r.name = %s; """ \ No newline at end of file diff --git a/src/git_logger.py b/src/git_logger.py index acbf4cfe..7a0ff886 100644 --- a/src/git_logger.py +++ b/src/git_logger.py @@ -41,13 +41,11 @@ def __init__(self, api_type: APITypes, auth_data: list[dict]): self.clients = [] self.token_map = {} - if api_type != APITypes.forgejo_db: - - for auth in auth_data: - client, token = RepositoryFactory.create_api(api_type, auth) - if client: - self.clients.append(client) - self.token_map[client] = token + for auth in auth_data: + client, token = RepositoryFactory.create_api(api_type, auth) + if client: + self.clients.append(client) + self.token_map[client] = token if not self.clients: raise Exception(f"No clients after login: type={api_type}") diff --git a/src/pull_requests_parser.py b/src/pull_requests_parser.py index b90400d4..53033734 100644 --- a/src/pull_requests_parser.py +++ b/src/pull_requests_parser.py @@ -123,7 +123,7 @@ def get_info(obj, attr): merged=pull.merged, source_branch=pull.head_ref, target_branch=pull.base_ref, - assignee_story=get_assignee_story(pull, client, token, repository), + assignee_story=None, #get_assignee_story(pull, client, token, repository), related_issues=( get_related_issues(pull._id, repository.owner, repository.name, token) if pull.issue_url is not None From 6c77ea68d641c7b6d83f49ffd487a1e77800dab3 Mon Sep 17 00:00:00 2001 From: Dmitry Ivanov Date: Sat, 26 Sep 2026 15:14:25 +0300 Subject: [PATCH 5/6] Rm default value for base_url --- main.py | 1 - 1 file changed, 1 deletion(-) diff --git a/main.py b/main.py index 70f1b447..3fdc821c 100644 --- a/main.py +++ b/main.py @@ -61,7 +61,6 @@ def parse_args(): "--base_url", type=str, required=False, - default="https://git.moevm.pro/api/v1", help="Base URL for Forgejo instance (if using Forgejo)", ) From 182a8194b7816773f7905ccdc3c28c6941a0a6a6 Mon Sep 17 00:00:00 2001 From: Dmitry Ivanov Date: Sat, 26 Sep 2026 20:38:42 +0300 Subject: [PATCH 6/6] update reponame getter for PR export --- src/api/forgejo/ForgejoRepoDB.py | 65 ++++++++++++++++++-------------- src/api/models.py | 1 + src/pull_requests_parser.py | 2 +- 3 files changed, 38 insertions(+), 30 deletions(-) diff --git a/src/api/forgejo/ForgejoRepoDB.py b/src/api/forgejo/ForgejoRepoDB.py index 3cd607df..c127f382 100644 --- a/src/api/forgejo/ForgejoRepoDB.py +++ b/src/api/forgejo/ForgejoRepoDB.py @@ -1,4 +1,3 @@ -import base64 import logging import psycopg2 @@ -7,14 +6,9 @@ from src.api.baseAPI import IRepositoryAPI from src.api.models import ( Branch, - Comment, - Commit, - Contributor, - Issue, PullRequest, Repository, User, - WikiPage, ) from src.utils import log_exceptions from src.api.forgejo.sql_queries import GET_FULL_PR_INFO, GET_USER_BY_ID, GET_REPO @@ -23,11 +17,11 @@ class ForgejoDBClient: def __init__(self, connect_info: dict): self.connection_params = dict( - host=connect_info.get("FORGEJO_DB_HOST", 'db'), + host=connect_info.get("FORGEJO_DB_HOST", "db"), port=int(connect_info.get("FORGEJO_DB_PORT", 5432)), - dbname=connect_info.get("FORGEJO_DB_NAME", 'forgejo'), + dbname=connect_info.get("FORGEJO_DB_NAME", "forgejo"), user=connect_info.get("FORGEJO_DB_USER", "forgejo"), - password=connect_info.get("FORGEJO_DB_PASSWORD", "moevmforgejopassword"), + password=connect_info.get("FORGEJO_DB_PASSWORD", ""), ) self.connection = None self.base_url = connect_info.get("base_url") @@ -47,7 +41,6 @@ def get_all_pull_requests(self): cur.execute(GET_FULL_PR_INFO) columns = [d[0] for d in cur.description] rows = [dict(zip(columns, r)) for r in cur.fetchall()] - logging.error(f"{rows}") return [self._row_to_pr(row) for row in rows] @staticmethod @@ -60,7 +53,9 @@ def _user_from_row(row: dict, prefix: str, base_url: str = "") -> User | None: return User( _id=uid, login=login, - username=row.get(f"{prefix}_full_name") or row.get(f"{prefix}_name") or "No name", + username=row.get(f"{prefix}_full_name") + or row.get(f"{prefix}_name") + or "No name", email=row.get(f"{prefix}_email") or "", html_url=f"{base_url.rstrip('/')}/{login}" if base_url and login else "", node_id=str(uid), @@ -72,11 +67,11 @@ def _user_from_row(row: dict, prefix: str, base_url: str = "") -> User | None: def _row_to_pr(self, row: dict) -> PullRequest: author = self._user_from_row(row, "creator", self.base_url) merged_by = ( - self._user_from_row(row, "merger", self.base_url) - if row["merged"] else None + self._user_from_row(row, "merger", self.base_url) if row["merged"] else None ) return PullRequest( + repository_name=row["repository_name"], _id=row["number"], title=row["title"], author=author, @@ -96,7 +91,8 @@ def _row_to_pr(self, row: dict) -> PullRequest: review_comments=0, ) -class ForgejoRepoDB(): + +class ForgejoRepoDB: # IRepositoryAPI _USER_TYPES = {0: "User", 1: "Organization", 2: "Bot"} def __init__(self, client: ForgejoDBClient): @@ -112,7 +108,9 @@ def get_user_data(self, row: dict) -> User: login=login, username=row.get("full_name") or row.get("name") or "No name", email=row.get("email") or "", - html_url=f"{self.client.base_url}/{login}" if self.client.base_url else None, + html_url=f"{self.client.base_url}/{login}" + if self.client.base_url + else None, node_id=row.get("id"), type=self._USER_TYPES.get(row.get("type") or 0, ""), bio=row.get("description") or "", @@ -130,9 +128,13 @@ def get_user_by_id(self, user_id: int) -> User | None: cols = [d[0] for d in cur.description] return self.get_user_data(dict(zip(cols, row))) - @log_exceptions(default_return=None, message="Failed to get repository from Forgejo DB") + @log_exceptions( + default_return=None, message="Failed to get repository from Forgejo DB" + ) def get_repository(self, id: str) -> Repository | None: - logging.info(f"{self} doesn't support get_repository (it parses all PR in DB), so returned Repo is Mock") + logging.info( + f"{self} doesn't support get_repository (it parses all PR in DB), so returned Repo is Mock" + ) with self.client: with self.client.connection.cursor() as cur: cur.execute(GET_REPO, (id,)) @@ -143,21 +145,25 @@ def get_repository(self, id: str) -> Repository | None: cols = [d[0] for d in cur.description] row = dict(zip(cols, row)) - owner = self.get_user_data({ - "id": row["owner_id"], - "name": row["owner_name_user"], - "lower_name": row["owner_login"], - "full_name": row["owner_full_name"], - "email": row["owner_email"], - "description": row["owner_description"], - "is_admin": row["owner_is_admin"], - "type": row["owner_type"], - }) + owner = self.get_user_data( + { + "id": row["owner_id"], + "name": row["owner_name_user"], + "lower_name": row["owner_login"], + "full_name": row["owner_full_name"], + "email": row["owner_email"], + "description": row["owner_description"], + "is_admin": row["owner_is_admin"], + "type": row["owner_type"], + } + ) return Repository( _id=f"{row['owner_name']}/{row['name']}", name=row["name"], - url=f"{self.client.base_url}/{row['owner_name']}/{row['name']}" if self.client.base_url else None, + url=f"{self.client.base_url}/{row['owner_name']}/{row['name']}" + if self.client.base_url + else None, default_branch=Branch(name=row["default_branch"], last_commit=None), owner=owner, ) @@ -171,7 +177,8 @@ def get_pull_requests(self, repo: Repository) -> list[PullRequest]: def get_rate_limiting(self) -> tuple[int, int]: import sys + return sys.maxsize, sys.maxsize def get_base_url(self): - return self.client.base_url \ No newline at end of file + return self.client.base_url diff --git a/src/api/models.py b/src/api/models.py index 5d442a29..07022dde 100644 --- a/src/api/models.py +++ b/src/api/models.py @@ -88,6 +88,7 @@ class PullRequest: milestone: str comments: int = 0 review_comments: int = 0 + repository_name: str = "" @dataclass diff --git a/src/pull_requests_parser.py b/src/pull_requests_parser.py index 53033734..3ead413a 100644 --- a/src/pull_requests_parser.py +++ b/src/pull_requests_parser.py @@ -106,7 +106,7 @@ def get_info(obj, attr): continue pr_data = PullRequestData( - repository_name=repository.name, + repository_name=pull.repository_name or repository.name, title=pull.title, id=pull._id, state=pull.state,