experiment: GitHub digest on PyGithub, Crowdin scripts on crowdin-api-client

crowdin/approve_strings.py

@@ -34,9 +34,13 @@
 import sys
 import time
 
-import requests
+from crowdin_api.api_resources.source_strings.enums import ScopeFilter
+from crowdin_api.exceptions import APIException, ValidationError
+
+sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
+sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
+import crowdin_sdk  # noqa: E402
 
-API = "https://api.crowdin.com/api/v2"
 DEFAULT_PROJECT = "618696"
 
 # libsecret attributes identifying the token in the keyring; must match the
@@ -128,23 +132,16 @@ def main():
     token = get_token()
 
     identifiers = args.identifiers
-    pid = args.project_id
-    s = requests.Session()
-    s.headers.update({"Authorization": f"Bearer {token}"})
-
-    def get(path, **params):
-        r = s.get(f"{API}{path}", params=params, timeout=30)
-        r.raise_for_status()
-        return r.json()
+    client = crowdin_sdk.client(token, args.project_id, attempts=6, timeout=30)
 
     # 1) Resolve identifiers -> string IDs
     string_ids = {}
     for ident in identifiers:
         found = None
         offset = 0
         while True:
-            data = get(f"/projects/{pid}/strings", filter=ident,
-                       scope="identifier", limit=500, offset=offset)["data"]
+            data = client.source_strings.list_strings(
+                filter=ident, scope=ScopeFilter.IDENTIFIER, limit=500, offset=offset)["data"]
             for row in data:
                 if row["data"]["identifier"] == ident:
                     found = row["data"]["id"]
@@ -158,7 +155,7 @@ def get(path, **params):
         print(f"  string '{ident}' -> id {found}")
 
     # 2) Target languages
-    proj = get(f"/projects/{pid}")["data"]
+    proj = client.projects.get_project()["data"]
     target_langs = proj["targetLanguageIds"]
     print(f"\n{len(target_langs)} target languages: {', '.join(target_langs)}\n")
 
@@ -169,15 +166,15 @@ def get(path, **params):
     approved, skipped, already, errors = 0, 0, 0, 0
     for ident, sid in string_ids.items():
         for lang in target_langs:
-            raw = get(f"/projects/{pid}/translations",
-                      stringId=sid, languageId=lang, limit=500)["data"]
+            raw = client.string_translations.list_string_translations(
+                stringId=sid, languageId=lang, limit=500)["data"]
             trans = [t["data"] for t in raw]
 
             if args.list:
                 # Show only unapproved translations (i.e. what a run would approve),
                 # honouring the --by-user filter if one was given.
-                approvals = get(f"/projects/{pid}/approvals",
-                                stringId=sid, languageId=lang, limit=500)["data"]
+                approvals = client.string_translations.list_translation_approvals(
+                    stringId=sid, languageId=lang, limit=500)["data"]
                 approved_tids = {a["data"]["translationId"] for a in approvals}
                 pending = [t for t in trans
                            if t["id"] not in approved_tids
@@ -209,17 +206,19 @@ def get(path, **params):
                 approved += 1
                 continue
 
-            r = s.post(f"{API}/projects/{pid}/approvals",
-                       json={"translationId": tid}, timeout=30)
-            if r.status_code == 201:
+            try:
+                client.string_translations.add_approval(translationId=tid)
+            except APIException as exc:
+                message = crowdin_sdk.error_message(exc)
+                if isinstance(exc, ValidationError) and "already" in message.lower():
+                    print(f"  [ -- ] {ident} / {lang}: already approved {info}")
+                    already += 1
+                else:
+                    print(f"  [ERR ] {ident} / {lang}: {exc.http_status} {message}")
+                    errors += 1
+            else:
                 print(f"  [ ok ] {ident} / {lang}: approved {info}")
                 approved += 1
-            elif r.status_code == 400 and "already" in r.text.lower():
-                print(f"  [ -- ] {ident} / {lang}: already approved {info}")
-                already += 1
-            else:
-                print(f"  [ERR ] {ident} / {lang}: {r.status_code} {r.text}")
-                errors += 1
             time.sleep(0.1)  # gentle on rate limits
 
     if args.list:

crowdin/crowdin_sdk.py

@@ -0,0 +1,89 @@
+"""crowdin-api-client with this repo's retry underneath it.
+
+The SDK's own retry is a fixed 100 ms sleep, five times, on 5xx only: a 429 raises
+at once whatever Retry-After says, and Crowdin throttles at ~40 req/s. So the SDK
+supplies the endpoints, parameter names and error mapping, and shared.retry the
+transport, by standing in for the requests.Session its requester talks to.
+"""
+import functools
+import json
+import threading
+
+import requests
+from crowdin_api import CrowdinClient
+from crowdin_api.requester import APIRequester
+
+from shared.retry import request_with_retry
+
+
+class RetryingSession:
+    """The SDK's session, routed through shared.retry.
+
+    One requests.Session per thread: the SDK shares its session across callers, and
+    requests.Session is not guaranteed thread-safe. `fixed` is one session for every
+    thread instead, which is how a test hands in a fake.
+    """
+
+    def __init__(self, attempts):
+        self.attempts = attempts
+        self.headers = {}
+        self.fixed = None
+        self._local = threading.local()
+
+    def _session(self):
+        if self.fixed is not None:
+            return self.fixed
+        session = getattr(self._local, "session", None)
+        if session is None:
+            session = self._local.session = requests.Session()
+            session.headers.update(self.headers)
+        return session
+
+    def request(self, method, url, **kwargs):
+        return request_with_retry(self._session(), method, url, attempts=self.attempts,
+                                  **kwargs)
+
+    def close(self):
+        session = getattr(self._local, "session", None)
+        if session is not None:
+            session.close()
+
+
+class RetryingRequester(APIRequester):
+    def __init__(self, *args, attempts, **kwargs):
+        super().__init__(*args, **kwargs)
+        # super().__init__ has already put the auth headers on the session it created.
+        plain, self._session = self._session, RetryingSession(attempts)
+        self._session.headers.update(plain.headers)
+        plain.close()
+
+
+def client(token, project_id, attempts, timeout):
+    """A CrowdinClient retrying like shared.retry: `attempts` tries, backing off on
+    429 and 5xx. max_retries=1 is the SDK's own loop switched off."""
+    crowdin = CrowdinClient(token=token, project_id=int(project_id), timeout=timeout,
+                            max_retries=1)
+    crowdin.API_REQUESTER_CLASS = functools.partial(RetryingRequester, attempts=attempts)
+    return crowdin
+
+
+def use_session(crowdin, session):
+    """Answer every request from `session` (a test fake) instead of the network."""
+    crowdin.get_api_requestor().session.fixed = session
+
+
+def error_message(exc):
+    """Crowdin's error message, or the start of the body when it is not that envelope."""
+    body = exc.context or b""
+    if isinstance(body, bytes):
+        body = body.decode("utf-8", "replace")
+    try:
+        return json.loads(body).get("error", {}).get("message", "Unknown error")
+    except (ValueError, AttributeError):
+        return body[:200] or "Unknown error"
+
+
+def fetch_all(resource, method, **params):
+    """Every item of a paginated list endpoint, unwrapped from the SDK's envelopes."""
+    listing = getattr(resource.with_fetch_all(), method)
+    return [row["data"] for row in listing(**params)["data"]]

crowdin/download_translations_from_crowdin.py

@@ -14,13 +14,15 @@
 from threading import Lock, Semaphore
 
 import requests
+from crowdin_api.api_resources.enums import ExportProjectTranslationFormat
+from crowdin_api.exceptions import APIException
 
 sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
 sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
+import crowdin_sdk  # noqa: E402
 from generate_shared import print_error, print_progress, print_success, run_main  # noqa: E402
 from shared.retry import request_with_retry  # noqa: E402
 
-API = "https://api.crowdin.com/api/v2"
 # Crowdin allows 20 simultaneous requests per account.
 MAX_CONCURRENT_REQUESTS = 20
 REQUEST_TIMEOUT_S = 30
@@ -31,35 +33,25 @@ class CrowdinError(Exception):
     pass
 
 
-def error_message(response):
-    """Crowdin's error message, or the start of the body when it is not that envelope."""
-    try:
-        return response.json().get("error", {}).get("message", "Unknown error")
-    except ValueError:
-        return response.text[:200] or "Unknown error"
-
-
 class Crowdin:
     """One project's API, gated to Crowdin's concurrency limit across the worker threads."""
 
     def __init__(self, token, project_id, max_workers):
         self.project_id = project_id
-        self.session = requests.Session()
-        self.session.headers["Authorization"] = f"Bearer {token}"
+        self.api = crowdin_sdk.client(token, project_id, attempts=MAX_ATTEMPTS,
+                                      timeout=REQUEST_TIMEOUT_S)
         # Exports are served from a signed URL on another host, which must not see the token.
         self.downloads = requests.Session()
         self.gate = Semaphore(min(max_workers, MAX_CONCURRENT_REQUESTS))
 
-    def request(self, method, path, context, **kwargs):
-        """One API response's JSON, or CrowdinError naming `context`."""
+    def call(self, context, method, **kwargs):
+        """One SDK call's JSON, or CrowdinError naming `context`."""
         with self.gate:
-            response = request_with_retry(self.session, method, f"{API}/{path}",
-                                          attempts=MAX_ATTEMPTS, timeout=REQUEST_TIMEOUT_S,
-                                          **kwargs)
-        if response.status_code != 200:
-            raise CrowdinError(f"{context}: {error_message(response)} "
-                               f"(Code: {response.status_code})")
-        return response.json()
+            try:
+                return method(**kwargs)
+            except APIException as exc:
+                raise CrowdinError(f"{context}: {crowdin_sdk.error_message(exc)} "
+                                   f"(Code: {exc.http_status})") from exc
 
     def download(self, url, output_path):
         response = request_with_retry(self.downloads, "GET", url, attempts=MAX_ATTEMPTS,
@@ -84,14 +76,12 @@ def export_and_download_language(client, language, directory, is_source, skip_un
                                  allow_unapproved):
     """Export one language and save it as <locale>.xliff. Returns the locale."""
     locale = language["locale"]
-    payload = {
-        "targetLanguageId": language["id"],
-        "format": "xliff",
-        "skipUntranslatedStrings": False if is_source else skip_untranslated,
-        "exportApprovedOnly": False if is_source else not allow_unapproved,
-    }
-    export = client.request("POST", f"projects/{client.project_id}/translations/exports",
-                            f"Export failed for {locale}", json=payload)
+    export = client.call(f"Export failed for {locale}",
+                         client.api.translations.export_project_translation,
+                         targetLanguageId=language["id"],
+                         format=ExportProjectTranslationFormat.XLIFF,
+                         skipUntranslatedStrings=False if is_source else skip_untranslated,
+                         exportApprovedOnly=False if is_source else not allow_unapproved)
     client.download(export["data"]["url"], os.path.join(directory, f"{locale}.xliff"))
     return locale
 
@@ -118,8 +108,7 @@ def main():
     client = Crowdin(args.api_token, args.project_id, args.max_workers)
 
     print_progress("Retrieving project details...")
-    project = client.request("GET", f"projects/{args.project_id}",
-                             "Failed to retrieve project details")
+    project = client.call("Failed to retrieve project details", client.api.projects.get_project)
     if args.verbose:
         print(json.dumps(project, indent=2))
     source_language = project["data"]["sourceLanguage"]
@@ -157,10 +146,9 @@ def main():
 
     if args.glossary_id is not None and args.concept_id is not None:
         print_progress("Retrieving non-translatable strings...")
-        terms = client.request(
-            "GET", f"glossaries/{args.glossary_id}/terms",
-            "Failed to retrieve non-translatable strings",
-            params={"conceptId": args.concept_id, "limit": 500})
+        terms = client.call("Failed to retrieve non-translatable strings",
+                            client.api.glossaries.list_terms,
+                            glossaryId=args.glossary_id, conceptId=args.concept_id, limit=500)
         if args.verbose:
             print(json.dumps(terms, indent=2))
         with open(os.path.join(args.download_directory, "_non_translatable_strings.json"), "w",