CEKA Sovereign Intelligence Sync #548
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: CEKA Sovereign Intelligence Sync | |
| on: | |
| schedule: | |
| # Every 3 hours (East Africa Time: 3, 6, 9, 12, 15, 18, 21, 00) | |
| # UTC: 00, 03, 06, 09, 12, 15, 18, 21 | |
| - cron: '0 0,3,6,9,12,15,18,21 * * 1-5' | |
| workflow_dispatch: | |
| inputs: | |
| force_news: | |
| description: 'Force news corroboration search' | |
| type: boolean | |
| default: false | |
| run_phase_a: | |
| description: 'Run Phase A: Sovereign Refresh (broad sweep, all bills)' | |
| type: boolean | |
| default: false | |
| run_phase_b: | |
| description: 'Run Phase B: Matrix Solvency (surgical NULL fill)' | |
| type: boolean | |
| default: false | |
| phase_b_column: | |
| description: 'Phase B Column Sweep target (e.g. constitutional_section). Leave blank for full matrix.' | |
| type: string | |
| default: '' | |
| phase_b_bill_id: | |
| description: 'Phase B Row Full-Ham target Bill UUID. Leave blank for all bills.' | |
| type: string | |
| default: '' | |
| phase_a_force: | |
| description: 'Phase A: Force re-process ALL bills even if data is complete' | |
| type: boolean | |
| default: false | |
| jobs: | |
| sync: | |
| runs-on: ubuntu-latest | |
| permissions: | |
| contents: write | |
| pull-requests: write | |
| env: | |
| # ── Supabase ─────────────────────────────────────────────────────────── | |
| SUPABASE_URL: ${{ secrets.SUPABASE_URL }} | |
| SUPABASE_SERVICE_ROLE_KEY: ${{ secrets.SUPABASE_SERVICE_ROLE_KEY }} | |
| # ── LLM Providers (10-provider orchestrator chain) ───────────────────── | |
| GEMINI_API_KEY: ${{ secrets.GEMINI_API_KEY }} | |
| GROQ_API_KEY: ${{ secrets.GROQ_API_KEY }} | |
| OPENAI_API_KEY_1: ${{ secrets.OPENAI_API_KEY_1 }} | |
| ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }} | |
| OPENROUTER_API_KEY: ${{ secrets.OPENROUTER_API_KEY }} | |
| DEEPSEEK_API_KEY: ${{ secrets.DEEPSEEK_API_KEY }} | |
| DEEPSEEK_API_KEY_SECONDARY: ${{ secrets.DEEPSEEK_API_KEY_SECONDARY }} | |
| CEREBRAS_API_KEY: ${{ secrets.CEREBRAS_API_KEY }} | |
| CEREBRAS_API_KEY_SECONDARY: ${{ secrets.CEREBRAS_API_KEY_SECONDARY }} | |
| COHERE_API_KEY: ${{ secrets.COHERE_API_KEY }} | |
| NVIDIA_API_KEY: ${{ secrets.NVIDIA_API_KEY }} | |
| HF_API_TOKEN: ${{ secrets.HF_API_TOKEN }} | |
| MANUS_API_KEY: ${{ secrets.MANUS_API_KEY }} | |
| # ── Storage & Scraping ───────────────────────────────────────────────── | |
| B2_S3_ENDPOINT: ${{ secrets.B2_S3_ENDPOINT }} | |
| B2_KEY_ID: ${{ secrets.B2_KEY_ID }} | |
| B2_APPLICATION_KEY: ${{ secrets.B2_APPLICATION_KEY }} | |
| OCR_SPACE_API_KEY: ${{ secrets.OCR_SPACE_API_KEY }} | |
| SCRAPINGROBOT_API_KEY: ${{ secrets.SCRAPINGROBOT_API_KEY }} | |
| SCRAPINGDOG_API_KEY: ${{ secrets.SCRAPINGDOG_API_KEY }} | |
| SERPAPI_API_KEY: ${{ secrets.SERPAPI_API_KEY }} | |
| # ── New Proxy Orchestration ────────────────────────────────────────── | |
| BRIGHTDATA_PROXY_URL: ${{ secrets.BRIGHTDATA_PROXY_URL }} | |
| BRIGHTDATA_MONTHLY_LIMIT: ${{ secrets.BRIGHTDATA_MONTHLY_LIMIT }} | |
| SCRAPERAPI_KEY: ${{ secrets.SCRAPERAPI_KEY }} | |
| SCRAPERAPI_MONTHLY_LIMIT: ${{ secrets.SCRAPERAPI_MONTHLY_LIMIT }} | |
| SCRAPERAPI_FIRST_MONTH_BONUS: ${{ secrets.SCRAPERAPI_FIRST_MONTH_BONUS }} | |
| WEBSHARE_PROXIES: ${{ secrets.WEBSHARE_PROXIES }} | |
| steps: | |
| - name: Checkout repository | |
| uses: actions/checkout@v4 | |
| - name: Set up Python | |
| uses: actions/setup-python@v5 | |
| with: | |
| python-version: '3.11' | |
| cache: 'pip' | |
| - name: Cache Playwright Browsers | |
| uses: actions/cache@v4 | |
| with: | |
| path: ~/.cache/ms-playwright | |
| key: ${{ runner.os }}-playwright-${{ hashFiles('**/requirements.txt') }} | |
| - name: Install system dependencies (Poppler + Tesseract for OCR) | |
| run: | | |
| sudo apt-get update -q | |
| sudo apt-get install -y poppler-utils tesseract-ocr tesseract-ocr-eng | |
| - name: Install Python dependencies | |
| run: | | |
| python -m pip install --upgrade pip | |
| pip install \ | |
| playwright supabase requests pymupdf pdfplumber python-dotenv \ | |
| beautifulsoup4 pillow pdf2image pytesseract \ | |
| google-genai google-search-results \ | |
| groq openai anthropic cohere \ | |
| boto3 aiohttp lxml | |
| # Install cerebras SDK (may fail gracefully if not available) | |
| pip install cerebras-cloud-sdk || echo "⚠️ cerebras-cloud-sdk not available — skipping" | |
| playwright install chromium --with-deps | |
| - name: Determine News Sync | |
| id: news_check | |
| run: | | |
| HOUR=$(date +%H) | |
| if [[ "$HOUR" == "06" || "$HOUR" == "12" || "$HOUR" == "18" || "${{ github.event.inputs.force_news }}" == "true" ]]; then | |
| echo "RUN_NEWS=true" >> $GITHUB_ENV | |
| echo "run_news=true" >> $GITHUB_OUTPUT | |
| echo "✅ News Corroboration ENABLED." | |
| else | |
| echo "RUN_NEWS=false" >> $GITHUB_ENV | |
| echo "run_news=false" >> $GITHUB_OUTPUT | |
| echo "⏳ News Corroboration SKIPPED (Tiered)." | |
| fi | |
| - name: Run Legislative Scraper | |
| run: python scripts/legislative_scraper.py | |
| continue-on-error: true | |
| env: | |
| PYTHONIOENCODING: utf-8 | |
| - name: Supabase Sync Bridge | |
| run: python scripts/sync_to_supabase.py | |
| continue-on-error: true | |
| env: | |
| PYTHONIOENCODING: utf-8 | |
| - name: Run Stage Detector | |
| run: python scripts/stage_detector.py | |
| continue-on-error: true | |
| env: | |
| PYTHONIOENCODING: utf-8 | |
| - name: Run News Intelligence | |
| if: steps.news_check.outputs.run_news == 'true' | |
| run: python scripts/news_intelligence.py | |
| continue-on-error: true | |
| env: | |
| PYTHONIOENCODING: utf-8 | |
| - name: Run Sovereign Corroborator (AI Synthesis) | |
| run: python scripts/sovereign_corroborator.py | |
| continue-on-error: true | |
| env: | |
| PYTHONIOENCODING: utf-8 | |
| - name: "Phase A: Sovereign Refresh (Weekly — All Bills Broad Sweep)" | |
| # Runs every Monday at 06:00 EAT (03:00 UTC) OR when manually triggered with run_phase_a=true | |
| if: | | |
| github.event.inputs.run_phase_a == 'true' || | |
| (github.event_name == 'schedule' && startsWith(github.event.schedule, '0 3')) | |
| run: | | |
| FORCE_FLAG="" | |
| if [ "${{ github.event.inputs.phase_a_force }}" == "true" ]; then | |
| FORCE_FLAG="--force" | |
| fi | |
| python scripts/sovereign_refresh.py $FORCE_FLAG | |
| continue-on-error: true | |
| env: | |
| PYTHONIOENCODING: utf-8 | |
| - name: "Phase B: Matrix Solvency (Weekly — Surgical NULL Fill)" | |
| # Runs every Monday at 08:00 EAT (05:00 UTC) OR when manually triggered with run_phase_b=true | |
| if: | | |
| github.event.inputs.run_phase_b == 'true' || | |
| (github.event_name == 'schedule' && startsWith(github.event.schedule, '0 5')) | |
| run: | | |
| COL_FLAG="" | |
| BILL_FLAG="" | |
| if [ -n "${{ github.event.inputs.phase_b_column }}" ]; then | |
| COL_FLAG="--column ${{ github.event.inputs.phase_b_column }}" | |
| fi | |
| if [ -n "${{ github.event.inputs.phase_b_bill_id }}" ]; then | |
| BILL_FLAG="--bill-id ${{ github.event.inputs.phase_b_bill_id }}" | |
| fi | |
| python scripts/matrix_solvency.py $COL_FLAG $BILL_FLAG | |
| continue-on-error: true | |
| env: | |
| PYTHONIOENCODING: utf-8 | |
| - name: Run Vault Generator | |
| run: python scripts/vault_generator.py | |
| continue-on-error: true | |
| env: | |
| PYTHONIOENCODING: utf-8 | |
| - name: Sync Summary | |
| if: always() | |
| run: | | |
| echo "### 🏛️ CEKA Sovereign Sync Complete" >> $GITHUB_STEP_SUMMARY | |
| echo "- **Run:** #${{ github.run_number }}" >> $GITHUB_STEP_SUMMARY | |
| echo "- **Time (UTC):** $(date -u)" >> $GITHUB_STEP_SUMMARY | |
| echo "- **News Sync:** $RUN_NEWS" >> $GITHUB_STEP_SUMMARY | |
| echo "- **Triggered by:** ${{ github.event_name }}" >> $GITHUB_STEP_SUMMARY | |
| - name: Persist Proxy Usage and Scraped Data | |
| if: always() | |
| run: | | |
| git config --global user.name "ceka-intelligence-bot" | |
| git config --global user.email "bot@ceka.online" | |
| # Ensure the target directory exists (git add on a non‑existent directory fails) | |
| mkdir -p scripts/processed_data/legislative | |
| # Add files – ignore errors if no files match the wildcard | |
| git add scripts/proxy_usage.json || true | |
| git add scripts/processed_data/legislative/*.json || true | |
| # Commit only if there are changes; ignore empty commit error | |
| git commit -m "chore: update proxy usage & legislative data [skip ci]" || echo "No changes to commit" | |
| git push || echo "Push failed (maybe nothing to push or auth issue)" | |
| - name: Upload Intelligence Logs | |
| if: always() | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: intelligence-logs-${{ github.run_number }} | |
| path: | | |
| scripts/*.log | |
| scripts/processed_data/ | |
| retention-days: 14 | |
| if-no-files-found: warn |