CEKA Sovereign Intelligence Sync #125
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Legislative Data Sync | |
| on: | |
| schedule: | |
| # 9AM EAT (06:00 UTC) and 4PM EAT (13:00 UTC) | |
| - cron: '0 6 * * *' | |
| - cron: '0 13 * * *' | |
| workflow_dispatch: | |
| jobs: | |
| scrape: | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: Checkout repository | |
| uses: actions/checkout@v4 | |
| - name: Set up Python | |
| uses: actions/setup-python@v5 | |
| with: | |
| python-version: '3.11' | |
| cache: 'pip' | |
| - name: Install dependencies | |
| run: | | |
| python -m pip install --upgrade pip | |
| pip install playwright pandas beautifulsoup4 requests supabase rapidfuzz | |
| playwright install chromium | |
| playwright install-deps | |
| - name: Run Scraper | |
| run: python scripts/legislative_scraper.py | |
| env: | |
| PYTHONIOENCODING: utf-8 | |
| - name: Verify Scrape Output | |
| run: | | |
| echo "### Scrape Results" >> $GITHUB_STEP_SUMMARY | |
| LATEST=$(ls -t processed_data/legislative/bills_*.json 2>/dev/null | head -1) | |
| if [ -z "$LATEST" ]; then | |
| echo "❌ No output file found" >> $GITHUB_STEP_SUMMARY | |
| exit 1 | |
| fi | |
| COUNT=$(python3 -c "import json; print(len(json.load(open('$LATEST'))))") | |
| echo "- File: $LATEST" >> $GITHUB_STEP_SUMMARY | |
| echo "- Bills scraped: $COUNT" >> $GITHUB_STEP_SUMMARY | |
| if [ "$COUNT" -eq 0 ]; then | |
| echo "⚠️ Warning: Scraper produced 0 bills. Website may have changed." >> $GITHUB_STEP_SUMMARY | |
| fi | |
| - name: Sync to Supabase | |
| env: | |
| SUPABASE_URL: ${{ secrets.SUPABASE_URL }} | |
| SUPABASE_SERVICE_ROLE_KEY: ${{ secrets.SUPABASE_SERVICE_ROLE_KEY }} | |
| run: python scripts/sync_to_supabase.py | |
| - name: Upload Artifacts | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: scraped-bills-${{ github.run_number }} | |
| path: processed_data/legislative/ | |
| retention-days: 30 | |
| - name: Summary | |
| run: | | |
| echo "### Legislative Sync Complete 🏛️" >> $GITHUB_STEP_SUMMARY | |
| echo "- Run: ${{ github.run_number }}" >> $GITHUB_STEP_SUMMARY | |
| echo "- Time: $(date)" >> $GITHUB_STEP_SUMMARY |