diff --git a/.github/workflows/data-ingestion.yml b/.github/workflows/data-ingestion.yml index c5bcc4ca..08fb284d 100644 --- a/.github/workflows/data-ingestion.yml +++ b/.github/workflows/data-ingestion.yml @@ -66,6 +66,11 @@ on: required: true type: boolean default: false + run_senators_indexer: + description: 'Senators: fetch and generate senators JSON -> S3' + required: true + type: boolean + default: false parliament_number: description: Optional parliament number override for search, lobbying, and bills required: false @@ -101,6 +106,7 @@ jobs: RUN_MEMBERS: ${{ inputs.run_members }} RUN_MEMBERS_INDEXER: ${{ inputs.run_members_indexer }} RUN_VOTES: ${{ inputs.run_votes }} + RUN_SENATORS_INDEXER: ${{ inputs.run_senators_indexer }} run: | set -euo pipefail @@ -129,6 +135,7 @@ jobs: case "$EVENT_SCHEDULE" in "0 12 * * *"|"0 3 * * *") add_ingestor "search" "Hansard search index" "hansard-search-index" "false" + add_ingestor "senators-indexer" "Senators indexer" "senators-indexer" "false" ;; "0 12 * * 1"|"0 12 1 */3 *") add_ingestor "lobbying" "Lobbying index" "lobbying-index" "false" @@ -145,6 +152,7 @@ jobs: [ "$RUN_MEMBERS" = "true" ] && add_ingestor "members" "Members artifacts" "members-publisher" "true" [ "$RUN_MEMBERS_INDEXER" = "true" ] && add_ingestor "members-indexer" "Members SQLite index" "members-indexer" "false" [ "$RUN_VOTES" = "true" ] && add_ingestor "votes" "Member votes artifacts" "member-votes-publisher" "true" + [ "$RUN_SENATORS_INDEXER" = "true" ] && add_ingestor "senators-indexer" "Senators indexer" "senators-indexer" "false" fi if [ "$(jq 'length' <<< "$items")" -gt 0 ]; then @@ -283,6 +291,11 @@ jobs: votes) (cd backend/member-votes-publisher && go run .) ;; + senators-indexer) + output_dir="$RUNNER_TEMP/artifacts/senators" + python3 scripts/artifacts/fetch_senators.py --output "$output_dir/v1/all.json" + aws s3 sync "$output_dir" "$(artifact_uri senators)" + ;; *) echo "Unknown ingestor: $INGESTOR" exit 1 diff --git a/backend/go.work b/backend/go.work index fb24ae60..138ec550 100644 --- a/backend/go.work +++ b/backend/go.work @@ -34,6 +34,7 @@ use ( ./openapi ./push-notification-dispatcher ./riding-boundary + ./senators ./sittings ./sittings-publisher ./telemetry diff --git a/backend/manifest/deployment-services.json b/backend/manifest/deployment-services.json index 60cfa202..4b249324 100644 --- a/backend/manifest/deployment-services.json +++ b/backend/manifest/deployment-services.json @@ -868,6 +868,40 @@ "artifact": false } } + }, + { + "name": "senators", + "deploy": { + "staging": true, + "production": true + }, + "http": { + "payload_format_version": "1.0", + "routes": { + "staging": [ + { + "method": "GET", + "path": "/api/v1/senators" + } + ], + "production": [ + { + "method": "GET", + "path": "/api/v1/senators" + } + ] + } + }, + "sync": { + "staging": { + "database": false, + "artifact": true + }, + "production": { + "database": false, + "artifact": true + } + } } ] } diff --git a/backend/openapi/openapi.json b/backend/openapi/openapi.json index 6ba7e5cc..7e3b5370 100644 --- a/backend/openapi/openapi.json +++ b/backend/openapi/openapi.json @@ -169,6 +169,24 @@ } } }, + "/api/v1/senators": { + "get": { + "tags": ["Senators"], + "summary": "List senators of Canada", + "operationId": "listSenators", + "responses": { + "200": { + "description": "Senator list", + "content": { + "application/json": { + "schema": { "$ref": "#/components/schemas/SenatorsResponse" } + } + } + }, + "429": { "$ref": "#/components/responses/RateLimit" } + } + } + }, "/api/v1/members": { "get": { "tags": ["Members"], @@ -1926,6 +1944,36 @@ "source_url": { "type": "string", "format": "uri" } } }, + "SenatorsResponse": { + "type": "object", + "required": ["items"], + "properties": { + "items": { "type": "array", "items": { "$ref": "#/components/schemas/OpenAPISenator" } } + } + }, + "OpenAPISenator": { + "type": "object", + "required": ["PersonOfficialFirstName", "PersonOfficialLastName", "ProvinceNameEn"], + "properties": { + "PersonOfficialFirstName": { "type": "string" }, + "PersonOfficialLastName": { "type": "string" }, + "ProvinceNameEn": { "type": "string" }, + "CaucusAbbreviationEn": { "type": "string" }, + "CaucusNameEn": { "type": "string" }, + "PersonPageUrl": { "type": "string", "format": "uri" }, + "appointment": { "$ref": "#/components/schemas/OpenAPISenateAppointment" } + } + }, + "OpenAPISenateAppointment": { + "type": "object", + "required": ["appointment_date", "declared_affiliation", "source_url"], + "properties": { + "appointment_date": { "type": "string", "format": "date" }, + "appointing_prime_minister": { "type": "string" }, + "declared_affiliation": { "type": "string" }, + "source_url": { "type": "string", "format": "uri" } + } + }, "MembersResponse": { "type": "object", "required": ["members"], diff --git a/backend/senators/go.mod b/backend/senators/go.mod new file mode 100644 index 00000000..bd0f0127 --- /dev/null +++ b/backend/senators/go.mod @@ -0,0 +1,36 @@ +module epac/senators + +go 1.24.0 + +require ( + epac/observability v0.0.0 + epac/shared v0.0.0 + github.com/aws/aws-lambda-go v1.54.0 +) + +require ( + github.com/aws/aws-sdk-go-v2 v1.41.7 // indirect + github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.10 // indirect + github.com/aws/aws-sdk-go-v2/config v1.32.1 // indirect + github.com/aws/aws-sdk-go-v2/credentials v1.19.1 // indirect + github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.14 // indirect + github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.23 // indirect + github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.23 // indirect + github.com/aws/aws-sdk-go-v2/internal/ini v1.8.4 // indirect + github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.24 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.9 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.15 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.23 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.23 // indirect + github.com/aws/aws-sdk-go-v2/service/s3 v1.101.0 // indirect + github.com/aws/aws-sdk-go-v2/service/signin v1.0.1 // indirect + github.com/aws/aws-sdk-go-v2/service/sso v1.30.4 // indirect + github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.9 // indirect + github.com/aws/aws-sdk-go-v2/service/sts v1.41.1 // indirect + github.com/aws/smithy-go v1.25.1 // indirect + github.com/stretchr/testify v1.11.1 // indirect +) + +replace epac/observability => ../observability + +replace epac/shared => ../_shared diff --git a/backend/senators/main.go b/backend/senators/main.go new file mode 100644 index 00000000..fdb598c6 --- /dev/null +++ b/backend/senators/main.go @@ -0,0 +1,52 @@ +// senators Lambda — GET /api/v1/senators +package main + +import ( + "context" + "net/http" + + "epac/observability" + "epac/shared/artifacts" + "github.com/aws/aws-lambda-go/events" + "github.com/aws/aws-lambda-go/lambda" +) + +const senatorsArtifactKey = "senators/v1/all.json" + +var newArtifactStore = artifacts.NewFromEnv + +func HandleRequest(ctx context.Context, _ events.APIGatewayProxyRequest) (events.APIGatewayProxyResponse, error) { + store, err := newArtifactStore(ctx) + if err != nil { + return jsonError(http.StatusServiceUnavailable, err.Error()), nil + } + data, err := store.Get(ctx, senatorsArtifactKey) + if err != nil { + status := http.StatusInternalServerError + if artifacts.IsNotFound(err) { + status = http.StatusNotFound + } + return jsonError(status, err.Error()), nil + } + + return events.APIGatewayProxyResponse{ + StatusCode: http.StatusOK, + Headers: map[string]string{ + "Content-Type": "application/json", + "Cache-Control": "public, max-age=300", + }, + Body: string(data), + }, nil +} + +func jsonError(status int, message string) events.APIGatewayProxyResponse { + return events.APIGatewayProxyResponse{ + StatusCode: status, + Headers: map[string]string{"Content-Type": "application/json"}, + Body: `{"error":"` + message + `"}`, + } +} + +func main() { + lambda.Start(observability.WrapAPIGateway("senators", HandleRequest)) +} diff --git a/backend/senators/main_test.go b/backend/senators/main_test.go new file mode 100644 index 00000000..93370b83 --- /dev/null +++ b/backend/senators/main_test.go @@ -0,0 +1,51 @@ +package main + +import ( + "context" + "net/http" + "os" + "path/filepath" + "strings" + "testing" + + "github.com/aws/aws-lambda-go/events" +) + +func TestHandleRequestReadsSenatorsArtifact(t *testing.T) { + dir := t.TempDir() + writeFixture(t, dir, senatorsArtifactKey, `{"items":[{"PersonOfficialFirstName":"Charles","PersonOfficialLastName":"Adler"}]}`) + t.Setenv("ARTIFACTS_DIR", dir) + + resp, err := HandleRequest(context.Background(), events.APIGatewayProxyRequest{}) + if err != nil { + t.Fatalf("HandleRequest error: %v", err) + } + if resp.StatusCode != http.StatusOK { + t.Fatalf("status = %d body = %s, want 200", resp.StatusCode, resp.Body) + } + if !strings.Contains(resp.Body, `"PersonOfficialFirstName":"Charles"`) { + t.Fatalf("unexpected body: %s", resp.Body) + } +} + +func TestHandleRequestMissingArtifactReturns404(t *testing.T) { + t.Setenv("ARTIFACTS_DIR", t.TempDir()) + resp, err := HandleRequest(context.Background(), events.APIGatewayProxyRequest{}) + if err != nil { + t.Fatalf("HandleRequest error: %v", err) + } + if resp.StatusCode != http.StatusNotFound { + t.Fatalf("status = %d body = %s, want 404", resp.StatusCode, resp.Body) + } +} + +func writeFixture(t *testing.T, root, key, body string) { + t.Helper() + path := filepath.Join(root, filepath.FromSlash(key)) + if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil { + t.Fatalf("mkdir fixture: %v", err) + } + if err := os.WriteFile(path, []byte(body), 0o644); err != nil { + t.Fatalf("write fixture: %v", err) + } +} diff --git a/docs/architecture/use-case-catalog.md b/docs/architecture/use-case-catalog.md index c76f34b9..72c1d87d 100644 --- a/docs/architecture/use-case-catalog.md +++ b/docs/architecture/use-case-catalog.md @@ -22,6 +22,8 @@ For the Clean Architecture shape this catalog assumes, see [`docs/architecture/` | `RecordedVote` | A House of Commons division record with date, bill reference, result, and member ballot. | | `MemberID` | The stable source identifier used to address a ParliamentMember across backend artifacts. | | `ParliamentMember` | An elected Member of Parliament with riding, party, and contact info. | +| `Senator` | An appointed Senator with province, declared caucus affiliation, Senate profile URL, and appointment facts when available. | +| `SenateAppointment` | Appointment facts for a Senator: date, appointing prime minister, represented province, declared affiliation, and Privy Council Office source URL. | | `Sitting` | A House sitting date with Parliament/session metadata and source URL. | | `Bill` | A Parliament of Canada bill with number, title, stage, sponsor, and LEGISinfo source URL. | | `BillVersion` | Backend-only bill publication/version row with source links for text, PDF, and XML artifacts. | @@ -92,6 +94,7 @@ to the issue that will build the missing artifact. | `MemberRepository` | backend Go | outbound | Implemented: `backend/members/internal/usecase/members.go`; adapter: `backend/members/internal/adapter/sqlite/repository.go`. | List members and load member-profile attendance rows from the verified members SQLite artifact. | | `MemberContentRepository` | backend Go | outbound | Implemented: `backend/member-speeches/internal/usecase/usecase.go` with adapter `backend/member-speeches/internal/adapter/artifact/artifact.go`; `backend/member-votes/main.go` has a local vote-feed interface implemented by `S3ArtifactMemberContentRepository`. | Load per-member append-only content feeds such as speeches and recorded votes. There is no iOS Swift protocol with this name today. | | `TopicPreferenceStore` | iOS Swift | outbound | Implemented: `ios/epac/Domain/Ports/TopicPreferenceStore.swift`; adapter: `ios/epac/Data/Adapters/TopicFollowStoreAdapter.swift`. | Read and persist followed topic IDs as a Domain-layer port. | +| `SenatorAppointmentQueryPort` | iOS Swift | outbound | Implemented by existing `HomeFeedRepository.fetchSenators(for:)`; adapter: `ios/epac/Data/Repositories/HomeFeedSwiftDataRepository.swift` using `ios/epac/Util/SenatorsService.swift`. | Load province-filtered senator appointment context for Home and My MP surfaces. | | `DeviceSubscriptionRepository` | backend Go | outbound | Implemented: `backend/push-notification-dispatcher/internal/usecase/dispatch_push_notification.go`; adapter: `backend/push-notification-dispatcher/internal/adapter/postgres/subscriptions.go`. | List device subscriptions eligible for the current internal notification fan-out. | | `PushNotificationClient` | backend Go | outbound | Implemented: `backend/push-notification-dispatcher/internal/usecase/dispatch_push_notification.go`; adapter: `backend/push-notification-dispatcher/internal/adapter/apns/client.go`. | Deliver a typed push payload to a subscribed device through APNs. | | `HansardSearchProviding` | iOS Swift | outbound | Implemented: `ios/epac/Util/HansardSearchService.swift`; conformer: `BackendHansardSearchService`. | Search Hansard through the backend search endpoint from iOS presentation code. | @@ -781,6 +784,51 @@ Current implementation: --- +### TrackSenateAppointments + +``` +Actor: Backend ingestion job / iOS app +Goal: Keep current Senator rows associated with appointment date, appointing prime minister, province, declared affiliation, and Orders in Council source. +Inputs: Senator roster/appointment payload from backend or existing Senate roster adapter. +Outputs: Senator values with optional SenateAppointment facts. +Entities / values: Senator, SenateAppointment. +Ports: iOS Swift: `SenatorAppointmentQueryPort`. +Primary adapters: SenatorsService, HomeFeedSwiftDataRepository, SenatorCard, MyMPView, HomeFeedView. +Current implementation: + ios/epac/Model/Senator.swift + ios/epac/Util/SenatorsService.swift + ios/epac/Data/Repositories/HomeFeedSwiftDataRepository.swift + ios/epac/Views/Senate/SenatorCard.swift + ios/epac/Views/MyMP/MyMPView.swift + ios/epac/Views/Home/HomeFeedView.swift +``` + +> **Boundary note:** iOS decodes typed appointment fields when the roster payload includes them and does not scrape PCO or Senate HTML. Backend ingestion/parsing remains outside the iOS adapter. + +--- + +### LoadAppointingPM + +``` +Actor: User (iOS app, Home / My MP) +Goal: See which prime minister appointed each province Senator and when, with their declared affiliation and PCO Orders in Council citation. +Inputs: User's saved MP province. +Outputs: Province-filtered Senator cards with appointment summary and source link. +Entities / values: Senator, SenateAppointment. +Ports: iOS Swift: `SenatorAppointmentQueryPort`. +Primary adapters: LoadHomeFeed, HomeFeedSwiftDataRepository, SenatorCard, MyMPView, HomeFeedView. +Current implementation: + ios/epac/Domain/UseCases/LoadHomeFeed.swift + ios/epac/Data/Repositories/HomeFeedSwiftDataRepository.swift + ios/epac/Views/Senate/SenatorCard.swift + ios/epac/Views/MyMP/MyMPView.swift + ios/epac/Views/Home/HomeFeedView.swift +``` + +> **Notification note:** Senate appointment notification eligibility uses the canonical `senate` topic ID added to `shared/topic-taxonomy/parliamentary_topics.json`; existing topic-follow storage carries that ID through device preferences. + +--- + ### FollowTopic ``` diff --git a/ios/epac/Model/ParliamentaryTopic.swift b/ios/epac/Model/ParliamentaryTopic.swift index 0b095116..9f746357 100644 --- a/ios/epac/Model/ParliamentaryTopic.swift +++ b/ios/epac/Model/ParliamentaryTopic.swift @@ -34,6 +34,7 @@ struct ParliamentaryTopic: Identifiable, Codable, Hashable, Sendable { ParliamentaryTopic(id: "childcare", nameKey: "topic.childcare", keywords: ["child care", "daycare", "family", "children", "services de garde", "enfant"]), ParliamentaryTopic(id: "energy", nameKey: "topic.energy", keywords: ["energy", "oil", "gas", "pipeline", "electricity", "LNG", "pétrole"]), ParliamentaryTopic(id: "naturalresources", nameKey: "topic.naturalResources", keywords: ["natural resources", "forestry", "forest", "lumber", "timber", "mining", "minerals", "mineral production", "potash", "wood harvest", "ressources naturelles", "foresterie", "mines"]), + ParliamentaryTopic(id: "senate", nameKey: "topic.senate", keywords: ["senate", "senator", "senators", "upper chamber", "sénat", "sénateur", "sénatrice"]), ParliamentaryTopic(id: "pharma", nameKey: "topic.pharma", keywords: ["drug", "pharmaceutical", "medication", "opioid", "naloxone", "médicament"]), ParliamentaryTopic(id: "digital", nameKey: "topic.digital", keywords: ["digital", "artificial intelligence", "AI", "online harms", "privacy", "cybersecurity", "numérique"]), ParliamentaryTopic(id: "labour", nameKey: "topic.labour", keywords: ["labour", "labor", "union", "strike", "wage", "employment", "travail", "grève"]), diff --git a/ios/epac/Model/Senator.swift b/ios/epac/Model/Senator.swift index c5b69108..44a519fc 100644 --- a/ios/epac/Model/Senator.swift +++ b/ios/epac/Model/Senator.swift @@ -10,8 +10,6 @@ // import Foundation -import SwiftUI -import UIKit struct Senator: Identifiable, Codable { let id: String @@ -23,14 +21,37 @@ struct Senator: Identifiable, Codable { let caucusFullName: String // e.g. "Independent Senators Group" let senateURL: URL // link to Senate profile page let appointedDate: Date? + let appointment: SenateAppointment? - var caucusColor: Color { - switch caucus.uppercased() { - case "CPC", "CONS": return Color(UIColor.systemBlue) - case "PSG": return Color(UIColor.systemRed) - case "ISG": return Color(UIColor.systemTeal) - case "CSG": return Color(UIColor.systemPurple) - default: return Color(UIColor.systemGray) - } + var appointmentDate: Date? { + appointment?.date ?? appointedDate + } + + var appointmentSourceURL: URL { + appointment?.sourceURL ?? SenateAppointment.defaultSourceURL + } +} + +struct SenateAppointment: Codable, Equatable { + static let defaultSourceURL = URL(string: "https://pco-bcp.gc.ca/oic-ddc")! + + let date: Date + let appointingPrimeMinister: String? + let province: String + let declaredAffiliation: String + let sourceURL: URL + + init( + date: Date, + appointingPrimeMinister: String?, + province: String, + declaredAffiliation: String, + sourceURL: URL = Self.defaultSourceURL + ) { + self.date = date + self.appointingPrimeMinister = appointingPrimeMinister + self.province = province + self.declaredAffiliation = declaredAffiliation + self.sourceURL = sourceURL } } diff --git a/ios/epac/Util/SenatorsService.swift b/ios/epac/Util/SenatorsService.swift index ba2327ba..b338a50f 100644 --- a/ios/epac/Util/SenatorsService.swift +++ b/ios/epac/Util/SenatorsService.swift @@ -10,6 +10,7 @@ // import Foundation +import UserNotifications struct SenatorsService { private static let cacheKey = "epac.senators.cache" @@ -35,17 +36,24 @@ struct SenatorsService { static func fetchSenators() async -> [Senator] { if let cached = loadFromCache() { return cached } + let previousSenators = loadFromCacheBypassingTTL() ?? [] + + var freshSenators: [Senator] = [] if let senators = await fetchFromOpenAPI(), !senators.isEmpty { - saveToCache(senators) - return senators + freshSenators = senators + } else if let senators = await fetchFromXML(), !senators.isEmpty { + freshSenators = senators } - if let senators = await fetchFromXML(), !senators.isEmpty { - saveToCache(senators) - return senators + if !freshSenators.isEmpty { + saveToCache(freshSenators) + if await TopicFollowStore.shared.isFollowing("senate") && !previousSenators.isEmpty { + notifyNewAppointments(fresh: freshSenators, previous: previousSenators) + } + return freshSenators } - return [] + return previousSenators.isEmpty ? [] : previousSenators } static func senators(for province: String, from senators: [Senator]) -> [Senator] { @@ -57,16 +65,18 @@ struct SenatorsService { // MARK: - OurCommons open API (primary) private static func fetchFromOpenAPI() async -> [Senator]? { - guard let url = URL(string: - "https://api.openparliament.ca/ocd/members/?parliament=45&chamber=Senate&pageSize=200&format=json" - ) else { return nil } + let url = BackendConfig.shared.baseURL.appendingPathComponent("api/v1/senators") guard let (data, response) = try? await NetworkService.shared.data(from: url), let http = response as? HTTPURLResponse, - Constants.successStatusCodes.contains(http.statusCode), - let json = try? JSONSerialization.jsonObject(with: data) as? [String: Any], - let items = json["items"] as? [[String: Any]] else { return nil } + Constants.successStatusCodes.contains(http.statusCode) else { return nil } + return parseOpenAPISenators(from: data) + } + + static func parseOpenAPISenators(from data: Data) -> [Senator]? { + guard let json = try? JSONSerialization.jsonObject(with: data) as? [String: Any], + let items = json["items"] as? [[String: Any]] else { return nil } return items.compactMap { item -> Senator? in guard let fn = item["PersonOfficialFirstName"] as? String, let ln = item["PersonOfficialLastName"] as? String else { return nil } @@ -90,9 +100,64 @@ struct SenatorsService { let senateURL = URL(string: urlStr) ?? URL(string: "https://sencanada.ca/en/senators/")! - var date: Date? - if let dateStr = item["StartDate"] as? String { - date = ISO8601DateFormatter().date(from: dateStr) + let appointmentPayload = item["appointment"] as? [String: Any] ?? item + let appointmentDateValue = stringValue( + forAnyKey: ["appointment_date", "appointmentDate", "appointed_date", "appointedDate", "date", "StartDate"], + in: appointmentPayload + ) ?? stringValue( + forAnyKey: ["appointment_date", "appointmentDate", "appointed_date", "appointedDate", "StartDate"], + in: item + ) + let date = parseDate(appointmentDateValue) + let primeMinisterKeys = [ + "appointing_prime_minister", + "appointingPrimeMinister", + "appointing_pm", + "appointingPM", + "appointed_by", + "appointedBy", + "prime_minister", + "primeMinister", + "prime_minister_name", + "primeMinisterName", + "PrimeMinisterName" + ] + let appointingPrimeMinister = stringValue(forAnyKey: primeMinisterKeys, in: appointmentPayload) + ?? stringValue(forAnyKey: primeMinisterKeys, in: item) + let sourceURLKeys = [ + "source_url", + "sourceURL", + "sourceUrl", + "orders_in_council_url", + "ordersInCouncilURL", + "ordersInCouncilUrl", + "order_in_council_url", + "orderInCouncilURL", + "orderInCouncilUrl", + "OrderInCouncilURL" + ] + let sourceURL = urlValue(forAnyKey: sourceURLKeys, in: appointmentPayload) + ?? urlValue(forAnyKey: sourceURLKeys, in: item) + ?? SenateAppointment.defaultSourceURL + let affiliationKeys = [ + "declared_affiliation", + "declaredAffiliation", + "affiliation", + "caucus_full_name", + "caucusFullName", + "CaucusNameEn" + ] + let declaredAffiliation = stringValue(forAnyKey: affiliationKeys, in: appointmentPayload) + ?? stringValue(forAnyKey: affiliationKeys, in: item) + ?? caucusFull + let appointment = date.map { + SenateAppointment( + date: $0, + appointingPrimeMinister: appointingPrimeMinister, + province: abbrev, + declaredAffiliation: declaredAffiliation, + sourceURL: sourceURL + ) } return Senator( @@ -103,7 +168,8 @@ struct SenatorsService { caucus: caucus, caucusFullName: caucusFull, senateURL: senateURL, - appointedDate: date + appointedDate: date, + appointment: appointment ) } } @@ -160,7 +226,8 @@ struct SenatorsService { caucus: caucus, caucusFullName: caucus, senateURL: senateURL, - appointedDate: nil + appointedDate: nil, + appointment: nil ) } @@ -175,6 +242,38 @@ struct SenatorsService { return String(block[range]) } + private static func stringValue(forAnyKey keys: [String], in item: [String: Any]) -> String? { + for key in keys { + if let value = item[key] as? String { + let trimmed = value.trimmingCharacters(in: .whitespacesAndNewlines) + if !trimmed.isEmpty { return trimmed } + } + } + return nil + } + + private static func urlValue(forAnyKey keys: [String], in item: [String: Any]) -> URL? { + guard let rawValue = stringValue(forAnyKey: keys, in: item) else { return nil } + return URL(string: rawValue) + } + + private static func parseDate(_ rawValue: String?) -> Date? { + guard let rawValue else { return nil } + if let date = ISO8601DateFormatter().date(from: rawValue) { + return date + } + return dateOnlyFormatter.date(from: rawValue) + } + + private static let dateOnlyFormatter: DateFormatter = { + let formatter = DateFormatter() + formatter.calendar = Calendar(identifier: .gregorian) + formatter.locale = Locale(identifier: "en_US_POSIX") + formatter.timeZone = TimeZone(secondsFromGMT: 0) + formatter.dateFormat = "yyyy-MM-dd" + return formatter + }() + // MARK: - Province mapping private static func provinceAbbrev(_ full: String) -> String { @@ -214,4 +313,43 @@ struct SenatorsService { UserDefaults.standard.set(data, forKey: cacheKey) UserDefaults.standard.set(Date(), forKey: cacheTimestampKey) } + + private static func loadFromCacheBypassingTTL() -> [Senator]? { + guard + let data = UserDefaults.standard.data(forKey: cacheKey), + let senators = try? JSONDecoder().decode([Senator].self, from: data) + else { return nil } + return senators + } + + private static func notifyNewAppointments(fresh: [Senator], previous: [Senator]) { + let previousIDs = Set(previous.map { $0.id }) + let newAppointments = fresh.filter { !previousIDs.contains($0.id) } + for senator in newAppointments { + triggerNotification(for: senator) + } + } + + private static func triggerNotification(for senator: Senator) { + let content = UNMutableNotificationContent() + content.title = NSLocalizedString("senate.notification.title", comment: "") + let bodyFormat = NSLocalizedString("senate.notification.body", comment: "") + let pm = senator.appointment?.appointingPrimeMinister ?? "" + content.body = String(format: bodyFormat, senator.name, senator.province, pm) + content.sound = UNNotificationSound.default + + let request = UNNotificationRequest( + identifier: "epac.senator-appointment.\(senator.id)", + content: content, + trigger: nil + ) + + UNUserNotificationCenter.current().add(request) { error in + if let error = error { + Log.error("Failed to post senator appointment notification: \(error)") + } else { + Log.debug("Posted senator appointment notification for \(senator.name)") + } + } + } } diff --git a/ios/epac/Views/Home/HomeFeedView.swift b/ios/epac/Views/Home/HomeFeedView.swift index 7798e5d6..5ba9f5c1 100644 --- a/ios/epac/Views/Home/HomeFeedView.swift +++ b/ios/epac/Views/Home/HomeFeedView.swift @@ -23,10 +23,6 @@ private enum HomeFeedLayout { static let compactVerticalPadding = EpacSpacing.xxs static let followedTopicsLimit = 6 static let senatorsLimit = 3 - static let senatorRowSpacing: CGFloat = 10 - static let senatorDotOpacity = EpacOpacity.tintStrong - static let senatorDotSize = EpacSpacing.s - static let senatorTextSpacing = EpacSpacing.xxs static let healthcarePreviewLimit = 2 static let reorderBillsThreshold = 5 static let unreadDotSize: CGFloat = 8 @@ -526,27 +522,7 @@ struct HomeFeedView: View { private var senatorsSection: some View { Section(header: Text(NSLocalizedString("senate.mySenators.title", comment: "")).accessibilityAddTraits(.isHeader)) { ForEach(mySenators.prefix(HomeFeedLayout.senatorsLimit)) { senator in - Link(destination: senator.senateURL) { - HStack(spacing: HomeFeedLayout.senatorRowSpacing) { - Circle() - .fill(senator.caucusColor.opacity(HomeFeedLayout.senatorDotOpacity)) - .frame(width: HomeFeedLayout.senatorDotSize, height: HomeFeedLayout.senatorDotSize) - VStack(alignment: .leading, spacing: HomeFeedLayout.senatorTextSpacing) { - Text(senator.name) - .font(.subheadline) - .foregroundStyle(.primary) - Text(senator.caucusFullName) - .font(.caption2) - .foregroundStyle(.secondary) - .fixedSize(horizontal: false, vertical: true) - } - Spacer() - Image(systemName: "arrow.up.right.square") - .font(.epacCaption) - .foregroundStyle(Color.epacText.tertiary) - } - } - .accessibilityLabel("\(senator.name), \(senator.caucusFullName)") + SenatorCard(senator: senator) } } } diff --git a/ios/epac/Views/Senate/SenatorCard.swift b/ios/epac/Views/Senate/SenatorCard.swift index 404325fd..c2eda6e1 100644 --- a/ios/epac/Views/Senate/SenatorCard.swift +++ b/ios/epac/Views/Senate/SenatorCard.swift @@ -8,6 +8,7 @@ // import SwiftUI +import UIKit private enum Layout { static let rowSpacing: CGFloat = 12 @@ -18,31 +19,90 @@ struct SenatorCard: View { let senator: Senator var body: some View { - Link(destination: senator.senateURL) { - HStack(spacing: Layout.rowSpacing) { - Circle() - .fill(senator.caucusColor) - .frame(width: Layout.caucusDotSize, height: Layout.caucusDotSize) - .accessibilityHidden(true) - VStack(alignment: .leading, spacing: EpacSpacing.xxs) { - Text(String(format: NSLocalizedString("senate.card.name", comment: ""), senator.name)) - .font(.subheadline.weight(.semibold)) - Text(senator.caucusFullName) + HStack(alignment: .top, spacing: Layout.rowSpacing) { + Circle() + .fill(caucusColor) + .frame(width: Layout.caucusDotSize, height: Layout.caucusDotSize) + .padding(.top, EpacSpacing.xxs) + .accessibilityHidden(true) + VStack(alignment: .leading, spacing: EpacSpacing.xxs) { + Text(String(format: NSLocalizedString("senate.card.name", comment: ""), senator.name)) + .font(.subheadline.weight(.semibold)) + Text(declaredAffiliation) + .font(.caption2) + .foregroundStyle(.secondary) + if let appointmentSummary { + Text(appointmentSummary) .font(.caption2) .foregroundStyle(.secondary) + .fixedSize(horizontal: false, vertical: true) + Link(destination: senator.appointmentSourceURL) { + Label(NSLocalizedString("senate.card.source", comment: ""), systemImage: "doc.text") + .font(.caption2) + } } - Spacer() + } + .fixedSize(horizontal: false, vertical: true) + Spacer() + Link(destination: senator.senateURL) { Image(systemName: "arrow.up.right.square") .foregroundStyle(.secondary) .font(.caption) - .accessibilityHidden(true) } - .padding(.vertical, EpacSpacing.xxs) + .accessibilityLabel(String(format: NSLocalizedString("senate.card.name", comment: ""), senator.name)) } + .padding(.vertical, EpacSpacing.xxs) .foregroundStyle(.primary) - .accessibilityLabel( - String(format: NSLocalizedString("senate.card.accessibility", comment: ""), - senator.name, senator.caucusFullName) + .accessibilityLabel(accessibilitySummary) + } + + private var declaredAffiliation: String { + senator.appointment?.declaredAffiliation ?? senator.caucusFullName + } + + private var appointmentSummary: String? { + guard let date = senator.appointmentDate else { return nil } + let formattedDate = Self.appointmentDateFormatter.string(from: date) + if let primeMinister = senator.appointment?.appointingPrimeMinister, !primeMinister.isEmpty { + return String( + format: NSLocalizedString("senate.card.appointed", comment: ""), + primeMinister, + formattedDate + ) + } + return String(format: NSLocalizedString("senate.card.appointedUnknownPM", comment: ""), formattedDate) + } + + private var caucusColor: Color { + switch senator.caucus.uppercased() { + case "CPC", "CONS": return Color(UIColor.systemBlue) + case "PSG": return Color(UIColor.systemRed) + case "ISG": return Color(UIColor.systemTeal) + case "CSG": return Color(UIColor.systemPurple) + default: return Color(UIColor.systemGray) + } + } + + private var accessibilitySummary: String { + if let appointmentSummary { + return String( + format: NSLocalizedString("senate.card.accessibilityWithAppointment", comment: ""), + senator.name, + declaredAffiliation, + appointmentSummary + ) + } + return String( + format: NSLocalizedString("senate.card.accessibility", comment: ""), + senator.name, + declaredAffiliation ) } + + private static let appointmentDateFormatter: DateFormatter = { + let formatter = DateFormatter() + formatter.dateStyle = .medium + formatter.timeStyle = .none + return formatter + }() } diff --git a/ios/epac/en.lproj/Localizable.strings b/ios/epac/en.lproj/Localizable.strings index 1019b490..b2d75a28 100644 --- a/ios/epac/en.lproj/Localizable.strings +++ b/ios/epac/en.lproj/Localizable.strings @@ -346,6 +346,7 @@ "topic.childcare" = "Child Care"; "topic.energy" = "Energy"; "topic.naturalResources" = "Natural Resources"; +"topic.senate" = "Senate"; "topic.pharma" = "Pharmaceuticals"; "topic.digital" = "Digital & AI"; "topic.labour" = "Labour"; @@ -462,7 +463,13 @@ "senate.mySenators.title" = "My Senators"; "senate.mySenators.empty" = "No senators found for this province."; "senate.card.name" = "Senator %@"; +"senate.card.appointed" = "Appointed by %@ on %@"; +"senate.card.appointedUnknownPM" = "Appointed on %@"; +"senate.card.source" = "Privy Council Office — Orders in Council"; "senate.card.accessibility" = "Senator %@, %@"; +"senate.card.accessibilityWithAppointment" = "Senator %@, %@, %@"; +"senate.notification.title" = "New Senate Appointment"; +"senate.notification.body" = "%@ has been appointed to the Senate for %@ by %@."; /* Ontario Legislature */ "ontario.debates.navTitle" = "Queen's Park Debates"; diff --git a/ios/epac/fr.lproj/Localizable.strings b/ios/epac/fr.lproj/Localizable.strings index 5ea907af..a61624e6 100644 --- a/ios/epac/fr.lproj/Localizable.strings +++ b/ios/epac/fr.lproj/Localizable.strings @@ -348,6 +348,7 @@ "topic.childcare" = "Services de garde"; "topic.energy" = "Énergie"; "topic.naturalResources" = "Ressources naturelles"; +"topic.senate" = "Sénat"; "topic.pharma" = "Produits pharmaceutiques"; "topic.digital" = "Numérique et IA"; "topic.labour" = "Travail"; @@ -464,7 +465,13 @@ "senate.mySenators.title" = "Mes sénateurs"; "senate.mySenators.empty" = "Aucun sénateur trouvé pour cette province."; "senate.card.name" = "Sénateur(trice) %@"; +"senate.card.appointed" = "Nommé(e) par %@ le %@"; +"senate.card.appointedUnknownPM" = "Nommé(e) le %@"; +"senate.card.source" = "Bureau du Conseil privé — décrets"; "senate.card.accessibility" = "Sénateur(trice) %@, %@"; +"senate.card.accessibilityWithAppointment" = "Sénateur(trice) %@, %@, %@"; +"senate.notification.title" = "Nouvelle nomination au Sénat"; +"senate.notification.body" = "%@ a été nommé(e) au Sénat pour %@ par %@."; /* Assemblée législative de l'Ontario */ "ontario.debates.navTitle" = "Débats de Queen's Park"; diff --git a/ios/epacTests/LoadHomeFeedTests.swift b/ios/epacTests/LoadHomeFeedTests.swift index 6ea6c7c8..b2b56d61 100644 --- a/ios/epacTests/LoadHomeFeedTests.swift +++ b/ios/epacTests/LoadHomeFeedTests.swift @@ -79,6 +79,57 @@ final class LoadHomeFeedTests: XCTestCase { XCTAssertEqual(snapshot.latestSpeechHighlight?.memberName, "Jane Smith") XCTAssertEqual(snapshot.latestSpeechHighlight?.subjectTitle, "Budget Debate") } + + func testLoadHomeFeedIncludesProvinceSenateAppointments() async throws { + let clock = MockClock(date: Date()) + let repository = MockHomeFeedRepository() + let followPrefs = MockFollowPreferenceReading(savedMemberName: "Jane Smith") + let appointmentDate = try XCTUnwrap(Self.dateFormatter.date(from: "2024-12-19")) + let appointment = SenateAppointment( + date: appointmentDate, + appointingPrimeMinister: "Justin Trudeau", + province: "ON", + declaredAffiliation: "Independent Senators Group" + ) + repository.membersToReturn = [ + HomeFollowedMember(memberID: 42, name: "Jane Smith", lastName: "Smith", provinceCode: "ON") + ] + repository.senatorsToReturn = [ + Senator( + id: "test-senator-on", + firstName: "Jane", + lastName: "Senator", + province: "ON", + caucus: "ISG", + caucusFullName: "Independent Senators Group", + senateURL: URL(string: "https://sencanada.ca/en/senators/test")!, + appointedDate: appointmentDate, + appointment: appointment + ) + ] + + let useCase = LoadHomeFeed( + repository: repository, + followPreferenceReading: followPrefs, + clock: clock + ) + + let snapshot = await useCase.execute() + + XCTAssertEqual(snapshot.civicContext.provinceAbbrev, "ON") + XCTAssertEqual(snapshot.civicContext.mySenators.count, 1) + XCTAssertEqual(snapshot.civicContext.mySenators.first?.appointment?.appointingPrimeMinister, "Justin Trudeau") + XCTAssertEqual(snapshot.civicContext.mySenators.first?.appointment?.declaredAffiliation, "Independent Senators Group") + } + + private static let dateFormatter: DateFormatter = { + let formatter = DateFormatter() + formatter.calendar = Calendar(identifier: .gregorian) + formatter.locale = Locale(identifier: "en_US_POSIX") + formatter.timeZone = TimeZone(secondsFromGMT: 0) + formatter.dateFormat = "yyyy-MM-dd" + return formatter + }() } // MARK: - Mocks @@ -95,6 +146,7 @@ class MockHomeFeedRepository: HomeFeedRepository { var hansardsToReturn: [HomeHansardRecord] = [] var voteToReturn: HomeVoteRecord? var memberVoteToReturn: HomeMemberVoteRecord? + var senatorsToReturn: [Senator] = [] func fetchSittingDates() async throws -> [Date] { [] } func fetchAllMembers() async throws -> [HomeFollowedMember] { membersToReturn } @@ -103,7 +155,7 @@ class MockHomeFeedRepository: HomeFeedRepository { func fetchHansards(between start: Date, and end: Date) async throws -> [HomeHansardRecord] { [] } func fetchLatestVote() async throws -> HomeVoteRecord? { voteToReturn } func fetchMemberVote(memberID: Int, voteID: Int) async throws -> HomeMemberVoteRecord? { memberVoteToReturn } - func fetchSenators(for provinceAbbrev: String) async throws -> [Senator] { [] } + func fetchSenators(for provinceAbbrev: String) async throws -> [Senator] { senatorsToReturn } } @MainActor diff --git a/ios/epacTests/ModelTests.swift b/ios/epacTests/ModelTests.swift index f141ea9e..9afcf8b4 100644 --- a/ios/epacTests/ModelTests.swift +++ b/ios/epacTests/ModelTests.swift @@ -31,4 +31,46 @@ struct ModelTests { #expect(Province(rawValue: "Quebec") == .Quebec) #expect(Province(rawValue: "Saskatchewan") == .Saskatchewan) } + + @Test func senateTopicAvailableForAppointmentNotifications() { + let senate = ParliamentaryTopic.all.first { $0.id == "senate" } + + #expect(senate?.nameKey == "topic.senate") + #expect(senate?.keywords.contains("senator") == true) + #expect(ParliamentaryTopic.matching("The PM has appointed a new senator").map(\.id).contains("senate")) + } + + @Test func senatorOpenAPIParserIncludesAppointmentFacts() throws { + let payload = """ + { + "items": [ + { + "PersonOfficialFirstName": "Jane", + "PersonOfficialLastName": "Senator", + "ProvinceName": "Ontario", + "CaucusAbbreviationEn": "ISG", + "CaucusNameEn": "Independent Senators Group", + "PersonPageUrl": "https://sencanada.ca/en/senators/jane-senator", + "appointment": { + "appointment_date": "2024-12-19", + "appointing_prime_minister": "Justin Trudeau", + "declared_affiliation": "Independent Senators Group", + "orders_in_council_url": "https://pco-bcp.gc.ca/oic-ddc.asp?lang=eng&Page=secretariats&txtOICID=2024-1300" + } + } + ] + } + """.data(using: .utf8) + let data = try #require(payload) + + let senator = try #require(SenatorsService.parseOpenAPISenators(from: data)?.first) + let appointment = try #require(senator.appointment) + + #expect(senator.province == "ON") + #expect(appointment.appointingPrimeMinister == "Justin Trudeau") + #expect(appointment.declaredAffiliation == "Independent Senators Group") + #expect(appointment.province == "ON") + #expect(appointment.sourceURL.absoluteString.contains("pco-bcp.gc.ca/oic-ddc.asp")) + #expect(senator.appointmentDate == appointment.date) + } } diff --git a/ios/epacTests/SnapshotTests.swift b/ios/epacTests/SnapshotTests.swift index b4755c9d..884259a0 100644 --- a/ios/epacTests/SnapshotTests.swift +++ b/ios/epacTests/SnapshotTests.swift @@ -79,6 +79,25 @@ final class SnapshotTests: XCTestCase { ) } + private static var senatorWithAppointment: Senator { + Senator( + id: "sample-senator-on", + firstName: "Jane", + lastName: "Senator", + province: "ON", + caucus: "ISG", + caucusFullName: "Independent Senators Group", + senateURL: URL(string: "https://sencanada.ca/en/senators/sample")!, + appointedDate: date("2024-12-19"), + appointment: SenateAppointment( + date: date("2024-12-19"), + appointingPrimeMinister: "Justin Trudeau", + province: "ON", + declaredAffiliation: "Independent Senators Group" + ) + ) + } + // MARK: - MemberRow func testMemberRow_liberal() { @@ -99,6 +118,19 @@ final class SnapshotTests: XCTestCase { ) } + // MARK: - SenatorCard + + func testSenatorCard_appointment() { + snapshot( + SenatorCard(senator: Self.senatorWithAppointment) + .frame(width: 375) + .padding() + .background(Color(.systemBackground)) + .frame(maxWidth: .infinity, maxHeight: .infinity, alignment: .topLeading), + name: "SenatorCard_appointment" + ) + } + // MARK: - PartyBadge func testPartyBadge_allParties() { diff --git a/ios/epacTests/__Snapshots__/SnapshotTests/testSenatorCard_appointment.SenatorCard_appointment_a11y.png b/ios/epacTests/__Snapshots__/SnapshotTests/testSenatorCard_appointment.SenatorCard_appointment_a11y.png new file mode 100644 index 00000000..40bd7add Binary files /dev/null and b/ios/epacTests/__Snapshots__/SnapshotTests/testSenatorCard_appointment.SenatorCard_appointment_a11y.png differ diff --git a/ios/epacTests/__Snapshots__/SnapshotTests/testSenatorCard_appointment.SenatorCard_appointment_dark.png b/ios/epacTests/__Snapshots__/SnapshotTests/testSenatorCard_appointment.SenatorCard_appointment_dark.png new file mode 100644 index 00000000..2c52fb03 Binary files /dev/null and b/ios/epacTests/__Snapshots__/SnapshotTests/testSenatorCard_appointment.SenatorCard_appointment_dark.png differ diff --git a/ios/epacTests/__Snapshots__/SnapshotTests/testSenatorCard_appointment.SenatorCard_appointment_light.png b/ios/epacTests/__Snapshots__/SnapshotTests/testSenatorCard_appointment.SenatorCard_appointment_light.png new file mode 100644 index 00000000..106b93e8 Binary files /dev/null and b/ios/epacTests/__Snapshots__/SnapshotTests/testSenatorCard_appointment.SenatorCard_appointment_light.png differ diff --git a/scripts/artifacts/fetch_senators.py b/scripts/artifacts/fetch_senators.py new file mode 100755 index 00000000..135e7cca --- /dev/null +++ b/scripts/artifacts/fetch_senators.py @@ -0,0 +1,195 @@ +#!/usr/bin/env python3 +import os +import sys +import re +import json +import argparse +import urllib.request +from html.parser import HTMLParser + +class SenateHTMLParser(HTMLParser): + def __init__(self): + super().__init__() + self.in_tbody = False + self.in_row = False + self.in_cell = False + self.current_cell_index = -1 + self.rows = [] + self.current_row = [] + self.current_cell_data = "" + self.current_link = "" + self.cell_attrs = {} + + def handle_starttag(self, tag, attrs): + attrs_dict = dict(attrs) + if tag == "tbody": + self.in_tbody = True + elif tag == "tr" and self.in_tbody: + self.in_row = True + self.current_row = [] + self.current_cell_index = -1 + elif tag == "td" and self.in_row: + self.in_cell = True + self.current_cell_index += 1 + self.current_cell_data = "" + self.current_link = "" + self.cell_attrs = attrs_dict + elif tag == "a" and self.in_cell: + if "href" in attrs_dict: + self.current_link = attrs_dict["href"] + + def handle_data(self, data): + if self.in_cell: + self.current_cell_data += data + + def handle_endtag(self, tag): + if tag == "tbody": + self.in_tbody = False + elif tag == "tr" and self.in_row: + self.in_row = False + self.rows.append(self.current_row) + elif tag == "td" and self.in_cell: + self.in_cell = False + text = self.current_cell_data.strip() + self.current_row.append({ + "text": text, + "link": self.current_link, + "attrs": self.cell_attrs + }) + +def split_name(name_str): + name_str = name_str.strip() + if ',' in name_str: + parts = name_str.split(',', 1) + last_name = parts[0].strip() + first_name = parts[1].strip() + else: + parts = name_str.rsplit(' ', 1) + if len(parts) == 2: + first_name = parts[0].strip() + last_name = parts[1].strip() + else: + first_name = name_str + last_name = "" + return first_name, last_name + +def clean_pm(pm_str): + pm_str = pm_str.strip() + # Remove party abbreviation in parentheses, e.g., (Lib.) or (PC) + pm_str = re.sub(r'\s*\([^)]*\)', '', pm_str).strip() + if ',' in pm_str: + parts = pm_str.split(',', 1) + last = parts[0].strip() + first = parts[1].strip() + return f"{first} {last}" + return pm_str + +def clean_province(prov_str): + # Remove anything in parentheses, e.g. "Quebec (Grandville)" -> "Quebec" + prov_str = re.sub(r'\s*\([^)]*\)', '', prov_str).strip() + known = { + "british columbia", "alberta", "saskatchewan", "manitoba", "ontario", + "quebec", "québec", "new brunswick", "nova scotia", "prince edward island", + "newfoundland and labrador", "northwest territories", "nunavut", "yukon" + } + for k in known: + if prov_str.lower() == k: + return prov_str + return prov_str.title() + +def main(): + parser = argparse.ArgumentParser(description="Fetch and parse Senate list of Canada.") + parser.add_argument("--output", "-o", required=True, help="Path to write senators/v1/all.json output") + args = parser.parse_args() + + url = "https://sencanada.ca/umbraco/surface/SenatorsAjax/GetSenators?displayFor=senatorslist&Lang=en" + req = urllib.request.Request( + url, + headers={"User-Agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36"} + ) + + try: + with urllib.request.urlopen(req, timeout=30) as response: + html = response.read().decode("utf-8") + except Exception as e: + print(f"Error fetching senators list: {e}", file=sys.stderr) + sys.exit(1) + + html_parser = SenateHTMLParser() + html_parser.feed(html) + + items = [] + for r in html_parser.rows: + if len(r) < 6: + continue + + name_info = r[0] + name_str = name_info["text"] + link_str = name_info["link"] + if link_str.startswith("/"): + link_str = "https://sencanada.ca" + link_str + + first_name, last_name = split_name(name_str) + caucus_short = r[1]["text"].strip() + if caucus_short == "C": + caucus_short = "CPC" + + caucus_full = caucus_short + caucus_map = { + "CPC": "Conservative Party of Canada", + "CSG": "Canadian Senators Group", + "ISG": "Independent Senators Group", + "PSG": "Progressive Senate Group", + "GRO": "Government Representative's Office", + "Non-affiliated": "Non-affiliated" + } + if caucus_short in caucus_map: + caucus_full = caucus_map[caucus_short] + + province_raw = r[2]["attrs"].get("data-order", r[2]["text"]).strip() + province_full = clean_province(province_raw) + + nom_date_str = r[3]["text"].strip() + match = re.match(r'^\d{4}-\d{2}-\d{2}$', nom_date_str) + if not match: + order_attr = r[3]["attrs"].get("data-order", "") + match = re.match(r'^(\d{4}-\d{2}-\d{2})', order_attr) + if match: + nom_date_str = match.group(1) + else: + nom_date_str = None + + pm_raw = r[5]["text"].strip() + appointing_pm = clean_pm(pm_raw) + + item = { + "PersonOfficialFirstName": first_name, + "PersonOfficialLastName": last_name, + "ProvinceNameEn": province_full, + "CaucusAbbreviationEn": caucus_short, + "CaucusNameEn": caucus_full, + "PersonPageUrl": link_str, + } + + if nom_date_str: + item["appointment"] = { + "appointment_date": nom_date_str, + "appointing_prime_minister": appointing_pm, + "source_url": "https://pco-bcp.gc.ca/oic-ddc", + "declared_affiliation": caucus_short + } + items.append(item) + + # Write output directory + output_dir = os.path.dirname(args.output) + if output_dir: + os.makedirs(output_dir, exist_ok=True) + + payload = {"items": items} + with open(args.output, "w", encoding="utf-8") as f: + json.dump(payload, f, indent=2, ensure_ascii=False) + + print(f"Ingested {len(items)} senators and saved to {args.output}") + +if __name__ == "__main__": + main() diff --git a/scripts/ci/check_backend_manifest_deployment.py b/scripts/ci/check_backend_manifest_deployment.py index 683d4c4a..c66fc5e2 100755 --- a/scripts/ci/check_backend_manifest_deployment.py +++ b/scripts/ci/check_backend_manifest_deployment.py @@ -39,6 +39,7 @@ class ArtifactContract: "members": ArtifactContract(("MEMBERS_INDEX_PREFIX", "EPAC_MEMBERS_INDEX_PREFIX"), "members/v1"), "hansard-search": ArtifactContract(("EPAC_HANSARD_SEARCH_PREFIX",), "hansard-search/v1"), "lobbying": ArtifactContract(("LOBBYING_INDEX_PREFIX",), "lobbying-index/v1", require_prefix_env=True), + "senators": ArtifactContract(("EPAC_SENATORS_PREFIX",), "senators/v1", required_files=("all.json",)), } diff --git a/scripts/ci/run_native_indexer.py b/scripts/ci/run_native_indexer.py index ae17d534..8064d411 100644 --- a/scripts/ci/run_native_indexer.py +++ b/scripts/ci/run_native_indexer.py @@ -58,6 +58,15 @@ "table_counts": "object[string,integer]", } +SENATORS_MANIFEST_FORMAT = { + "version": "string", + "built_at": "rfc3339", + "senator_count": "integer", + "sqlite_key": "string", + "sqlite_size_bytes": "integer", + "sqlite_sha256": "sha256", +} + def build_parser() -> argparse.ArgumentParser: parser = argparse.ArgumentParser( @@ -70,6 +79,7 @@ def build_parser() -> argparse.ArgumentParser: "lobbying-index", "bills-indexer", "members-indexer", + "senators-indexer", ), help="Indexer pipeline to run.", ) @@ -176,6 +186,24 @@ def planned_payload(args: argparse.Namespace, summary_path: Path | None) -> dict ], } + if args.pipeline == "senators-indexer": + return { + "pipeline": args.pipeline, + "environment": args.environment, + "mode": "dry-run", + "status": "planned", + "summary_markdown_path": str(summary_path) if summary_path else None, + "manifest_format": SENATORS_MANIFEST_FORMAT, + "parameters": {}, + "commands": [ + { + "working_directory": "backend/senators", + "argv": ["python3", "../../scripts/artifacts/fetch_senators.py", "--output", "../../build/artifacts/senators/v1/all.json"], + "env": {}, + } + ], + } + return { "pipeline": args.pipeline, "environment": args.environment, diff --git a/scripts/ci/tests/test_native_indexer_cli.py b/scripts/ci/tests/test_native_indexer_cli.py index 4deaa6ef..6fbabe83 100644 --- a/scripts/ci/tests/test_native_indexer_cli.py +++ b/scripts/ci/tests/test_native_indexer_cli.py @@ -70,6 +70,18 @@ "table_counts": "object[string,integer]", }, ), + ( + "senators-indexer", + [], + { + "version": "string", + "built_at": "rfc3339", + "senator_count": "integer", + "sqlite_key": "string", + "sqlite_size_bytes": "integer", + "sqlite_sha256": "sha256", + }, + ), ], ) def test_native_indexer_cli_dry_run_reports_expected_output_formats( diff --git a/scripts/topic_taxonomy/generate_topic_taxonomy.py b/scripts/topic_taxonomy/generate_topic_taxonomy.py index 1a5b92f9..f61e0867 100755 --- a/scripts/topic_taxonomy/generate_topic_taxonomy.py +++ b/scripts/topic_taxonomy/generate_topic_taxonomy.py @@ -59,7 +59,7 @@ def render_swift(payload: dict) -> str: "", "import Foundation", "", - "struct ParliamentaryTopic: Identifiable, Codable, Hashable {", + "struct ParliamentaryTopic: Identifiable, Codable, Hashable, Sendable {", " let id: String // stable lowercase slug, e.g. \"housing\"", " let nameKey: String // localization key, e.g. \"topic.housing\"", " let keywords: [String] // case-insensitive substrings to match in titles", diff --git a/shared/topic-taxonomy/parliamentary_topics.json b/shared/topic-taxonomy/parliamentary_topics.json index f8d5fc01..fa69ceea 100644 --- a/shared/topic-taxonomy/parliamentary_topics.json +++ b/shared/topic-taxonomy/parliamentary_topics.json @@ -249,6 +249,20 @@ "mines" ] }, + { + "id": "senate", + "name_key": "topic.senate", + "backend_name": "Senate", + "keywords": [ + "senate", + "senator", + "senators", + "upper chamber", + "sénat", + "sénateur", + "sénatrice" + ] + }, { "id": "pharma", "name_key": "topic.pharma",