Merge pull request #93 from MohamedKamal000/migrate-to-oba_gtfs

Migrate to oba gtfs

Author: 0xaboomar

go.mod

@@ -3,10 +3,10 @@ module watchdog.onebusaway.org
 go 1.23.5
 
 require (
+	github.com/OneBusAway/go-gtfs v1.1.1
 	github.com/OneBusAway/go-sdk v0.1.0-alpha.13
 	github.com/getsentry/sentry-go v0.33.0
 	github.com/golang/geo v0.0.0-20250707181242-c5087ca84cf4
-	github.com/jamespfennell/gtfs v0.1.24
 	github.com/julienschmidt/httprouter v1.3.0
 	github.com/prometheus/client_golang v1.20.5
 	github.com/prometheus/client_model v0.6.1

go.sum

@@ -1,3 +1,5 @@
+github.com/OneBusAway/go-gtfs v1.1.1 h1:JWl0ndXHBED6PAh8v3w0UgSDYWBg2OmHvAJb5RXX3Ss=
+github.com/OneBusAway/go-gtfs v1.1.1/go.mod h1:MJqNyFOJs+iE1R6uerTyfBY6g3/sxvTvVdRhDeN1bu8=
 github.com/OneBusAway/go-sdk v0.1.0-alpha.13 h1:xQdZjREPJTON4XKoQpUf9YTm8KCVsLJyOW9LkldyquY=
 github.com/OneBusAway/go-sdk v0.1.0-alpha.13/go.mod h1:h1TnOvie6gN5gi0no/0w6nPg1jbidz2D+Osyq72R60Q=
 github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM=
@@ -15,12 +17,6 @@ github.com/golang/geo v0.0.0-20250707181242-c5087ca84cf4 h1:vCeHcs8N7MOccOOsOVIy
 github.com/golang/geo v0.0.0-20250707181242-c5087ca84cf4/go.mod h1:AN0OjM34c3PbjAsX+QNma1nYtJtRxl+s9MZNV7S+efw=
 github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8=
 github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU=
-github.com/golang/geo v0.0.0-20250707181242-c5087ca84cf4 h1:vCeHcs8N7MOccOOsOVIy1xcYu+kBkA4J5urTgigww7c=
-github.com/golang/geo v0.0.0-20250707181242-c5087ca84cf4/go.mod h1:AN0OjM34c3PbjAsX+QNma1nYtJtRxl+s9MZNV7S+efw=
-github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8=
-github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU=
-github.com/jamespfennell/gtfs v0.1.24 h1:em/W8Bg88xZwX70r7FoghMiuZDn3m5oj0EfqmikU1Io=
-github.com/jamespfennell/gtfs v0.1.24/go.mod h1:nj9QFIc695IUKM5djZXzyPtCIAYxufafb0sQNbiVMOs=
 github.com/julienschmidt/httprouter v1.3.0 h1:U0609e9tgbseu3rBINet9P48AI/D3oJs4dN7jwJOQ1U=
 github.com/julienschmidt/httprouter v1.3.0/go.mod h1:JR6WtHb+2LUe8TCKY3cZOxFyyO8IZAc4RVcycCCAKdM=
 github.com/klauspost/compress v1.17.11 h1:In6xLpyWOi1+C7tXUUWv2ot1QvBjxevKAaI6IXrJmUc=

internal/app/test_helpers.go

@@ -7,7 +7,7 @@ import (
 	"testing"
 	"time"
 
-	remoteGtfs "github.com/jamespfennell/gtfs"
+	remoteGtfs "github.com/OneBusAway/go-gtfs"
 	"github.com/prometheus/client_golang/prometheus"
 	"watchdog.onebusaway.org/internal/config"
 	"watchdog.onebusaway.org/internal/geo"

internal/geo/geo_cluster.go

@@ -3,8 +3,8 @@ package geo
 import (
 	"fmt"
 
+	remoteGtfs "github.com/OneBusAway/go-gtfs"
 	"github.com/golang/geo/s2"
-	remoteGtfs "github.com/jamespfennell/gtfs"
 )
 
 const s2Level = 13 // S2 cell level with 850–1225 m spatial resolution

internal/geo/geo_service.go

@@ -1,7 +1,7 @@
 package geo
 
 import (
-	remoteGtfs "github.com/jamespfennell/gtfs"
+	remoteGtfs "github.com/OneBusAway/go-gtfs"
 )
 
 // For now geo package only exposes helper functions to be used by other packages.

internal/geo/geo_utils.go

@@ -8,8 +8,8 @@ import (
 	"math"
 	"sync"
 
+	remoteGtfs "github.com/OneBusAway/go-gtfs"
 	"github.com/golang/geo/s2"
-	remoteGtfs "github.com/jamespfennell/gtfs"
 )
 
 // BoundingBox defines the geographic boundaries of a rectangular area.
@@ -41,8 +41,8 @@ func computeBoundingBox(stops []remoteGtfs.Stop) (BoundingBox, error) {
 
 	for _, stop := range stops {
 		if stop.Latitude != nil && stop.Longitude != nil {
-			lat := float64(*stop.Latitude)
-			lon := float64(*stop.Longitude)
+			lat := *stop.Latitude
+			lon := *stop.Longitude
 			if lat < minLat {
 				minLat = lat
 			}

internal/gtfs/gtfs_bundles.go

@@ -11,8 +11,8 @@ import (
 	"sync"
 	"time"
 
+	remoteGtfs "github.com/OneBusAway/go-gtfs"
 	"github.com/getsentry/sentry-go"
-	remoteGtfs "github.com/jamespfennell/gtfs"
 	"watchdog.onebusaway.org/internal/config"
 	"watchdog.onebusaway.org/internal/geo"
 	"watchdog.onebusaway.org/internal/models"

internal/gtfs/gtfs_bundles_test.go

@@ -10,7 +10,7 @@ import (
 	"testing"
 	"time"
 
-	remoteGtfs "github.com/jamespfennell/gtfs"
+	remoteGtfs "github.com/OneBusAway/go-gtfs"
 	"watchdog.onebusaway.org/internal/geo"
 	"watchdog.onebusaway.org/internal/models"
 )
@@ -56,7 +56,6 @@ func TestDownloadGTFSBundle(t *testing.T) {
 		if staticBundle == nil {
 			t.Fatal("static data retrieved from the store is nil; expected non-nil value")
 		}
-
 		data := readFixture(t, "gtfs.zip")
 		expectedStaticData, err := remoteGtfs.ParseStatic(data, remoteGtfs.ParseStaticOptions{})
 		if err != nil {
@@ -109,6 +108,151 @@ func TestDownloadGTFSBundle(t *testing.T) {
 
 }
 
+func TestAgencyParsing(t *testing.T) {
+	data := readFixture(t, "gtfs.zip")
+	staticBundle, err := remoteGtfs.ParseStatic(data, remoteGtfs.ParseStaticOptions{})
+	if err != nil {
+		t.Fatalf("failed to parse expected GTFS static data from fixture: %v", err)
+	}
+
+	expectedStaticAgencies := []remoteGtfs.Agency{
+		{
+			Id:       "40",
+			Name:     "Sound Transit",
+			Url:      "https://www.soundtransit.org",
+			Timezone: "America/Los_Angeles",
+			Language: "en",
+			Phone:    "1-888-889-6368",
+			FareUrl:  "https://www.soundtransit.org/ride-with-us/how-to-pay/fares",
+			Email:    "main@soundtransit.org",
+		},
+	}
+
+	if staticBundle.Agencies == nil {
+		t.Fatal("stored static data has nil Agencies slice; expected it to be populated")
+	}
+	if len(staticBundle.Agencies) == 0 {
+		t.Fatal("stored Agencies slice is empty; static data likely not parsed correctly")
+	}
+
+	if len(expectedStaticAgencies) != len(staticBundle.Agencies) {
+		t.Fatalf("expected %d agencies, got %d", len(expectedStaticAgencies), len(staticBundle.Agencies))
+	}
+
+	expectedAgencies := make(map[string]remoteGtfs.Agency)
+
+	for _, agency := range expectedStaticAgencies {
+		expectedAgencies[agency.Id] = remoteGtfs.Agency{
+			Id:       agency.Id,
+			Name:     agency.Name,
+			Timezone: agency.Timezone,
+			Url:      agency.Url,
+			FareUrl:  agency.FareUrl,
+			Language: agency.Language,
+			Phone:    agency.Phone,
+			Email:    agency.Email,
+		}
+	}
+
+	for _, agency := range staticBundle.Agencies {
+		expectedAgency, ok := expectedAgencies[agency.Id]
+		if !ok {
+			t.Fatalf("unexpected agency ID %s", agency.Id)
+		}
+
+		if expectedAgency != agency {
+			t.Errorf(
+				"agency mismatch for ID %s expected: %+v got %+v",
+				agency.Id,
+				expectedAgency,
+				agency,
+			)
+		}
+	}
+}
+
+func TestStopsParsing(t *testing.T) {
+	server := models.ObaServer{ID: 1, Name: "test"}
+
+	data := readFixture(t, "gtfs.zip")
+	staticBundle, err := remoteGtfs.ParseStatic(data, remoteGtfs.ParseStaticOptions{})
+	if err != nil {
+		t.Fatal("failed to parse gtfs static data")
+	}
+	staticData := models.NewStaticData(staticBundle)
+	staticStore := NewStaticStore()
+	staticStore.Set(server.ID, staticData)
+	stopIDs := []string{"11060", "1108"} // Make sure these exist in your test GTFS
+	stopsData := map[string]struct {
+		stopName string
+		lat      float64
+		long     float64
+	}{
+		"11060": {stopName: "Broadway & E Denny Way", lat: 47.618425, long: -122.320940},
+		"1108":  {stopName: "Westlake", lat: 47.611450, long: -122.337532},
+	}
+
+	stops, err := getStopLocationsByIDs(server.ID, stopIDs, staticStore)
+	if err != nil {
+		t.Fatalf("unexpected error: %v", err)
+	}
+	if len(stops) == 0 {
+		t.Fatalf("expected some matched stops, got 0")
+	}
+
+	for _, stop := range stops {
+		expected, ok := stopsData[stop.Id]
+		if !ok {
+			t.Fatalf("unexpected stop ID returned: %s", stop.Id)
+		}
+		if stop.Latitude == nil || stop.Longitude == nil {
+			t.Fatalf("stop %s missing coordinates", stop.Id)
+		}
+
+		if stop.Name != expected.stopName {
+			t.Errorf("stop %s name mismatch: expected %s, got %s",
+				stop.Id, expected.stopName, stop.Name)
+		}
+
+		const epsilon = 1e-5
+		if diff := *stop.Latitude - expected.lat; diff > epsilon || diff < -epsilon {
+			t.Errorf("stop %s latitude mismatch: expected %f, got %f",
+				stop.Id, expected.lat, *stop.Latitude)
+		}
+		if diff := *stop.Longitude - expected.long; diff > epsilon || diff < -epsilon {
+			t.Errorf("stop %s longitude mismatch: expected %f, got %f",
+				stop.Id, expected.long, *stop.Longitude)
+		}
+	}
+}
+
+func TestGetEarliestAndLatestServiceDates(t *testing.T) {
+	server := models.ObaServer{ID: 1, Name: "test"}
+	data := readFixture(t, "gtfs.zip")
+	staticBundle, err := remoteGtfs.ParseStatic(data, remoteGtfs.ParseStaticOptions{})
+	if err != nil {
+		t.Fatal("failed to parse gtfs static data")
+	}
+	staticData := models.NewStaticData(staticBundle)
+	staticStore := NewStaticStore()
+	staticStore.Set(server.ID, staticData)
+	loc, _ := time.LoadLocation("America/Los_Angeles")
+	// make sure these already exist in the test data
+	expectedEarliestEndDate := time.Date(2024, 11, 22, 0, 0, 0, 0, loc)
+	expectedLatestEndDate := time.Date(2025, 03, 28, 0, 0, 0, 0, loc)
+	actualEarliest, actualLatest, err := getEarliestAndLatestServiceDates(staticData)
+	if err != nil {
+		t.Fatalf("unexpected error: %v", err)
+	}
+
+	if !expectedEarliestEndDate.Equal(actualEarliest) {
+		t.Fatalf("earliest date mismatch: expected %s, got %s", expectedEarliestEndDate.Format("2006-01-02"), actualEarliest.Format("2006-01-02"))
+	}
+	if !expectedLatestEndDate.Equal(actualLatest) {
+		t.Fatalf("latest date mismatch: expected %s, got %s", expectedLatestEndDate.Format("2006-01-02"), actualLatest.Format("2006-01-02"))
+	}
+}
+
 func TestGetStopLocationsByIDs(t *testing.T) {
 	server := models.ObaServer{ID: 1, Name: "test"}
 
@@ -121,7 +265,7 @@ func TestGetStopLocationsByIDs(t *testing.T) {
 	staticStore := NewStaticStore()
 	staticStore.Set(server.ID, staticData)
 
-	t.Run("Valid stop IDs", func(t *testing.T) {
+	t.Run("Valid stops IDs", func(t *testing.T) {
 		stopIDs := []string{"11060", "1108"} // Make sure these exist in your test GTFS
 		stops, err := getStopLocationsByIDs(server.ID, stopIDs, staticStore)
 		if err != nil {
@@ -187,19 +331,21 @@ func TestFetchAndStoreGTFSRTFeed(t *testing.T) {
 			t.Fatalf("Expected %d vehicles, got %d", len(expectedRtData.Vehicles), len(realtimeData.Vehicles))
 		}
 
-		expectedMap := make(map[string]struct{})
+		expectedMap := make(map[string]remoteGtfs.Vehicle)
 		for _, vehicle := range expectedRtData.Vehicles {
 			if vehicle.ID != nil {
-				expectedMap[vehicle.ID.ID] = struct{}{}
+				expectedMap[vehicle.ID.ID] = vehicle
 			}
 		}
 		countExpectedNilIDs := len(expectedRtData.Vehicles) - len(expectedMap)
 		countNilIDs := 0
 		for _, vehicle := range realtimeData.Vehicles {
 			if vehicle.ID != nil {
-				if _, exists := expectedMap[vehicle.ID.ID]; !exists {
+				expectedVehicle, exists := expectedMap[vehicle.ID.ID]
+				if !exists {
 					t.Errorf("Unexpected vehicle ID %s found in GTFS-RT data", vehicle.ID.ID)
 				}
+				assertVehicle(t, &vehicle, &expectedVehicle)
 			} else {
 				countNilIDs++
 			}

internal/gtfs/gtfs_service.go

@@ -6,7 +6,7 @@ import (
 	"net/http"
 	"time"
 
-	remoteGtfs "github.com/jamespfennell/gtfs"
+	remoteGtfs "github.com/OneBusAway/go-gtfs"
 	"watchdog.onebusaway.org/internal/geo"
 	"watchdog.onebusaway.org/internal/models"
 )

internal/gtfs/test_helpers.go

@@ -6,6 +6,9 @@ import (
 	"os"
 	"path/filepath"
 	"testing"
+	"time"
+
+	remoteGtfs "github.com/OneBusAway/go-gtfs"
 )
 
 func setupGtfsServer(t *testing.T, fixturePath string) *httptest.Server {
@@ -55,3 +58,81 @@ func readFixture(t *testing.T, fixturePath string) []byte {
 
 	return data
 }
+
+func assertPtr[T comparable](t *testing.T, expected *T, actual *T, field string, equal func(a, b T) bool) {
+	t.Helper()
+
+	if expected == nil {
+		if actual != nil {
+			t.Errorf("%s mismatch:\nexpected: nil\ngot %v", field, *actual)
+		}
+		return
+	}
+
+	if actual == nil {
+		t.Errorf("%s mismatch:\nexpected: %v\ngot nil", field, *expected)
+		return
+	}
+	if !equal(*expected, *actual) {
+		t.Errorf("%s mismatch:\nexpected %v, got %v", field, *expected, *actual)
+	}
+}
+
+func assertVehicle(t *testing.T, actual *remoteGtfs.Vehicle, expected *remoteGtfs.Vehicle) {
+	t.Helper()
+	if expected == nil {
+		t.Fatal("expected vehicle must not be nil")
+	}
+	if actual == nil {
+		t.Errorf("vehicle mismatch:\nexpected: %v\ngot nil", expected)
+		return
+	}
+
+	if expected.ID == nil {
+		if actual.ID != nil {
+			t.Errorf("vehicle ID mismatch:\nexpected: nil\ngot %v", actual.ID)
+		}
+	} else {
+		if actual.ID == nil {
+			t.Errorf("vehicle ID mismatch:\nexpected: %v\ngot nil", expected.ID)
+		} else if expected.ID.ID != actual.ID.ID {
+			t.Errorf("vehicle ID mismatch:\nexpected: %s\ngot %s", expected.ID.ID, actual.ID.ID)
+		}
+	}
+
+	assertPtr(t, expected.StopID, actual.StopID, "vehicle StopID", func(a, b string) bool {
+		return a == b
+	})
+	assertPtr(t, expected.CurrentStopSequence, actual.CurrentStopSequence, "vehicle StopSequence", func(a, b uint32) bool {
+		return a == b
+	})
+	assertPtr(t, expected.Timestamp, actual.Timestamp, "vehicle Timestamp", func(a, b time.Time) bool {
+		return a.Equal(b)
+	})
+
+	if expected.Position == nil {
+		if actual.Position != nil {
+			t.Errorf("vehicle Position mismatch:\nexpected: nil\ngot %v", actual.Position)
+		}
+	} else {
+		if actual.Position == nil {
+			t.Errorf("vehicle Position mismatch:\nexpected: %v\ngot nil", expected.Position)
+		} else {
+			const eps = 1e-5
+
+			floatEq := func(a, b float32) bool {
+				diff := a - b
+				return diff <= eps && diff >= -eps
+			}
+
+			assertPtr(t, expected.Position.Latitude, actual.Position.Latitude, "vehicle latitude", floatEq)
+			assertPtr(t, expected.Position.Longitude, actual.Position.Longitude, "vehicle longitude", floatEq)
+			assertPtr(t, expected.Position.Speed, actual.Position.Speed, "vehicle speed", floatEq)
+		}
+	}
+
+	if expected.IsEntityInMessage != actual.IsEntityInMessage {
+		t.Errorf("IsEntityInMessage mismatch:\nexpected: %v\ngot %v",
+			expected.IsEntityInMessage, actual.IsEntityInMessage)
+	}
+}