[EXP] GTFS protobuf experiment

go.mod

@@ -14,6 +14,7 @@ require (
 	github.com/aws/aws-sdk-go-v2/credentials v1.17.67
 	github.com/aws/aws-sdk-go-v2/service/location v1.44.2
 	github.com/aws/aws-sdk-go-v2/service/s3 v1.79.3
+	github.com/bufbuild/protocompile v0.14.1
 	github.com/deckarep/golang-set/v2 v2.6.0
 	github.com/dimchansky/utfbom v1.1.1
 	github.com/flopp/go-staticmaps v0.0.0-20220221183018-c226716bec53

go.sum

@@ -76,6 +76,8 @@ github.com/aybabtme/iocontrol v0.0.0-20150809002002-ad15bcfc95a0 h1:0NmehRCgyk5r
 github.com/aybabtme/iocontrol v0.0.0-20150809002002-ad15bcfc95a0/go.mod h1:6L7zgvqo0idzI7IO8de6ZC051AfXb5ipkIJ7bIA2tGA=
 github.com/benbjohnson/clock v1.1.0 h1:Q92kusRqC1XV2MjkWETPvjJVqKetz1OzxZB7mHJLju8=
 github.com/benbjohnson/clock v1.1.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA=
+github.com/bufbuild/protocompile v0.14.1 h1:iA73zAf/fyljNjQKwYzUHD6AD4R8KMasmwa/FBatYVw=
+github.com/bufbuild/protocompile v0.14.1/go.mod h1:ppVdAIhbr2H8asPk6k4pY7t9zB1OU5DoEw9xY/FUi1c=
 github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs=
 github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
 github.com/coreos/go-systemd/v22 v22.5.0/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc=

internal/tlpb/gen.go

@@ -0,0 +1,156 @@
+package main
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"os"
+	"strings"
+
+	"github.com/bufbuild/protocompile"
+	"github.com/interline-io/transitland-lib/tlcli"
+	"github.com/spf13/cobra"
+	"github.com/spf13/pflag"
+	"google.golang.org/protobuf/reflect/protoreflect"
+)
+
+func main() {
+	cmd := tlcli.CobraHelper(&GenGtfsCommand{}, "", "")
+	cmd.Execute()
+}
+
+type GenGtfsCommand struct {
+	Protopath string
+	Outpath   string
+	Command   *cobra.Command
+}
+
+func (cmd *GenGtfsCommand) AddFlags(fl *pflag.FlagSet) {
+}
+
+func (cmd *GenGtfsCommand) HelpDesc() (string, string) {
+	return "Generate GTFS entities", ""
+}
+
+func (cmd *GenGtfsCommand) Parse(args []string) error {
+	fl := tlcli.NewNArgs(args)
+	if fl.NArg() < 2 {
+		return errors.New("<proto> <outppath>")
+	}
+	cmd.Protopath = fl.Arg(0)
+	cmd.Outpath = fl.Arg(1)
+	return nil
+}
+
+func (cmd *GenGtfsCommand) Run(ctx context.Context) error {
+	compiler := protocompile.Compiler{
+		Resolver: &protocompile.SourceResolver{},
+	}
+	files, err := compiler.Compile(ctx, cmd.Protopath)
+	if err != nil {
+		return err
+	}
+	outf, err := os.Create(cmd.Outpath)
+	if err != nil {
+		return err
+	}
+	defer outf.Close()
+
+	// Go
+	outf.WriteString(`package gtfs` + "\n\n")
+	outf.WriteString(`import ( "github.com/interline-io/transitland-lib/tt" )` + "\n\n")
+
+	ttKinds := map[string]string{
+		"Url":       "tt.Url",
+		"Date":      "tt.Date",
+		"Time":      "tt.Time",
+		"Color":     "tt.Color",
+		"Key":       "tt.Key",
+		"Phone":     "tt.Phone",
+		"Email":     "tt.Email",
+		"Reference": "tt.Reference",
+		"Currency":  "tt.Currency",
+		"Language":  "tt.Language",
+		"Int":       "tt.Int",
+		"Bool":      "tt.Bool",
+		"Float":     "tt.Float",
+		"String":    "tt.String",
+		"Timezone":  "tt.Timezone",
+		"Timestamp": "tt.Timestamp",
+		"Seconds":   "tt.Seconds",
+	}
+
+	for _, lf := range files {
+		enums := lf.Enums()
+		for i := 0; i < enums.Len(); i++ {
+			en := enums.Get(i)
+			outf.WriteString(fmt.Sprintf("type %s int32\n\n", en.Name()))
+		}
+		msgs := lf.Messages()
+		for i := 0; i < msgs.Len(); i++ {
+			msg := msgs.Get(i)
+			fields := msg.Fields()
+			if _, ok := ttKinds[string(msg.Name())]; ok {
+				continue
+			}
+			if fields.Len() == 1 && fields.Get(0).Name() == "val" {
+				field := fields.Get(0)
+				outf.WriteString(fmt.Sprintf(
+					"type %s struct { tt.Option[%s] }\n\n",
+					msg.Name(),
+					mapKind(field)),
+				)
+				continue
+			}
+
+			outf.WriteString(fmt.Sprintf("type %s struct {\n", msg.Name()))
+			for j := 0; j < fields.Len(); j++ {
+				field := fields.Get(j)
+				fieldName := toCamelCase(string(field.Name()))
+				fieldKind := mapKind(field)
+				if ttKind, ok := ttKinds[fieldKind]; ok {
+					outf.WriteString(fmt.Sprintf("\t%s %s\n", fieldName, ttKind))
+					continue
+				}
+				switch fieldKind {
+				case "DatabaseEntity":
+					outf.WriteString("\tDatabaseEntity\n")
+				default:
+					outf.WriteString(fmt.Sprintf("\t%s %s\n", fieldName, fieldKind))
+				}
+			}
+			outf.WriteString("}\n\n")
+		}
+	}
+	return nil
+}
+
+func mapKind(field protoreflect.FieldDescriptor) string {
+	fieldKind := field.Kind().String()
+	switch fieldKind {
+	case "enum":
+		fieldKind = string(field.Enum().Name())
+	case "double":
+		fieldKind = "float64"
+	case "float":
+		fieldKind = "float32"
+	}
+	if fmsg := field.Message(); fmsg != nil {
+		fieldKind = string(fmsg.Name())
+	}
+	return fieldKind
+}
+
+func toCamelCase(v string) string {
+	a := strings.Split(v, "_")
+	for i := 0; i < len(a); i++ {
+		s := a[i]
+		if s == "id" {
+			s = "ID"
+		} else {
+			s = strings.ToUpper(s[0:1]) + s[1:]
+		}
+		a[i] = s
+	}
+	return strings.Join(a, "")
+}

internal/tlpb/gen_test.go

@@ -0,0 +1,135 @@
+package main
+
+// func printFirst(v []any) {
+// 	if len(v) == 0 {
+// 		return
+// 	}
+// 	fmt.Println(toJson(v[0]))
+// }
+// func printAll(v []any) {
+// 	for _, ent := range v {
+// 		fmt.Println(toJson(ent))
+// 	}
+// }
+
+// func pbJson(v protoreflect.ProtoMessage) string {
+// 	jj, _ := protojson.Marshal(v)
+// 	return string(jj)
+// }
+
+// func toJson(v any) string {
+// 	jj, _ := json.Marshal(v)
+// 	return string(jj)
+// }
+
+// var TESTFILE = ""
+// var TESTTABLE = ""
+
+// func init() {
+// 	TESTFILE = testpath.RelPath("test/data/external/bart.zip")
+// 	TESTTABLE = "stops.txt"
+// }
+
+//////////////////
+
+// func TestReadPB(t *testing.T) {
+// 	ents, err := ReadPB(TESTFILE)
+// 	if err != nil {
+// 		t.Fatal(err)
+// 	}
+// 	for _, ent := range ents {
+// 		fmt.Println(ent)
+// 	}
+// }
+
+// func BenchmarkReadPB(b *testing.B) {
+// 	for n := 0; n < b.N; n++ {
+// 		ReadPB(TESTFILE)
+// 	}
+// }
+
+// func ReadPB(fn string) ([]any, error) {
+// 	a := tlcsv.NewZipAdapter(fn)
+// 	if err := a.Open(); err != nil {
+// 		panic(err)
+// 	}
+// 	var ret []any
+// 	err := a.ReadRows(TESTTABLE, func(row tlcsv.Row) {
+// 		ent := &pb.Stop{}
+// 		if errs := tlcsv.LoadRow(ent, row); errs != nil {
+// 			for _, err := range errs {
+// 				panic(err)
+// 			}
+// 		}
+// 		ret = append(ret, ent)
+// 	})
+// 	return ret, err
+// }
+
+//////////////////
+
+// func TestReadTT(t *testing.T) {
+// 	ents, err := ReadTT(TESTFILE)
+// 	assert.NoError(t, err)
+// 	printAll(ents)
+// }
+
+// func BenchmarkReadTT(b *testing.B) {
+// 	for n := 0; n < b.N; n++ {
+// 		a, _ := ReadTT(TESTFILE)
+// 		_ = a
+// 		// printFirst(a)
+// 	}
+// }
+
+// func ReadTT(fn string) ([]any, error) {
+// 	a := tlcsv.NewZipAdapter(fn)
+// 	if err := a.Open(); err != nil {
+// 		panic(err)
+// 	}
+// 	var ret []any
+// 	err := a.ReadRows(TESTTABLE, func(row tlcsv.Row) {
+// 		ent := gtfs.Stop{}
+// 		if errs := tlcsv.LoadRow(&ent, row); errs != nil {
+// 			for _, err := range errs {
+// 				panic(err)
+// 			}
+// 		}
+// 		ret = append(ret, ent)
+// 	})
+// 	return ret, err
+// }
+
+//////////////////
+
+// func TestReadG(t *testing.T) {
+// 	ents, err := ReadG(TESTFILE)
+// 	assert.NoError(t, err)
+// 	printAll(ents)
+// }
+
+// func BenchmarkReadG(b *testing.B) {
+// 	for n := 0; n < b.N; n++ {
+// 		a, _ := ReadG(TESTFILE)
+// 		_ = a
+// 		// printFirst(a)
+// 	}
+// }
+
+// func ReadG(fn string) ([]any, error) {
+// 	a := tlcsv.NewZipAdapter(fn)
+// 	if err := a.Open(); err != nil {
+// 		panic(err)
+// 	}
+// 	var ret []any
+// 	err := a.ReadRows(TESTTABLE, func(row tlcsv.Row) {
+// 		ent := gtfs.Stop{}
+// 		if errs := tlcsv.LoadRow(&ent, row); errs != nil {
+// 			for _, err := range errs {
+// 				panic(err)
+// 			}
+// 		}
+// 		ret = append(ret, ent)
+// 	})
+// 	return ret, err
+// }