diff --git a/README.md b/README.md index b59ef0d..93c9c70 100644 --- a/README.md +++ b/README.md @@ -40,6 +40,9 @@ $EDITOR config.yaml # 3. Run ./freefall + +# 4. Run in OpenAI-compatible mode +./freefall -openai ``` --- @@ -50,6 +53,7 @@ $EDITOR config.yaml # config.yaml backend: - name: free-key + type: gemini # (Default) Native Gemini API key: "AIza..." # free-tier Gemini API key models: - gemini-2.5-pro @@ -57,10 +61,30 @@ backend: - gemini-2.5-flash-lite - name: paid-key + type: gemini key: "AIza..." # paid Gemini API key models: - gemini-2.5-flash + - name: openai-backend + type: openai # Uses default OpenAI endpoint (requires -openai flag) + key: "sk-proj..." + models: + - gpt-4o + + - name: anthropic-backend + type: anthropic # Uses Anthropic's OpenAI-compatible endpoint (requires -openai flag) + key: "sk-ant..." + models: + - claude-3-5-sonnet-20241022 + + - name: local-qwen + type: custom_openai + base_url: "http://192.168.1.50:8000/v1" + key: "sk-dummy" + models: + - qwen-14b + config: port: 8080 proxy: @@ -78,6 +102,15 @@ config: Each entry under `proxy` is a named **path** — use the path name as your `x-goog-api-key`. Targets are `.` and are tried left-to-right. +### The `-openai` Flag + +Because `gemini-freefall` operates as a zero-memory envelope router, it **does not** parse or translate JSON payloads. +If a client sends an OpenAI-formatted payload, it will fail if routed to a native Gemini backend, and vice versa. + +To prevent misconfigurations: +- Running without flags allows **only** `type: gemini` backends. Non-Gemini targets are dropped from routing at startup. +- Running with `-openai` allows **only** OpenAI-compatible backends (`openai`, `anthropic`, `custom_openai`, `gemini_openai`). Native Gemini targets are dropped. + --- ## Usage diff --git a/cmd/server/main.go b/cmd/server/main.go index 6fb5291..73328e2 100644 --- a/cmd/server/main.go +++ b/cmd/server/main.go @@ -1,13 +1,17 @@ package main import ( + "flag" "gemini-freefall/internal/proxy" "log/slog" "os" ) func main() { - cfg, err := proxy.LoadConfig("config.yaml") + openaiMode := flag.Bool("openai", false, "Enable OpenAI-compatible routing mode") + flag.Parse() + + cfg, err := proxy.LoadConfig("config.yaml", *openaiMode) if err != nil { slog.Error("failed to load config", slog.Any("err", err)) os.Exit(1) diff --git a/internal/proxy/config.go b/internal/proxy/config.go index 4dc7b7b..57eb89f 100644 --- a/internal/proxy/config.go +++ b/internal/proxy/config.go @@ -2,6 +2,7 @@ package proxy import ( "errors" + "log/slog" "os" "strings" @@ -17,21 +18,24 @@ var ( ) type Config struct { - Backend []Backend `yaml:"backend"` - Config struct { + BaseURLs map[string]string `yaml:"base_urls"` + Backend []Backend `yaml:"backend"` + Config struct { Port int `yaml:"port"` Proxy map[string][]string `yaml:"proxy"` } `yaml:"config"` } type Backend struct { - Name string `yaml:"name"` - Key string `yaml:"key"` - Models []string `yaml:"models"` + Name string `yaml:"name"` + Type string `yaml:"type"` + BaseURL string `yaml:"base_url"` + Key string `yaml:"key"` + Models []string `yaml:"models"` } // LoadConfig . -func LoadConfig(fname string) (*Config, error) { +func LoadConfig(fname string, openaiMode bool) (*Config, error) { f, err := os.Open(fname) if err != nil { return nil, err @@ -46,6 +50,68 @@ func LoadConfig(fname string) (*Config, error) { return nil, err } + if cfg.BaseURLs == nil { + cfg.BaseURLs = make(map[string]string) + } + if cfg.BaseURLs["openai"] == "" { + cfg.BaseURLs["openai"] = "https://api.openai.com/v1" + } + if cfg.BaseURLs["anthropic"] == "" { + cfg.BaseURLs["anthropic"] = "https://api.anthropic.com/v1" + } + if cfg.BaseURLs["gemini_openai"] == "" { + cfg.BaseURLs["gemini_openai"] = "https://generativelanguage.googleapis.com/v1beta/openai" + } + + var filteredBackends []Backend + for i, b := range cfg.Backend { + if b.Type == "" { + b.Type = "gemini" + } + cfg.Backend[i] = b + + if openaiMode { + if b.Type == "gemini" { + // We drop gemini backends in openai mode + slog.Warn("dropping gemini backend in openai mode", slog.String("backend", b.Name)) + continue + } + } else { + if b.Type != "gemini" { + // We drop non-gemini backends in non-openai mode + slog.Warn("dropping non-gemini backend in default (gemini) mode", slog.String("backend", b.Name), slog.String("type", b.Type)) + continue + } + } + filteredBackends = append(filteredBackends, b) + } + cfg.Backend = filteredBackends + + // Drop proxy routes that refer to dropped backends + filteredProxy := make(map[string][]string) + for pName, pRoutes := range cfg.Config.Proxy { + var validRoutes []string + for _, route := range pRoutes { + parts := strings.SplitN(route, ".", 2) + if len(parts) >= 1 { + found := false + for _, b := range cfg.Backend { + if b.Name == parts[0] { + found = true + break + } + } + if found { + validRoutes = append(validRoutes, route) + } + } + } + if len(validRoutes) > 0 { + filteredProxy[pName] = validRoutes + } + } + cfg.Config.Proxy = filteredProxy + return &cfg, checkConfig(&cfg) } @@ -63,6 +129,10 @@ func checkConfig(cfg *Config) error { return ErrInvalidBackend } + if b.Type != "gemini" && b.BaseURL == "" && cfg.BaseURLs[b.Type] == "" { + return ErrInvalidBackend + } + if len(b.Models) == 0 { return ErrNoModels } diff --git a/internal/proxy/config_mixed_test.yaml b/internal/proxy/config_mixed_test.yaml new file mode 100644 index 0000000..0295353 --- /dev/null +++ b/internal/proxy/config_mixed_test.yaml @@ -0,0 +1,31 @@ +backend: + - name: "gemini-native" + type: "gemini" + key: "native-key" + models: + - "gemini-pro" + - name: "openai-backend" + type: "openai" + key: "openai-key" + models: + - "gpt-4" + - name: "anthropic-backend" + type: "anthropic" + key: "anthropic-key" + models: + - "claude-3" + - name: "custom-local" + type: "custom" + base_url: "http://localhost:11434/v1" + key: "local-key" + models: + - "llama-3" + +config: + port: 8080 + proxy: + mixed-path: + - "gemini-native.gemini-pro" + - "openai-backend.gpt-4" + - "anthropic-backend.claude-3" + - "custom-local.llama-3" diff --git a/internal/proxy/config_test.go b/internal/proxy/config_test.go index 1123b58..ec65d84 100644 --- a/internal/proxy/config_test.go +++ b/internal/proxy/config_test.go @@ -10,20 +10,50 @@ import ( func TestLoadConfig(t *testing.T) { t.Parallel() - c, err := LoadConfig("../../config.yaml.example") - require.NoError(t, err) - - require.Len(t, c.Backend, 2) - assert.Equal(t, "backend1", c.Backend[0].Name) - require.Len(t, c.Backend[0].Models, 3) - assert.Equal(t, "gemini-3.1-pro", c.Backend[0].Models[0]) - - assert.True(t, isValidBackend("backend1.gemini-3.1-flash", c)) - assert.True(t, isValidBackend("backend2.gemini-3.1-flash", c)) - assert.False(t, isValidBackend("backend1.gemini-3.1-nope", c)) - assert.False(t, isValidBackend("backend66.gemini-3.1-flash", c)) - - testB, err := getBackend("backend2.gemini-3.1-flash", c) - require.NoError(t, err) - assert.Equal(t, "", testB.Key) + t.Run("Default Gemini Mode (from example)", func(t *testing.T) { + c, err := LoadConfig("../../config.yaml.example", false) + require.NoError(t, err) + + require.Len(t, c.Backend, 2) + assert.Equal(t, "backend1", c.Backend[0].Name) + require.Len(t, c.Backend[0].Models, 3) + assert.Equal(t, "gemini-3.1-pro", c.Backend[0].Models[0]) + + assert.True(t, isValidBackend("backend1.gemini-3.1-flash", c)) + assert.True(t, isValidBackend("backend2.gemini-3.1-flash", c)) + assert.False(t, isValidBackend("backend1.gemini-3.1-nope", c)) + assert.False(t, isValidBackend("backend66.gemini-3.1-flash", c)) + + testB, err := getBackend("backend2.gemini-3.1-flash", c) + require.NoError(t, err) + assert.Equal(t, "", testB.Key) + }) + + t.Run("OpenAI Mode Filters Gemini", func(t *testing.T) { + c, err := LoadConfig("config_mixed_test.yaml", true) + require.NoError(t, err) + + // OpenAI mode should drop the gemini-native backend + require.Len(t, c.Backend, 3) + assert.Equal(t, "openai-backend", c.Backend[0].Name) + assert.Equal(t, "anthropic-backend", c.Backend[1].Name) + assert.Equal(t, "custom-local", c.Backend[2].Name) + + // The proxy route should only contain non-gemini targets + require.Len(t, c.Config.Proxy["mixed-path"], 3) + assert.Equal(t, "openai-backend.gpt-4", c.Config.Proxy["mixed-path"][0]) + }) + + t.Run("Gemini Mode Filters OpenAI", func(t *testing.T) { + c, err := LoadConfig("config_mixed_test.yaml", false) + require.NoError(t, err) + + // Gemini mode should drop the non-gemini backends + require.Len(t, c.Backend, 1) + assert.Equal(t, "gemini-native", c.Backend[0].Name) + + // The proxy route should only contain gemini targets + require.Len(t, c.Config.Proxy["mixed-path"], 1) + assert.Equal(t, "gemini-native.gemini-pro", c.Config.Proxy["mixed-path"][0]) + }) } diff --git a/internal/proxy/proxy.go b/internal/proxy/proxy.go index e6cf777..d9ede2a 100644 --- a/internal/proxy/proxy.go +++ b/internal/proxy/proxy.go @@ -7,6 +7,7 @@ import ( "io" "log/slog" "net/http" + "strings" ) const ( @@ -66,13 +67,33 @@ func (l *Logic) handler(w http.ResponseWriter, r *http.Request) { continue } + // Construct the final URL based on backend type + var targetURL string + if backend.Type == "gemini" { + targetURL = fmt.Sprintf(targetURLPattern, modelName(c)) + } else { + baseURL := backend.BaseURL + if baseURL == "" { + baseURL = l.config.BaseURLs[backend.Type] + } + + // Append incoming path, unconditionally stripping "/v1" if it's there + // to avoid duplicating API versions or creating invalid endpoints + incomingPath := r.URL.Path + if strings.HasPrefix(incomingPath, "/v1/") { + incomingPath = strings.TrimPrefix(incomingPath, "/v1") + } + targetURL = baseURL + incomingPath + } + // Forward the request to the backend - slog.Info("Forwarding request to backend", slog.String("backend", backend.Name), slog.String("proxyPath", c)) - req, err := http.NewRequest(r.Method, fmt.Sprintf(targetURLPattern, modelName(c)), bytes.NewReader(bodyBuffer)) + slog.Info("Forwarding request to backend", slog.String("backend", backend.Name), slog.String("proxyPath", c), slog.String("targetURL", targetURL)) + req, err := http.NewRequest(r.Method, targetURL, bytes.NewReader(bodyBuffer)) if err != nil { slog.Error("failed to create new request", slog.Any("err", err), slog.String("proxyPath", c)) continue } + // Copy headers from the original request for key, values := range r.Header { for _, value := range values { @@ -81,7 +102,16 @@ func (l *Logic) handler(w http.ResponseWriter, r *http.Request) { } // Re-set the header for the given backend's api key - req.Header.Set(headerAPIKey, backend.Key) + if backend.Type == "gemini" { + req.Header.Set(headerAPIKey, backend.Key) + } else if backend.Type == "anthropic" { + req.Header.Set("x-api-key", backend.Key) + req.Header.Del(headerAPIKey) + req.Header.Del("Authorization") + } else { + req.Header.Set("Authorization", "Bearer "+backend.Key) + req.Header.Del(headerAPIKey) + } resp, err := l.httpClient.Do(req) if err != nil { @@ -101,6 +131,7 @@ func (l *Logic) handler(w http.ResponseWriter, r *http.Request) { } size, _ := io.Copy(w, resp.Body) + _ = resp.Body.Close() // Close the response body to avoid resource leaks slog.Info("successfully forwarded request", slog.Int64("responseSize", size), slog.String("backend", backend.Name), slog.String("proxyPath", c)) // We are done, exit the handler