From a62bd7deb59c79df698cba904e77d61aaf6fe475 Mon Sep 17 00:00:00 2001 From: Jochen Delabie Date: Tue, 22 Sep 2026 14:21:52 +0200 Subject: [PATCH] feat(cloud): add TestingBot real device cloud provider Adds TestingBot as a sixth cloud device farm alongside BrowserStack, Sauce Labs, LambdaTest, AWS Device Farm and Firebase Test Lab. It implements the existing CloudProvider interface, so nothing on the shared cloud path in internal/cli/test.go changes. - UploadApp: POST /v1/storage (multipart "file"), returns the tb:// URL used as the appium:app capability. - ListDevices: GET /v1/devices, mapping platform_name/version onto the common Device shape. - StartSession/StopSession: Appium W3C hub at hub.testingbot.com, with credentials passed in tb:options as TestingBot documents. - GetSessionArtifacts: GET /v1/tests//assets for video and screenshots, polled up to 30s since assets are processed asynchronously. Credentials resolve from "key"/"secret" (TestingBot's own naming), falling back to the generic "username"/"access_key" so --cloud-key and --cloud-secret keep working without touching the CLI credential map. Like Sauce Labs and LambdaTest, the provider runs in relay mode; direct port forwarding would require the TestingBot Tunnel binary and returns an error pointing at --relay. Endpoints and payloads were taken from the TestingBot OpenAPI spec (api.testingbot.com/v1/openapi.json) and the Appium support docs. Signed-off-by: Jochen Delabie --- README.md | 1 + internal/cli/test.go | 2 +- internal/cloud/provider.go | 6 +- internal/cloud/testingbot.go | 336 +++++++++++++++++++++++++++ internal/cloud/testingbot_test.go | 370 ++++++++++++++++++++++++++++++ internal/config/config.go | 2 +- probe.yaml.example | 2 +- vscode/README.md | 9 +- vscode/package.json | 3 +- vscode/src/views/runProfile.ts | 1 + 10 files changed, 725 insertions(+), 7 deletions(-) create mode 100644 internal/cloud/testingbot.go create mode 100644 internal/cloud/testingbot_test.go diff --git a/README.md b/README.md index efc95b3c..4c4ab56e 100644 --- a/README.md +++ b/README.md @@ -721,6 +721,7 @@ Run tests on real devices without managing your own device lab. Bring your own a | BrowserStack App Automate | `browserstack` | | Sauce Labs Real Device Cloud | `saucelabs` | | LambdaTest Real Devices | `lambdatest` | +| TestingBot Real Device Cloud | `testingbot` | | AWS Device Farm | `aws` | | Firebase Test Lab | `firebase` | diff --git a/internal/cli/test.go b/internal/cli/test.go index cac0aca8..b8d7b227 100644 --- a/internal/cli/test.go +++ b/internal/cli/test.go @@ -137,7 +137,7 @@ func init() { f.String("cloud-url", "", "cloud API base URL (must be set via this flag or cloud.url in probe.yaml)") // Cloud device farm providers - f.String("cloud-provider", "", "cloud device farm provider (browserstack, aws, firebase, saucelabs, lambdatest)") + f.String("cloud-provider", "", "cloud device farm provider (browserstack, aws, firebase, saucelabs, lambdatest, testingbot)") f.String("cloud-app", "", "path to app binary (.apk/.ipa) to upload to cloud provider") f.String("cloud-device", "", "target device name on the cloud provider") f.String("cloud-key", "", "cloud provider API key or username") diff --git a/internal/cloud/provider.go b/internal/cloud/provider.go index 42428ea6..04f48833 100644 --- a/internal/cloud/provider.go +++ b/internal/cloud/provider.go @@ -78,8 +78,10 @@ func NewProvider(name string, creds map[string]string) (CloudProvider, error) { return newSauceLabs(creds) case "lambdatest": return newLambdaTest(creds) + case "testingbot": + return newTestingBot(creds) default: - return nil, fmt.Errorf("unknown cloud provider %q (supported: browserstack, aws, firebase, saucelabs, lambdatest)", name) + return nil, fmt.Errorf("unknown cloud provider %q (supported: browserstack, aws, firebase, saucelabs, lambdatest, testingbot)", name) } } @@ -107,7 +109,7 @@ func DetectPlatform(device string) string { } // validProviders lists all supported provider names for validation and help text. -var validProviders = []string{"browserstack", "aws", "firebase", "saucelabs", "lambdatest"} +var validProviders = []string{"browserstack", "aws", "firebase", "saucelabs", "lambdatest", "testingbot"} // ValidProviders returns the list of supported cloud provider names. func ValidProviders() []string { diff --git a/internal/cloud/testingbot.go b/internal/cloud/testingbot.go new file mode 100644 index 00000000..250c290a --- /dev/null +++ b/internal/cloud/testingbot.go @@ -0,0 +1,336 @@ +package cloud + +import ( + "bytes" + "context" + "encoding/json" + "fmt" + "io" + "mime/multipart" + "net/http" + "os" + "path/filepath" + "strings" + "time" +) + +const ( + tbBaseURL = "https://api.testingbot.com" + // tbAppiumHubURL is the TestingBot Appium W3C WebDriver endpoint. + tbAppiumHubURL = "https://hub.testingbot.com/wd/hub/session" +) + +// testingBot implements CloudProvider for the TestingBot Real Device Cloud. +// API docs: https://testingbot.com/support/api +type testingBot struct { + key string + secret string + // baseURL and hubURL default to the TestingBot endpoints above; tests + // point them at a local server. + baseURL string + hubURL string + http *http.Client +} + +// newTestingBot creates a TestingBot provider. +// Requires "key" and "secret" in creds. The generic "username"/"access_key" +// names are also accepted so --cloud-key/--cloud-secret work unchanged. +func newTestingBot(creds map[string]string) (*testingBot, error) { + key := creds["key"] + if key == "" { + key = creds["username"] + } + secret := creds["secret"] + if secret == "" { + secret = creds["access_key"] + } + if key == "" || secret == "" { + return nil, fmt.Errorf("testingbot: credentials require 'key' and 'secret' (set via --cloud-key/--cloud-secret or probe.yaml cloud.credentials)") + } + + return &testingBot{ + key: key, + secret: secret, + baseURL: tbBaseURL, + hubURL: tbAppiumHubURL, + http: &http.Client{ + Timeout: 5 * time.Minute, + }, + }, nil +} + +func (p *testingBot) Name() string { return "testingbot" } + +// UploadApp uploads the app binary to TestingBot storage and returns the +// tb:// URL to pass as the appium:app capability. +func (p *testingBot) UploadApp(ctx context.Context, appPath string) (string, error) { + file, err := os.Open(appPath) + if err != nil { + return "", fmt.Errorf("testingbot: opening app: %w", err) + } + defer file.Close() + + var body bytes.Buffer + writer := multipart.NewWriter(&body) + part, err := writer.CreateFormFile("file", filepath.Base(appPath)) + if err != nil { + return "", fmt.Errorf("testingbot: creating form: %w", err) + } + if _, err := io.Copy(part, file); err != nil { + return "", fmt.Errorf("testingbot: copying file: %w", err) + } + writer.Close() + + url := p.baseURL + "/v1/storage" + req, err := http.NewRequestWithContext(ctx, http.MethodPost, url, &body) + if err != nil { + return "", fmt.Errorf("testingbot: creating upload request: %w", err) + } + req.Header.Set("Content-Type", writer.FormDataContentType()) + req.SetBasicAuth(p.key, p.secret) + + resp, err := p.http.Do(req) + if err != nil { + return "", fmt.Errorf("testingbot: upload failed: %w", err) + } + defer resp.Body.Close() + + respBody, _ := io.ReadAll(resp.Body) + if resp.StatusCode != http.StatusOK && resp.StatusCode != http.StatusCreated { + return "", fmt.Errorf("testingbot: upload failed (HTTP %d): %s", resp.StatusCode, string(respBody)) + } + + var result struct { + AppURL string `json:"app_url"` // e.g. "tb://a1b2c3" + Error string `json:"error,omitempty"` + } + if err := json.Unmarshal(respBody, &result); err != nil { + return "", fmt.Errorf("testingbot: invalid upload response: %w", err) + } + if result.Error != "" { + return "", fmt.Errorf("testingbot: upload error: %s", result.Error) + } + if result.AppURL == "" { + return "", fmt.Errorf("testingbot: upload response contained no app_url") + } + + return result.AppURL, nil +} + +// ListDevices returns the real Android and iOS devices in the TestingBot grid. +func (p *testingBot) ListDevices(ctx context.Context) ([]Device, error) { + url := p.baseURL + "/v1/devices" + req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil) + if err != nil { + return nil, fmt.Errorf("testingbot: creating list request: %w", err) + } + req.SetBasicAuth(p.key, p.secret) + + resp, err := p.http.Do(req) + if err != nil { + return nil, fmt.Errorf("testingbot: list devices failed: %w", err) + } + defer resp.Body.Close() + + body, _ := io.ReadAll(resp.Body) + if resp.StatusCode != http.StatusOK { + return nil, fmt.Errorf("testingbot: list devices failed (HTTP %d): %s", resp.StatusCode, string(body)) + } + + var raw []struct { + Name string `json:"name"` + PlatformName string `json:"platform_name"` + Version string `json:"version"` + } + if err := json.Unmarshal(body, &raw); err != nil { + return nil, fmt.Errorf("testingbot: invalid devices response: %w", err) + } + + devices := make([]Device, 0, len(raw)) + for _, d := range raw { + osName := "android" + if strings.EqualFold(d.PlatformName, "ios") { + osName = "ios" + } + devices = append(devices, Device{ + Name: d.Name, + OS: osName, + Version: d.Version, + Provider: "testingbot", + }) + } + return devices, nil +} + +// StartSession starts a real device session on TestingBot via the Appium W3C +// WebDriver hub. Credentials travel in tb:options as TestingBot documents. +func (p *testingBot) StartSession(ctx context.Context, appID string, device string) (Session, error) { + deviceName, osVersion := ParseDeviceString(device) + platformName := DetectPlatform(deviceName) + + tbOpts := map[string]interface{}{ + "key": p.key, + "secret": p.secret, + "name": "probe-test", + "build": fmt.Sprintf("probe-%s", time.Now().Format("2006-01-02")), + } + + alwaysMatch := map[string]interface{}{ + "appium:app": appID, + "appium:deviceName": deviceName, + "platformName": platformName, + "appium:autoGrantPermissions": true, + "tb:options": tbOpts, + } + if osVersion != "" { + alwaysMatch["appium:platformVersion"] = osVersion + } + + if strings.EqualFold(platformName, "Android") { + alwaysMatch["appium:automationName"] = "UiAutomator2" + } else { + alwaysMatch["appium:automationName"] = "XCUITest" + } + + payload := map[string]interface{}{ + "capabilities": map[string]interface{}{ + "firstMatch": []map[string]interface{}{{}}, + "alwaysMatch": alwaysMatch, + }, + } + + data, _ := json.Marshal(payload) + req, err := http.NewRequestWithContext(ctx, http.MethodPost, p.hubURL, bytes.NewReader(data)) + if err != nil { + return Session{}, fmt.Errorf("testingbot: creating session request: %w", err) + } + req.Header.Set("Content-Type", "application/json") + req.SetBasicAuth(p.key, p.secret) + + resp, err := p.http.Do(req) + if err != nil { + return Session{}, fmt.Errorf("testingbot: start session failed: %w", err) + } + defer resp.Body.Close() + + body, _ := io.ReadAll(resp.Body) + if resp.StatusCode != http.StatusOK { + return Session{}, fmt.Errorf("testingbot: start session failed (HTTP %d): %s", resp.StatusCode, string(body)) + } + + var result struct { + Value struct { + SessionID string `json:"sessionId"` + Error string `json:"error,omitempty"` + Message string `json:"message,omitempty"` + } `json:"value"` + } + if err := json.Unmarshal(body, &result); err != nil { + return Session{}, fmt.Errorf("testingbot: invalid session response: %w", err) + } + if result.Value.Error != "" { + return Session{}, fmt.Errorf("testingbot: session error: %s — %s", result.Value.Error, result.Value.Message) + } + + return Session{ + ID: result.Value.SessionID, + DeviceName: device, + Provider: "testingbot", + }, nil +} + +// ForwardPort is a no-op for TestingBot when using relay mode. +// +// In relay mode, the ProbeAgent connects outbound to the ProbeRelay server. +// Direct mode requires the TestingBot Tunnel binary — not yet supported. +func (p *testingBot) ForwardPort(ctx context.Context, session Session, devicePort int) (int, error) { + if session.RelayURL != "" { + return devicePort, nil + } + return 0, fmt.Errorf("testingbot: direct port forwarding requires TestingBot Tunnel (not yet supported) — use relay mode with --relay flag") +} + +// GetSessionArtifacts retrieves the video and screenshot URLs for a session. +// TestingBot processes assets asynchronously, so this polls up to 30s for the +// video to appear. The returned URLs are presigned and expire (video after an +// hour, screenshots after seven days). +func (p *testingBot) GetSessionArtifacts(ctx context.Context, sessionID string) (*SessionArtifacts, error) { + var last *SessionArtifacts + deadline := time.Now().Add(30 * time.Second) + for { + arts, err := p.sessionAssets(ctx, sessionID) + if err != nil { + return nil, err + } + last = arts + if arts.VideoURL != "" || !time.Now().Before(deadline) { + break + } + select { + case <-ctx.Done(): + return nil, ctx.Err() + case <-time.After(3 * time.Second): + } + } + return last, nil +} + +// sessionAssets fetches a single snapshot of a session's assets. The assets +// endpoint accepts either the numeric test ID or the WebDriver session ID. +func (p *testingBot) sessionAssets(ctx context.Context, sessionID string) (*SessionArtifacts, error) { + url := fmt.Sprintf("%s/v1/tests/%s/assets", p.baseURL, sessionID) + req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil) + if err != nil { + return nil, fmt.Errorf("testingbot: creating assets request: %w", err) + } + req.SetBasicAuth(p.key, p.secret) + + resp, err := p.http.Do(req) + if err != nil { + return nil, fmt.Errorf("testingbot: get assets failed: %w", err) + } + defer resp.Body.Close() + + body, _ := io.ReadAll(resp.Body) + if resp.StatusCode != http.StatusOK { + return nil, fmt.Errorf("testingbot: get assets failed (HTTP %d): %s", resp.StatusCode, string(body)) + } + + var result struct { + Video string `json:"video"` + Screenshots []string `json:"screenshots"` + } + if err := json.Unmarshal(body, &result); err != nil { + return nil, fmt.Errorf("testingbot: invalid assets response: %w", err) + } + return &SessionArtifacts{ + VideoURL: result.Video, + ScreenshotURLs: result.Screenshots, + }, nil +} + +// StopSession terminates a TestingBot Appium session via WebDriver DELETE. +func (p *testingBot) StopSession(ctx context.Context, session Session) error { + if session.ID == "" { + return nil + } + + url := fmt.Sprintf("%s/%s", p.hubURL, session.ID) + req, err := http.NewRequestWithContext(ctx, http.MethodDelete, url, nil) + if err != nil { + return fmt.Errorf("testingbot: creating stop request: %w", err) + } + req.SetBasicAuth(p.key, p.secret) + + resp, err := p.http.Do(req) + if err != nil { + return fmt.Errorf("testingbot: stop session failed: %w", err) + } + resp.Body.Close() + + if resp.StatusCode != http.StatusOK && resp.StatusCode != http.StatusNoContent { + return fmt.Errorf("testingbot: stop session failed (HTTP %d)", resp.StatusCode) + } + + return nil +} diff --git a/internal/cloud/testingbot_test.go b/internal/cloud/testingbot_test.go new file mode 100644 index 00000000..13a24e70 --- /dev/null +++ b/internal/cloud/testingbot_test.go @@ -0,0 +1,370 @@ +package cloud + +import ( + "context" + "encoding/json" + "net/http" + "net/http/httptest" + "os" + "path/filepath" + "sync/atomic" + "testing" +) + +// newTestTestingBot returns a provider pointed at srv for both the REST API +// and the Appium hub. +func newTestTestingBot(t *testing.T, srv *httptest.Server) *testingBot { + t.Helper() + p, err := newTestingBot(map[string]string{"key": "k", "secret": "s"}) + if err != nil { + t.Fatalf("newTestingBot: %v", err) + } + p.baseURL = srv.URL + p.hubURL = srv.URL + "/wd/hub/session" + return p +} + +func TestNewTestingBot_Credentials(t *testing.T) { + tests := []struct { + name string + creds map[string]string + wantErr bool + wantKey string + wantSecret string + }{ + { + name: "native key/secret names", + creds: map[string]string{"key": "K", "secret": "S"}, + wantKey: "K", + wantSecret: "S", + }, + { + name: "generic names from --cloud-key/--cloud-secret", + creds: map[string]string{"username": "K", "access_key": "S"}, + wantKey: "K", + wantSecret: "S", + }, + { + name: "missing secret", + creds: map[string]string{"key": "K"}, + wantErr: true, + }, + { + name: "empty", + creds: map[string]string{}, + wantErr: true, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + p, err := newTestingBot(tt.creds) + if tt.wantErr { + if err == nil { + t.Fatal("expected error, got nil") + } + return + } + if err != nil { + t.Fatalf("newTestingBot: %v", err) + } + if p.key != tt.wantKey || p.secret != tt.wantSecret { + t.Errorf("key/secret = %q/%q, want %q/%q", p.key, p.secret, tt.wantKey, tt.wantSecret) + } + if p.Name() != "testingbot" { + t.Errorf("Name() = %q, want testingbot", p.Name()) + } + }) + } +} + +func TestTestingBot_UploadApp(t *testing.T) { + appPath := filepath.Join(t.TempDir(), "app.apk") + if err := os.WriteFile(appPath, []byte("not-really-an-apk"), 0o600); err != nil { + t.Fatalf("write app: %v", err) + } + + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path != "/v1/storage" { + t.Errorf("unexpected path: %s", r.URL.Path) + } + user, pass, ok := r.BasicAuth() + if !ok || user != "k" || pass != "s" { + t.Errorf("basic auth = %q/%q (ok=%v), want k/s", user, pass, ok) + } + file, header, err := r.FormFile("file") + if err != nil { + t.Fatalf("FormFile(\"file\"): %v", err) + } + defer file.Close() + if header.Filename != "app.apk" { + t.Errorf("filename = %q, want app.apk", header.Filename) + } + w.WriteHeader(http.StatusCreated) + json.NewEncoder(w).Encode(map[string]string{"app_url": "tb://abc123"}) + })) + defer srv.Close() + + got, err := newTestTestingBot(t, srv).UploadApp(context.Background(), appPath) + if err != nil { + t.Fatalf("UploadApp: %v", err) + } + if got != "tb://abc123" { + t.Errorf("appID = %q, want tb://abc123", got) + } +} + +func TestTestingBot_UploadApp_Error(t *testing.T) { + appPath := filepath.Join(t.TempDir(), "app.apk") + if err := os.WriteFile(appPath, []byte("x"), 0o600); err != nil { + t.Fatalf("write app: %v", err) + } + + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusUnauthorized) + w.Write([]byte("Authentication required")) + })) + defer srv.Close() + + if _, err := newTestTestingBot(t, srv).UploadApp(context.Background(), appPath); err == nil { + t.Fatal("expected error for 401 response") + } +} + +func TestTestingBot_ListDevices(t *testing.T) { + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path != "/v1/devices" { + t.Errorf("unexpected path: %s", r.URL.Path) + } + json.NewEncoder(w).Encode([]map[string]interface{}{ + {"name": "Galaxy S21", "platform_name": "Android", "version": "14"}, + {"name": "iPhone 15 Pro", "platform_name": "iOS", "version": "17.4"}, + }) + })) + defer srv.Close() + + devices, err := newTestTestingBot(t, srv).ListDevices(context.Background()) + if err != nil { + t.Fatalf("ListDevices: %v", err) + } + if len(devices) != 2 { + t.Fatalf("got %d devices, want 2", len(devices)) + } + want := []Device{ + {Name: "Galaxy S21", OS: "android", Version: "14", Provider: "testingbot"}, + {Name: "iPhone 15 Pro", OS: "ios", Version: "17.4", Provider: "testingbot"}, + } + for i, w := range want { + if devices[i] != w { + t.Errorf("device[%d] = %+v, want %+v", i, devices[i], w) + } + } +} + +func TestTestingBot_StartSession(t *testing.T) { + var caps map[string]interface{} + + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path != "/wd/hub/session" { + t.Errorf("unexpected path: %s", r.URL.Path) + } + var payload struct { + Capabilities struct { + AlwaysMatch map[string]interface{} `json:"alwaysMatch"` + } `json:"capabilities"` + } + if err := json.NewDecoder(r.Body).Decode(&payload); err != nil { + t.Fatalf("decode request: %v", err) + } + caps = payload.Capabilities.AlwaysMatch + json.NewEncoder(w).Encode(map[string]interface{}{ + "value": map[string]string{"sessionId": "sess-1"}, + }) + })) + defer srv.Close() + + sess, err := newTestTestingBot(t, srv).StartSession(context.Background(), "tb://abc123", "Galaxy S21-14") + if err != nil { + t.Fatalf("StartSession: %v", err) + } + if sess.ID != "sess-1" { + t.Errorf("session ID = %q, want sess-1", sess.ID) + } + if sess.Provider != "testingbot" { + t.Errorf("provider = %q, want testingbot", sess.Provider) + } + if sess.DeviceName != "Galaxy S21-14" { + t.Errorf("device = %q, want Galaxy S21-14", sess.DeviceName) + } + + if caps["appium:app"] != "tb://abc123" { + t.Errorf("appium:app = %v, want tb://abc123", caps["appium:app"]) + } + if caps["appium:deviceName"] != "Galaxy S21" { + t.Errorf("appium:deviceName = %v, want Galaxy S21", caps["appium:deviceName"]) + } + if caps["appium:platformVersion"] != "14" { + t.Errorf("appium:platformVersion = %v, want 14", caps["appium:platformVersion"]) + } + if caps["platformName"] != "Android" { + t.Errorf("platformName = %v, want Android", caps["platformName"]) + } + if caps["appium:automationName"] != "UiAutomator2" { + t.Errorf("appium:automationName = %v, want UiAutomator2", caps["appium:automationName"]) + } + tbOpts, ok := caps["tb:options"].(map[string]interface{}) + if !ok { + t.Fatalf("tb:options missing or wrong type: %T", caps["tb:options"]) + } + if tbOpts["key"] != "k" || tbOpts["secret"] != "s" { + t.Errorf("tb:options credentials = %v/%v, want k/s", tbOpts["key"], tbOpts["secret"]) + } +} + +func TestTestingBot_StartSession_iOS(t *testing.T) { + var caps map[string]interface{} + + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + var payload struct { + Capabilities struct { + AlwaysMatch map[string]interface{} `json:"alwaysMatch"` + } `json:"capabilities"` + } + json.NewDecoder(r.Body).Decode(&payload) + caps = payload.Capabilities.AlwaysMatch + json.NewEncoder(w).Encode(map[string]interface{}{ + "value": map[string]string{"sessionId": "sess-ios"}, + }) + })) + defer srv.Close() + + if _, err := newTestTestingBot(t, srv).StartSession(context.Background(), "tb://abc", "iPhone 15 Pro-17.4"); err != nil { + t.Fatalf("StartSession: %v", err) + } + if caps["platformName"] != "iOS" { + t.Errorf("platformName = %v, want iOS", caps["platformName"]) + } + if caps["appium:automationName"] != "XCUITest" { + t.Errorf("appium:automationName = %v, want XCUITest", caps["appium:automationName"]) + } +} + +func TestTestingBot_StartSession_WebDriverError(t *testing.T) { + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + json.NewEncoder(w).Encode(map[string]interface{}{ + "value": map[string]string{ + "error": "session not created", + "message": "no device available", + }, + }) + })) + defer srv.Close() + + if _, err := newTestTestingBot(t, srv).StartSession(context.Background(), "tb://abc", "Galaxy S21"); err == nil { + t.Fatal("expected error when WebDriver reports a session error") + } +} + +func TestTestingBot_ForwardPort(t *testing.T) { + p, err := newTestingBot(map[string]string{"key": "k", "secret": "s"}) + if err != nil { + t.Fatalf("newTestingBot: %v", err) + } + + // Relay mode: the agent dials out, so the device port passes through. + got, err := p.ForwardPort(context.Background(), Session{RelayURL: "wss://relay.example/x"}, 48686) + if err != nil { + t.Fatalf("ForwardPort (relay): %v", err) + } + if got != 48686 { + t.Errorf("port = %d, want 48686", got) + } + + // Direct mode is not supported yet. + if _, err := p.ForwardPort(context.Background(), Session{}, 48686); err == nil { + t.Fatal("expected error for direct port forwarding") + } +} + +func TestTestingBot_GetSessionArtifacts(t *testing.T) { + var calls atomic.Int32 + + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path != "/v1/tests/sess-1/assets" { + t.Errorf("unexpected path: %s", r.URL.Path) + } + // First poll: video still processing. + if calls.Add(1) == 1 { + json.NewEncoder(w).Encode(map[string]interface{}{"video": ""}) + return + } + json.NewEncoder(w).Encode(map[string]interface{}{ + "video": "https://testingbot.example/video.mp4", + "screenshots": []string{"https://testingbot.example/1.png"}, + }) + })) + defer srv.Close() + + arts, err := newTestTestingBot(t, srv).GetSessionArtifacts(context.Background(), "sess-1") + if err != nil { + t.Fatalf("GetSessionArtifacts: %v", err) + } + if arts.VideoURL != "https://testingbot.example/video.mp4" { + t.Errorf("video = %q", arts.VideoURL) + } + if len(arts.ScreenshotURLs) != 1 { + t.Errorf("got %d screenshots, want 1", len(arts.ScreenshotURLs)) + } + if calls.Load() < 2 { + t.Errorf("expected polling until video appeared, got %d calls", calls.Load()) + } +} + +func TestTestingBot_StopSession(t *testing.T) { + var deleted atomic.Bool + + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.Method != http.MethodDelete { + t.Errorf("expected DELETE, got %s", r.Method) + } + if r.URL.Path != "/wd/hub/session/sess-1" { + t.Errorf("unexpected path: %s", r.URL.Path) + } + deleted.Store(true) + w.WriteHeader(http.StatusOK) + })) + defer srv.Close() + + p := newTestTestingBot(t, srv) + if err := p.StopSession(context.Background(), Session{ID: "sess-1"}); err != nil { + t.Fatalf("StopSession: %v", err) + } + if !deleted.Load() { + t.Error("expected DELETE to be issued") + } + + // No session ID: nothing to stop, no error. + if err := p.StopSession(context.Background(), Session{}); err != nil { + t.Errorf("StopSession with empty ID: %v", err) + } +} + +func TestNewProvider_TestingBot(t *testing.T) { + p, err := NewProvider("testingbot", map[string]string{"key": "k", "secret": "s"}) + if err != nil { + t.Fatalf("NewProvider: %v", err) + } + if p.Name() != "testingbot" { + t.Errorf("Name() = %q, want testingbot", p.Name()) + } + + found := false + for _, name := range ValidProviders() { + if name == "testingbot" { + found = true + } + } + if !found { + t.Errorf("testingbot missing from ValidProviders(): %v", ValidProviders()) + } +} diff --git a/internal/config/config.go b/internal/config/config.go index 499d6df6..6e630cb4 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -52,7 +52,7 @@ type Config struct { type CloudConfig struct { URL string `yaml:"url"` // cloud API base URL (must be configured by the user) Token string `yaml:"token"` // API key for cloud authentication (supports ${ENV_VAR} syntax) - Provider string `yaml:"provider"` // cloud device farm provider: browserstack, aws, firebase, saucelabs, lambdatest + Provider string `yaml:"provider"` // cloud device farm provider: browserstack, aws, firebase, saucelabs, lambdatest, testingbot Credentials map[string]string `yaml:"credentials"` // provider-specific credentials (e.g. username, access_key) App string `yaml:"app"` // path to .apk/.ipa to upload to the cloud provider Devices []string `yaml:"devices"` // target device names for cloud testing diff --git a/probe.yaml.example b/probe.yaml.example index 7e793585..7ab0555b 100644 --- a/probe.yaml.example +++ b/probe.yaml.example @@ -31,7 +31,7 @@ device: # token: ${PROBE_CLOUD_TOKEN} # Cloud device farm provider (optional — bring your own account) -# Supported: browserstack | saucelabs | lambdatest | aws | firebase +# Supported: browserstack | saucelabs | lambdatest | testingbot | aws | firebase # # cloud: # provider: saucelabs diff --git a/vscode/README.md b/vscode/README.md index c33a0b6a..4e1ceaa0 100644 --- a/vscode/README.md +++ b/vscode/README.md @@ -190,7 +190,7 @@ All settings are under `flutterprobe.*` in VS Code settings. They follow the res | Setting | Default | Description | |---|---|---| -| `cloudProvider` | | Provider: `browserstack`, `saucelabs`, `lambdatest`, `aws`, `firebase` | +| `cloudProvider` | | Provider: `browserstack`, `saucelabs`, `lambdatest`, `testingbot`, `aws`, `firebase` | | `cloudDevice` | | Device string (e.g., `Google Pixel 7-13.0`) | | `cloudApp` | | Path to `.apk` or `.ipa` binary | | `cloudKey` | | Username or Access Key ID | @@ -235,6 +235,13 @@ All settings are under `flutterprobe.*` in VS Code settings. They follow the res 2. Go to **Settings** > **Account Settings** for credentials 3. Configure with `"flutterprobe.cloudProvider": "lambdatest"` +### TestingBot + +1. Sign up at [testingbot.com](https://testingbot.com) +2. Go to **Account** > **Credentials** for your key and secret +3. Configure with `"flutterprobe.cloudProvider": "testingbot"`, putting the key + in `cloudKey` and the secret in `cloudSecret` + ### AWS Device Farm 1. Create an IAM user with `AWSDeviceFarmFullAccess` diff --git a/vscode/package.json b/vscode/package.json index 12d726f7..56e880dd 100644 --- a/vscode/package.json +++ b/vscode/package.json @@ -375,12 +375,13 @@ "flutterprobe.cloudProvider": { "type": "string", "default": "", - "enum": ["", "browserstack", "saucelabs", "lambdatest", "aws", "firebase"], + "enum": ["", "browserstack", "saucelabs", "lambdatest", "testingbot", "aws", "firebase"], "enumDescriptions": [ "None (local testing)", "BrowserStack App Automate", "Sauce Labs Real Device Cloud", "LambdaTest Real Devices", + "TestingBot Real Device Cloud", "AWS Device Farm", "Firebase Test Lab" ], diff --git a/vscode/src/views/runProfile.ts b/vscode/src/views/runProfile.ts index ff85f3f6..4acff785 100644 --- a/vscode/src/views/runProfile.ts +++ b/vscode/src/views/runProfile.ts @@ -102,6 +102,7 @@ export class RunProfilePanel { +