feat(jshttp): optional Firefox/WebKit browser engine + ExecutablePath override

CHANGELOG.md

@@ -7,6 +7,16 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ## [Unreleased]
 
+### Added
+
+- `jshttp`: optional Firefox/WebKit browser engine for JS rendering via
+  `JSFetcherOptions.BrowserType` (`"chromium"` default, `"firefox"`, `"webkit"`)
+  and `JSFetcherOptions.ExecutablePath` to override the browser binary. Exposed
+  through `scrapemateapp.WithJSBrowserType(...)` and
+  `scrapemateapp.WithJSExecutablePath(...)` as `WithJS` sub-options. Chromium
+  launch flags are only applied to Chromium; Firefox/WebKit use the Playwright
+  engine defaults. Backward-compatible — the empty default keeps Chromium.
+
 ### Removed
 
 - Rod browser support, build tags, and related fetcher/page implementations

adapters/fetchers/jshttp/jshttp.go

@@ -21,12 +21,21 @@ type JSFetcherOptions struct {
 	PageReuseLimit     int
 	BrowserReuseLimit  int
 	UserAgent          string
+	// BrowserType selects the Playwright browser engine. Accepted values are
+	// "chromium" (the default, also for the empty string), "firefox" and
+	// "webkit". Callers that never set this field keep the existing Chromium
+	// behaviour unchanged.
+	BrowserType string
+	// ExecutablePath, when non-empty, overrides the Playwright-managed browser
+	// binary (for example a custom Firefox build). Empty uses the bundled binary.
+	ExecutablePath string
 }
 
+//nolint:gocritic // Keep value parameter to preserve the public constructor API.
 func New(params JSFetcherOptions) (scrapemate.HTTPFetcher, error) {
 	opts := []*playwright.RunOptions{
 		{
-			Browsers: []string{"chromium"},
+			Browsers: browsersToInstall(params.BrowserType),
 			Verbose:  true,
 		},
 	}
@@ -75,11 +84,13 @@ func New(params JSFetcherOptions) (scrapemate.HTTPFetcher, error) {
 			poolSize:           params.PoolSize,
 			maxPagesPerBrowser: maxPagesPerBrowser,
 			factory: playwrightSlotFactory{
-				pw:            pw,
-				headless:      params.Headless,
-				disableImages: params.DisableImages,
-				proxyPool:     pool,
-				ua:            params.UserAgent,
+				pw:             pw,
+				headless:       params.Headless,
+				disableImages:  params.DisableImages,
+				proxyPool:      pool,
+				ua:             params.UserAgent,
+				browserType:    params.BrowserType,
+				executablePath: params.ExecutablePath,
 			},
 		})
 		if err != nil {
@@ -94,11 +105,13 @@ func New(params JSFetcherOptions) (scrapemate.HTTPFetcher, error) {
 	ans.slots = make(chan *sessionSlot, params.PoolSize)
 
 	sessionFactory := &playwrightRuntimeFactory{
-		pw:            pw,
-		headless:      params.Headless,
-		disableImages: params.DisableImages,
-		proxyPool:     pool,
-		ua:            params.UserAgent,
+		pw:             pw,
+		headless:       params.Headless,
+		disableImages:  params.DisableImages,
+		proxyPool:      pool,
+		ua:             params.UserAgent,
+		browserType:    params.BrowserType,
+		executablePath: params.ExecutablePath,
 	}
 	ans.factory = sessionFactory
 
@@ -254,39 +267,85 @@ func (o *browser) Close() {
 	_ = o.browser.Close()
 }
 
-func newBrowser(pw *playwright.Playwright, headless, disableImages bool, proxyPool *ProxyPool, ua string) (*browser, error) {
+// browsersToInstall maps a BrowserType value to the Playwright install list.
+// An empty string or "chromium" both install Chromium so that callers that
+// never set BrowserType are unaffected.
+func browsersToInstall(browserType string) []string {
+	switch browserType {
+	case "firefox":
+		return []string{"firefox"}
+	case "webkit":
+		return []string{"webkit"}
+	default:
+		return []string{"chromium"}
+	}
+}
+
+// chromiumLaunchArgs are the Chromium-specific command-line flags. They are only
+// passed when launching Chromium: Firefox and WebKit reject or mishandle these
+// flags, and forwarding them causes Firefox to hang on the first NewPage call.
+func chromiumLaunchArgs(disableImages bool) []string {
+	args := []string{
+		`--start-maximized`,
+		`--no-default-browser-check`,
+		`--disable-dev-shm-usage`,
+		`--no-sandbox`,
+		`--disable-setuid-sandbox`,
+		`--no-zygote`,
+		`--disable-gpu`,
+		`--mute-audio`,
+		`--disable-extensions`,
+		`--single-process`,
+		`--disable-breakpad`,
+		`--disable-features=TranslateUI,BlinkGenPropertyTrees`,
+		`--disable-ipc-flooding-protection`,
+		`--enable-features=NetworkService,NetworkServiceInProcess`,
+		"--enable-features=NetworkService",
+		`--disable-default-apps`,
+		`--disable-notifications`,
+		`--disable-webgl`,
+		`--disable-blink-features=AutomationControlled`,
+		"--ignore-certificate-errors",
+		"--ignore-certificate-errors-spki-list",
+		"--disable-web-security",
+	}
+	if disableImages {
+		args = append(args, `--blink-settings=imagesEnabled=false`)
+	}
+
+	return args
+}
+
+// browserTypeFor returns the playwright.BrowserType for the configured engine.
+// Empty or "chromium" return Chromium so existing callers are unaffected.
+func browserTypeFor(pw *playwright.Playwright, browserType string) playwright.BrowserType {
+	switch browserType {
+	case "firefox":
+		return pw.Firefox
+	case "webkit":
+		return pw.WebKit
+	default:
+		return pw.Chromium
+	}
+}
+
+func newBrowser(pw *playwright.Playwright, headless, disableImages bool, proxyPool *ProxyPool, ua, browserType, executablePath string) (*browser, error) {
 	opts := playwright.BrowserTypeLaunchOptions{
 		Headless: playwright.Bool(headless),
-		Args: []string{
-			`--start-maximized`,
-			`--no-default-browser-check`,
-			`--disable-dev-shm-usage`,
-			`--no-sandbox`,
-			`--disable-setuid-sandbox`,
-			`--no-zygote`,
-			`--disable-gpu`,
-			`--mute-audio`,
-			`--disable-extensions`,
-			`--single-process`,
-			`--disable-breakpad`,
-			`--disable-features=TranslateUI,BlinkGenPropertyTrees`,
-			`--disable-ipc-flooding-protection`,
-			`--enable-features=NetworkService,NetworkServiceInProcess`,
-			"--enable-features=NetworkService",
-			`--disable-default-apps`,
-			`--disable-notifications`,
-			`--disable-webgl`,
-			`--disable-blink-features=AutomationControlled`,
-			"--ignore-certificate-errors",
-			"--ignore-certificate-errors-spki-list",
-			"--disable-web-security",
-		},
 	}
-	if disableImages {
-		opts.Args = append(opts.Args, `--blink-settings=imagesEnabled=false`)
+
+	// Chromium launch flags only apply to Chromium. Firefox/WebKit use the
+	// Playwright engine defaults; forwarding Chromium flags hangs Firefox at
+	// the first NewPage.
+	if browserType == "" || browserType == "chromium" {
+		opts.Args = chromiumLaunchArgs(disableImages)
+	}
+
+	if executablePath != "" {
+		opts.ExecutablePath = playwright.String(executablePath)
 	}
 
-	br, err := pw.Chromium.Launch(opts)
+	br, err := browserTypeFor(pw, browserType).Launch(opts)
 	if err != nil {
 		return nil, err
 	}

adapters/fetchers/jshttp/jshttp_browsertype_test.go

@@ -0,0 +1,49 @@
+package jshttp //nolint:testpackage // Need access to unexported browser selection helpers.
+
+import "testing"
+
+func TestBrowsersToInstall(t *testing.T) {
+	cases := map[string]string{
+		"":         "chromium",
+		"chromium": "chromium",
+		"firefox":  "firefox",
+		"webkit":   "webkit",
+		"unknown":  "chromium", // unknown values fall back to chromium
+	}
+
+	for in, want := range cases {
+		got := browsersToInstall(in)
+		if len(got) != 1 || got[0] != want {
+			t.Errorf("browsersToInstall(%q) = %v; want [%q]", in, got, want)
+		}
+	}
+}
+
+func TestChromiumLaunchArgs_DisableImages(t *testing.T) {
+	const imgFlag = "--blink-settings=imagesEnabled=false"
+
+	with := chromiumLaunchArgs(true)
+	if !contains(with, imgFlag) {
+		t.Errorf("chromiumLaunchArgs(true) missing %q", imgFlag)
+	}
+
+	without := chromiumLaunchArgs(false)
+	if contains(without, imgFlag) {
+		t.Errorf("chromiumLaunchArgs(false) unexpectedly contains %q", imgFlag)
+	}
+
+	// The core Chromium flags must always be present.
+	if !contains(without, "--no-sandbox") {
+		t.Error("chromiumLaunchArgs missing --no-sandbox")
+	}
+}
+
+func contains(s []string, v string) bool {
+	for _, x := range s {
+		if x == v {
+			return true
+		}
+	}
+
+	return false
+}

adapters/fetchers/jshttp/page_slot_pool.go

@@ -110,15 +110,17 @@ func (p *pageSlotPool) close() {
 }
 
 type playwrightSlotFactory struct {
-	pw            *playwright.Playwright
-	headless      bool
-	disableImages bool
-	proxyPool     *ProxyPool
-	ua            string
+	pw             *playwright.Playwright
+	headless       bool
+	disableImages  bool
+	proxyPool      *ProxyPool
+	ua             string
+	browserType    string
+	executablePath string
 }
 
 func (f playwrightSlotFactory) newSlot() (*pageSlot, error) {
-	b, err := newBrowser(f.pw, f.headless, f.disableImages, f.proxyPool, f.ua)
+	b, err := newBrowser(f.pw, f.headless, f.disableImages, f.proxyPool, f.ua, f.browserType, f.executablePath)
 	if err != nil {
 		return nil, err
 	}

adapters/fetchers/jshttp/session_slot.go

@@ -96,36 +96,42 @@ func (s *sessionSlot) release(ctx context.Context) error {
 }
 
 type playwrightRuntimeFactory struct {
-	pw            *playwright.Playwright
-	headless      bool
-	disableImages bool
-	proxyPool     *ProxyPool
-	ua            string
+	pw             *playwright.Playwright
+	headless       bool
+	disableImages  bool
+	proxyPool      *ProxyPool
+	ua             string
+	browserType    string
+	executablePath string
 }
 
 func (f *playwrightRuntimeFactory) create(context.Context) (slotRuntime, error) {
-	b, err := newBrowser(f.pw, f.headless, f.disableImages, f.proxyPool, f.ua)
+	b, err := newBrowser(f.pw, f.headless, f.disableImages, f.proxyPool, f.ua, f.browserType, f.executablePath)
 	if err != nil {
 		return nil, err
 	}
 
 	return &playwrightRuntime{
-		browser:       b,
-		pw:            f.pw,
-		headless:      f.headless,
-		disableImages: f.disableImages,
-		proxyPool:     f.proxyPool,
-		ua:            f.ua,
+		browser:        b,
+		pw:             f.pw,
+		headless:       f.headless,
+		disableImages:  f.disableImages,
+		proxyPool:      f.proxyPool,
+		ua:             f.ua,
+		browserType:    f.browserType,
+		executablePath: f.executablePath,
 	}, nil
 }
 
 type playwrightRuntime struct {
-	browser       *browser
-	pw            *playwright.Playwright
-	headless      bool
-	disableImages bool
-	proxyPool     *ProxyPool
-	ua            string
+	browser        *browser
+	pw             *playwright.Playwright
+	headless       bool
+	disableImages  bool
+	proxyPool      *ProxyPool
+	ua             string
+	browserType    string
+	executablePath string
 }
 
 func (r *playwrightRuntime) pageCount() int {
@@ -194,7 +200,7 @@ func (r *playwrightRuntime) recreateContext() error {
 func (r *playwrightRuntime) recreateBrowser() error {
 	r.browser.Close()
 
-	b, err := newBrowser(r.pw, r.headless, r.disableImages, r.proxyPool, r.ua)
+	b, err := newBrowser(r.pw, r.headless, r.disableImages, r.proxyPool, r.ua, r.browserType, r.executablePath)
 	if err != nil {
 		return err
 	}

scrapemateapp/config.go

@@ -9,9 +9,11 @@ import (
 )
 
 type jsOptions struct {
-	Headfull      bool
-	DisableImages bool
-	UA            string
+	Headfull       bool
+	DisableImages  bool
+	UA             string
+	BrowserType    string
+	ExecutablePath string
 }
 
 type Config struct {
@@ -194,6 +196,28 @@ func WithUA(ua string) func(*jsOptions) {
 	}
 }
 
+// WithJSBrowserType selects the Playwright browser engine for JS rendering.
+// Accepted values are "chromium" (the default), "firefox" and "webkit". The
+// empty string keeps the default Chromium behaviour.
+//
+// Example: WithJS(WithJSBrowserType("firefox"))
+func WithJSBrowserType(browserType string) func(*jsOptions) {
+	return func(o *jsOptions) {
+		o.BrowserType = browserType
+	}
+}
+
+// WithJSExecutablePath overrides the Playwright-managed browser binary with the
+// one at the given path (for example a custom Firefox build). Empty uses the
+// bundled binary.
+//
+// Example: WithJS(WithJSBrowserType("firefox"), WithJSExecutablePath("/opt/firefox/firefox"))
+func WithJSExecutablePath(path string) func(*jsOptions) {
+	return func(o *jsOptions) {
+		o.ExecutablePath = path
+	}
+}
+
 func WithBrowserEngine(_ string) func(*jsOptions) {
 	return func(_ *jsOptions) {
 	}