diff --git a/go.mod b/go.mod
index 7be9fdc2..a9ac027e 100644
--- a/go.mod
+++ b/go.mod
@@ -35,7 +35,7 @@ require (
github.com/projectdiscovery/retryablehttp-go v1.3.23
github.com/projectdiscovery/tlsx v1.3.2
github.com/projectdiscovery/useragent v0.0.108
- github.com/projectdiscovery/utils v0.11.1
+ github.com/projectdiscovery/utils v0.11.2-0.20260815171005-eb8925425716
github.com/projectdiscovery/wappalyzergo v0.2.94
github.com/rs/xid v1.6.0
github.com/spaolacci/murmur3 v1.1.0
diff --git a/go.sum b/go.sum
index fd88badb..8a9e61dc 100644
--- a/go.sum
+++ b/go.sum
@@ -296,8 +296,8 @@ github.com/projectdiscovery/tlsx v1.3.2 h1:1Lh2ith79o8R+rOOSBZiMq8EiZ8j4D3T/8ENO
github.com/projectdiscovery/tlsx v1.3.2/go.mod h1:2wgTGC/sourHvoR+RX8cAok0Sa8YK6QbEP3xo5SDzPI=
github.com/projectdiscovery/useragent v0.0.108 h1:fb+uLuFJvC+MHZjCtxQJxtvp1X6A8n98CUGPyFcg3NE=
github.com/projectdiscovery/useragent v0.0.108/go.mod h1:XdNRrlvtDmYfVL1Oybat4uMe+W6cLwsK9S18ond17CI=
-github.com/projectdiscovery/utils v0.11.1 h1:PWj1KjIASxt8icxommH72C0TQqNOvGkcSODRkiq0SQw=
-github.com/projectdiscovery/utils v0.11.1/go.mod h1:yktGrHGk2CTjNiccXovnvGrLHX9sV2bqz9nSnbA3V8M=
+github.com/projectdiscovery/utils v0.11.2-0.20260815171005-eb8925425716 h1:zoDnj6xAqCvIQuP/bV5COuje2kjZV7THKTuDRdvyXgI=
+github.com/projectdiscovery/utils v0.11.2-0.20260815171005-eb8925425716/go.mod h1:RtBO9urHlfN3aAEzhZz5m1/ZeJDq6U1Ka34lGWleuS8=
github.com/projectdiscovery/wappalyzergo v0.2.94 h1:uHyNIb5OFLzyvEd6v64JLoc7Dlg3dBSjlPyjCIz0+T8=
github.com/projectdiscovery/wappalyzergo v0.2.94/go.mod h1:E2p8L90ysTTUkU5FUOgGoCQ7ucEUgqJ/tvYgiGGAlEM=
github.com/refraction-networking/utls v1.8.2 h1:j4Q1gJj0xngdeH+Ox/qND11aEfhpgoEvV+S9iJ2IdQo=
diff --git a/runner/headless.go b/runner/headless.go
index 3d295d82..37afb036 100644
--- a/runner/headless.go
+++ b/runner/headless.go
@@ -11,6 +11,7 @@ import (
"github.com/go-rod/rod/lib/launcher/flags"
"github.com/go-rod/rod/lib/proto"
"github.com/pkg/errors"
+ "github.com/projectdiscovery/utils/chromeshell"
fileutil "github.com/projectdiscovery/utils/file"
mapsutil "github.com/projectdiscovery/utils/maps"
osutils "github.com/projectdiscovery/utils/os"
@@ -61,6 +62,26 @@ func NewBrowser(proxy string, useLocal bool, optionalArgs map[string]string) (*B
Set("window-size", fmt.Sprintf("%d,%d", 1080, 1920)).
Set("mute-audio", "true").
Set("incognito", "true").
+ // Performance: keep background/occluded tabs running at full speed so
+ // many concurrent screenshot pages don't get render-throttled.
+ Set("disable-background-timer-throttling", "true").
+ Set("disable-backgrounding-occluded-windows", "true").
+ Set("disable-renderer-backgrounding", "true").
+ Set("disable-ipc-flooding-protection", "true").
+ Set("disable-hang-monitor", "true").
+ // Performance: strip background chrome services we never use.
+ Set("disable-background-networking", "true").
+ Set("disable-client-side-phishing-detection", "true").
+ Set("disable-component-update", "true").
+ Set("disable-default-apps", "true").
+ Set("disable-domain-reliability", "true").
+ Set("disable-extensions", "true").
+ Set("disable-sync", "true").
+ Set("no-first-run", "true").
+ Set("no-default-browser-check", "true").
+ Set("metrics-recording-only", "true").
+ Set("safebrowsing-disable-auto-update", "true").
+ Set("disable-features", "Translate,BackForwardCache,AcceptCHFrame,MediaRouter,OptimizationHints,site-per-process").
Delete("use-mock-keychain").
Headless(true).
UserDataDir(dataStore)
@@ -82,6 +103,12 @@ func NewBrowser(proxy string, useLocal bool, optionalArgs map[string]string) (*B
} else {
return nil, errors.New("the chrome browser is not installed")
}
+ } else if chromeshell.Supported() {
+ // Prefer chrome-headless-shell on linux/amd64: smaller download and
+ // faster headless screenshots than full Chromium snapshots.
+ if shellPath, err := ensureChromeShell(); err == nil {
+ chromeLauncher.Bin(shellPath)
+ }
}
if proxy == "" {
@@ -119,13 +146,13 @@ func NewBrowser(proxy string, useLocal bool, optionalArgs map[string]string) (*B
}
func (b *Browser) ScreenshotWithBody(url string, timeout time.Duration, idle time.Duration, headers []string, fullPage bool, jsCodes []string) ([]byte, string, []NetworkRequest, error) {
- page, networkRequests, err := b.setupPageAndNavigate(url, timeout, headers, jsCodes)
+ page, networkRequests, err := b.setupPageAndNavigate(url, timeout, idle, headers, jsCodes)
if err != nil {
return nil, "", []NetworkRequest{}, err
}
defer b.closePage(page)
- screenshot, body, err := b.takeScreenshotAndGetBody(page, idle, fullPage)
+ screenshot, body, err := b.takeScreenshotAndGetBody(page, fullPage)
if err != nil {
return nil, "", networkRequests, err
}
@@ -134,7 +161,7 @@ func (b *Browser) ScreenshotWithBody(url string, timeout time.Duration, idle tim
}
// setupPageAndNavigate opens a page, performs all adaptive actions including JS injection
-func (b *Browser) setupPageAndNavigate(url string, timeout time.Duration, headers []string, jsCodes []string) (*rod.Page, []NetworkRequest, error) {
+func (b *Browser) setupPageAndNavigate(url string, timeout time.Duration, idle time.Duration, headers []string, jsCodes []string) (*rod.Page, []NetworkRequest, error) {
page, err := b.engine.Page(proto.TargetCreateTarget{})
if err != nil {
return nil, []NetworkRequest{}, err
@@ -208,6 +235,13 @@ func (b *Browser) setupPageAndNavigate(url string, timeout time.Duration, header
}
page = page.Timeout(timeout)
+ var waitReqIdle func()
+ if idle > 0 {
+ // Register before Navigate so the first SPA XHRs are tracked by the
+ // request-idle waiter. Zero/negative idle skips this (WaitRequestIdle
+ // and WaitDOMStable reject non-positive durations).
+ waitReqIdle = page.WaitRequestIdle(idle, nil, nil, nil)
+ }
if err := page.Navigate(url); err != nil {
return page, networkRequests.Slice, err
@@ -220,19 +254,58 @@ func (b *Browser) setupPageAndNavigate(url string, timeout time.Duration, header
}
}
- page.Timeout(5 * time.Second).WaitNavigation(proto.PageLifecycleEventNameFirstMeaningfulPaint)()
+ b.waitPageReady(page, idle, waitReqIdle)
return page, networkRequests.Slice, nil
}
-// takeScreenshotAndGetBody performs the screenshot actions
-func (b *Browser) takeScreenshotAndGetBody(page *rod.Page, idle time.Duration, fullPage bool) ([]byte, string, error) {
- if err := page.WaitLoad(); err != nil {
- return nil, "", err
+// waitPageReady blocks until the page is visually settled so we don't capture a
+// half-rendered SPA. It gates on three independent signals, each bounded by the
+// page timeout:
+// - window.onload
+// - network request-idle (a quiet network window)
+// - DOM stability (the rendered tree stops mutating)
+//
+// SPAs paint late: the network can briefly go quiet before hydration starts, so
+// request-idle alone can fire on the boot screen. Requiring DOM stability on top
+// closes that gap. DOM stability is re-checked after the network settles so the
+// first snapshot is taken post-hydration rather than on the boot screen.
+func (b *Browser) waitPageReady(page *rod.Page, idle time.Duration, waitReqIdle func()) {
+ _ = page.WaitLoad()
+ if idle <= 0 || waitReqIdle == nil {
+ return
+ }
+ waitReqIdle()
+ _ = page.WaitDOMStable(idle, 0)
+}
+
+const chromeShellEnsureTimeout = 2 * time.Minute
+
+func ensureChromeShell() (string, error) {
+ type result struct {
+ path string
+ err error
+ }
+ done := make(chan result, 1)
+ go func() {
+ path, err := chromeshell.Ensure()
+ done <- result{path: path, err: err}
+ }()
+ select {
+ case r := <-done:
+ return r.path, r.err
+ case <-time.After(chromeShellEnsureTimeout):
+ return "", errors.New("chrome-headless-shell download timed out")
}
- _ = page.WaitIdle(idle)
+}
+
+// takeScreenshotAndGetBody performs the screenshot actions
+func (b *Browser) takeScreenshotAndGetBody(page *rod.Page, fullPage bool) ([]byte, string, error) {
+ _ = page.WaitRepaint()
- screenshot, err := page.Screenshot(fullPage, &proto.PageCaptureScreenshot{})
+ screenshot, err := page.Screenshot(fullPage, &proto.PageCaptureScreenshot{
+ OptimizeForSpeed: true,
+ })
if err != nil {
return nil, "", err
}
diff --git a/runner/headless_screenshot_test.go b/runner/headless_screenshot_test.go
new file mode 100644
index 00000000..e21297db
--- /dev/null
+++ b/runner/headless_screenshot_test.go
@@ -0,0 +1,139 @@
+package runner
+
+import (
+ "fmt"
+ "net/http"
+ "net/http/httptest"
+ "strings"
+ "sync"
+ "testing"
+ "time"
+)
+
+const spaHydratedMarker = "hydrated-ok"
+
+// spaHTML is a tiny SPA-style page: it paints a boot screen first, then
+// fetches /app.js which hydrates #root after a short delay. Early captures
+// only contain "boot"; a correct wait must include spaHydratedMarker.
+const spaHTML = `
+spa-bench
+
+boot
+
+`
+
+const spaJS = `
+setTimeout(function(){
+ document.getElementById("root").textContent = "` + spaHydratedMarker + `";
+}, 80);
+`
+
+func startSPAScreenshotServer(t testing.TB) *httptest.Server {
+ t.Helper()
+ mux := http.NewServeMux()
+ mux.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) {
+ w.Header().Set("Content-Type", "text/html; charset=utf-8")
+ _, _ = w.Write([]byte(spaHTML))
+ })
+ mux.HandleFunc("/app.js", func(w http.ResponseWriter, r *http.Request) {
+ time.Sleep(50 * time.Millisecond)
+ w.Header().Set("Content-Type", "application/javascript")
+ _, _ = w.Write([]byte(spaJS))
+ })
+ srv := httptest.NewServer(mux)
+ t.Cleanup(srv.Close)
+ return srv
+}
+
+func newScreenshotBrowser(t testing.TB, useLocal bool) *Browser {
+ t.Helper()
+ browser, err := NewBrowser("", useLocal, nil)
+ if err != nil {
+ t.Skipf("headless browser unavailable: %v", err)
+ }
+ t.Cleanup(browser.Close)
+ return browser
+}
+
+func captureSPA(t testing.TB, browser *Browser, url string) (screenshot []byte, body string) {
+ t.Helper()
+ screenshot, body, _, err := browser.ScreenshotWithBody(url, 15*time.Second, 200*time.Millisecond, nil, false, nil)
+ if err != nil {
+ t.Fatalf("screenshot: %v", err)
+ }
+ return screenshot, body
+}
+
+func TestScreenshotSPAWaitsForHydration(t *testing.T) {
+ srv := startSPAScreenshotServer(t)
+ browser := newScreenshotBrowser(t, false)
+
+ screenshot, body := captureSPA(t, browser, srv.URL)
+ if len(screenshot) == 0 {
+ t.Fatal("empty screenshot")
+ }
+ if !strings.Contains(body, spaHydratedMarker) {
+ t.Fatalf("early capture: body missing %q\n%s", spaHydratedMarker, body)
+ }
+}
+
+func TestScreenshotSPAConcurrentNoEarlyCapture(t *testing.T) {
+ const workers = 8
+ srv := startSPAScreenshotServer(t)
+ browser := newScreenshotBrowser(t, false)
+
+ var wg sync.WaitGroup
+ errs := make(chan error, workers)
+ for i := 0; i < workers; i++ {
+ wg.Add(1)
+ go func() {
+ defer wg.Done()
+ screenshot, body, _, err := browser.ScreenshotWithBody(srv.URL, 15*time.Second, 200*time.Millisecond, nil, false, nil)
+ if err != nil {
+ errs <- err
+ return
+ }
+ if len(screenshot) == 0 {
+ errs <- fmt.Errorf("empty screenshot")
+ return
+ }
+ if !strings.Contains(body, spaHydratedMarker) {
+ errs <- fmt.Errorf("early capture: body missing %q", spaHydratedMarker)
+ }
+ }()
+ }
+ wg.Wait()
+ close(errs)
+ for err := range errs {
+ t.Error(err)
+ }
+}
+
+// BenchmarkScreenshotSPA times concurrent captures of the local SPA.
+//
+// On linux/amd64, /default uses chrome-headless-shell when available and
+// /local forces a system Chrome via NewBrowser(..., useLocal=true). Compare:
+//
+// go test -bench=BenchmarkScreenshotSPA -benchtime=20s -count=5 ./runner
+func BenchmarkScreenshotSPA(b *testing.B) {
+ for _, useLocal := range []bool{false, true} {
+ name := "default"
+ if useLocal {
+ name = "local"
+ }
+ b.Run(name, func(b *testing.B) {
+ srv := startSPAScreenshotServer(b)
+ browser := newScreenshotBrowser(b, useLocal)
+ b.ResetTimer()
+ for i := 0; i < b.N; i++ {
+ screenshot, body, _, err := browser.ScreenshotWithBody(srv.URL, 15*time.Second, 200*time.Millisecond, nil, false, nil)
+ if err != nil {
+ b.Fatalf("screenshot: %v", err)
+ }
+ if len(screenshot) == 0 || !strings.Contains(body, spaHydratedMarker) {
+ b.Fatal("early or empty capture")
+ }
+ }
+ })
+ }
+}