From 509088ef0415c7415dab567ab8c4900093b9596c Mon Sep 17 00:00:00 2001 From: Mzack9999 Date: Sat, 15 Aug 2026 19:22:28 +0400 Subject: [PATCH 1/9] speed screenshots --- runner/headless.go | 40 +++++++++++++++++++++++++++++++++------- 1 file changed, 33 insertions(+), 7 deletions(-) diff --git a/runner/headless.go b/runner/headless.go index 3d295d82..a622cede 100644 --- a/runner/headless.go +++ b/runner/headless.go @@ -10,6 +10,7 @@ import ( "github.com/go-rod/rod/lib/launcher" "github.com/go-rod/rod/lib/launcher/flags" "github.com/go-rod/rod/lib/proto" + "github.com/go-rod/rod/lib/utils" "github.com/pkg/errors" fileutil "github.com/projectdiscovery/utils/file" mapsutil "github.com/projectdiscovery/utils/maps" @@ -61,6 +62,26 @@ func NewBrowser(proxy string, useLocal bool, optionalArgs map[string]string) (*B Set("window-size", fmt.Sprintf("%d,%d", 1080, 1920)). Set("mute-audio", "true"). Set("incognito", "true"). + // Performance: keep background/occluded tabs running at full speed so + // many concurrent screenshot pages don't get render-throttled. + Set("disable-background-timer-throttling", "true"). + Set("disable-backgrounding-occluded-windows", "true"). + Set("disable-renderer-backgrounding", "true"). + Set("disable-ipc-flooding-protection", "true"). + Set("disable-hang-monitor", "true"). + // Performance: strip background chrome services we never use. + Set("disable-background-networking", "true"). + Set("disable-client-side-phishing-detection", "true"). + Set("disable-component-update", "true"). + Set("disable-default-apps", "true"). + Set("disable-domain-reliability", "true"). + Set("disable-extensions", "true"). + Set("disable-sync", "true"). + Set("no-first-run", "true"). + Set("no-default-browser-check", "true"). + Set("metrics-recording-only", "true"). + Set("safebrowsing-disable-auto-update", "true"). + Set("disable-features", "Translate,BackForwardCache,AcceptCHFrame,MediaRouter,OptimizationHints,site-per-process"). Delete("use-mock-keychain"). Headless(true). UserDataDir(dataStore) @@ -220,19 +241,24 @@ func (b *Browser) setupPageAndNavigate(url string, timeout time.Duration, header } } - page.Timeout(5 * time.Second).WaitNavigation(proto.PageLifecycleEventNameFirstMeaningfulPaint)() - return page, networkRequests.Slice, nil } // takeScreenshotAndGetBody performs the screenshot actions func (b *Browser) takeScreenshotAndGetBody(page *rod.Page, idle time.Duration, fullPage bool) ([]byte, string, error) { - if err := page.WaitLoad(); err != nil { - return nil, "", err - } - _ = page.WaitIdle(idle) + // Wait for the page to be visually ready. WaitLoad returns immediately if + // window.onload already fired, and WaitIdle resolves on the next idle + // callback (or after idle). Running them concurrently, both bounded by the + // page timeout set by the caller, avoids the slow serial navigation waits. + utils.All(func() { + _ = page.WaitLoad() + }, func() { + _ = page.WaitIdle(idle) + })() - screenshot, err := page.Screenshot(fullPage, &proto.PageCaptureScreenshot{}) + screenshot, err := page.Screenshot(fullPage, &proto.PageCaptureScreenshot{ + OptimizeForSpeed: true, + }) if err != nil { return nil, "", err } From 6e09825a3652c877bfdd66bb331f75c8172ac9d6 Mon Sep 17 00:00:00 2001 From: Mzack9999 Date: Sat, 15 Aug 2026 19:29:58 +0400 Subject: [PATCH 2/9] fix waits --- runner/headless.go | 15 +++++---------- 1 file changed, 5 insertions(+), 10 deletions(-) diff --git a/runner/headless.go b/runner/headless.go index a622cede..8c2be060 100644 --- a/runner/headless.go +++ b/runner/headless.go @@ -10,7 +10,6 @@ import ( "github.com/go-rod/rod/lib/launcher" "github.com/go-rod/rod/lib/launcher/flags" "github.com/go-rod/rod/lib/proto" - "github.com/go-rod/rod/lib/utils" "github.com/pkg/errors" fileutil "github.com/projectdiscovery/utils/file" mapsutil "github.com/projectdiscovery/utils/maps" @@ -246,15 +245,11 @@ func (b *Browser) setupPageAndNavigate(url string, timeout time.Duration, header // takeScreenshotAndGetBody performs the screenshot actions func (b *Browser) takeScreenshotAndGetBody(page *rod.Page, idle time.Duration, fullPage bool) ([]byte, string, error) { - // Wait for the page to be visually ready. WaitLoad returns immediately if - // window.onload already fired, and WaitIdle resolves on the next idle - // callback (or after idle). Running them concurrently, both bounded by the - // page timeout set by the caller, avoids the slow serial navigation waits. - utils.All(func() { - _ = page.WaitLoad() - }, func() { - _ = page.WaitIdle(idle) - })() + // WaitLoad returns immediately if onload already fired. Keep WaitIdle + // serial after it: concurrent rod evaluates on the same page can race and + // screenshot before SPA hydration finishes. + _ = page.WaitLoad() + _ = page.WaitIdle(idle) screenshot, err := page.Screenshot(fullPage, &proto.PageCaptureScreenshot{ OptimizeForSpeed: true, From 9137eecec7ab96f1b739d78b5f0b14c5dd35dab4 Mon Sep 17 00:00:00 2001 From: Mzack9999 Date: Sat, 15 Aug 2026 19:31:02 +0400 Subject: [PATCH 3/9] wait network --- runner/headless.go | 19 ++++++++++--------- 1 file changed, 10 insertions(+), 9 deletions(-) diff --git a/runner/headless.go b/runner/headless.go index 8c2be060..59057981 100644 --- a/runner/headless.go +++ b/runner/headless.go @@ -139,13 +139,13 @@ func NewBrowser(proxy string, useLocal bool, optionalArgs map[string]string) (*B } func (b *Browser) ScreenshotWithBody(url string, timeout time.Duration, idle time.Duration, headers []string, fullPage bool, jsCodes []string) ([]byte, string, []NetworkRequest, error) { - page, networkRequests, err := b.setupPageAndNavigate(url, timeout, headers, jsCodes) + page, networkRequests, err := b.setupPageAndNavigate(url, timeout, idle, headers, jsCodes) if err != nil { return nil, "", []NetworkRequest{}, err } defer b.closePage(page) - screenshot, body, err := b.takeScreenshotAndGetBody(page, idle, fullPage) + screenshot, body, err := b.takeScreenshotAndGetBody(page, fullPage) if err != nil { return nil, "", networkRequests, err } @@ -154,7 +154,7 @@ func (b *Browser) ScreenshotWithBody(url string, timeout time.Duration, idle tim } // setupPageAndNavigate opens a page, performs all adaptive actions including JS injection -func (b *Browser) setupPageAndNavigate(url string, timeout time.Duration, headers []string, jsCodes []string) (*rod.Page, []NetworkRequest, error) { +func (b *Browser) setupPageAndNavigate(url string, timeout time.Duration, idle time.Duration, headers []string, jsCodes []string) (*rod.Page, []NetworkRequest, error) { page, err := b.engine.Page(proto.TargetCreateTarget{}) if err != nil { return nil, []NetworkRequest{}, err @@ -228,6 +228,9 @@ func (b *Browser) setupPageAndNavigate(url string, timeout time.Duration, header } page = page.Timeout(timeout) + // Register before Navigate so SPA XHRs are tracked. WaitIdle(rIC) can fire + // between hydration gaps; request-idle waits for a quiet network window. + wait := page.WaitRequestIdle(idle, nil, nil, nil) if err := page.Navigate(url); err != nil { return page, networkRequests.Slice, err @@ -240,16 +243,14 @@ func (b *Browser) setupPageAndNavigate(url string, timeout time.Duration, header } } + wait() + return page, networkRequests.Slice, nil } // takeScreenshotAndGetBody performs the screenshot actions -func (b *Browser) takeScreenshotAndGetBody(page *rod.Page, idle time.Duration, fullPage bool) ([]byte, string, error) { - // WaitLoad returns immediately if onload already fired. Keep WaitIdle - // serial after it: concurrent rod evaluates on the same page can race and - // screenshot before SPA hydration finishes. - _ = page.WaitLoad() - _ = page.WaitIdle(idle) +func (b *Browser) takeScreenshotAndGetBody(page *rod.Page, fullPage bool) ([]byte, string, error) { + _ = page.WaitRepaint() screenshot, err := page.Screenshot(fullPage, &proto.PageCaptureScreenshot{ OptimizeForSpeed: true, From a7bd8b6c870665bc4c4870ccb139cf4b31ee3029 Mon Sep 17 00:00:00 2001 From: Mzack9999 Date: Sat, 15 Aug 2026 19:40:39 +0400 Subject: [PATCH 4/9] add chrome-shell --- runner/chrome_shell.go | 78 +++++++++++++++++++++++++++++++++++++ runner/chrome_shell_test.go | 33 ++++++++++++++++ runner/headless.go | 6 +++ 3 files changed, 117 insertions(+) create mode 100644 runner/chrome_shell.go create mode 100644 runner/chrome_shell_test.go diff --git a/runner/chrome_shell.go b/runner/chrome_shell.go new file mode 100644 index 00000000..03e1642b --- /dev/null +++ b/runner/chrome_shell.go @@ -0,0 +1,78 @@ +package runner + +import ( + "fmt" + "os" + "path/filepath" + "runtime" + + "github.com/go-rod/rod/lib/launcher" + "github.com/ysmood/leakless" +) + +const ( + // chromeShellVersion pins Chrome for Testing's chrome-headless-shell build. + // Smaller and faster than full Chromium for headless screenshots on Linux. + chromeShellVersion = "152.0.7977.42" + // chromeShellRevision is a synthetic cache key for launcher.Browser.Dir. + chromeShellRevision = 1520797742 +) + +// HostChromeShell returns the Chrome for Testing chrome-headless-shell zip for +// linux/amd64. Other platforms should not use this host. +func HostChromeShell(_ int) string { + return fmt.Sprintf( + "https://storage.googleapis.com/chrome-for-testing-public/%s/linux64/chrome-headless-shell-linux64.zip", + chromeShellVersion, + ) +} + +func supportsChromeShellDownload() bool { + return runtime.GOOS == "linux" && runtime.GOARCH == "amd64" +} + +// ensureLinuxChromeShell downloads chrome-headless-shell once into the rod +// browser cache and returns the executable path. +func ensureLinuxChromeShell() (string, error) { + b := launcher.NewBrowser() + b.Revision = chromeShellRevision + b.Hosts = []launcher.Host{HostChromeShell} + + defer leakless.LockPort(b.LockPort)() + + if p := chromeShellBin(b.Dir()); p != "" { + return p, nil + } + + _ = os.RemoveAll(b.Dir()) + if err := b.Download(); err != nil { + return "", err + } + + p := chromeShellBin(b.Dir()) + if p == "" { + return "", fmt.Errorf("chrome-headless-shell binary missing after download in %s", b.Dir()) + } + + // go-rod's linux BinPath expects "chrome"; keep a symlink for Validate(). + chrome := b.BinPath() + if p != chrome { + _ = os.Remove(chrome) + if err := os.Symlink(filepath.Base(p), chrome); err != nil { + // Non-fatal: we still launch via the real shell path. + _ = err + } + } + + return p, nil +} + +func chromeShellBin(dir string) string { + for _, name := range []string{"chrome-headless-shell", "headless_shell", "chrome"} { + p := filepath.Join(dir, name) + if fi, err := os.Stat(p); err == nil && !fi.IsDir() { + return p + } + } + return "" +} diff --git a/runner/chrome_shell_test.go b/runner/chrome_shell_test.go new file mode 100644 index 00000000..7b8da0c4 --- /dev/null +++ b/runner/chrome_shell_test.go @@ -0,0 +1,33 @@ +package runner + +import ( + "os" + "path/filepath" + "strings" + "testing" +) + +func TestHostChromeShell(t *testing.T) { + u := HostChromeShell(0) + if !strings.Contains(u, "chrome-for-testing-public/"+chromeShellVersion+"/") { + t.Fatalf("unexpected host url: %s", u) + } + if !strings.HasSuffix(u, "/linux64/chrome-headless-shell-linux64.zip") { + t.Fatalf("unexpected zip path: %s", u) + } +} + +func TestChromeShellBin(t *testing.T) { + dir := t.TempDir() + if got := chromeShellBin(dir); got != "" { + t.Fatalf("expected empty, got %q", got) + } + + shell := filepath.Join(dir, "chrome-headless-shell") + if err := os.WriteFile(shell, []byte("x"), 0o755); err != nil { + t.Fatal(err) + } + if got := chromeShellBin(dir); got != shell { + t.Fatalf("got %q want %q", got, shell) + } +} diff --git a/runner/headless.go b/runner/headless.go index 59057981..e5df2e53 100644 --- a/runner/headless.go +++ b/runner/headless.go @@ -102,6 +102,12 @@ func NewBrowser(proxy string, useLocal bool, optionalArgs map[string]string) (*B } else { return nil, errors.New("the chrome browser is not installed") } + } else if supportsChromeShellDownload() { + // Prefer chrome-headless-shell on linux/amd64: smaller download and + // faster headless screenshots than full Chromium snapshots. + if shellPath, err := ensureLinuxChromeShell(); err == nil { + chromeLauncher.Bin(shellPath) + } } if proxy == "" { From 973542f0743f0359854b4de2840c940a43505d0e Mon Sep 17 00:00:00 2001 From: Mzack9999 Date: Sat, 15 Aug 2026 20:08:31 +0400 Subject: [PATCH 5/9] gate render --- runner/headless.go | 25 +++++++++++++++++++++---- 1 file changed, 21 insertions(+), 4 deletions(-) diff --git a/runner/headless.go b/runner/headless.go index e5df2e53..41b83581 100644 --- a/runner/headless.go +++ b/runner/headless.go @@ -234,9 +234,9 @@ func (b *Browser) setupPageAndNavigate(url string, timeout time.Duration, idle t } page = page.Timeout(timeout) - // Register before Navigate so SPA XHRs are tracked. WaitIdle(rIC) can fire - // between hydration gaps; request-idle waits for a quiet network window. - wait := page.WaitRequestIdle(idle, nil, nil, nil) + // Register before Navigate so the first SPA XHRs are tracked by the + // request-idle waiter. + waitReqIdle := page.WaitRequestIdle(idle, nil, nil, nil) if err := page.Navigate(url); err != nil { return page, networkRequests.Slice, err @@ -249,11 +249,28 @@ func (b *Browser) setupPageAndNavigate(url string, timeout time.Duration, idle t } } - wait() + b.waitPageReady(page, idle, waitReqIdle) return page, networkRequests.Slice, nil } +// waitPageReady blocks until the page is visually settled so we don't capture a +// half-rendered SPA. It gates on three independent signals, each bounded by the +// page timeout: +// - window.onload +// - network request-idle (a quiet network window) +// - DOM stability (the rendered tree stops mutating) +// +// SPAs paint late: the network can briefly go quiet before hydration starts, so +// request-idle alone can fire on the boot screen. Requiring DOM stability on top +// closes that gap. DOM stability is re-checked after the network settles so the +// first snapshot is taken post-hydration rather than on the boot screen. +func (b *Browser) waitPageReady(page *rod.Page, idle time.Duration, waitReqIdle func()) { + _ = page.WaitLoad() + waitReqIdle() + _ = page.WaitDOMStable(idle, 0) +} + // takeScreenshotAndGetBody performs the screenshot actions func (b *Browser) takeScreenshotAndGetBody(page *rod.Page, fullPage bool) ([]byte, string, error) { _ = page.WaitRepaint() From 4e98d97e8fb9d2bc3c232aa0b7362bf62d8ae943 Mon Sep 17 00:00:00 2001 From: Mzack9999 Date: Sat, 15 Aug 2026 21:11:26 +0400 Subject: [PATCH 6/9] use chromeshell --- go.mod | 2 + go.sum | 2 - runner/chrome_shell.go | 78 ------------------------------------- runner/chrome_shell_test.go | 33 ---------------- runner/headless.go | 5 ++- 5 files changed, 5 insertions(+), 115 deletions(-) delete mode 100644 runner/chrome_shell.go delete mode 100644 runner/chrome_shell_test.go diff --git a/go.mod b/go.mod index 18d04dc3..db39ab4f 100644 --- a/go.mod +++ b/go.mod @@ -172,3 +172,5 @@ require ( golang.org/x/tools v0.47.0 // indirect gopkg.in/ini.v1 v1.67.0 // indirect ) + +replace github.com/projectdiscovery/utils => ../utils diff --git a/go.sum b/go.sum index 4c1afc11..1c4ca0a4 100644 --- a/go.sum +++ b/go.sum @@ -297,8 +297,6 @@ github.com/projectdiscovery/tlsx v1.2.2 h1:Y96QBqeD2anpzEtBl4kqNbwzXh2TrzJuXfgiB github.com/projectdiscovery/tlsx v1.2.2/go.mod h1:ZJl9F1sSl0sdwE+lR0yuNHVX4Zx6tCSTqnNxnHCFZB4= github.com/projectdiscovery/useragent v0.0.108 h1:fb+uLuFJvC+MHZjCtxQJxtvp1X6A8n98CUGPyFcg3NE= github.com/projectdiscovery/useragent v0.0.108/go.mod h1:XdNRrlvtDmYfVL1Oybat4uMe+W6cLwsK9S18ond17CI= -github.com/projectdiscovery/utils v0.11.1 h1:PWj1KjIASxt8icxommH72C0TQqNOvGkcSODRkiq0SQw= -github.com/projectdiscovery/utils v0.11.1/go.mod h1:yktGrHGk2CTjNiccXovnvGrLHX9sV2bqz9nSnbA3V8M= github.com/projectdiscovery/wappalyzergo v0.2.89 h1:z4TMGcX2iq1+wF94srp5F/TzyUL18Od/l0B+b7PBfqM= github.com/projectdiscovery/wappalyzergo v0.2.89/go.mod h1:gMH0o5lBp65sKMwHx/tuUdOtW2RjodC6Ti+9QDsYMkY= github.com/refraction-networking/utls v1.8.2 h1:j4Q1gJj0xngdeH+Ox/qND11aEfhpgoEvV+S9iJ2IdQo= diff --git a/runner/chrome_shell.go b/runner/chrome_shell.go deleted file mode 100644 index 03e1642b..00000000 --- a/runner/chrome_shell.go +++ /dev/null @@ -1,78 +0,0 @@ -package runner - -import ( - "fmt" - "os" - "path/filepath" - "runtime" - - "github.com/go-rod/rod/lib/launcher" - "github.com/ysmood/leakless" -) - -const ( - // chromeShellVersion pins Chrome for Testing's chrome-headless-shell build. - // Smaller and faster than full Chromium for headless screenshots on Linux. - chromeShellVersion = "152.0.7977.42" - // chromeShellRevision is a synthetic cache key for launcher.Browser.Dir. - chromeShellRevision = 1520797742 -) - -// HostChromeShell returns the Chrome for Testing chrome-headless-shell zip for -// linux/amd64. Other platforms should not use this host. -func HostChromeShell(_ int) string { - return fmt.Sprintf( - "https://storage.googleapis.com/chrome-for-testing-public/%s/linux64/chrome-headless-shell-linux64.zip", - chromeShellVersion, - ) -} - -func supportsChromeShellDownload() bool { - return runtime.GOOS == "linux" && runtime.GOARCH == "amd64" -} - -// ensureLinuxChromeShell downloads chrome-headless-shell once into the rod -// browser cache and returns the executable path. -func ensureLinuxChromeShell() (string, error) { - b := launcher.NewBrowser() - b.Revision = chromeShellRevision - b.Hosts = []launcher.Host{HostChromeShell} - - defer leakless.LockPort(b.LockPort)() - - if p := chromeShellBin(b.Dir()); p != "" { - return p, nil - } - - _ = os.RemoveAll(b.Dir()) - if err := b.Download(); err != nil { - return "", err - } - - p := chromeShellBin(b.Dir()) - if p == "" { - return "", fmt.Errorf("chrome-headless-shell binary missing after download in %s", b.Dir()) - } - - // go-rod's linux BinPath expects "chrome"; keep a symlink for Validate(). - chrome := b.BinPath() - if p != chrome { - _ = os.Remove(chrome) - if err := os.Symlink(filepath.Base(p), chrome); err != nil { - // Non-fatal: we still launch via the real shell path. - _ = err - } - } - - return p, nil -} - -func chromeShellBin(dir string) string { - for _, name := range []string{"chrome-headless-shell", "headless_shell", "chrome"} { - p := filepath.Join(dir, name) - if fi, err := os.Stat(p); err == nil && !fi.IsDir() { - return p - } - } - return "" -} diff --git a/runner/chrome_shell_test.go b/runner/chrome_shell_test.go deleted file mode 100644 index 7b8da0c4..00000000 --- a/runner/chrome_shell_test.go +++ /dev/null @@ -1,33 +0,0 @@ -package runner - -import ( - "os" - "path/filepath" - "strings" - "testing" -) - -func TestHostChromeShell(t *testing.T) { - u := HostChromeShell(0) - if !strings.Contains(u, "chrome-for-testing-public/"+chromeShellVersion+"/") { - t.Fatalf("unexpected host url: %s", u) - } - if !strings.HasSuffix(u, "/linux64/chrome-headless-shell-linux64.zip") { - t.Fatalf("unexpected zip path: %s", u) - } -} - -func TestChromeShellBin(t *testing.T) { - dir := t.TempDir() - if got := chromeShellBin(dir); got != "" { - t.Fatalf("expected empty, got %q", got) - } - - shell := filepath.Join(dir, "chrome-headless-shell") - if err := os.WriteFile(shell, []byte("x"), 0o755); err != nil { - t.Fatal(err) - } - if got := chromeShellBin(dir); got != shell { - t.Fatalf("got %q want %q", got, shell) - } -} diff --git a/runner/headless.go b/runner/headless.go index 41b83581..24ee6313 100644 --- a/runner/headless.go +++ b/runner/headless.go @@ -11,6 +11,7 @@ import ( "github.com/go-rod/rod/lib/launcher/flags" "github.com/go-rod/rod/lib/proto" "github.com/pkg/errors" + "github.com/projectdiscovery/utils/chromeshell" fileutil "github.com/projectdiscovery/utils/file" mapsutil "github.com/projectdiscovery/utils/maps" osutils "github.com/projectdiscovery/utils/os" @@ -102,10 +103,10 @@ func NewBrowser(proxy string, useLocal bool, optionalArgs map[string]string) (*B } else { return nil, errors.New("the chrome browser is not installed") } - } else if supportsChromeShellDownload() { + } else if chromeshell.Supported() { // Prefer chrome-headless-shell on linux/amd64: smaller download and // faster headless screenshots than full Chromium snapshots. - if shellPath, err := ensureLinuxChromeShell(); err == nil { + if shellPath, err := chromeshell.Ensure(); err == nil { chromeLauncher.Bin(shellPath) } } From ac4f888abcdb3c2585189d1657001455ca80481b Mon Sep 17 00:00:00 2001 From: Mzack9999 Date: Sat, 15 Aug 2026 21:21:01 +0400 Subject: [PATCH 7/9] pin utils --- go.mod | 4 +--- go.sum | 2 ++ 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index db39ab4f..25b46f27 100644 --- a/go.mod +++ b/go.mod @@ -35,7 +35,7 @@ require ( github.com/projectdiscovery/retryablehttp-go v1.3.18 github.com/projectdiscovery/tlsx v1.2.2 github.com/projectdiscovery/useragent v0.0.108 - github.com/projectdiscovery/utils v0.11.1 + github.com/projectdiscovery/utils v0.11.2-0.20260815171005-eb8925425716 github.com/projectdiscovery/wappalyzergo v0.2.89 github.com/rs/xid v1.6.0 github.com/spaolacci/murmur3 v1.1.0 @@ -172,5 +172,3 @@ require ( golang.org/x/tools v0.47.0 // indirect gopkg.in/ini.v1 v1.67.0 // indirect ) - -replace github.com/projectdiscovery/utils => ../utils diff --git a/go.sum b/go.sum index 1c4ca0a4..d979a7d6 100644 --- a/go.sum +++ b/go.sum @@ -297,6 +297,8 @@ github.com/projectdiscovery/tlsx v1.2.2 h1:Y96QBqeD2anpzEtBl4kqNbwzXh2TrzJuXfgiB github.com/projectdiscovery/tlsx v1.2.2/go.mod h1:ZJl9F1sSl0sdwE+lR0yuNHVX4Zx6tCSTqnNxnHCFZB4= github.com/projectdiscovery/useragent v0.0.108 h1:fb+uLuFJvC+MHZjCtxQJxtvp1X6A8n98CUGPyFcg3NE= github.com/projectdiscovery/useragent v0.0.108/go.mod h1:XdNRrlvtDmYfVL1Oybat4uMe+W6cLwsK9S18ond17CI= +github.com/projectdiscovery/utils v0.11.2-0.20260815171005-eb8925425716 h1:zoDnj6xAqCvIQuP/bV5COuje2kjZV7THKTuDRdvyXgI= +github.com/projectdiscovery/utils v0.11.2-0.20260815171005-eb8925425716/go.mod h1:RtBO9urHlfN3aAEzhZz5m1/ZeJDq6U1Ka34lGWleuS8= github.com/projectdiscovery/wappalyzergo v0.2.89 h1:z4TMGcX2iq1+wF94srp5F/TzyUL18Od/l0B+b7PBfqM= github.com/projectdiscovery/wappalyzergo v0.2.89/go.mod h1:gMH0o5lBp65sKMwHx/tuUdOtW2RjodC6Ti+9QDsYMkY= github.com/refraction-networking/utls v1.8.2 h1:j4Q1gJj0xngdeH+Ox/qND11aEfhpgoEvV+S9iJ2IdQo= From da14e2015fc6e84612b50caab853f6bcff0dbbc7 Mon Sep 17 00:00:00 2001 From: Mzack9999 Date: Wed, 26 Aug 2026 14:13:16 +0400 Subject: [PATCH 8/9] add bench --- runner/headless_screenshot_test.go | 142 +++++++++++++++++++++++++++++ 1 file changed, 142 insertions(+) create mode 100644 runner/headless_screenshot_test.go diff --git a/runner/headless_screenshot_test.go b/runner/headless_screenshot_test.go new file mode 100644 index 00000000..b8b74ce7 --- /dev/null +++ b/runner/headless_screenshot_test.go @@ -0,0 +1,142 @@ +package runner + +import ( + "fmt" + "net/http" + "net/http/httptest" + "strings" + "sync" + "testing" + "time" +) + +const spaHydratedMarker = "hydrated-ok" + +// spaHTML is a tiny SPA-style page: it paints a boot screen first, then +// fetches /app.js which hydrates #root after a short delay. Early captures +// only contain "boot"; a correct wait must include spaHydratedMarker. +const spaHTML = ` +spa-bench + +
boot
+ +` + +const spaJS = ` +setTimeout(function(){ + document.getElementById("root").textContent = "` + spaHydratedMarker + `"; +}, 80); +` + +func startSPAScreenshotServer(t testing.TB) *httptest.Server { + t.Helper() + mux := http.NewServeMux() + mux.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "text/html; charset=utf-8") + _, _ = w.Write([]byte(spaHTML)) + }) + mux.HandleFunc("/app.js", func(w http.ResponseWriter, r *http.Request) { + time.Sleep(50 * time.Millisecond) + w.Header().Set("Content-Type", "application/javascript") + _, _ = w.Write([]byte(spaJS)) + }) + srv := httptest.NewServer(mux) + t.Cleanup(srv.Close) + return srv +} + +func newScreenshotBrowser(t testing.TB, useLocal bool) *Browser { + t.Helper() + browser, err := NewBrowser("", useLocal, nil) + if err != nil { + t.Skipf("headless browser unavailable: %v", err) + } + t.Cleanup(browser.Close) + return browser +} + +func captureSPA(t testing.TB, browser *Browser, url string) (screenshot []byte, body string) { + t.Helper() + screenshot, body, _, err := browser.ScreenshotWithBody(url, 15*time.Second, 200*time.Millisecond, nil, false, nil) + if err != nil { + t.Fatalf("screenshot: %v", err) + } + return screenshot, body +} + +func TestScreenshotSPAWaitsForHydration(t *testing.T) { + srv := startSPAScreenshotServer(t) + browser := newScreenshotBrowser(t, false) + + screenshot, body := captureSPA(t, browser, srv.URL) + if len(screenshot) == 0 { + t.Fatal("empty screenshot") + } + if !strings.Contains(body, spaHydratedMarker) { + t.Fatalf("early capture: body missing %q\n%s", spaHydratedMarker, body) + } +} + +func TestScreenshotSPAConcurrentNoEarlyCapture(t *testing.T) { + const workers = 8 + srv := startSPAScreenshotServer(t) + browser := newScreenshotBrowser(t, false) + + var wg sync.WaitGroup + errs := make(chan error, workers) + for i := 0; i < workers; i++ { + wg.Add(1) + go func() { + defer wg.Done() + screenshot, body, _, err := browser.ScreenshotWithBody(srv.URL, 15*time.Second, 200*time.Millisecond, nil, false, nil) + if err != nil { + errs <- err + return + } + if len(screenshot) == 0 { + errs <- fmt.Errorf("empty screenshot") + return + } + if !strings.Contains(body, spaHydratedMarker) { + errs <- fmt.Errorf("early capture: body missing %q", spaHydratedMarker) + } + }() + } + wg.Wait() + close(errs) + for err := range errs { + t.Error(err) + } +} + +// BenchmarkScreenshotSPA times concurrent captures of the local SPA. +// +// On linux/amd64, /default uses chrome-headless-shell when available and +// /local forces a system Chrome via NewBrowser(..., useLocal=true). Compare: +// +// go test -bench=BenchmarkScreenshotSPA -benchtime=20s -count=5 ./runner +func BenchmarkScreenshotSPA(b *testing.B) { + for _, useLocal := range []bool{false, true} { + name := "default" + if useLocal { + name = "local" + } + b.Run(name, func(b *testing.B) { + srv := startSPAScreenshotServer(b) + browser := newScreenshotBrowser(b, useLocal) + b.ResetTimer() + b.RunParallel(func(pb *testing.PB) { + for pb.Next() { + screenshot, body, _, err := browser.ScreenshotWithBody(srv.URL, 15*time.Second, 200*time.Millisecond, nil, false, nil) + if err != nil { + b.Errorf("screenshot: %v", err) + continue + } + if len(screenshot) == 0 || !strings.Contains(body, spaHydratedMarker) { + b.Errorf("early or empty capture") + } + } + }) + }) + } +} From 0dbc874d419eaeaecdeb092d8950b9683ad35426 Mon Sep 17 00:00:00 2001 From: Mzack9999 Date: Wed, 26 Aug 2026 14:30:06 +0400 Subject: [PATCH 9/9] fix bench --- runner/headless_screenshot_test.go | 19 ++++++++----------- 1 file changed, 8 insertions(+), 11 deletions(-) diff --git a/runner/headless_screenshot_test.go b/runner/headless_screenshot_test.go index b8b74ce7..e21297db 100644 --- a/runner/headless_screenshot_test.go +++ b/runner/headless_screenshot_test.go @@ -125,18 +125,15 @@ func BenchmarkScreenshotSPA(b *testing.B) { srv := startSPAScreenshotServer(b) browser := newScreenshotBrowser(b, useLocal) b.ResetTimer() - b.RunParallel(func(pb *testing.PB) { - for pb.Next() { - screenshot, body, _, err := browser.ScreenshotWithBody(srv.URL, 15*time.Second, 200*time.Millisecond, nil, false, nil) - if err != nil { - b.Errorf("screenshot: %v", err) - continue - } - if len(screenshot) == 0 || !strings.Contains(body, spaHydratedMarker) { - b.Errorf("early or empty capture") - } + for i := 0; i < b.N; i++ { + screenshot, body, _, err := browser.ScreenshotWithBody(srv.URL, 15*time.Second, 200*time.Millisecond, nil, false, nil) + if err != nil { + b.Fatalf("screenshot: %v", err) } - }) + if len(screenshot) == 0 || !strings.Contains(body, spaHydratedMarker) { + b.Fatal("early or empty capture") + } + } }) } }