Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 2 additions & 0 deletions src/ciberwebscan/core/client/http_client.py
Original file line number Diff line number Diff line change
Expand Up @@ -263,6 +263,7 @@ def __init__(
verify: bool | None = None,
follow_redirects: bool | None = None,
default_headers: dict[str, str] | None = None,
cookies: dict[str, str] | None = None,
metrics_callback: MetricsCallback | None = None,
proxy: str | None = None,
retryable_status_codes: set[int] | None = None,
Expand Down Expand Up @@ -374,6 +375,7 @@ def __init__(
follow_redirects=resolved_follow_redirects,
max_redirects=resolved_max_redirects,
headers=default_headers,
cookies=cookies,
proxy=resolved_proxy,
)

Expand Down
6 changes: 4 additions & 2 deletions src/ciberwebscan/services/analyze_service.py
Original file line number Diff line number Diff line change
Expand Up @@ -501,9 +501,10 @@ def _analyze_headers(
with HTTPClient(
timeout=timeout,
default_headers=default_headers or None,
cookies=options.cookies or None,
proxy=self._resolve_proxy(options.proxy),
) as client:
resp = client.get(url, cookies=options.cookies or None)
resp = client.get(url)

response_headers = dict(resp.headers)
analysis = self.headers_analyzer.analyze(response_headers)
Expand Down Expand Up @@ -570,9 +571,10 @@ def _fingerprint(
with HTTPClient(
timeout=timeout,
default_headers=default_headers or None,
cookies=options.cookies or None,
proxy=self._resolve_proxy(options.proxy),
) as client:
resp = client.get(url, cookies=options.cookies or None)
resp = client.get(url)
headers = dict(resp.headers)
html = resp.text

Expand Down
1 change: 1 addition & 0 deletions src/ciberwebscan/services/attack_service.py
Original file line number Diff line number Diff line change
Expand Up @@ -364,6 +364,7 @@ def attack(self, options: AttackOptions) -> ServiceResult[AttackResult]:
verify=http_config.verify_ssl,
follow_redirects=http_config.follow_redirects,
default_headers=default_headers or None,
cookies=options.cookies or None,
proxy=self._resolve_proxy(options.proxy),
)

Expand Down
22 changes: 16 additions & 6 deletions src/ciberwebscan/services/scrape_service.py
Original file line number Diff line number Diff line change
Expand Up @@ -119,6 +119,7 @@ def _build_http_client(
*,
proxy: str | None = None,
verify: bool | None = None,
cookies: dict[str, str] | None = None,
):
from ciberwebscan.core.client import HTTPClient

Expand All @@ -140,6 +141,7 @@ def _build_http_client(
http2=http_config.http2,
verify=http_config.verify_ssl if verify is None else verify,
follow_redirects=http_config.follow_redirects,
cookies=cookies,
proxy=proxy,
default_headers=default_headers,
)
Expand Down Expand Up @@ -199,11 +201,18 @@ def static_scraper(self) -> StaticScraper:
return self._static_scraper

def _get_static_scraper_with_proxy(
self, proxy: str | None, verify_ssl: bool = True
self,
proxy: str | None,
verify_ssl: bool = True,
cookies: dict[str, str] | None = None,
) -> StaticScraper:
"""Get static scraper with specific proxy configuration."""
if proxy or not verify_ssl:
client = self._build_http_client(proxy=proxy, verify=verify_ssl)
"""Get static scraper with specific proxy/cookies configuration."""
if proxy or not verify_ssl or cookies:
client = self._build_http_client(
proxy=proxy,
verify=verify_ssl,
cookies=cookies,
)
return StaticScraper(
client,
proxy_rotator=self._proxy_rotator,
Expand Down Expand Up @@ -363,13 +372,14 @@ def _scrape_static(self, url: str, options: ScrapeOptions) -> ScrapeResult:
schema=options.schema,
timeout=timeout,
headers=options.headers or None,
cookies=options.cookies or None,
check_robots=options.check_robots,
extract_forms=options.extract_forms,
)

core_result: CoreScrapeResult = self._get_static_scraper_with_proxy(
options.proxy, config.verify_ssl
options.proxy,
config.verify_ssl,
cookies=options.cookies or None,
).scrape(url, config)

# Map core ScrapeResult to export ScrapeResult
Expand Down
1 change: 1 addition & 0 deletions tests/unit/services/test_scrape_service.py
Original file line number Diff line number Diff line change
Expand Up @@ -220,6 +220,7 @@ def test_static_scraper_uses_http_config(
http2=False,
verify=False,
follow_redirects=False,
cookies=None,
proxy=None,
default_headers={"User-Agent": "TestAgent"},
)
Expand Down
Loading