diff --git a/packages/basic-crawler/src/internals/basic-crawler.ts b/packages/basic-crawler/src/internals/basic-crawler.ts index 6663e784d29d..0b48a3080d95 100644 --- a/packages/basic-crawler/src/internals/basic-crawler.ts +++ b/packages/basic-crawler/src/internals/basic-crawler.ts @@ -1764,7 +1764,8 @@ export class BasicCrawler { expect(requests).toHaveLength(2); expect(requests[0]).toMatchObject({ url: 'https://example.com/1/', crawlDepth: 3 }); }); + + it.each([ + null, + undefined, + false, + ] as const)('should skip requests when transformRequestFunction returns %s', async (returnValue) => { + const transformRequestFunction = vi.fn(() => returnValue); + const optionsWithTransform = { ...options, transformRequestFunction }; + + await crawler.exposedEnqueueLinksWithCrawlDepth(optionsWithTransform, request, requestQueue); + + const requests = addRequestsBatchedMock.mock.calls[0][0]; + expect(requests).toHaveLength(0); + + const skippedRequests = onSkippedRequestMock.mock.calls.map((call) => call[0]); + expect(skippedRequests).toHaveLength(2); + expect(skippedRequests[0]).toStrictEqual({ url: 'https://example.com/1/', reason: 'filters' }); + expect(skippedRequests[1]).toStrictEqual({ url: 'https://example.com/2/', reason: 'filters' }); + }); }); it('addCrawlDepthRequestGenerator() should generate requests with maxCrawlDepth', async () => { diff --git a/website/yarn.lock b/website/yarn.lock index d24bb82df687..3fcbc94c6abf 100644 --- a/website/yarn.lock +++ b/website/yarn.lock @@ -6480,11 +6480,11 @@ __metadata: linkType: hard "brace-expansion@npm:^2.0.2": - version: 2.1.2 - resolution: "brace-expansion@npm:2.1.2" + version: 2.1.3 + resolution: "brace-expansion@npm:2.1.3" dependencies: balanced-match: "npm:^1.0.0" - checksum: 10c0/5442ecab84045d21826268bc56c81a6ef4215327ee1f5ee153f67928e93f7a915d130ecec9966623143277fa3cce036ebb50020024bcc4d78853cdd6af9a19f8 + checksum: 10c0/62bef43c5755b4978662d66d0844635cc171610644da7d0266db8c0180c21c6e8ff7b969a27d2f2ec7893b6efd9c33dc73383d84a467669832f2da1fd3771f99 languageName: node linkType: hard