[pull] master from apify:master

packages/basic-crawler/src/internals/basic-crawler.ts

@@ -1764,7 +1764,8 @@ export class BasicCrawler<Context extends CrawlingContext = BasicCrawlingContext
             }
 
             // After injecting the crawlDepth, we call the user-provided transform function, if there is one.
-            return options.transformRequestFunction?.(newRequest) ?? newRequest;
+            // Its return value is passed through as is, so a falsy one still skips the request.
+            return options.transformRequestFunction ? options.transformRequestFunction(newRequest) : newRequest;
         };
 
         // Create a request-scoped callback that logs enqueueLimit once per request handler call

packages/core/src/enqueue_links/shared.ts

@@ -299,7 +299,7 @@ function createPatternObjectMatcher(urlPatternObject: UrlPatternObject) {
 export interface RequestTransform {
     /**
      * @param original Request options to be modified.
-     * @returns The modified request options to enqueue.
+     * @returns The modified request options to enqueue, or any falsy value to skip the request.
      */
     (original: RequestOptions): RequestOptions | false | undefined | null;
 }

test/core/crawlers/basic_crawler.test.ts

@@ -307,6 +307,25 @@ describe('BasicCrawler', () => {
             expect(requests).toHaveLength(2);
             expect(requests[0]).toMatchObject({ url: 'https://example.com/1/', crawlDepth: 3 });
         });
+
+        it.each([
+            null,
+            undefined,
+            false,
+        ] as const)('should skip requests when transformRequestFunction returns %s', async (returnValue) => {
+            const transformRequestFunction = vi.fn(() => returnValue);
+            const optionsWithTransform = { ...options, transformRequestFunction };
+
+            await crawler.exposedEnqueueLinksWithCrawlDepth(optionsWithTransform, request, requestQueue);
+
+            const requests = addRequestsBatchedMock.mock.calls[0][0];
+            expect(requests).toHaveLength(0);
+
+            const skippedRequests = onSkippedRequestMock.mock.calls.map((call) => call[0]);
+            expect(skippedRequests).toHaveLength(2);
+            expect(skippedRequests[0]).toStrictEqual({ url: 'https://example.com/1/', reason: 'filters' });
+            expect(skippedRequests[1]).toStrictEqual({ url: 'https://example.com/2/', reason: 'filters' });
+        });
     });
 
     it('addCrawlDepthRequestGenerator() should generate requests with maxCrawlDepth', async () => {

website/yarn.lock

@@ -6480,11 +6480,11 @@ __metadata:
   linkType: hard
 
 "brace-expansion@npm:^2.0.2":
-  version: 2.1.2
-  resolution: "brace-expansion@npm:2.1.2"
+  version: 2.1.3
+  resolution: "brace-expansion@npm:2.1.3"
   dependencies:
     balanced-match: "npm:^1.0.0"
-  checksum: 10c0/5442ecab84045d21826268bc56c81a6ef4215327ee1f5ee153f67928e93f7a915d130ecec9966623143277fa3cce036ebb50020024bcc4d78853cdd6af9a19f8
+  checksum: 10c0/62bef43c5755b4978662d66d0844635cc171610644da7d0266db8c0180c21c6e8ff7b969a27d2f2ec7893b6efd9c33dc73383d84a467669832f2da1fd3771f99
   languageName: node
   linkType: hard