fix: respect autoscaledPoolOptions.isTaskReadyFunction option (#2948)

Closes #2922

Author: B4nan

packages/basic-crawler/src/internals/basic-crawler.ts

@@ -54,8 +54,6 @@ import type { Awaitable, BatchAddRequestsResult, Dictionary, SetStatusMessageOpt
 import { RobotsTxtFile, ROTATE_PROXY_ERRORS } from '@crawlee/utils';
 import { stringify } from 'csv-stringify/sync';
 import { ensureDir, writeFile, writeJSON } from 'fs-extra';
-// @ts-expect-error This throws a compilation error due to got-scraping being ESM only but we only import types, so its alllll gooooood
-import type { GotResponse, Method, OptionsInit } from 'got-scraping';
 import ow, { ArgumentError } from 'ow';
 import { getDomain } from 'tldts';
 import type { SetRequired } from 'type-fest';
@@ -741,7 +739,8 @@ export class BasicCrawler<Context extends CrawlingContext = BasicCrawlingContext
         let shouldLogMaxPagesExceeded = true;
         const isMaxPagesExceeded = () => maxRequestsPerCrawl && maxRequestsPerCrawl <= this.handledRequestsCount;
 
-        let { isFinishedFunction } = autoscaledPoolOptions;
+        // eslint-disable-next-line prefer-const
+        let { isFinishedFunction, isTaskReadyFunction } = autoscaledPoolOptions;
 
         // override even if `isFinishedFunction` provided by user - `keepAlive` has higher priority
         if (keepAlive) {
@@ -765,7 +764,7 @@ export class BasicCrawler<Context extends CrawlingContext = BasicCrawlingContext
                     return false;
                 }
 
-                return this._isTaskReadyFunction();
+                return isTaskReadyFunction ? await isTaskReadyFunction() : await this._isTaskReadyFunction();
             },
             isFinishedFunction: async () => {
                 if (isMaxPagesExceeded()) {

test/core/crawlers/basic_crawler.test.ts

@@ -734,20 +734,27 @@ describe('BasicCrawler', () => {
         expect(await crawler._isTaskReadyFunction()).toBe(false);
     });
 
-    test('should be possible to override isFinishedFunction of underlying AutoscaledPool', async () => {
+    test('should be possible to override isFinishedFunction and isTaskReadyFunction of underlying AutoscaledPool', async () => {
         const requestQueue = new RequestQueue({ id: 'xxx', client: Configuration.getStorageClient() });
         const processed: Request[] = [];
         const queue: Request[] = [];
         let isFinished = false;
+        let isFinishedFunctionCalled = false;
+        let isTaskReadyFunctionCalled = false;
 
         const basicCrawler = new BasicCrawler({
             requestQueue,
             autoscaledPoolOptions: {
                 minConcurrency: 1,
                 maxConcurrency: 1,
                 isFinishedFunction: async () => {
+                    isFinishedFunctionCalled = true;
                     return Promise.resolve(isFinished);
                 },
+                isTaskReadyFunction: async () => {
+                    isTaskReadyFunctionCalled = true;
+                    return Promise.resolve(!isFinished);
+                },
             },
             requestHandler: async ({ request }) => {
                 await sleep(10);
@@ -783,6 +790,8 @@ describe('BasicCrawler', () => {
         expect(markRequestHandled).toBeCalledWith(request0);
         expect(markRequestHandled).toBeCalledWith(request1);
         expect(isFinishedOrig).not.toBeCalled();
+        expect(isFinishedFunctionCalled).toBe(true);
+        expect(isTaskReadyFunctionCalled).toBe(true);
 
         // TODO: see why the request1 was passed as a second parameter to includes
         expect(processed.includes(request0)).toBe(true);