{ "data": { "id": "moJRLRc85AitArpNN", "userId": "ZscMwFR5H7eCtWtyh", "name": "web-scraper", "username": "apify", "description": "Crawls arbitrary websites using a web browser and extracts structured data from web pages using a provided JavaScript function. The Actor supports both recursive crawling and lists of URLs, and automatically manages concurrency for maximum performance.", "restartOnError": true, "isPublic": true, "createdAt": "2019-03-07T11:28:01.600Z", "modifiedAt": "2026-10-27T13:02:03.726Z", "taggedBuilds": { "beta": { "buildId": "zyywHJwZgZtuqwKWL", "finishedAt": "2019-12-19T17:24:27.466Z", "buildNumberInt": 42, "buildNumber": "0.0.42" }, "experimental": { "buildId": "NTLDiFYjFvcoDxWm6", "finishedAt": "2019-05-02T11:16:09.179Z", "buildNumberInt": 990000003, "buildNumber": "99.0.3" }, "version-2": { "buildId": "in9Da6X2czfcGXu3x", "finishedAt": "2022-05-09T12:58:46.469Z", "buildNumberInt": 20000008, "buildNumber": "2.0.8" }, "version-1": { "buildId": "VezLeaxKZT0ZmuguN", "finishedAt": "2020-11-11T13:44:07.316Z", "buildNumberInt": 100044, "buildNumber": "0.1.44" }, "version-3": { "buildId": "20g7OuP9h8wyKU9jC", "finishedAt": "2026-10-27T13:02:03.726Z", "buildNumberInt": 30000027, "buildNumber": "3.0.27" }, "development": { "buildId": "knTUheCvlrqwfM8hO", "finishedAt": "2026-09-03T08:01:40.638Z", "buildNumberInt": 74, "buildNumber": "0.0.74" } }, "stats": { "totalBuilds": 169, "totalRuns": 181845308, "totalUsers": 126397, "lastRunStartedAt": "2026-10-27T21:42:34.625Z", "totalMetamorphs": 309071, "totalUsers30Days": 1337, "totalUsers7Days": 698, "totalUsers90Days": 2776, "publicActorRunStats30Days": { "ABORTED": 1042, "FAILED": 1807, "SUCCEEDED": 287060, "TIMED-OUT": 20781, "TOTAL": 310690 }, "actorReviewCount": 100, "actorReviewRating": 4.7115238426034765, "bookmarkCount": 1404 }, "versions": [ { "versionNumber": "0.0", "buildTag": "development", "sourceType": "GIT_REPO", "envVars": [], "applyEnvVarsToBuild": false, "gitRepoUrl": "https://github.com/apify/actor-scraper#master:packages/actor-scraper/web-scraper", "sourceFiles": [ { "name": ".actor/actor.json", "format": "TEXT", "content": "{\n \"actorSpecification\": 1,\n \"name\": \"web-scraper\",\n \"version\": \"0.1\",\n \"buildTag\": \"latest\",\n \"storages\": {\n \"dataset\": {\n \"actorSpecification\": 1,\n \"fields\": {},\n \"views\": {}\n }\n }\n}" }, { "name": "test/bundle.browser.test.ts", "format": "TEXT", "content": "import { launchPuppeteer } from '@crawlee/puppeteer';\nimport type { Browser, Page } from 'puppeteer';\n\nimport { createBundle } from '../src/internals/bundle.browser';\n\nconst NAMESPACE = 'Apify';\n\ndescribe('Bundle', () => {\n let browser: Browser;\n let page: Page;\n\n beforeAll(async () => {\n browser = await launchPuppeteer({ launchOptions: { headless: true } });\n });\n\n afterAll(async () => {\n await browser.close();\n });\n\n beforeEach(async () => {\n page = await browser.newPage();\n await page.evaluateOnNewDocument(createBundle, NAMESPACE);\n });\n\n afterEach(async () => {\n await page.close();\n });\n\n describe('Context', () => {\n const CONTEXT_OPTIONS = {\n crawlerSetup: {\n rawInput: '{}',\n },\n browserHandles: {\n apify: {},\n globalStore: {},\n log: {},\n },\n pageFunctionArguments: {\n request: {},\n },\n };\n\n beforeEach(async () => {\n await page.goto('about:chrome');\n await page.waitForFunction(\n (namespace: string) => !!window[namespace],\n {},\n NAMESPACE,\n );\n await page.evaluate(\n (namespace: string, contextOptions) => {\n window.contextInstance =\n window[namespace].createContext(contextOptions);\n },\n NAMESPACE,\n CONTEXT_OPTIONS,\n );\n });\n\n describe('waitFor', () => {\n it('should work with a number', async () => {\n const millis = await page.evaluate(async () => {\n const ctx = window.contextInstance;\n const start = Date.now();\n await ctx.waitFor(10);\n return Date.now() - start;\n });\n expect(millis).toBeGreaterThan(9);\n });\n\n it('should work with a selector', async () => {\n const millis = await page.evaluate(async () => {\n const ctx = window.contextInstance;\n const start = Date.now();\n setTimeout(() => {\n const el = document.createElement('div');\n el.id = 'very-unique-id';\n document.body.appendChild(el);\n }, 10);\n await ctx.waitFor('#very-unique-id');\n return Date.now() - start;\n });\n expect(millis).toBeGreaterThan(9);\n });\n\n it('should work with a function', async () => {\n const millis = await page.evaluate(async () => {\n const ctx = window.contextInstance;\n let done = false;\n const start = Date.now();\n setTimeout(() => {\n done = true;\n }, 10);\n await ctx.waitFor(() => done);\n return Date.now() - start;\n });\n expect(millis).toBeGreaterThan(9);\n });\n });\n });\n});" }, { "name": "test/devtools-server.regression.test.ts", "format": "TEXT", "content": "import { describe, expect, it, vi } from 'vitest';\n\ndescribe('DevToolsServer (regression)', () => {\n it('starts only once even if preLaunchHook calls it multiple times', async () => {\n const oldEnv = process.env;\n const actorOnMock = vi.fn();\n\n const startMock = vi.fn(async () => {});\n const stopMock = vi.fn(() => {});\n const DevToolsCtorMock = vi.fn(() => ({\n start: startMock,\n stop: stopMock,\n }));\n\n try {\n process.env = { ...oldEnv };\n process.env.ACTOR_WEB_SERVER_URL = 'http://127.0.0.1:4321';\n process.env.ACTOR_WEB_SERVER_PORT = '4321';\n\n vi.resetModules();\n vi.doMock('devtools-server', () => ({ default: DevToolsCtorMock }));\n vi.doMock('apify', () => ({ Actor: { on: actorOnMock } }));\n\n const mod = await import(\n new URL('../src/internals/crawler_setup.ts', import.meta.url)\n .href\n );\n const { CrawlerSetup } = mod as any;\n\n CrawlerSetup.devToolsServerPromise = null;\n\n const fn =\n CrawlerSetup.getDevToolsServer ??\n CrawlerSetup.startDevToolsServerOnce;\n\n await Promise.all(\n Array.from({ length: 10 }, () => fn.call(CrawlerSetup)),\n );\n\n expect(DevToolsCtorMock).toHaveBeenCalledTimes(1);\n expect(startMock).toHaveBeenCalledTimes(1);\n expect(actorOnMock).toHaveBeenCalledTimes(1);\n expect(actorOnMock).toHaveBeenCalledWith(\n 'exit',\n expect.any(Function),\n );\n } finally {\n process.env = oldEnv;\n vi.resetModules();\n }\n });\n});" }, { "name": "src/internals/bundle.browser.ts", "format": "TEXT", "content": "/* eslint-disable max-classes-per-file */\n\nimport type {\n Dictionary,\n KeyValueStore,\n RecordOptions,\n Request,\n RequestOptions,\n RequestQueue,\n RequestQueueOperationOptions,\n} from '@crawlee/puppeteer';\nimport type { ApifyEnv } from 'apify';\n\nimport type { Log } from '@apify/log';\nimport type {\n constants,\n CrawlerSetupOptions,\n RequestMetadata,\n} from '@apify/scraper-tools';\n\nimport type { Input } from './consts.ts';\nimport type { GlobalStore } from './global_store.ts';\n\ninterface PoolOptions {\n pollingIntervalMillis?: number;\n timeoutMillis?: number;\n}\n\ninterface InternalState {\n browserHandles: Dictionary<\n | string\n | Record<\n string,\n { value: unknown; type: 'METHOD' | 'VALUE' | 'GETTER' }\n >\n >;\n requestQueue: RequestQueue | null;\n keyValueStore: KeyValueStore | null;\n}\n\ninterface ProvidedResponse {\n status: number;\n headers: Dictionary;\n}\n\ninterface BrowserCrawlerSetup extends CrawlerSetupOptions {\n injectJQuery?: boolean;\n META_KEY: typeof constants.META_KEY;\n}\n\ninterface ContextOptions {\n crawlerSetup: BrowserCrawlerSetup;\n browserHandles: InternalState['browserHandles'];\n pageFunctionArguments: Dictionary;\n}\n\n/**\n * Command to be evaluated for Browser side code injection.\n * @param apifyNamespace\n */\nexport function createBundle(apifyNamespace: string) {\n (function (global, namespace) {\n if (typeof window[namespace] !== 'object') window[namespace] = {};\n /**\n * Takes a configuration object with function references\n * as an input and creates a dummy class that proxies\n * function invocations from Browser context to Node context.\n *\n * The configuration is to be provided by the\n * tools.createBrowserHandlesForObject function.\n */\n class NodeProxy {\n constructor(\n config: Dictionary<{\n value: unknown;\n type: 'METHOD' | 'VALUE' | 'GETTER';\n }>,\n ) {\n if (!config || typeof config !== 'object') {\n throw new Error(\n 'NodeProxy: Parameter config of type Object must be provided.',\n );\n }\n\n Object.entries(config).forEach(([key, { value, type }]) => {\n if (type === 'METHOD') {\n // @ts-expect-error\n this[key] = (...args: unknown[]) =>\n // @ts-expect-error\n global[value](...args);\n } else if (type === 'GETTER') {\n Object.defineProperty(this, key, {\n // @ts-expect-error\n get: () => global[value](),\n });\n } else if (type === 'VALUE') {\n // @ts-expect-error\n this[key] = value;\n } else {\n throw new Error(\n `Unsupported function type: ${type} for function: ${key}.`,\n );\n }\n });\n }\n }\n\n /**\n * Exposed factory.\n * @param config\n */\n global[namespace].createNodeProxy = (\n config: Dictionary<{\n value: unknown;\n type: 'METHOD' | 'VALUE' | 'GETTER';\n }>,\n ) => new NodeProxy(config);\n\n const setup = Symbol('crawler-setup');\n const internalState = Symbol('request-internal-state');\n\n /**\n * Context represents everything that is available to the user\n * via Page Function. A class is used instead of a simple object\n * to avoid having to create new instances of functions with each\n * request.\n *\n * Some properties need to be accessible to the Context,\n * but should not be exposed to the user thus they are hidden\n * using a Symbol to prevent the user from easily accessing\n * and manipulating them.\n */\n class Context {\n [setup]: BrowserCrawlerSetup;\n [internalState]: InternalState;\n input: Input;\n env: ApifyEnv;\n customData: unknown;\n response: ProvidedResponse;\n request: Request;\n globalStore: GlobalStore;\n log: Log;\n\n jQuery: any;\n\n constructor(options: ContextOptions) {\n const { crawlerSetup, browserHandles, pageFunctionArguments } =\n options;\n\n const createProxy = global[namespace].createNodeProxy;\n\n // Private\n this[setup] = crawlerSetup;\n this[internalState] = {\n browserHandles,\n requestQueue: browserHandles.requestQueue\n ? createProxy(browserHandles.requestQueue)\n : null,\n keyValueStore: browserHandles.keyValueStore\n ? createProxy(browserHandles.keyValueStore)\n : null,\n };\n\n // Copies of Node objects\n this.input = JSON.parse(crawlerSetup.rawInput);\n this.env = { ...crawlerSetup.env };\n this.customData = crawlerSetup.customData;\n this.response =\n pageFunctionArguments.response as ProvidedResponse;\n this.request = pageFunctionArguments.request as Request;\n // Functions are not converted so we need to add them this way\n // to not be enumerable and thus not polluting the object.\n Reflect.defineProperty(this.request, 'pushErrorMessage', {\n value(this: Request, errorOrMessage: Error) {\n // It's a simplified fake of the original function.\n const msg =\n (errorOrMessage && errorOrMessage.message) ||\n `${errorOrMessage}`;\n this.errorMessages.push(msg);\n },\n enumerable: false,\n });\n\n // Proxied Node objects\n this.globalStore = createProxy(browserHandles.globalStore);\n this.log = createProxy(browserHandles.log);\n\n // Browser side libraries\n if (this[setup].injectJQuery)\n this.jQuery = global.jQuery.noConflict(true);\n\n // Bind this to allow destructuring off context in pageFunction.\n this.getValue = this.getValue.bind(this);\n this.setValue = this.setValue.bind(this);\n this.saveSnapshot = this.saveSnapshot.bind(this);\n this.skipLinks = this.skipLinks.bind(this);\n this.enqueueRequest = this.enqueueRequest.bind(this);\n this.waitFor = this.waitFor.bind(this);\n }\n\n async getValue(...args: Parameters) {\n return this[internalState].keyValueStore!.getValue(\n ...args,\n ) as Promise;\n }\n\n async setValue(...args: Parameters) {\n return this[internalState].keyValueStore!.setValue(\n ...(args as [\n key: string,\n value: T | null,\n options?: RecordOptions,\n ]),\n );\n }\n\n async saveSnapshot() {\n const handle = this[internalState].browserHandles\n .saveSnapshot as string;\n return global[handle]();\n }\n\n async skipLinks() {\n const handle = this[internalState].browserHandles\n .skipLinks as string;\n return global[handle]();\n }\n\n async enqueueRequest(\n requestOpts: RequestOptions = {} as RequestOptions,\n options: RequestQueueOperationOptions = {},\n ) {\n const defaultRequestOpts = {\n useExtendedUniqueKey: true,\n keepUrlFragment: this.input.keepUrlFragments,\n };\n\n const newRequest = { ...defaultRequestOpts, ...requestOpts };\n\n const metaKey = this[setup].META_KEY;\n const defaultUserData = {\n [metaKey]: {\n parentRequestId:\n this.request.id || this.request.uniqueKey,\n depth:\n (this.request.userData![metaKey] as RequestMetadata)\n .depth + 1,\n },\n };\n\n newRequest.userData = {\n ...defaultUserData,\n ...requestOpts.userData,\n };\n\n return this[internalState].requestQueue!.addRequest(\n newRequest,\n options,\n );\n }\n\n async waitFor(\n selectorOrNumberOrFunction:\n | string\n | number\n | ((...args: unknown[]) => boolean),\n options = {},\n ) {\n if (!options || typeof options !== 'object')\n throw new Error('Parameter options must be an Object');\n const type = typeof selectorOrNumberOrFunction;\n if (type === 'string') {\n return this._waitForSelector(\n selectorOrNumberOrFunction as string,\n options,\n );\n }\n if (type === 'number') {\n return this._waitForMillis(\n selectorOrNumberOrFunction as number,\n );\n }\n if (type === 'function') {\n return this._waitForFunction(\n selectorOrNumberOrFunction as (\n ...args: unknown[]\n ) => boolean,\n options,\n );\n }\n throw new Error(\n 'Parameter selectorOrNumberOrFunction must be one of the said types.',\n );\n }\n\n async _waitForSelector(selector: string, options = {}) {\n try {\n await this._poll(() => {\n return !!global.document.querySelector(selector);\n }, options);\n } catch (err) {\n const casted = err as Error;\n if (/timeout of \\d+ms exceeded/.test(casted.message)) {\n throw new Error(\n `Timeout Error: waiting for selector failed: ${casted.message}`,\n );\n }\n throw err;\n }\n }\n\n async _waitForMillis(millis: number) {\n return new Promise((res) => {\n setTimeout(res, millis);\n });\n }\n\n async _waitForFunction(\n predicate: () => boolean,\n options: PoolOptions = {},\n ) {\n try {\n await this._poll(predicate, options);\n } catch (err) {\n const casted = err as Error;\n if (/timeout of \\d+ms exceeded/.test(casted.message)) {\n throw new Error(\n `Timeout Error: waiting for function failed: ${casted.message}`,\n );\n }\n throw err;\n }\n }\n\n async _poll(predicate: () => boolean, options: PoolOptions = {}) {\n const { pollingIntervalMillis = 50, timeoutMillis = 20000 } =\n options;\n return new Promise((resolve, reject) => {\n const handler = (): void => {\n if (predicate()) {\n resolve();\n } else {\n setTimeout(handler);\n }\n };\n const pollTimeout = setTimeout(\n handler,\n pollingIntervalMillis,\n );\n setTimeout(() => {\n clearTimeout(pollTimeout);\n return reject(\n new Error(\n `timeout of ${timeoutMillis}ms exceeded.`,\n ),\n );\n }, timeoutMillis);\n });\n }\n }\n\n /**\n * Exposed factory.\n */\n global[namespace].createContext = (options: ContextOptions) => {\n return new Context(options);\n };\n })(window, apifyNamespace);\n}" }, { "name": "src/internals/consts.ts", "format": "TEXT", "content": "import type {\n Dictionary,\n GlobInput,\n ProxyConfigurationOptions,\n PseudoUrlInput,\n RegExpInput,\n RequestOptions,\n Session,\n} from '@crawlee/puppeteer';\nimport type { PuppeteerLifeCycleEvent } from 'puppeteer';\n\n/**\n * Port where the remote debugging interface in Chrome\n * should be started.\n */\nexport const CHROME_DEBUGGER_PORT = 9222;\n\nexport const enum RunMode {\n Production = 'PRODUCTION',\n Development = 'DEVELOPMENT',\n}\n\nexport const enum ProxyRotation {\n Recommended = 'RECOMMENDED',\n PerRequest = 'PER_REQUEST',\n UntilFailure = 'UNTIL_FAILURE',\n}\n\nexport const enum BreakpointLocation {\n None = 'NONE',\n BeforeGoto = 'BEFORE_GOTO',\n BeforePageFunction = 'BEFORE_PAGE_FUNCTION',\n AfterPageFunction = 'AFTER_PAGE_FUNCTION',\n}\n\ndeclare global {\n interface Window {\n [K: string]: any;\n }\n // eslint-disable-next-line vars-on-top\n var window: Window & typeof globalThis;\n // eslint-disable-next-line vars-on-top\n var document: Document;\n}\n\n/**\n * Replicates the INPUT_SCHEMA with TypeScript types for quick reference\n * and IDE type check integration.\n */\nexport interface Input {\n startUrls: RequestOptions[];\n pageFunction: string;\n runMode: RunMode;\n keepUrlFragments: boolean;\n respectRobotsTxtFile: boolean;\n linkSelector?: string;\n globs: GlobInput[];\n regexps: RegExpInput[];\n pseudoUrls: PseudoUrlInput[];\n excludes: GlobInput[];\n preNavigationHooks?: string;\n postNavigationHooks?: string;\n injectJQuery: boolean;\n proxyConfiguration: ProxyConfigurationOptions;\n proxyRotation: ProxyRotation;\n sessionPoolName?: string;\n initialCookies: Parameters[0];\n useChrome: boolean;\n maxScrollHeightPixels: number;\n ignoreSslErrors: boolean;\n ignoreCorsAndCsp: boolean;\n closeCookieModals: boolean;\n downloadMedia: boolean;\n downloadCss: boolean;\n maxRequestRetries: number;\n maxPagesPerCrawl: number;\n maxResultsPerCrawl: number;\n maxCrawlingDepth: number;\n maxConcurrency: number;\n pageLoadTimeoutSecs: number;\n pageFunctionTimeoutSecs: number;\n waitUntil: PuppeteerLifeCycleEvent[];\n breakpointLocation: BreakpointLocation;\n debugLog: boolean;\n browserLog: boolean;\n customData: Dictionary;\n datasetName?: string;\n keyValueStoreName?: string;\n requestQueueName?: string;\n headless: boolean;\n}" }, { "name": "src/internals/crawler_setup.ts", "format": "TEXT", "content": "import { readFile } from 'node:fs/promises';\nimport { dirname } from 'node:path';\nimport { setTimeout } from 'node:timers/promises';\nimport { fileURLToPath, URL } from 'node:url';\n\nimport type {\n AutoscaledPool,\n Awaitable,\n Dictionary,\n ProxyConfiguration,\n PuppeteerCrawlerOptions,\n PuppeteerCrawlingContext,\n Request,\n} from '@crawlee/puppeteer';\nimport {\n Dataset,\n KeyValueStore,\n log,\n PuppeteerCrawler,\n puppeteerUtils,\n RequestList,\n RequestQueueV2,\n} from '@crawlee/puppeteer';\nimport type { ApifyEnv } from 'apify';\nimport { Actor } from 'apify';\nimport contentType from 'content-type';\n// @ts-expect-error no typings\nimport DevToolsServer from 'devtools-server';\nimport { getInjectableScript } from 'idcac-playwright';\nimport type { HTTPResponse, Page } from 'puppeteer';\n\nimport type { CrawlerSetupOptions, createContext } from '@apify/scraper-tools';\nimport {\n browserTools,\n constants as scraperToolsConstants,\n tools,\n} from '@apify/scraper-tools';\n\nimport { createBundle } from './bundle.browser.js';\nimport type { Input } from './consts.js';\nimport {\n BreakpointLocation,\n CHROME_DEBUGGER_PORT,\n ProxyRotation,\n RunMode,\n} from './consts.js';\nimport { GlobalStore } from './global_store.js';\n\nconst SESSION_STORE_NAME = 'APIFY-WEB-SCRAPER-SESSION-STORE';\nconst REQUEST_QUEUE_INIT_FLAG_KEY = 'REQUEST_QUEUE_INITIALIZED';\n\nconst MAX_CONCURRENCY_IN_DEVELOPMENT = 1;\nconst {\n SESSION_MAX_USAGE_COUNTS,\n DEFAULT_VIEWPORT,\n DEVTOOLS_TIMEOUT_SECS,\n META_KEY,\n} = scraperToolsConstants;\nconst SCHEMA = JSON.parse(\n await readFile(new URL('../../INPUT_SCHEMA.json', import.meta.url), 'utf8'),\n);\n\ninterface PageContext {\n apifyNamespace: string;\n skipLinks: boolean;\n timers: {\n start: [number, number];\n navStart?: [number, number];\n };\n\n browserHandles?: Awaited>;\n}\n\ninterface Output {\n pageFunctionResult?: Dictionary;\n pageFunctionError?: tools.ErrorLike;\n requestFromBrowser: Request;\n}\n\n/**\n * Holds all the information necessary for constructing a crawler\n * instance and creating a context for a pageFunction invocation.\n */\nexport class CrawlerSetup implements CrawlerSetupOptions {\n name = 'Web Scraper';\n rawInput: string;\n env: ApifyEnv;\n /**\n * Used to store data that persist navigations\n */\n globalStore = new GlobalStore();\n requestQueue: RequestQueueV2;\n keyValueStore: KeyValueStore;\n customData: unknown;\n input: Input;\n maxSessionUsageCount: number;\n evaledPreNavigationHooks: ((...args: unknown[]) => Awaitable)[];\n evaledPostNavigationHooks: ((...args: unknown[]) => Awaitable)[];\n\n /**\n * Used to store page specific data.\n */\n pageContexts = new WeakMap();\n\n blockedUrlPatterns: string[] = [];\n isDevRun: boolean;\n datasetName?: string;\n keyValueStoreName?: string;\n requestQueueName?: string;\n\n crawler!: PuppeteerCrawler;\n dataset!: Dataset;\n pagesOutputted!: number;\n private initPromise: Promise;\n\n constructor(input: Input) {\n // Set log level early to prevent missed messages.\n if (input.debugLog) log.setLevel(log.LEVELS.DEBUG);\n\n // Keep this as string to be immutable.\n this.rawInput = JSON.stringify(input);\n\n // Attempt to load page function from disk if not present on input.\n tools.maybeLoadPageFunctionFromDisk(\n input,\n dirname(fileURLToPath(import.meta.url)),\n );\n\n // Validate INPUT if not running on Apify Cloud Platform.\n if (!Actor.isAtHome()) tools.checkInputOrThrow(input, SCHEMA);\n\n this.input = input;\n this.env = Actor.getEnv();\n\n // Validations\n this.input.pseudoUrls.forEach((purl) => {\n if (!tools.isPlainObject(purl)) {\n throw new Error(\n 'The pseudoUrls Array must only contain Objects.',\n );\n }\n if (purl.userData && !tools.isPlainObject(purl.userData)) {\n throw new Error(\n 'The userData property of a pseudoUrl must be an Object.',\n );\n }\n });\n\n this.input.initialCookies.forEach((cookie) => {\n if (!tools.isPlainObject(cookie)) {\n throw new Error(\n 'The initialCookies Array must only contain Objects.',\n );\n }\n });\n\n this.input.waitUntil.forEach((event) => {\n if (\n !/^(domcontentloaded|load|networkidle2|networkidle0)$/.test(\n event,\n )\n ) {\n throw new Error(\n 'Navigation wait until events must be valid. See tooltip.',\n );\n }\n });\n\n // solving proxy rotation settings\n this.maxSessionUsageCount =\n SESSION_MAX_USAGE_COUNTS[this.input.proxyRotation];\n\n tools.evalFunctionOrThrow(this.input.pageFunction);\n\n if (this.input.preNavigationHooks) {\n this.evaledPreNavigationHooks = tools.evalFunctionArrayOrThrow(\n this.input.preNavigationHooks,\n 'preNavigationHooks',\n );\n } else {\n this.evaledPreNavigationHooks = [];\n }\n\n if (this.input.postNavigationHooks) {\n this.evaledPostNavigationHooks = tools.evalFunctionArrayOrThrow(\n this.input.postNavigationHooks,\n 'postNavigationHooks',\n );\n } else {\n this.evaledPostNavigationHooks = [];\n }\n\n // Excluded resources\n if (!this.input.downloadMedia) {\n this.blockedUrlPatterns = [\n '.jpg',\n '.jpeg',\n '.png',\n '.svg',\n '.gif',\n '.webp',\n '.webm',\n '.ico',\n '.woff',\n '.eot',\n ];\n }\n\n if (!this.input.downloadCss) {\n this.blockedUrlPatterns.push('.css');\n }\n\n this.isDevRun = this.input.runMode === RunMode.Development;\n\n // Named storages\n this.datasetName = this.input.datasetName;\n this.keyValueStoreName = this.input.keyValueStoreName;\n this.requestQueueName = this.input.requestQueueName;\n\n // Initialize async operations.\n this.crawler = null!;\n this.requestQueue = null!;\n this.dataset = null!;\n this.keyValueStore = null!;\n this.initPromise = this._initializeAsync();\n }\n\n private static devToolsStartPromise: Promise | null =\n null;\n\n private static async getDevToolsServer(): Promise {\n if (this.devToolsStartPromise) return this.devToolsStartPromise;\n\n this.devToolsStartPromise = (async () => {\n const url = process.env.ACTOR_WEB_SERVER_URL;\n const portRaw = process.env.ACTOR_WEB_SERVER_PORT;\n if (!url || !portRaw) return null;\n\n const port = Number(portRaw);\n if (!Number.isFinite(port)) return null;\n\n const server = new DevToolsServer({\n containerHost: new URL(url).host,\n devToolsServerPort: port,\n chromeRemoteDebuggingPort: CHROME_DEBUGGER_PORT,\n });\n\n await server.start();\n\n Actor.on('exit', () => {\n server.stop?.();\n });\n return server;\n })();\n\n return this.devToolsStartPromise;\n }\n\n private async _initializeAsync() {\n // RequestList\n const startUrls = this.input.startUrls.map((req) => {\n req.useExtendedUniqueKey = true;\n req.keepUrlFragment = this.input.keepUrlFragments;\n return req;\n });\n\n // KeyValueStore\n this.keyValueStore = await KeyValueStore.open(this.keyValueStoreName);\n\n // RequestQueue\n this.requestQueue = await RequestQueueV2.open(this.requestQueueName);\n\n if (\n !(await this.keyValueStore.recordExists(\n REQUEST_QUEUE_INIT_FLAG_KEY,\n ))\n ) {\n const requests: Request[] = [];\n for await (const request of await RequestList.open(\n null,\n startUrls,\n )) {\n if (\n this.input.maxResultsPerCrawl > 0 &&\n requests.length >= 1.5 * this.input.maxResultsPerCrawl\n ) {\n break;\n }\n requests.push(request);\n }\n\n const { waitForAllRequestsToBeAdded } =\n await this.requestQueue.addRequestsBatched(requests);\n\n void waitForAllRequestsToBeAdded.then(async () => {\n await this.keyValueStore.setValue(\n REQUEST_QUEUE_INIT_FLAG_KEY,\n '1',\n );\n });\n }\n\n // Dataset\n this.dataset = await Dataset.open(this.datasetName);\n const info = await this.dataset.getInfo();\n this.pagesOutputted = info?.itemCount ?? 0;\n }\n\n /**\n * Resolves to a `PuppeteerCrawler` instance.\n * constructor.\n */\n async createCrawler() {\n await this.initPromise;\n\n const args = ['--disable-dev-shm-usage'];\n if (this.input.ignoreCorsAndCsp) args.push('--disable-web-security');\n if (this.isDevRun)\n args.push(`--remote-debugging-port=${CHROME_DEBUGGER_PORT}`);\n\n const options: PuppeteerCrawlerOptions = {\n requestHandler: this._requestHandler.bind(this),\n requestQueue: this.requestQueue,\n requestHandlerTimeoutSecs: this.isDevRun\n ? DEVTOOLS_TIMEOUT_SECS\n : this.input.pageFunctionTimeoutSecs,\n preNavigationHooks: [],\n postNavigationHooks: [],\n failedRequestHandler: this._failedRequestHandler.bind(this),\n respectRobotsTxtFile: this.input.respectRobotsTxtFile,\n maxConcurrency: this.isDevRun\n ? MAX_CONCURRENCY_IN_DEVELOPMENT\n : this.input.maxConcurrency,\n maxRequestRetries: this.input.maxRequestRetries,\n maxRequestsPerCrawl:\n this.input.maxPagesPerCrawl === 0\n ? undefined\n : this.input.maxPagesPerCrawl,\n proxyConfiguration: (await Actor.createProxyConfiguration(\n this.input.proxyConfiguration,\n )) as any as ProxyConfiguration,\n browserPoolOptions: {\n preLaunchHooks: [\n async () => {\n if (!this.isDevRun) return;\n await CrawlerSetup.getDevToolsServer();\n },\n ],\n },\n launchContext: {\n useChrome: this.input.useChrome,\n launchOptions: {\n acceptInsecureCerts: this.input.ignoreSslErrors,\n defaultViewport: DEFAULT_VIEWPORT,\n args,\n headless: this.input.headless,\n },\n },\n useSessionPool: !this.isDevRun,\n persistCookiesPerSession: !this.isDevRun,\n sessionPoolOptions: {\n persistStateKeyValueStoreId: this.input.sessionPoolName\n ? SESSION_STORE_NAME\n : undefined,\n persistStateKey: this.input.sessionPoolName,\n sessionOptions: {\n maxUsageCount: this.maxSessionUsageCount,\n },\n },\n experiments: {\n requestLocking: true,\n },\n };\n\n this._createNavigationHooks(options);\n\n if (this.input.proxyRotation === ProxyRotation.UntilFailure) {\n options.sessionPoolOptions!.maxPoolSize = 1;\n }\n if (this.isDevRun) {\n options.browserPoolOptions!.retireBrowserAfterPageCount = Infinity;\n }\n\n this.crawler = new PuppeteerCrawler(options);\n\n if (this.isDevRun) logDevRunWarning();\n return this.crawler;\n }\n\n private _createNavigationHooks(options: PuppeteerCrawlerOptions) {\n options.preNavigationHooks!.push(\n async ({ request, page, session }, gotoOptions) => {\n const start = process.hrtime();\n\n // Create a new page context with a new random key for Apify namespace.\n const pageContext: PageContext = {\n apifyNamespace: await tools.createRandomHash(),\n skipLinks: false,\n timers: { start },\n };\n this.pageContexts.set(page, pageContext);\n\n // Attach a console listener to get all logs as soon as possible.\n if (this.input.browserLog) browserTools.dumpConsole(page);\n\n // Prevent download of stylesheets and media, unless selected otherwise\n if (this.blockedUrlPatterns.length) {\n await puppeteerUtils.blockRequests(page, {\n urlPatterns: this.blockedUrlPatterns,\n });\n }\n\n // Add initial cookies, if any.\n if (\n this.input.initialCookies &&\n this.input.initialCookies.length\n ) {\n const cookiesToSet = session\n ? tools.getMissingCookiesFromSession(\n session,\n this.input.initialCookies,\n request.url,\n )\n : this.input.initialCookies;\n if (cookiesToSet && cookiesToSet.length) {\n // setting initial cookies that are not already in the session and page\n // TODO: We can remove the condition when there is an option to define blocked status codes in sessionPool\n session?.setCookies(cookiesToSet, request.url);\n await page.setCookie(...cookiesToSet);\n }\n }\n\n // Disable content security policy.\n if (this.input.ignoreCorsAndCsp) await page.setBypassCSP(true);\n\n tools.logPerformance(request, 'gotoFunction INIT', start);\n const handleStart = process.hrtime();\n pageContext.browserHandles = await this._injectBrowserHandles(\n page,\n pageContext,\n );\n tools.logPerformance(\n request,\n 'gotoFunction INJECTION HANDLES',\n handleStart,\n );\n\n const evalStart = process.hrtime();\n await Promise.all([\n page.evaluateOnNewDocument(\n createBundle,\n pageContext.apifyNamespace,\n ),\n page.evaluateOnNewDocument(\n browserTools.wrapPageFunction(\n this.input.pageFunction,\n pageContext.apifyNamespace,\n ),\n ),\n ]);\n tools.logPerformance(\n request,\n 'gotoFunction INJECTION EVAL',\n evalStart,\n );\n\n if (this.isDevRun) {\n const cdpClient = await page.target().createCDPSession();\n await cdpClient.send('Debugger.enable');\n if (\n this.input.breakpointLocation ===\n BreakpointLocation.BeforeGoto\n ) {\n await cdpClient.send('Debugger.pause');\n }\n }\n\n pageContext.timers.navStart = process.hrtime();\n if (gotoOptions) {\n gotoOptions.timeout = this.input.pageLoadTimeoutSecs * 1000;\n gotoOptions.waitUntil = this.input.waitUntil;\n }\n },\n );\n\n options.preNavigationHooks!.push(\n ...this._runHookWithEnhancedContext(this.evaledPreNavigationHooks),\n );\n options.postNavigationHooks!.push(\n ...this._runHookWithEnhancedContext(this.evaledPostNavigationHooks),\n );\n\n options.postNavigationHooks!.push(\n async ({ request, page, response }) => {\n await this._waitForLoadEventWhenXml(page, response);\n const pageContext = this.pageContexts.get(page)!;\n tools.logPerformance(\n request,\n 'gotoFunction NAVIGATION',\n pageContext.timers.navStart!,\n );\n\n const delayStart = process.hrtime();\n await this._assertNamespace(page, pageContext.apifyNamespace);\n\n // Inject selected libraries\n if (this.input.injectJQuery)\n await puppeteerUtils.injectJQuery(page);\n\n tools.logPerformance(\n request,\n 'gotoFunction INJECTION DELAY',\n delayStart,\n );\n tools.logPerformance(\n request,\n 'gotoFunction EXECUTION',\n pageContext.timers.start,\n );\n },\n );\n }\n\n private _runHookWithEnhancedContext(\n hooks: ((...args: unknown[]) => Awaitable)[],\n ) {\n return hooks.map((hook) => (ctx: Dictionary, ...args: unknown[]) => {\n const { customData } = this.input;\n return hook({ ...ctx, customData }, ...args);\n });\n }\n\n private async _failedRequestHandler({ request }: PuppeteerCrawlingContext) {\n const lastError =\n request.errorMessages[request.errorMessages.length - 1];\n const errorMessage = lastError ? lastError.split('\\n')[0] : 'no error';\n log.error(\n `Request ${request.url} failed and will not be retried anymore. Marking as failed.\\nLast Error Message: ${errorMessage}`,\n );\n return this._handleResult(request, undefined, undefined, true);\n }\n\n /**\n * First of all, it initializes the state that is exposed to the user via\n * `pageFunction` context.\n *\n * Then it invokes the user provided `pageFunction` with the prescribed context\n * and saves its return value.\n *\n * Finally, it makes decisions based on the current state and post-processes\n * the data returned from the `pageFunction`.\n */\n private async _requestHandler(crawlingContext: PuppeteerCrawlingContext) {\n const { request, response, page, crawler, proxyInfo } = crawlingContext;\n const start = process.hrtime();\n const pageContext = this.pageContexts.get(page)!;\n\n /**\n * PRE-PROCESSING\n */\n // Make sure that an object containing internal metadata\n // is present on every request.\n tools.ensureMetaData(request);\n\n // Abort the crawler if the maximum number of results was reached.\n const aborted = await this._handleMaxResultsPerCrawl(\n crawler.autoscaledPool,\n );\n if (aborted) return;\n\n // Setup Context and pass the configuration down to Browser.\n const contextOptions = {\n crawlerSetup: {\n rawInput: this.rawInput,\n env: this.env,\n customData: this.input.customData,\n injectJQuery: this.input.injectJQuery,\n META_KEY,\n },\n browserHandles: pageContext.browserHandles,\n pageFunctionArguments: {\n request,\n proxyInfo,\n response: {\n status: response && response.status(),\n headers: response && response.headers(),\n },\n },\n };\n\n /**\n * USER FUNCTION EXECUTION\n */\n tools.logPerformance(request, 'requestHandler PREPROCESSING', start);\n\n if (\n this.isDevRun &&\n this.input.breakpointLocation ===\n BreakpointLocation.BeforePageFunction\n ) {\n await page.evaluate(async () => {\n // eslint-disable-next-line no-debugger -- Debugger is enabled in dev run\n debugger;\n });\n }\n\n if (this.input.closeCookieModals) {\n await setTimeout(500);\n await page.evaluate(getInjectableScript());\n await setTimeout(2000);\n }\n\n if (this.input.maxScrollHeightPixels > 0) {\n await crawlingContext.infiniteScroll({\n maxScrollHeight: this.input.maxScrollHeightPixels,\n });\n }\n\n const startUserFn = process.hrtime();\n\n const namespace = pageContext.apifyNamespace;\n const output = await page.evaluate(\n async (ctxOpts: typeof contextOptions, namespc: string) => {\n const context: ReturnType['context'] =\n window[namespc].createContext(ctxOpts);\n // eslint-disable-next-line @typescript-eslint/no-shadow\n const output = {} as Output;\n try {\n output.pageFunctionResult =\n await window[namespc].pageFunction(context);\n } catch (err) {\n const casted = err as Error;\n output.pageFunctionError = Object.getOwnPropertyNames(\n casted,\n ).reduce((memo, name) => {\n memo[name] = casted[name as keyof Error];\n return memo;\n }, {} as Dictionary);\n }\n\n // This needs to be added after pageFunction has run.\n output.requestFromBrowser = context.request as Request;\n\n /**\n * Since Dates cannot travel back to Node and Puppeteer does not use .toJSON\n * to stringify, they come back as empty objects. We could use JSON.stringify\n * ourselves, but that exposes us to overridden .toJSON in the target websites.\n * This hack is not ideal, but it avoids both problems.\n */\n function replaceAllDatesInObjectWithISOStrings(obj: Output) {\n for (const [key, value] of Object.entries(obj)) {\n if (\n value instanceof Date &&\n typeof value.toISOString === 'function'\n ) {\n Reflect.set(obj, key, value.toISOString());\n } else if (value && typeof value === 'object') {\n replaceAllDatesInObjectWithISOStrings(\n value as Output,\n );\n }\n }\n return obj;\n }\n\n return replaceAllDatesInObjectWithISOStrings(output);\n },\n contextOptions,\n namespace,\n );\n\n tools.logPerformance(\n request,\n 'requestHandler USER FUNCTION',\n startUserFn,\n );\n const finishUserFn = process.hrtime();\n\n /**\n * POST-PROCESSING\n */\n const { pageFunctionResult, requestFromBrowser, pageFunctionError } =\n output;\n // Merge requestFromBrowser into request to preserve modifications that\n // may have been made in browser context.\n Object.assign(request, requestFromBrowser);\n\n // Throw error from pageFunction, if any.\n if (pageFunctionError) throw tools.createError(pageFunctionError);\n\n // Enqueue more links if a link selector is available,\n // unless the user invoked the `skipLinks()` context function\n // or maxCrawlingDepth would be exceeded.\n if (!pageContext.skipLinks) {\n await this._handleLinks(crawlingContext);\n }\n\n // Save the `pageFunction`s result (or just metadata) to the default dataset.\n await this._handleResult(request, response, pageFunctionResult);\n\n tools.logPerformance(\n request,\n 'requestHandler POSTPROCESSING',\n finishUserFn,\n );\n tools.logPerformance(request, 'requestHandler EXECUTION', start);\n\n if (\n this.isDevRun &&\n this.input.breakpointLocation ===\n BreakpointLocation.AfterPageFunction\n ) {\n await page.evaluate(async () => {\n // eslint-disable-next-line no-debugger -- Debugger is enabled in dev run\n debugger;\n });\n }\n }\n\n private async _handleMaxResultsPerCrawl(autoscaledPool?: AutoscaledPool) {\n if (\n !this.input.maxResultsPerCrawl ||\n this.pagesOutputted < this.input.maxResultsPerCrawl\n )\n return false;\n if (!autoscaledPool) return false;\n log.info(\n `User set limit of ${this.input.maxResultsPerCrawl} results was reached. Finishing the crawl.`,\n );\n await autoscaledPool.abort();\n return true;\n }\n\n private async _handleLinks({\n request,\n enqueueLinks,\n }: PuppeteerCrawlingContext) {\n if (!(this.input.linkSelector && this.requestQueue)) return;\n const start = process.hrtime();\n\n const currentDepth = (\n request.userData![META_KEY] as tools.RequestMetadata\n ).depth;\n const hasReachedMaxDepth =\n this.input.maxCrawlingDepth &&\n currentDepth >= this.input.maxCrawlingDepth;\n if (hasReachedMaxDepth) {\n log.debug(\n `Request ${request.url} reached the maximum crawling depth of ${currentDepth}.`,\n );\n return;\n }\n\n await enqueueLinks({\n selector: this.input.linkSelector,\n globs: this.input.globs,\n pseudoUrls: this.input.pseudoUrls,\n exclude: this.input.excludes,\n transformRequestFunction: (requestOptions) => {\n requestOptions.userData ??= {};\n requestOptions.userData[META_KEY] = {\n parentRequestId: request.id || request.uniqueKey,\n depth: currentDepth + 1,\n };\n\n requestOptions.useExtendedUniqueKey = true;\n requestOptions.keepUrlFragment = this.input.keepUrlFragments;\n return requestOptions;\n },\n });\n\n tools.logPerformance(request, 'handleLinks EXECUTION', start);\n }\n\n private async _handleResult(\n request: Request,\n response?: HTTPResponse,\n pageFunctionResult?: Dictionary,\n isError?: boolean,\n ) {\n const start = process.hrtime();\n const payload = tools.createDatasetPayload(\n request,\n response,\n pageFunctionResult,\n isError,\n );\n await this.dataset.pushData(payload);\n this.pagesOutputted++;\n tools.logPerformance(request, 'handleResult EXECUTION', start);\n }\n\n private async _assertNamespace(page: Page, namespace: string) {\n try {\n await page.waitForFunction(\n (nmspc: string) => !!window[nmspc],\n { timeout: this.input.pageLoadTimeoutSecs * 1000 },\n namespace,\n );\n } catch (err) {\n const casted = err as Error;\n if (casted.stack!.startsWith('TimeoutError')) {\n throw new Error(\n 'Injection of environment into the browser context timed out. ' +\n 'If this persists even after retries, try increasing the Page load timeout input setting.',\n );\n } else {\n throw err;\n }\n }\n }\n\n private async _waitForLoadEventWhenXml(\n page: Page,\n response?: HTTPResponse,\n ) {\n // Response can sometimes be null.\n if (!response) return;\n\n const cTypeHeader = response.headers()['content-type'];\n try {\n const { type } = contentType.parse(cTypeHeader);\n if (!/^(text|application)\\/xml$|\\+xml$/.test(type)) return;\n } catch (err) {\n // Invalid type is not XML.\n return;\n }\n\n try {\n const timeout = this.input.pageLoadTimeoutSecs * 1000;\n await page.waitForFunction(\n () => document.readyState === 'complete',\n { timeout },\n );\n } catch (err) {\n const casted = err as Error;\n if (casted.stack!.startsWith('TimeoutError')) {\n throw new Error(\n \"Parsing of XML in the page timed out. If you're expecting a large XML file, \" +\n ' such as a site map, try increasing the Page load timeout input setting.',\n );\n } else {\n throw err;\n }\n }\n }\n\n private async _injectBrowserHandles(page: Page, pageContext: PageContext) {\n const saveSnapshotP = browserTools.createBrowserHandle(page, async () =>\n browserTools.saveSnapshot({ page }),\n );\n const skipLinksP = browserTools.createBrowserHandle(page, () => {\n pageContext.skipLinks = true;\n });\n const globalStoreP = browserTools.createBrowserHandlesForObject(\n page,\n this.globalStore,\n [\n 'size',\n 'clear',\n 'delete',\n 'entries',\n 'get',\n 'has',\n 'keys',\n 'set',\n 'values',\n ],\n ['size'],\n );\n const logP = browserTools.createBrowserHandlesForObject(page, log, [\n 'LEVELS',\n 'setLevel',\n 'getLevel',\n 'debug',\n 'info',\n 'warning',\n 'error',\n 'exception',\n ]);\n const requestQueueP = this.requestQueue\n ? browserTools.createBrowserHandlesForObject(\n page,\n this.requestQueue,\n ['addRequest'],\n )\n : null;\n const keyValueStoreP = this.keyValueStore\n ? browserTools.createBrowserHandlesForObject(\n page,\n this.keyValueStore,\n ['getValue', 'setValue'],\n )\n : null;\n\n const [\n saveSnapshot,\n skipLinks,\n globalStore,\n logHandle,\n requestQueue,\n keyValueStore,\n ] = await Promise.all([\n saveSnapshotP,\n skipLinksP,\n globalStoreP,\n logP,\n\n requestQueueP,\n\n keyValueStoreP,\n ]);\n\n return {\n saveSnapshot,\n skipLinks,\n globalStore,\n log: logHandle,\n keyValueStore,\n requestQueue,\n };\n }\n}\n\nfunction logDevRunWarning() {\n log.warning(`\n*****************************************************************\n* Web Scraper is running in DEVELOPMENT MODE! *\n* Concurrency is limited, sessionPool is not available, *\n* timeouts are increased and debugger is enabled. *\n* If you want full control and performance switch *\n* Run type to PRODUCTION! *\n*****************************************************************\n`);\n}" }, { "name": "src/internals/global_store.ts", "format": "TEXT", "content": "/**\n * GlobalStore is a trivial storage that resembles a Map to be used from Browser contexts\n * to retain data through page navigations and browser instances. It limits Map's functionality\n * because it's currently impossible for functions and object references to cross Node-Browser threshold.\n */\nexport class GlobalStore extends Map {\n override get(key: string) {\n if (typeof key !== 'string') {\n throw new Error(\n 'GlobalStore#get parameter \"key\" must be a string.',\n );\n }\n return super.get(key);\n }\n\n override set(key: string, value: V) {\n if (typeof key !== 'string') {\n throw new Error(\n 'GlobalStore#set parameter \"key\" must be a string.',\n );\n }\n return super.set(key, value);\n }\n\n override forEach(): never {\n throw new Error(\n 'GlobalStore#forEach function is not available due to underlying technology limitations.',\n );\n }\n\n override values() {\n return Array.from(super.values()) as any;\n }\n\n override keys() {\n return Array.from(super.keys()) as any;\n }\n\n override entries() {\n return Array.from(super.entries()) as any;\n }\n}" }, { "name": "src/main.ts", "format": "TEXT", "content": "import { runActor } from '@apify/scraper-tools';\n\nimport { CrawlerSetup } from './internals/crawler_setup.js';\n\nrunActor(CrawlerSetup);" }, { "name": ".dockerignore", "format": "TEXT", "content": "# configurations\n.idea\n\n# crawlee and apify storage folders\napify_storage\ncrawlee_storage\nstorage\n\n# installed files\nnode_modules" }, { "name": "CHANGELOG.md", "format": "TEXT", "content": "# Change Log\n\n## 3.0.18 (2024-10-25)\n\n- Updated Crawlee version to v3.11.5 and SDK v3.2.6\n- Updated Node to v22\n\n## 3.0.17 (2024-04-09)\n\n- Updated Crawlee version to v3.8.0.\n- Updated to use new request queue in scraper\n\n## 3.0.14 (2023-08-22)\n\n- Updated Crawlee version to v3.5.2.\n- Updated Node.js version to v18.\n- Added new options:\n - **Dismiss cookie modals** (`closeCookieModals`): Using the [I don't care about cookies](https://addons.mozilla.org/en-US/firefox/addon/i-dont-care-about-cookies/) browser extension. When on, the crawler will automatically try to dismiss cookie consent modals. This can be useful when crawling European websites that show cookie consent modals.\n - **Maximum scrolling distance in pixels** (`maxScrollHeightPixels`): The crawler will scroll down the page until all content is loaded or the maximum scrolling distance is reached. Setting this to `0` disables scrolling altogether.\n - **Exclude Glob Patterns** (`excludes`): Glob patterns to match links in the page that you want to exclude from being enqueued.\n\n## 3.0 (`version-3`)\n\n- Rewrite from Apify SDK to Crawlee, see the [v3 migration guide](https://sdk.apify.com/docs/upgrading/upgrading-to-v3) for more details.\n- Proxy usage is now required.\n\n## 2.0 (`version-2`)\n\nMain difference between v1 and v2 of the scrapers is the upgrade of SDK to v2, which requires node v15.10+. SDK v2 uses http2 to do the requests with cheerio-scraper, and the http2 support in older node versions were too buggy, so we decided to drop support for those. If you need to run on older node version, use SDK v1.\n\nPlease refer to the SDK 1.0 migration guide for more details about functional changes in the SDK. SDK v2 basically only changes the required node version and has no other breaking changes.\n\n- deprecated `useRequestQueue` option has been removed\n - `RequestQueue` will be always used\n- deprecated `context.html` getter from the `cheerio-scraper` has been removed\n - use `context.body` instead\n- deprecated prepareRequestFunction input option\n - use `pre/postNavigationHooks` instead\n- removed `puppeteerPool/autoscaledPool` from the crawlingContext object\n - `puppeteerPool` was replaced by `browserPool`\n - `autoscaledPool` and `browserPool` and available on the `crawler` property of `crawlingContext` object\n- custom \"Key-value store name\" option in Advanced configuration is now fixed, previously the default store was always used" }, { "name": "Dockerfile", "format": "TEXT", "content": "FROM apify/actor-node-puppeteer-chrome:22 AS builder\n\nCOPY --chown=myuser package*.json ./\n\nRUN npm install --include=dev --audit=false\n\nCOPY --chown=myuser . ./\n\nRUN npm run build\n\nFROM apify/actor-node-puppeteer-chrome:22\n\nCOPY --from=builder --chown=myuser /home/myuser/dist ./dist\n\nCOPY --chown=myuser package*.json ./\n\nRUN npm --quiet set progress=false \\\n && npm install --omit=dev --omit=optional \\\n && echo \"Installed NPM packages:\" \\\n && (npm list --omit=dev --all || true) \\\n && echo \"Node.js version:\" \\\n && node --version \\\n && echo \"NPM version:\" \\\n && npm --version \\\n && rm -r ~/.npm\n\nCOPY --chown=myuser . ./\n\nENV APIFY_DISABLE_OUTDATED_WARNING=1\n\nCMD ./start_xvfb_and_run_cmd.sh && npm run start:prod --silent" }, { "name": "INPUT_SCHEMA.json", "format": "TEXT", "content": "{\n \"title\": \"Web Scraper Input\",\n \"type\": \"object\",\n \"description\": \"Web Scraper loads Start URLs in the Chrome browser and executes Page function on each page to extract data from it. To follow links and scrape additional pages, set Link selector with Pseudo-URLs and/or Glob patterns to specify which links to follow. Alternatively, you can manually enqueue new links in Page function. For details, see Actor's README or Web scraping tutorial in the Apify documentation.\",\n \"schemaVersion\": 1,\n \"properties\": {\n \"runMode\": {\n \"sectionCaption\": \"Basic configuration\",\n \"title\": \"Run mode\",\n \"type\": \"string\",\n \"description\": \"This property indicates the scraper's mode of operation. In DEVELOPMENT mode, the scraper ignores page timeouts, doesn't use sessionPool, opens pages one by one and enables debugging via Chrome DevTools. Open the live view tab or the container URL to access the debugger. Further debugging options can be configured in the Advanced configuration section. PRODUCTION mode disables debugging and enables timeouts and concurrency.

For details, see Run mode in README.\",\n \"default\": \"PRODUCTION\",\n \"prefill\": \"DEVELOPMENT\",\n \"editor\": \"select\",\n \"enum\": [\"PRODUCTION\", \"DEVELOPMENT\"]\n },\n \"startUrls\": {\n \"title\": \"Start URLs\",\n \"type\": \"array\",\n \"description\": \"A static list of URLs to scrape.

For details, see Start URLs in README.\",\n \"prefill\": [{ \"url\": \"https://crawlee.dev/js\" }],\n \"editor\": \"requestListSources\"\n },\n \"keepUrlFragments\": {\n \"title\": \"URL #fragments identify unique pages\",\n \"type\": \"boolean\",\n \"description\": \"Indicates that URL fragments (e.g. http://example.com#fragment) should be included when checking whether a URL has already been visited or not. Typically, URL fragments are used for page navigation only and therefore they should be ignored, as they don't identify separate pages. However, some single-page websites use URL fragments to display different pages; in such a case, this option should be enabled.\",\n \"default\": false,\n \"groupCaption\": \"Options\"\n },\n \"respectRobotsTxtFile\": {\n \"title\": \"Respect the robots.txt file\",\n \"type\": \"boolean\",\n \"description\": \"If enabled, the crawler will consult the robots.txt file for the target website before crawling each page. At the moment, the crawler does not use any specific user agent identifier. The crawl-delay directive is also not supported yet.\",\n \"default\": false,\n \"prefill\": true\n },\n \"linkSelector\": {\n \"title\": \"Link selector\",\n \"type\": \"string\",\n \"description\": \"A CSS selector saying which links on the page (<a> elements with href attribute) shall be followed and added to the request queue. To filter the links added to the queue, use the Pseudo-URLs and/or Glob patterns setting.

If Link selector is empty, the page links are ignored.

For details, see Link selector in README.\",\n \"editor\": \"textfield\",\n \"prefill\": \"a[href]\"\n },\n \"globs\": {\n \"title\": \"Glob Patterns\",\n \"type\": \"array\",\n \"description\": \"Glob patterns to match links in the page that you want to enqueue. Combine with Link selector to tell the scraper where to find links. Omitting the Glob patterns will cause the scraper to enqueue all links matched by the Link selector.\",\n \"editor\": \"globs\",\n \"default\": [],\n \"prefill\": [\n {\n \"glob\": \"https://crawlee.dev/js/*/*\"\n }\n ]\n },\n \"pseudoUrls\": {\n \"title\": \"Pseudo-URLs\",\n \"type\": \"array\",\n \"description\": \"Specifies what kind of URLs found by Link selector should be added to the request queue. A pseudo-URL is a URL with regular expressions enclosed in [] brackets, e.g. http://www.example.com/[.*].

If Pseudo-URLs are omitted, the Actor enqueues all links matched by the Link selector.

For details, see Pseudo-URLs in README.\",\n \"editor\": \"pseudoUrls\",\n \"default\": [],\n \"prefill\": []\n },\n \"excludes\": {\n \"title\": \"Exclude Glob Patterns\",\n \"type\": \"array\",\n \"description\": \"Glob patterns to match links in the page that you want to exclude from being enqueued.\",\n \"editor\": \"globs\",\n \"default\": [],\n \"prefill\": [\n {\n \"glob\": \"/**/*.{png,jpg,jpeg,pdf}\"\n }\n ]\n },\n \"pageFunction\": {\n \"title\": \"Page function\",\n \"type\": \"string\",\n \"description\": \"JavaScript (ES6) function that is executed in the context of every page loaded in the Chrome browser. Use it to scrape data from the page, perform actions or add new URLs to the request queue.

For details, see Page function in README.\",\n \"prefill\": \"// The function accepts a single argument: the \\\"context\\\" object.\\n// For a complete list of its properties and functions,\\n// see https://apify.com/apify/web-scraper#page-function \\nasync function pageFunction(context) {\\n // This statement works as a breakpoint when you're trying to debug your code. Works only with Run mode: DEVELOPMENT!\\n // debugger; \\n\\n // jQuery is handy for finding DOM elements and extracting data from them.\\n // To use it, make sure to enable the \\\"Inject jQuery\\\" option.\\n const $ = context.jQuery;\\n const pageTitle = $('title').first().text();\\n const h1 = $('h1').first().text();\\n const first_h2 = $('h2').first().text();\\n const random_text_from_the_page = $('p').first().text();\\n\\n\\n // Print some information to Actor log\\n context.log.info(`URL: ${context.request.url}, TITLE: ${pageTitle}`);\\n\\n // Manually add a new page to the queue for scraping.\\n await context.enqueueRequest({ url: 'http://www.example.com' });\\n\\n // Return an object with the data extracted from the page.\\n // It will be stored to the resulting dataset.\\n return {\\n url: context.request.url,\\n pageTitle,\\n h1,\\n first_h2,\\n random_text_from_the_page\\n };\\n}\",\n \"editor\": \"javascript\"\n },\n \"injectJQuery\": {\n \"title\": \"Inject jQuery\",\n \"type\": \"boolean\",\n \"description\": \"If enabled, the scraper will inject the jQuery library into every web page loaded, before Page function is invoked. Note that the jQuery object ($) will not be registered into global namespace in order to avoid conflicts with libraries used by the web page. It can only be accessed through context.jQuery in Page function.\",\n \"default\": true\n },\n \"proxyConfiguration\": {\n \"sectionCaption\": \"Proxy and browser configuration\",\n \"title\": \"Proxy configuration\",\n \"type\": \"object\",\n \"description\": \"Specifies proxy servers that will be used by the scraper in order to hide its origin.

For details, see Proxy configuration in README.\",\n \"prefill\": { \"useApifyProxy\": true },\n \"default\": { \"useApifyProxy\": true },\n \"editor\": \"proxy\"\n },\n \"proxyRotation\": {\n \"title\": \"Proxy rotation\",\n \"type\": \"string\",\n \"description\": \"This property indicates the strategy of proxy rotation and can only be used in conjunction with Apify Proxy. The recommended setting automatically picks the best proxies from your available pool and rotates them evenly, discarding proxies that become blocked or unresponsive. If this strategy does not work for you for any reason, you may configure the scraper to either use a new proxy for each request, or to use one proxy as long as possible, until the proxy fails. IMPORTANT: This setting will only use your available Apify Proxy pool, so if you don't have enough proxies for a given task, no rotation setting will produce satisfactory results.\",\n \"default\": \"RECOMMENDED\",\n \"editor\": \"select\",\n \"enum\": [\"RECOMMENDED\", \"PER_REQUEST\", \"UNTIL_FAILURE\"],\n \"enumTitles\": [\n \"Use recommended settings\",\n \"Rotate proxy after each request\",\n \"Use one proxy until failure\"\n ]\n },\n \"sessionPoolName\": {\n \"title\": \"Session pool name\",\n \"type\": \"string\",\n \"description\": \"Use only english alphanumeric characters dashes and underscores. A session is a representation of a user. It has it's own IP and cookies which are then used together to emulate a real user. Usage of the sessions is controlled by the Proxy rotation option. By providing a session pool name, you enable sharing of those sessions across multiple Actor runs. This is very useful when you need specific cookies for accessing the websites or when a lot of your proxies are already blocked. Instead of trying randomly, a list of working sessions will be saved and a new Actor run can reuse those sessions. Note that the IP lock on sessions expires after 24 hours, unless the session is used again in that window.\",\n \"editor\": \"textfield\",\n \"minLength\": 3,\n \"maxLength\": 200,\n \"pattern\": \"[0-9A-z-]\"\n },\n \"initialCookies\": {\n \"title\": \"Initial cookies\",\n \"type\": \"array\",\n \"description\": \"A JSON array with cookies that will be set to every Chrome browser tab opened before loading the page, in the format accepted by Puppeteer's Page.setCookie() function. This option is useful for transferring a logged-in session from an external web browser.\",\n \"default\": [],\n \"prefill\": [],\n \"editor\": \"json\"\n },\n \"useChrome\": {\n \"title\": \"Use Chrome\",\n \"type\": \"boolean\",\n \"description\": \"If enabled, the scraper will use a real Chrome browser instead of Chromium bundled with Puppeteer. This option may help bypass certain anti-scraping protections, but might make the scraper unstable. Use at your own risk \\uD83D\\uDE42\",\n \"default\": false,\n \"groupCaption\": \"Browser masking\"\n },\n \"headless\": {\n \"title\": \"Run browsers in headless mode\",\n \"type\": \"boolean\",\n \"description\": \"By default, browsers run in headless mode. You can toggle this off to run them in headful mode, which can help with certain rare anti-scraping protections but is slower and more costly.\",\n \"default\": true\n },\n \"ignoreSslErrors\": {\n \"title\": \"Ignore SSL errors\",\n \"type\": \"boolean\",\n \"description\": \"If enabled, the scraper will ignore SSL/TLS certificate errors. Use at your own risk.\",\n \"default\": false,\n \"groupCaption\": \"Security\"\n },\n \"ignoreCorsAndCsp\": {\n \"title\": \"Ignore CORS and CSP\",\n \"type\": \"boolean\",\n \"description\": \"If enabled, the scraper will ignore Content Security Policy (CSP) and Cross-Origin Resource Sharing (CORS) settings of visited pages and requested domains. This enables you to freely use XHR/Fetch to make HTTP requests from Page function.\",\n \"default\": false\n },\n \"downloadMedia\": {\n \"sectionCaption\": \"Performance and limits\",\n \"title\": \"Download media files\",\n \"type\": \"boolean\",\n \"description\": \"If enabled, the scraper will download media such as images, fonts, videos and sound files, as usual. Disabling this option might speed up the scrape, but certain websites could stop working correctly.\",\n \"default\": true,\n \"groupCaption\": \"Page resources\"\n },\n \"downloadCss\": {\n \"title\": \"Download CSS files\",\n \"type\": \"boolean\",\n \"description\": \"If enabled, the scraper will download CSS files with stylesheets, as usual. Disabling this option may speed up the scrape, but certain websites could stop working correctly, and the live view will not look as cool.\",\n \"default\": true\n },\n \"maxRequestRetries\": {\n \"title\": \"Max page retries\",\n \"type\": \"integer\",\n \"description\": \"The maximum number of times the scraper will retry to load each web page on error, in case of a page load error or an exception thrown by Page function.

If set to 0, the page will be considered failed right after the first error.\",\n \"minimum\": 0,\n \"default\": 3\n },\n \"maxPagesPerCrawl\": {\n \"title\": \"Max pages per run\",\n \"type\": \"integer\",\n \"description\": \"The maximum number of pages that the scraper will load. The scraper will stop when this limit is reached. It's always a good idea to set this limit in order to prevent excess platform usage for misconfigured scrapers. Note that the actual number of pages loaded might be slightly higher than this value.

If set to 0, there is no limit.\",\n \"minimum\": 0,\n \"default\": 0\n },\n \"maxResultsPerCrawl\": {\n \"title\": \"Max result records\",\n \"type\": \"integer\",\n \"description\": \"The maximum number of records that will be saved to the resulting dataset. The scraper will stop when this limit is reached.

If set to 0, there is no limit.\",\n \"minimum\": 0,\n \"default\": 0\n },\n \"maxCrawlingDepth\": {\n \"title\": \"Max crawling depth\",\n \"type\": \"integer\",\n \"description\": \"Specifies how many links away from Start URLs the scraper will descend. This value is a safeguard against infinite crawling depths for misconfigured scrapers. Note that pages added using context.enqueuePage() in Page function are not subject to the maximum depth constraint.

If set to 0, there is no limit. To crawl only the pages specified by the Start URLs, set linkSelector empty instead.\",\n \"minimum\": 0,\n \"default\": 0\n },\n \"maxConcurrency\": {\n \"title\": \"Max concurrency\",\n \"type\": \"integer\",\n \"description\": \"Specified the maximum number of pages that can be processed by the scraper in parallel. The scraper automatically increases and decreases concurrency based on available system resources. This option enables you to set an upper limit, for example to reduce the load on a target web server.\",\n \"minimum\": 1,\n \"default\": 50\n },\n \"pageLoadTimeoutSecs\": {\n \"title\": \"Page load timeout\",\n \"type\": \"integer\",\n \"description\": \"The maximum amount of time the scraper will wait for a web page to load, in seconds. If the web page does not load in this timeframe, it is considered to have failed and will be retried (subject to Max page retries), similarly as with other page load errors.\",\n \"minimum\": 1,\n \"default\": 60,\n \"unit\": \"seconds\"\n },\n \"pageFunctionTimeoutSecs\": {\n \"title\": \"Page function timeout\",\n \"type\": \"integer\",\n \"description\": \"The maximum amount of time the scraper will wait for Page function to execute, in seconds. It's a good idea to set this limit, to ensure that unexpected behavior in page function will not get the scraper stuck.\",\n \"minimum\": 1,\n \"default\": 60,\n \"unit\": \"seconds\"\n },\n \"waitUntil\": {\n \"title\": \"Navigation waits until\",\n \"type\": \"array\",\n \"description\": \"Contains a JSON array with names of page events to wait, before considering a web page fully loaded. The scraper will wait until all of the events are triggered in the web page before executing Page function. Available events are domcontentloaded, load, networkidle2 and networkidle0.

For details, see waitUntil option in Puppeteer's Page.goto() function documentation.\",\n \"default\": [\"networkidle2\"],\n \"prefill\": [\"networkidle2\"],\n \"editor\": \"json\"\n },\n \"preNavigationHooks\": {\n \"sectionCaption\": \"Advanced configuration\",\n \"title\": \"Pre-navigation hooks\",\n \"type\": \"string\",\n \"description\": \"Async functions that are sequentially evaluated before the navigation. Good for setting additional cookies or browser properties before navigation. The function accepts two parameters, `crawlingContext` and `gotoOptions`, which are passed to the `page.goto()` function the crawler calls to navigate.\",\n \"prefill\": \"// We need to return array of (possibly async) functions here.\\n// The functions accept two arguments: the \\\"crawlingContext\\\" object\\n// and \\\"gotoOptions\\\".\\n[\\n async (crawlingContext, gotoOptions) => {\\n // ...\\n },\\n]\\n\",\n \"editor\": \"javascript\"\n },\n \"postNavigationHooks\": {\n \"title\": \"Post-navigation hooks\",\n \"type\": \"string\",\n \"description\": \"Async functions that are sequentially evaluated after the navigation. Good for checking if the navigation was successful. The function accepts `crawlingContext` as the only parameter.\",\n \"prefill\": \"// We need to return array of (possibly async) functions here.\\n// The functions accept a single argument: the \\\"crawlingContext\\\" object.\\n[\\n async (crawlingContext) => {\\n // ...\\n },\\n]\",\n \"editor\": \"javascript\"\n },\n \"breakpointLocation\": {\n \"title\": \"Insert breakpoint\",\n \"type\": \"string\",\n \"description\": \"This property has no effect if Run mode is set to PRODUCTION. When set to DEVELOPMENT it inserts a breakpoint at the selected location in every page the scraper visits. Execution of code stops at the breakpoint until manually resumed in the DevTools window accessible via Live View tab or Container URL. Additional breakpoints can be added by adding debugger; statements within your Page function.

See Run mode in README for details.\",\n \"default\": \"NONE\",\n \"prefill\": \"NONE\",\n \"editor\": \"select\",\n \"enum\": [\n \"NONE\",\n \"BEFORE_GOTO\",\n \"BEFORE_PAGE_FUNCTION\",\n \"AFTER_PAGE_FUNCTION\"\n ],\n \"enumTitles\": [\n \"Nowhere. Break only on debugger; statements\",\n \"Before navigation to URL\",\n \"Before Page function invocation\",\n \"After Page function invocation\"\n ]\n },\n \"closeCookieModals\": {\n \"title\": \"Dismiss cookie modals\",\n \"type\": \"boolean\",\n \"description\": \"Using the [I don't care about cookies](https://addons.mozilla.org/en-US/firefox/addon/i-dont-care-about-cookies/) browser extension. When on, the crawler will automatically try to dismiss cookie consent modals. This can be useful when crawling European websites that show cookie consent modals.\",\n \"default\": false\n },\n \"maxScrollHeightPixels\": {\n \"title\": \"Maximum scrolling distance in pixels\",\n \"type\": \"integer\",\n \"description\": \"The crawler will scroll down the page until all content is loaded or the maximum scrolling distance is reached. Setting this to `0` disables scrolling altogether.\",\n \"default\": 5000\n },\n \"debugLog\": {\n \"title\": \"Debug log\",\n \"type\": \"boolean\",\n \"description\": \"If enabled, the Actor log will include debug messages. Beware that this can be quite verbose. Use context.log.debug('message') to log your own debug messages from Page function.\",\n \"default\": false,\n \"groupCaption\": \"Logging\"\n },\n \"browserLog\": {\n \"title\": \"Browser log\",\n \"type\": \"boolean\",\n \"description\": \"If enabled, the Actor log will include console messages produced by JavaScript executed by the web pages (e.g. using console.log()). Beware that this may result in the log being flooded by error messages, warnings and other messages of little value, especially with high concurrency.\",\n \"default\": false\n },\n \"customData\": {\n \"title\": \"Custom data\",\n \"type\": \"object\",\n \"description\": \"A custom JSON object that is passed to Page function as context.customData. This setting is useful when invoking the scraper via API, in order to pass some arbitrary parameters to your code.\",\n \"default\": {},\n \"prefill\": {},\n \"editor\": \"json\"\n },\n \"datasetName\": {\n \"title\": \"Dataset name\",\n \"type\": \"string\",\n \"description\": \"Name or ID of the dataset that will be used for storing results. If left empty, the default dataset of the run will be used.\",\n \"editor\": \"textfield\"\n },\n \"keyValueStoreName\": {\n \"title\": \"Key-value store name\",\n \"type\": \"string\",\n \"description\": \"Name or ID of the key-value store that will be used for storing records. If left empty, the default key-value store of the run will be used.\",\n \"editor\": \"textfield\"\n },\n \"requestQueueName\": {\n \"title\": \"Request queue name\",\n \"type\": \"string\",\n \"description\": \"Name of the request queue that will be used for storing requests. If left empty, the default request queue of the run will be used.\",\n \"editor\": \"textfield\"\n }\n },\n \"required\": [\"startUrls\", \"pageFunction\", \"proxyConfiguration\"]\n}" }, { "name": "README.md", "format": "TEXT", "content": "# Web Scraper\n\n## What is Web Scraper?\n\nWeb Scraper is a tool for extracting data from any website. It can navigate pages, render JavaScript, and extract structured data using a few simple commands. Whether you need to scrape product prices, real estate data, or social media profiles, this Actor turns any web page into an API.\n\n- Configurable with an **intuitive user interface**\n- Can handle almost **any website** and can scrape dynamic content\n- Scrape a list of **URLs or crawl an entire website** by following links\n- Runs entirely on the **Apify platform**; no need to manage servers or proxies\n- Set your scraper to **run on a schedule** and get data delivered automatically\n- Can be used as a template to **create your own scraper**\n\n## What can Web Scraper data be used for?\n\nWeb Scraper can extract almost any data from any site, effectively turning any site into a data source. All data can be exported into **JSON, CSV, HTML, and Excel** formats.\n\nHere are some examples:\n\n- **Extract reviews** from sites like Yelp or Amazon\n- Gather **real estate data** from Zillow or local realtor pages\n- Get **contact details** and social media accounts from local businesses\n- **Monitor mentions** of a brand or person on specific sites\n- **Collect and monitor product prices** on e-commerce websites\n\nAs a generic tool, Web Scraper can also serve as a template to **build your own scraper** which you can then market on Apify Store.\n\n## How much does the Web Scraper cost?\n\nWeb Scraper is free to use, but you do pay for Apify platform usage, which is calculated in [compute units](https://help.apify.com/en/articles/3490384-what-is-a-compute-unit?ref=apify) (CU). On the free plan, these are charged at $0.04 per CU. CUs get cheaper with higher subscription plans - [see our pricing page](https://apify.com/pricing) for more details.\n\nWith our free plan, you get **$5 in platform credits every month**, which is enough to scrape from 500 to 1,000 **web pages**. If you sign up to our Starter plan, you can expect to scrape thousands.\n\n## How to use Web Scraper\n\n1. [Create](https://console.apify.com/actors/moJRLRc85AitArpNN?addFromActorId=moJRLRc85AitArpNN) a free Apify account using your email and open [Web Scraper](https://apify.com/apify/web-scraper)\n2. Add one or more URLs you want to scrape\n3. Set paths that you’d like to include or exclude from crawling by configuring glob patterns or pseudo-URLs\n4. Configure the page function that determines the data that needs to be scraped\n5. Click the “Start” button and wait for the data to be extracted\n6. Download your data in JSON, XML, CSV, Excel, or HTML\n\nFor more in-depth instructions, please read our article on [scraping with Web Scraper](https://docs.apify.com/tutorials/apify-scrapers/web-scraper), which features step-by-step instructions on how to use Web Scraper on the basis of real-life examples. We also have a video tutorial you can follow along with:\n\nhttps://www.youtube.com/watch?v=5kcaHAuGxmY\n\n## Using Web Scraper with the Apify API\n\nThe Apify API gives you programmatic access to the Apify platform. The API is organized around RESTful HTTP endpoints that enable you to manage, schedule, and run Apify Actors. The API also lets you access any datasets, monitor actor performance, fetch results, create and update versions, and more.\n\nTo access the API using Node.js, use the `apify-client` [NPM package](https://apify.com/apify/web-scraper/api/javascript). To access the API using Python, use the `apify-client` [PyPI package](https://apify.com/apify/web-scraper/api/python).\n\nClick on the [API tab](https://apify.com/apify/web-scraper/api/python) for code examples, or check out the [Apify API reference](https://docs.apify.com/api/v2) docs for all the details.\n\n## Web Scraper and MCP Server\n\nWith Apify API, you can use almost any Actor in conjunction with an MCP server. You can connect to the MCP server using clients like ClaudeDesktop and LibreChat, or even build your own. Read all about how you can [set up Apify Actors with MCP](https://blog.apify.com/how-to-use-mcp/).\n\nFor Web Scraper, go to the [MCP tab](https://apify.com/apify/web-scraper/api/mcp) and then go through the following steps:\n\n1. Start a Server-Sent Events (SSE) session to receive a `sessionId`\n2. Send API messages using that `sessionId` to trigger the scraper\n3. The message starts the Web Scraper with the provided input\n4. The response should be: `Accepted`\n\n## Integrating Web Scraper into your workflows\n\nYou can integrate Web Scraper with almost any cloud service or web app. We offer integrations with **Make, Zapier, Slack, Airbyte, GitHub, Google Sheets, Google Drive**, [and plenty more](https://docs.apify.com/integrations).\n\nAlternatively, you could use [webhooks](https://docs.apify.com/integrations/webhooks) to carry out an action whenever an event occurs, such as getting a notification whenever Web Scraper successfully finishes a run.\n\n## Advanced configuration settings\n\nBelow you’ll find detailed instructions on more advanced configuration settings for Web Scraper.\n\n## Input configurations\n\nOn input, the Web Scraper Actor accepts a number of configuration settings. These can be entered either manually in the user interface in [Apify Console](https://console.apify.com/), or programmatically in a JSON object using the [Apify API](https://docs.apify.com/api/v2#/reference/actors/run-collection/run-actor).\n\nFor a complete list of input fields and their type, please see the [input tab](https://apify.com/apify/web-scraper/input-schema).\n\n### Run mode\n\nRun mode allows you to switch between two modes of operation for Web Scraper.\n\n**PRODUCTION** mode gives you full control and full performance. You should always switch Web Scraper to production mode once you're done making changes to your scraper.\n\nWhen starting to develop your Scraper, you want to be able to inspect what's happening in the browser and debug your code. You can do that with the scraper's **DEVELOPMENT** mode. It allows you to directly control the browser using Chrome DevTools. Open the Live View tab to access the DevTools. It will also limit concurrency and prevent timeouts to improve your DevTools experience. Other debugging related options can be configured in the **Advanced configuration** section.\n\n### Start URLs\n\nThe **Start URLs** (`startUrls`) field represent the initial list of URLs of pages that the scraper will visit. You can either enter these URLs manually one by one, upload them in a CSV file or\n[link URLs from the Google Sheets](https://help.apify.com/en/articles/2906022-scraping-a-list-of-urls-from-a-google-sheets-document) document. Each URL must start with either a `http://` or `https://` protocol prefix.\n\nThe scraper supports adding new URLs to scrape on the fly, either using the [**Link selector**](#link-selector) and [**Glob Patterns**](#glob-patterns)/[**Pseudo-URLs**](#pseudo-urls) options or by calling `await context.enqueueRequest()` inside [**Page function**](#page-function).\n\nOptionally, each URL can be associated with custom user data - a JSON object that can be referenced from your JavaScript code in [**Page function**](#page-function) under `context.request.userData`. This is useful for determining which start URL is currently loaded, in order to perform some page-specific actions. For example, when crawling an online store, you might want to perform different\nactions on a page listing the products vs. a product detail page. For details, see our [web scraping tutorial](https://docs.apify.com/tutorials/apify-scrapers/getting-started#the-start-url).\n\n\n\n### Link selector\n\nThe **Link selector** (`linkSelector`) field contains a CSS selector that is used to find links to other web pages, i.e. `` elements with the `href` attribute.\n\nOn every page loaded, the scraper looks for all links matching **Link selector**, checks that the target URL matches one of the [**Glob Patterns**](#glob-patterns)/[**Pseudo-URLs**](#pseudo-urls), and if so then adds the URL to the request queue, so that it's loaded by the scraper later.\n\nBy default, new scrapers are created with the following selector that matches all links:\n\n```\na[href]\n```\n\nIf **Link selector** is empty, the page links are ignored, and the scraper only loads pages that were specified in [**Start URLs**](#start-urls) or that were manually added to the request queue by calling `await context.enqueueRequest()` in [**Page function**](#page-function).\n\n### Glob Patterns\n\nThe **Glob Patterns** (`globs`) field specifies which types of URLs found by [**Link selector**](#link-selector) should be added to the request queue.\n\nA glob pattern is simply a string with wildcard characters.\n\nFor example, a glob pattern `http://www.example.com/pages/**/*` will match all the\nfollowing URLs:\n\n- `http://www.example.com/pages/deeper-level/page`\n- `http://www.example.com/pages/my-awesome-page`\n- `http://www.example.com/pages/something`\n\nNote that you don't need to use the **Glob Patterns** setting at all, because you can completely control which pages the scraper will access by calling `await context.enqueueRequest()` from the [**Page function**](#page-function).\n\n### Pseudo-URLs\n\nThe **Pseudo-URLs** (`pseudoUrls`) field specifies what kind of URLs found by [**Link selector**](#link-selector) should be added to the request queue.\n\nA pseudo-URL is simply a URL with special directives enclosed in `[]` brackets. Currently, the only supported directive is `[regexp]`, which defines a JavaScript-style regular expression to match against the URL.\n\nFor example, a pseudo-URL `http://www.example.com/pages/[(\\w|-)*]` will match all the\nfollowing URLs:\n\n- `http://www.example.com/pages/`\n- `http://www.example.com/pages/my-awesome-page`\n- `http://www.example.com/pages/something`\n\nIf either `[` or `]` is part of the normal query string, it must be encoded as `[\\x5B]` or `[\\x5D]`, respectively. For example, the following pseudo-URL:\n\n```\nhttp://www.example.com/search?do[\\x5B]load[\\x5D]=1\n```\n\nwill match the URL:\n\n```\nhttp://www.example.com/search?do[load]=1\n```\n\nOptionally, each pseudo-URL can be associated with user data that can be referenced from\nyour [**Page function**](#page-function) using `context.request.label` to determine which kind of page is currently loaded in the browser.\n\nNote that you don't need to use the **Pseudo-URLs** setting at all, because you can completely control which pages the scraper will access by calling `await context.enqueueRequest()` from [**Page function**](#page-function).\n\n### Page function\n\nThe **Page function** (`pageFunction`) field contains a JavaScript function that is executed in the context of every page loaded in the Chromium browser. The purpose of this function is to extract\ndata from the web page, manipulate the DOM by clicking elements, add new URLs to the request queue and otherwise control Web Scraper's operation.\n\nExample:\n\n```javascript\nasync function pageFunction(context) {\n // jQuery is handy for finding DOM elements and extracting data from them.\n // To use it, make sure to enable the \"Inject jQuery\" option.\n const $ = context.jQuery;\n const pageTitle = $('title').first().text();\n\n // Print some information to Actor log\n context.log.info(`URL: ${context.request.url}, TITLE: ${pageTitle}`);\n\n // Manually add a new page to the scraping queue.\n await context.enqueueRequest({ url: 'http://www.example.com' });\n\n // Return an object with the data extracted from the page.\n // It will be stored to the resulting dataset.\n return {\n url: context.request.url,\n pageTitle,\n };\n}\n```\n\nThe page function accepts a single argument, the `context` object, whose properties are listed in the table below. Since the function is executed in the context of the web page, it can access the DOM, e.g. using the `window` or `document` global variables.\n\nThe return value of the page function is an object (or an array of objects) representing the data extracted from the web page. The return value must be stringify-able to JSON, i.e. it can only contain basic types and no circular references. If you don't want to extract any data from the page and skip it in the clean results, simply return `null` or `undefined`.\n\nThe page function supports the JavaScript ES6 syntax and is asynchronous, which means you can use the `await` keyword to wait for background operations to finish. To learn more about `async` functions, see Mozilla documentation.\n\n**Properties of the `context` object:**\n\n- **`customData: Object`**\n Contains the object provided in the **Custom data** (`customData`) input setting. This is useful for passing dynamic parameters to your Web Scraper using API.\n- **`enqueueRequest(request, [options]): AsyncFunction`**\n Adds a new URL to the request queue, if it wasn't already there. The `request` parameter is an object containing details of the request, with properties such as `url`, `label`, `userData`, `headers` etc. For the full list of the supported properties, see the `Request` object's constructor in Crawlee documentation.\n The optional `options` parameter is an object with additional options. Currently, it only supports the `forefront` boolean flag. If it's `true`, the request is added to the beginning of the queue. By default, requests are added to the end.\n Example:\n ```javascript\n await context.enqueueRequest({ url: 'https://www.example.com' });\n await context.enqueueRequest(\n { url: 'https://www.example.com/first' },\n { forefront: true },\n );\n ```\n- **`env: Object`**\n A map of all relevant values set by the Apify platform to the Actor run via the `APIFY_` environment variables. For example, you can find here information such as Actor run ID, timeouts, Actor run memory, etc.\n For the full list of available values, see\n `ApifyEnv` interface in Apify SDK.\n Example:\n ```javascript\n console.log(`Actor run ID: ${context.env.actorRunId}`);\n ```\n- **`getValue(key): AsyncFunction`**\n Gets a value from the default key-value store associated with the Actor run. The key-value store is useful for persisting named data records, such as state objects, files, etc. The function is very similar to `Actor.getValue()` function in Apify SDK.\n To set the value, use the dual function `context.setValue(key, value)`.\n Example:\n ```javascript\n const value = await context.getValue('my-key');\n console.dir(value);\n ```\n- **`globalStore: Object`**\n Represents an in-memory store that can be used to share data across page function invocations, e.g. state variables, API responses or other data. The `globalStore` object has an equivalent interface as JavaScript's `Map` object, with a few important differences:\n - All functions of `globalStore` are `async`; use `await` when calling them.\n - Keys must be strings and values need to be JSON stringify-able.\n - `forEach()` function is not supported.\n Note that the stored data is not persisted. If the Actor run is restarted or migrated to another worker server, the content of `globalStore` is reset. Therefore, never depend on a specific value to be present in the store.\n Example:\n ```javascript\n let movies = await context.globalStore.get('cached-movies');\n if (!movies) {\n movies = await fetch('http://example.com/movies.json');\n await context.globalStore.set('cached-movies', movies);\n }\n console.dir(movies);\n ```\n- **`input: Object`**\n An object containing the Actor run input, i.e. the Web Scraper's configuration. Each page function invocation gets a fresh copy of the `input` object, so changing its properties has no effect.\n- **`jQuery: Function`**\n A reference to the `jQuery` library, which is extremely useful for DOM traversing, manipulation, querying and data extraction. This field is only available if the **Inject jQuery** option is enabled.\n Typically, the jQuery function is registered under a global variable called $.\n However, the web page might use this global variable for something else. To avoid conflicts, the jQuery object is not registered globally and is only available through the `context.jQuery` property.\n Example:\n ```javascript\n const $ = context.jQuery;\n const pageTitle = $('title').first().text();\n ```\n- **`log: Object`**\n An object containing logging functions, with the same interface as provided by the `Crawlee.utils.log` object in Crawlee. The log messages are written directly to the Actor run log, which is useful for monitoring and debugging. Note that `log.debug()` only prints messages to the log if the **Enable debug log** input setting is set.\n Example:\n ```javascript\n const log = context.log;\n log.debug('Debug message', { hello: 'world!' });\n log.info('Information message', { all: 'good' });\n log.warning('Warning message');\n log.error('Error message', { details: 'This is bad!' });\n try {\n throw new Error('Not good!');\n } catch (e) {\n log.exception(e, 'Exception occurred', {\n details: 'This is really bad!',\n });\n }\n ```\n- **`request: Object`**\n An object containing information about the currently loaded web page, such as the URL, number of retries, a unique key, etc. Its properties are equivalent to the `Request` object in Crawlee.\n- **`response: Object`**\n An object containing information about the HTTP response from the web server. Currently, it only contains the `status` and `headers` properties. For example:\n\n ```javascript\n {\n // HTTP status code\n status: 200,\n\n // HTTP headers\n headers: {\n 'content-type': 'text/html; charset=utf-8',\n 'date': 'Wed, 06 Nov 2019 16:01:53 GMT',\n 'cache-control': 'no-cache',\n 'content-encoding': 'gzip',\n },\n }\n\n ```\n\n- **`saveSnapshot(): AsyncFunction`**\n Saves a screenshot and full HTML of the current page to the key-value store\n associated with the Actor run, under the `SNAPSHOT-SCREENSHOT` and `SNAPSHOT-HTML` keys, respectively. This feature is useful when debugging your scraper.\n Note that each snapshot overwrites the previous one and the `saveSnapshot()` calls are throttled to at most one call in two seconds, in order to avoid excess consumption of resources and slowdown of the Actor.\n- **`setValue(key, data, options): AsyncFunction`**\n Sets a value to the default key-value store associated with the Actor run. The key-value store is useful for persisting named data records, such as state objects, files, etc. The function is very similar to `KeyValueStore.setValue()` function in Crawlee.\n To get the value, use the dual function `await context.getValue(key)`.\n Example:\n ```javascript\n await context.setValue('my-key', { hello: 'world' });\n ```\n- **`skipLinks(): AsyncFunction`**\n Calling this function ensures that page links from the current page will not be added to the request queue, even if they match the [**Link selector**](#link-selector) and/or [**Glob Patterns**](#glob-patterns)/[**Pseudo-URLs**](#pseudo-urls) settings. This is useful to programmatically stop recursive crawling, e.g. if you know there are no more interesting links on the current page to follow.\n- **`waitFor(task, options): AsyncFunction`**\n A helper function that waits either a specific amount of time (in milliseconds), for an element specified using a CSS selector to appear in the DOM or for a provided function to return `true`.\n This is useful for extracting data from web pages with dynamic content, where the content might not be available at the time when the page function is called.\n The `options` parameter is an object with the following properties and default values:\n\n ```javascript\n {\n // Maximum time to wait\n timeoutMillis: 20000,\n\n // How often to check if the condition changes\n pollingIntervalMillis: 50,\n }\n ```\n\n Example:\n\n ```javascript\n // Wait for selector\n await context.waitFor('.foo');\n // Wait for 1 second\n await context.waitFor(1000);\n // Wait for predicate\n await context.waitFor(() => !!document.querySelector('.foo'), {\n timeoutMillis: 5000,\n });\n ```\n\n## Proxy configuration\n\nThe **Proxy configuration** (`proxyConfiguration`) option enables you to set proxies that will be used by the scraper in order to prevent its detection by target websites. You can use both [Apify Proxy](https://apify.com/proxy) and custom HTTP or SOCKS5 proxy servers.\n\nProxy is required to run the scraper. The following table lists the available options of the proxy configuration setting:\n\n\n \n \n \n \n \n \n \n \n \n \n \n
Apify Proxy (automatic)\n \n The scraper will load all web pages using Apify Proxy in the automatic mode. In this mode, the proxy uses all proxy groups that are available to the user, and for each new web page it automatically selects the proxy that hasn't been used in the longest time for the specific hostname, in order to reduce the chance of detection by the website. You can view the list of available proxy groups on the Proxy page in Apify Console.\n
Apify Proxy (selected groups)\n \n The scraper will load all web pages using Apify Proxy with specific groups of target proxy servers.\n
Custom proxies\n \n

\n The scraper will use a custom list of proxy servers. The proxies must be specified in the `scheme://user:password@host:port` format, multiple proxies should be separated by a space or new line. The URL scheme can be either `HTTP` or `SOCKS5`. User and password might be omitted, but the port must always be present.\n

\n

\n Example:\n

\n 
http://bob:password@proxy1.example.com:8000\nhttp://bob:password@proxy2.example.com:8000
\n
\n\nThe proxy configuration can be set programmatically when calling the Actor using the API\nby setting the `proxyConfiguration` field. It accepts a JSON object with the following structure:\n\n```javascript\n{\n // Indicates whether to use Apify Proxy or not.\n \"useApifyProxy\": Boolean,\n\n // Array of Apify Proxy groups, only used if \"useApifyProxy\" is true.\n // If missing or null, Apify Proxy will use the automatic mode.\n \"apifyProxyGroups\": String[],\n\n // Array of custom proxy URLs, in \"scheme://user:password@host:port\" format.\n // If missing or null, custom proxies are not used.\n \"proxyUrls\": String[],\n}\n```\n\n### Logging into websites with Web Scraper\n\nThe **Initial cookies** field allows you to set cookies that will be used by the scraper to log into websites. Cookies are small text files that are stored on your computer by your web browser. Various websites use cookies to store information about your current session. By transferring this information to the scraper, it will be able to log into websites using your credentials. To learn more about logging into websites by transferring cookies, check out our [tutorial](https://docs.apify.com/tutorials/log-in-by-transferring-cookies).\n\nBe aware that cookies usually have a limited lifespan and will expire after a certain period of time. This means that you will have to update the cookies periodically in order to keep the scraper logged in. Alternative approach is to make the scraper actively log in to the website in the Page function. For more info about this approach, check out our [tutorial](https://docs.apify.com/tutorials/log-into-a-website-using-puppeteer) on logging into websites using Puppeteer.\n\nThe scraper expects the cookies in the **Initial cookies** field to be stored as separate JSON objects in a JSON array, see example below:\n\n```json\n[\n {\n \"name\": \" ga\",\n \"value\": \"GA1.1.689972112. 1627459041\",\n \"domain\": \".apify.com\",\n \"hostOnly\": false,\n \"path\": \"/\",\n \"secure\": false,\n \"httpOnly\": false,\n \"sameSite\": \"no_restriction\",\n \"session\": false,\n \"firstPartyDomain\": \"\",\n \"expirationDate\": 1695304183,\n \"storelId\": \"firefox-default\",\n \"id\": 1\n }\n]\n```\n\n## Advanced configuration\n\n### Pre-navigation hooks\n\nThis is an array of functions that will be executed **BEFORE** the main `pageFunction` is run. A similar `context` object is passed into each of these functions as is passed into the `pageFunction`; however, a second \"[DirectNavigationOptions](https://crawlee.dev/api/puppeteer-crawler/namespace/puppeteerUtils#DirectNavigationOptions)\" object is also passed in.\n\nThe available options can be seen here:\n\n```javascript\npreNavigationHooks: [\n async (\n { id, request, session, proxyInfo },\n { timeout, waitUntil, referer },\n ) => {},\n];\n```\n\n> Unlike with playwright, puppeteer and cheerio scrapers, in web scraper we don't have the Actor object available in the hook parameters, as the hook is executed inside the browser.\n\nCheck out the docs for [Pre-navigation hooks](https://crawlee.dev/api/puppeteer-crawler/interface/PuppeteerCrawlerOptions#preNavigationHooks) and the [PuppeteerHook type](https://crawlee.dev/api/puppeteer-crawler/interface/PuppeteerHook) for more info regarding the objects passed into these functions.\n\n### Post-navigation hooks\n\nAn array of functions that will be executed **AFTER** the main `pageFunction` is run. The only available parameter is the `CrawlingContext` object.\n\n```javascript\npostNavigationHooks: [\n async ({ id, request, session, proxyInfo, response }) => {},\n],\n```\n\n> Unlike with playwright, puppeteer and cheerio scrapers, in web scraper we don't have the Actor object available in the hook parameters, as the hook is executed inside the browser.\n\nCheck out the docs for [Post-navigation hooks](https://crawlee.dev/api/puppeteer-crawler/interface/PuppeteerCrawlerOptions#postNavigationHooks) and the [PuppeteerHook type](https://crawlee.dev/api/puppeteer-crawler/interface/PuppeteerHook) for more info regarding the objects passed into these functions.\n\n### Insert breakpoint\n\nThis property has no effect if [Run mode](#run-mode) is set to **PRODUCTION**. When set to **DEVELOPMENT** it inserts a breakpoint at the selected location in every page the scraper visits. Execution of code stops at the breakpoint until manually resumed in the DevTools window accessible via Live View tab or Container URL. Additional breakpoints can be added by adding debugger; statements within your Page function.\n\n### Debug log\n\nWhen set to true, debug messages will be included in the log. Use `context.log.debug('message')` to log your own debug messages.\n\n### Browser log\n\nWhen set to true, console messages from the browser will be included in the Actor's log. This may result in the log being flooded by error messages, warnings and other messages of little value (especially with a high concurrency).\n\n### Custom data\n\nSince the input UI is fixed, it does not support adding of other fields that may be needed for all specific use cases. If you need to pass arbitrary data to the scraper, use the [Custom data](#custom-data) input field within [Advanced configuration](#advanced-configuration) and its contents will be available under the `customData` context key as an object within the [pageFunction](#page-function).\n\n### Custom names\n\nWith the final three options in the **Advanced configuration**, you can set custom names for the following:\n\n- Dataset\n- Key-value store\n- Request queue\n\nLeave the storage unnamed if you only want the data within it to be persisted on the Apify platform for a number of days corresponding to your [plan](https://apify.com/pricing) (after which it will expire). Named storages are retained indefinitely. Additionally, using a named storage allows you to share it across multiple runs (e.g. instead of having 10 different unnamed datasets for 10 different runs, all the data from all 10 runs can be accumulated into a single named dataset). Learn more [here](https://docs.apify.com/storage#named-and-unnamed-storages).\n\n## Results\n\nAll scraping results returned by [**Page function**](#page-function) are stored in the default dataset associated with the Actor run, and can be saved in several different formats, such as JSON, XML, CSV or Excel. For each object returned by [**Page function**](#page-function), Web Scraper pushes one record into the dataset, and extends it with metadata such as the URL of the web page where the results come from.\n\nFor example, if your page function returned the following object:\n\n```javascript\n{\n message: 'Hello world!',\n}\n```\n\nThe full object stored in the dataset will look as follows\n(in JSON format, including the metadata fields `#error` and `#debug`):\n\n```json\n{\n \"message\": \"Hello world!\",\n \"#error\": false,\n \"#debug\": {\n \"requestId\": \"fvwscO2UJLdr10B\",\n \"url\": \"https://www.example.com/\",\n \"loadedUrl\": \"https://www.example.com/\",\n \"method\": \"GET\",\n \"retryCount\": 0,\n \"errorMessages\": null,\n \"statusCode\": 200\n }\n}\n```\n\nTo download the results, call the [Get dataset items](https://docs.apify.com/api/v2#/reference/datasets/item-collection) API endpoint:\n\n```\nhttps://api.apify.com/v2/datasets/[DATASET_ID]/items?format=json\n```\n\nwhere `[DATASET_ID]` is the ID of the Actor's run dataset, in which you can find the Run object returned when starting the Actor. Alternatively, you'll find the download links for the results in Apify Console.\n\nTo skip the `#error` and `#debug` metadata fields from the results and not include empty result records, simply add the `clean=true` query parameter to the API URL, or select the **Clean items** option when downloading the dataset in Apify Console.\n\nTo get the results in other formats, set the `format` query parameter to `xml`, `xlsx`, `csv`, `html`, etc. For more information, see [Datasets](https://apify.com/docs/storage#dataset) in documentation or the [Get dataset items](https://docs.apify.com/api/v2#/reference/datasets/item-collection) endpoint in Apify API reference.\n\n## Additional resources\n\nIf you’d like to learn more about Web Scraper or Apify’s other Actors and tools, check out these resources:\n\n- [Cheerio Scraper](https://apify.com/apify/cheerio-scraper), another web scraping Actor that downloads and processes pages in raw HTML for much higher performance.\n- [Playwright Scraper](https://apify.com/apify/playwright-scraper), a similar web scraping Actor to Web Scraper, which provides lower-level control of the underlying [Playwright](https://github.com/microsoft/playwright) library and the ability to use server-side libraries.\n- [Puppeteer Scraper](https://apify.com/apify/puppeteer-scraper), an Actor similar to Web Scraper, which provides lower-level control of the underlying [Puppeteer](https://github.com/puppeteer/puppeteer) library and the ability to use server-side libraries.\n- [Actors documentation](https://apify.com/docs/actor) for the Apify cloud computing platform.\n- [Apify SDK documentation](https://sdk.apify.com/), where you can learn more about the tools required to run your own Apify Actors.\n- [Crawlee documentation](https://crawlee.dev/?__hstc=160404322.4ff1f55e48512a0b19aa0955767abc98.1753772621636.1753781308751.1753783813114.4&__hssc=160404322.2.1753783813114&__hsfp=3081399490), how to build a new web scraping project from scratch using the world's most popular web crawling and scraping library for Node.js.\n\n## Frequently asked questions\n\n### Are there any limitations to using Web Scraper?\n\nWeb Scraper is designed to be user-friendly and generic, which may affect its performance and flexibility compared to more specialized solutions. It uses a resource-intensive Chromium browser to supports client-side JavaScript code.\n\n### Is web scraping legal?\n\nIt is legal to scrape any non-personal data. Personal data is protected by the [GDPR](https://en.wikipedia.org/wiki/General_Data_Protection_Regulation) in the European Union and by other regulations around the world. You should not scrape personal data unless you have a legitimate reason to do so. If you're unsure whether your reason is legitimate, consult your lawyers. You can also read our blog post on the [legality of web scraping](https://blog.apify.com/is-web-scraping-legal/).\n\n### Can I control the crawling behavior of Web Scraper?\n\nYes, you can control the crawling behavior of Web Scraper. You can specify start URLs, define link selectors, enter pseudo-URLs to guide the scraper in following specific page links, and plenty of other configurations options. This allows recursive crawling of websites or targeted extraction of data.\n\n### Is it possible to use proxies with Web Scraper?\n\nYes, you can configure proxies for Web Scraper. You have the option to use [Apify Proxy](https://apify.com/proxy), which under the free plan is set up for you. On paid plans, you can configure them yourself, or even set up your own.\n\n### How can I access and export the data scraped by Web Scraper?\n\nThe data scraped by Web Scraper is stored in a dataset which you can access and export in various formats such as JSON, XML, CSV, or as an Excel spreadsheet. The results can be downloaded using the Apify API or through the Apify Console.\n\n### Your feedback\n\nWe’re always working on improving the performance of our Actors. If you have any technical feedback for Web Scraper or found a bug, please create an issue in the [Issues tab](https://apify.com/apify/web-scraper/issues/open)." }, { "name": "package.json", "format": "TEXT", "content": "{\n \"name\": \"actor-web-scraper\",\n \"version\": \"3.1.0\",\n \"private\": true,\n \"description\": \"Crawl web pages using Apify, headless Chrome and Puppeteer\",\n \"main\": \"dist/main.js\",\n \"type\": \"module\",\n \"dependencies\": {\n \"@apify/scraper-tools\": \"^1.1.4\",\n \"@crawlee/puppeteer\": \"^3.16.0\",\n \"apify\": \"^3.2.6\",\n \"content-type\": \"^1.0.5\",\n \"crawlee\": \"^3.16.0\",\n \"devtools-server\": \"^0.0.2\",\n \"idcac-playwright\": \"^0.2.0\",\n \"puppeteer\": \"*\"\n },\n \"devDependencies\": {\n \"@apify/tsconfig\": \"^0.1.0\",\n \"@types/content-type\": \"^1.1.8\",\n \"@types/node\": \"^24.0.0\",\n \"tsx\": \"^4.19.1\",\n \"typescript\": \"~5.9.0\"\n },\n \"scripts\": {\n \"start\": \"npm run start:dev\",\n \"start:prod\": \"node dist/main.js\",\n \"start:dev\": \"tsx src/main.ts\",\n \"build\": \"tsc\"\n },\n \"repository\": {\n \"type\": \"git\",\n \"url\": \"https://github.com/apify/apify-sdk-js\"\n },\n \"author\": {\n \"name\": \"Apify Technologies\",\n \"email\": \"support@apify.com\",\n \"url\": \"https://apify.com\"\n },\n \"contributors\": [\n \"Marek Trunkat \",\n \"Ondra Urban \"\n ],\n \"license\": \"Apache-2.0\",\n \"homepage\": \"https://github.com/apify/apify-sdk-js\"\n}" }, { "name": "tsconfig.json", "format": "TEXT", "content": "{\n \"extends\": \"@apify/tsconfig\",\n \"compilerOptions\": {\n \"outDir\": \"dist\",\n \"module\": \"ESNext\",\n \"allowJs\": true,\n \"skipLibCheck\": true,\n \"lib\": [\"dom\"]\n },\n \"include\": [\"src\"]\n}" } ] }, { "versionNumber": "0.1", "buildTag": "version-1", "sourceType": "GIT_REPO", "envVars": [ { "name": "DISABLE_OUTDATED_WARNING", "value": "1" } ], "applyEnvVarsToBuild": false, "gitRepoUrl": "https://github.com/apifytech/actor-scraper.git#master:web-scraper" }, { "versionNumber": "2.0", "buildTag": "version-2", "sourceType": "GIT_REPO", "envVars": [], "applyEnvVarsToBuild": false, "gitRepoUrl": "https://github.com/apifytech/actor-scraper.git#master:web-scraper" }, { "versionNumber": "3.0", "buildTag": "version-3", "sourceType": "GIT_REPO", "envVars": [], "applyEnvVarsToBuild": false, "gitRepoUrl": "https://github.com/apify/actor-scraper#master:packages/actor-scraper/web-scraper", "sourceFiles": [ { "name": ".actor/actor.json", "format": "TEXT", "content": "{\n \"actorSpecification\": 1,\n \"name\": \"web-scraper\",\n \"version\": \"0.1\",\n \"buildTag\": \"latest\",\n \"storages\": {\n \"dataset\": {\n \"actorSpecification\": 1,\n \"fields\": {},\n \"views\": {}\n }\n }\n}" }, { "name": "src/internals/bundle.browser.ts", "format": "TEXT", "content": "/* eslint-disable max-classes-per-file */\n\nimport type {\n Dictionary,\n KeyValueStore,\n RecordOptions,\n Request,\n RequestOptions,\n RequestQueue,\n RequestQueueOperationOptions,\n} from '@crawlee/puppeteer';\nimport type { ApifyEnv } from 'apify';\n\nimport type { Log } from '@apify/log';\nimport type {\n constants,\n CrawlerSetupOptions,\n RequestMetadata,\n} from '@apify/scraper-tools';\n\nimport type { Input } from './consts.ts';\nimport type { GlobalStore } from './global_store.ts';\n\ninterface PoolOptions {\n pollingIntervalMillis?: number;\n timeoutMillis?: number;\n}\n\ninterface InternalState {\n browserHandles: Dictionary<\n | string\n | Record<\n string,\n { value: unknown; type: 'METHOD' | 'VALUE' | 'GETTER' }\n >\n >;\n requestQueue: RequestQueue | null;\n keyValueStore: KeyValueStore | null;\n}\n\ninterface ProvidedResponse {\n status: number;\n headers: Dictionary;\n}\n\ninterface BrowserCrawlerSetup extends CrawlerSetupOptions {\n injectJQuery?: boolean;\n META_KEY: typeof constants.META_KEY;\n}\n\ninterface ContextOptions {\n crawlerSetup: BrowserCrawlerSetup;\n browserHandles: InternalState['browserHandles'];\n pageFunctionArguments: Dictionary;\n}\n\n/**\n * Command to be evaluated for Browser side code injection.\n * @param apifyNamespace\n */\nexport function createBundle(apifyNamespace: string) {\n (function (global, namespace) {\n if (typeof window[namespace] !== 'object') window[namespace] = {};\n /**\n * Takes a configuration object with function references\n * as an input and creates a dummy class that proxies\n * function invocations from Browser context to Node context.\n *\n * The configuration is to be provided by the\n * tools.createBrowserHandlesForObject function.\n */\n class NodeProxy {\n constructor(\n config: Dictionary<{\n value: unknown;\n type: 'METHOD' | 'VALUE' | 'GETTER';\n }>,\n ) {\n if (!config || typeof config !== 'object') {\n throw new Error(\n 'NodeProxy: Parameter config of type Object must be provided.',\n );\n }\n\n Object.entries(config).forEach(([key, { value, type }]) => {\n if (type === 'METHOD') {\n // @ts-expect-error\n this[key] = (...args: unknown[]) =>\n // @ts-expect-error\n global[value](...args);\n } else if (type === 'GETTER') {\n Object.defineProperty(this, key, {\n // @ts-expect-error\n get: () => global[value](),\n });\n } else if (type === 'VALUE') {\n // @ts-expect-error\n this[key] = value;\n } else {\n throw new Error(\n `Unsupported function type: ${type} for function: ${key}.`,\n );\n }\n });\n }\n }\n\n /**\n * Exposed factory.\n * @param config\n */\n global[namespace].createNodeProxy = (\n config: Dictionary<{\n value: unknown;\n type: 'METHOD' | 'VALUE' | 'GETTER';\n }>,\n ) => new NodeProxy(config);\n\n const setup = Symbol('crawler-setup');\n const internalState = Symbol('request-internal-state');\n\n /**\n * Context represents everything that is available to the user\n * via Page Function. A class is used instead of a simple object\n * to avoid having to create new instances of functions with each\n * request.\n *\n * Some properties need to be accessible to the Context,\n * but should not be exposed to the user thus they are hidden\n * using a Symbol to prevent the user from easily accessing\n * and manipulating them.\n */\n class Context {\n [setup]: BrowserCrawlerSetup;\n [internalState]: InternalState;\n input: Input;\n env: ApifyEnv;\n customData: unknown;\n response: ProvidedResponse;\n request: Request;\n globalStore: GlobalStore;\n log: Log;\n\n jQuery: any;\n\n constructor(options: ContextOptions) {\n const { crawlerSetup, browserHandles, pageFunctionArguments } =\n options;\n\n const createProxy = global[namespace].createNodeProxy;\n\n // Private\n this[setup] = crawlerSetup;\n this[internalState] = {\n browserHandles,\n requestQueue: browserHandles.requestQueue\n ? createProxy(browserHandles.requestQueue)\n : null,\n keyValueStore: browserHandles.keyValueStore\n ? createProxy(browserHandles.keyValueStore)\n : null,\n };\n\n // Copies of Node objects\n this.input = JSON.parse(crawlerSetup.rawInput);\n this.env = { ...crawlerSetup.env };\n this.customData = crawlerSetup.customData;\n this.response =\n pageFunctionArguments.response as ProvidedResponse;\n this.request = pageFunctionArguments.request as Request;\n // Functions are not converted so we need to add them this way\n // to not be enumerable and thus not polluting the object.\n Reflect.defineProperty(this.request, 'pushErrorMessage', {\n value(this: Request, errorOrMessage: Error) {\n // It's a simplified fake of the original function.\n const msg =\n (errorOrMessage && errorOrMessage.message) ||\n `${errorOrMessage}`;\n this.errorMessages.push(msg);\n },\n enumerable: false,\n });\n\n // Proxied Node objects\n this.globalStore = createProxy(browserHandles.globalStore);\n this.log = createProxy(browserHandles.log);\n\n // Browser side libraries\n if (this[setup].injectJQuery)\n this.jQuery = global.jQuery.noConflict(true);\n\n // Bind this to allow destructuring off context in pageFunction.\n this.getValue = this.getValue.bind(this);\n this.setValue = this.setValue.bind(this);\n this.saveSnapshot = this.saveSnapshot.bind(this);\n this.skipLinks = this.skipLinks.bind(this);\n this.enqueueRequest = this.enqueueRequest.bind(this);\n this.waitFor = this.waitFor.bind(this);\n }\n\n async getValue(...args: Parameters) {\n return this[internalState].keyValueStore!.getValue(\n ...args,\n ) as Promise;\n }\n\n async setValue(...args: Parameters) {\n return this[internalState].keyValueStore!.setValue(\n ...(args as [\n key: string,\n value: T | null,\n options?: RecordOptions,\n ]),\n );\n }\n\n async saveSnapshot() {\n const handle = this[internalState].browserHandles\n .saveSnapshot as string;\n return global[handle]();\n }\n\n async skipLinks() {\n const handle = this[internalState].browserHandles\n .skipLinks as string;\n return global[handle]();\n }\n\n async enqueueRequest(\n requestOpts: RequestOptions = {} as RequestOptions,\n options: RequestQueueOperationOptions = {},\n ) {\n const defaultRequestOpts = {\n useExtendedUniqueKey: true,\n keepUrlFragment: this.input.keepUrlFragments,\n };\n\n const newRequest = { ...defaultRequestOpts, ...requestOpts };\n\n const metaKey = this[setup].META_KEY;\n const defaultUserData = {\n [metaKey]: {\n parentRequestId:\n this.request.id || this.request.uniqueKey,\n depth:\n (this.request.userData![metaKey] as RequestMetadata)\n .depth + 1,\n },\n };\n\n newRequest.userData = {\n ...defaultUserData,\n ...requestOpts.userData,\n };\n\n return this[internalState].requestQueue!.addRequest(\n newRequest,\n options,\n );\n }\n\n async waitFor(\n selectorOrNumberOrFunction:\n | string\n | number\n | ((...args: unknown[]) => boolean),\n options = {},\n ) {\n if (!options || typeof options !== 'object')\n throw new Error('Parameter options must be an Object');\n const type = typeof selectorOrNumberOrFunction;\n if (type === 'string') {\n return this._waitForSelector(\n selectorOrNumberOrFunction as string,\n options,\n );\n }\n if (type === 'number') {\n return this._waitForMillis(\n selectorOrNumberOrFunction as number,\n );\n }\n if (type === 'function') {\n return this._waitForFunction(\n selectorOrNumberOrFunction as (\n ...args: unknown[]\n ) => boolean,\n options,\n );\n }\n throw new Error(\n 'Parameter selectorOrNumberOrFunction must be one of the said types.',\n );\n }\n\n async _waitForSelector(selector: string, options = {}) {\n try {\n await this._poll(() => {\n return !!global.document.querySelector(selector);\n }, options);\n } catch (err) {\n const casted = err as Error;\n if (/timeout of \\d+ms exceeded/.test(casted.message)) {\n throw new Error(\n `Timeout Error: waiting for selector failed: ${casted.message}`,\n );\n }\n throw err;\n }\n }\n\n async _waitForMillis(millis: number) {\n return new Promise((res) => {\n setTimeout(res, millis);\n });\n }\n\n async _waitForFunction(\n predicate: () => boolean,\n options: PoolOptions = {},\n ) {\n try {\n await this._poll(predicate, options);\n } catch (err) {\n const casted = err as Error;\n if (/timeout of \\d+ms exceeded/.test(casted.message)) {\n throw new Error(\n `Timeout Error: waiting for function failed: ${casted.message}`,\n );\n }\n throw err;\n }\n }\n\n async _poll(predicate: () => boolean, options: PoolOptions = {}) {\n const { pollingIntervalMillis = 50, timeoutMillis = 20000 } =\n options;\n return new Promise((resolve, reject) => {\n const handler = (): void => {\n if (predicate()) {\n resolve();\n } else {\n setTimeout(handler);\n }\n };\n const pollTimeout = setTimeout(\n handler,\n pollingIntervalMillis,\n );\n setTimeout(() => {\n clearTimeout(pollTimeout);\n return reject(\n new Error(\n `timeout of ${timeoutMillis}ms exceeded.`,\n ),\n );\n }, timeoutMillis);\n });\n }\n }\n\n /**\n * Exposed factory.\n */\n global[namespace].createContext = (options: ContextOptions) => {\n return new Context(options);\n };\n })(window, apifyNamespace);\n}" }, { "name": "src/internals/consts.ts", "format": "TEXT", "content": "import type {\n Dictionary,\n GlobInput,\n ProxyConfigurationOptions,\n PseudoUrlInput,\n RegExpInput,\n RequestOptions,\n Session,\n} from '@crawlee/puppeteer';\nimport type { PuppeteerLifeCycleEvent } from 'puppeteer';\n\n/**\n * Port where the remote debugging interface in Chrome\n * should be started.\n */\nexport const CHROME_DEBUGGER_PORT = 9222;\n\nexport const enum RunMode {\n Production = 'PRODUCTION',\n Development = 'DEVELOPMENT',\n}\n\nexport const enum ProxyRotation {\n Recommended = 'RECOMMENDED',\n PerRequest = 'PER_REQUEST',\n UntilFailure = 'UNTIL_FAILURE',\n}\n\nexport const enum BreakpointLocation {\n None = 'NONE',\n BeforeGoto = 'BEFORE_GOTO',\n BeforePageFunction = 'BEFORE_PAGE_FUNCTION',\n AfterPageFunction = 'AFTER_PAGE_FUNCTION',\n}\n\ndeclare global {\n interface Window {\n [K: string]: any;\n }\n // eslint-disable-next-line vars-on-top\n var window: Window & typeof globalThis;\n // eslint-disable-next-line vars-on-top\n var document: Document;\n}\n\n/**\n * Replicates the INPUT_SCHEMA with TypeScript types for quick reference\n * and IDE type check integration.\n */\nexport interface Input {\n startUrls: RequestOptions[];\n pageFunction: string;\n runMode: RunMode;\n keepUrlFragments: boolean;\n respectRobotsTxtFile: boolean;\n linkSelector?: string;\n globs: GlobInput[];\n regexps: RegExpInput[];\n pseudoUrls: PseudoUrlInput[];\n excludes: GlobInput[];\n preNavigationHooks?: string;\n postNavigationHooks?: string;\n injectJQuery: boolean;\n proxyConfiguration: ProxyConfigurationOptions;\n proxyRotation: ProxyRotation;\n sessionPoolName?: string;\n initialCookies: Parameters[0];\n useChrome: boolean;\n maxScrollHeightPixels: number;\n ignoreSslErrors: boolean;\n ignoreCorsAndCsp: boolean;\n closeCookieModals: boolean;\n downloadMedia: boolean;\n downloadCss: boolean;\n maxRequestRetries: number;\n maxPagesPerCrawl: number;\n maxResultsPerCrawl: number;\n maxCrawlingDepth: number;\n maxConcurrency: number;\n pageLoadTimeoutSecs: number;\n pageFunctionTimeoutSecs: number;\n waitUntil: PuppeteerLifeCycleEvent[];\n breakpointLocation: BreakpointLocation;\n debugLog: boolean;\n browserLog: boolean;\n customData: Dictionary;\n datasetName?: string;\n keyValueStoreName?: string;\n requestQueueName?: string;\n headless: boolean;\n}" }, { "name": "src/internals/crawler_setup.ts", "format": "TEXT", "content": "import { readFile } from 'node:fs/promises';\nimport { dirname } from 'node:path';\nimport { setTimeout } from 'node:timers/promises';\nimport { fileURLToPath, URL } from 'node:url';\n\nimport type {\n AutoscaledPool,\n Awaitable,\n Dictionary,\n ProxyConfiguration,\n PuppeteerCrawlerOptions,\n PuppeteerCrawlingContext,\n Request,\n} from '@crawlee/puppeteer';\nimport {\n Dataset,\n KeyValueStore,\n log,\n PuppeteerCrawler,\n puppeteerUtils,\n RequestList,\n RequestQueueV2,\n} from '@crawlee/puppeteer';\nimport type { ApifyEnv } from 'apify';\nimport { Actor } from 'apify';\nimport contentType from 'content-type';\n// @ts-expect-error no typings\nimport DevToolsServer from 'devtools-server';\nimport { getInjectableScript } from 'idcac-playwright';\nimport type { HTTPResponse, Page } from 'puppeteer';\n\nimport type { CrawlerSetupOptions, createContext } from '@apify/scraper-tools';\nimport {\n browserTools,\n constants as scraperToolsConstants,\n tools,\n} from '@apify/scraper-tools';\n\nimport { createBundle } from './bundle.browser.js';\nimport type { Input } from './consts.js';\nimport {\n BreakpointLocation,\n CHROME_DEBUGGER_PORT,\n ProxyRotation,\n RunMode,\n} from './consts.js';\nimport { GlobalStore } from './global_store.js';\n\nconst SESSION_STORE_NAME = 'APIFY-WEB-SCRAPER-SESSION-STORE';\nconst REQUEST_QUEUE_INIT_FLAG_KEY = 'REQUEST_QUEUE_INITIALIZED';\n\nconst MAX_CONCURRENCY_IN_DEVELOPMENT = 1;\nconst {\n SESSION_MAX_USAGE_COUNTS,\n DEFAULT_VIEWPORT,\n DEVTOOLS_TIMEOUT_SECS,\n META_KEY,\n} = scraperToolsConstants;\nconst SCHEMA = JSON.parse(\n await readFile(new URL('../../INPUT_SCHEMA.json', import.meta.url), 'utf8'),\n);\n\ninterface PageContext {\n apifyNamespace: string;\n skipLinks: boolean;\n timers: {\n start: [number, number];\n navStart?: [number, number];\n };\n\n browserHandles?: Awaited>;\n}\n\ninterface Output {\n pageFunctionResult?: Dictionary;\n pageFunctionError?: tools.ErrorLike;\n requestFromBrowser: Request;\n}\n\n/**\n * Holds all the information necessary for constructing a crawler\n * instance and creating a context for a pageFunction invocation.\n */\nexport class CrawlerSetup implements CrawlerSetupOptions {\n name = 'Web Scraper';\n rawInput: string;\n env: ApifyEnv;\n /**\n * Used to store data that persist navigations\n */\n globalStore = new GlobalStore();\n requestQueue: RequestQueueV2;\n keyValueStore: KeyValueStore;\n customData: unknown;\n input: Input;\n maxSessionUsageCount: number;\n evaledPreNavigationHooks: ((...args: unknown[]) => Awaitable)[];\n evaledPostNavigationHooks: ((...args: unknown[]) => Awaitable)[];\n\n /**\n * Used to store page specific data.\n */\n pageContexts = new WeakMap();\n\n blockedUrlPatterns: string[] = [];\n isDevRun: boolean;\n datasetName?: string;\n keyValueStoreName?: string;\n requestQueueName?: string;\n\n crawler!: PuppeteerCrawler;\n dataset!: Dataset;\n pagesOutputted!: number;\n private initPromise: Promise;\n\n constructor(input: Input) {\n // Set log level early to prevent missed messages.\n if (input.debugLog) log.setLevel(log.LEVELS.DEBUG);\n\n // Keep this as string to be immutable.\n this.rawInput = JSON.stringify(input);\n\n // Attempt to load page function from disk if not present on input.\n tools.maybeLoadPageFunctionFromDisk(\n input,\n dirname(fileURLToPath(import.meta.url)),\n );\n\n // Validate INPUT if not running on Apify Cloud Platform.\n if (!Actor.isAtHome()) tools.checkInputOrThrow(input, SCHEMA);\n\n this.input = input;\n this.env = Actor.getEnv();\n\n // Validations\n this.input.pseudoUrls.forEach((purl) => {\n if (!tools.isPlainObject(purl)) {\n throw new Error(\n 'The pseudoUrls Array must only contain Objects.',\n );\n }\n if (purl.userData && !tools.isPlainObject(purl.userData)) {\n throw new Error(\n 'The userData property of a pseudoUrl must be an Object.',\n );\n }\n });\n\n this.input.initialCookies.forEach((cookie) => {\n if (!tools.isPlainObject(cookie)) {\n throw new Error(\n 'The initialCookies Array must only contain Objects.',\n );\n }\n });\n\n this.input.waitUntil.forEach((event) => {\n if (\n !/^(domcontentloaded|load|networkidle2|networkidle0)$/.test(\n event,\n )\n ) {\n throw new Error(\n 'Navigation wait until events must be valid. See tooltip.',\n );\n }\n });\n\n // solving proxy rotation settings\n this.maxSessionUsageCount =\n SESSION_MAX_USAGE_COUNTS[this.input.proxyRotation];\n\n tools.evalFunctionOrThrow(this.input.pageFunction);\n\n if (this.input.preNavigationHooks) {\n this.evaledPreNavigationHooks = tools.evalFunctionArrayOrThrow(\n this.input.preNavigationHooks,\n 'preNavigationHooks',\n );\n } else {\n this.evaledPreNavigationHooks = [];\n }\n\n if (this.input.postNavigationHooks) {\n this.evaledPostNavigationHooks = tools.evalFunctionArrayOrThrow(\n this.input.postNavigationHooks,\n 'postNavigationHooks',\n );\n } else {\n this.evaledPostNavigationHooks = [];\n }\n\n // Excluded resources\n if (!this.input.downloadMedia) {\n this.blockedUrlPatterns = [\n '.jpg',\n '.jpeg',\n '.png',\n '.svg',\n '.gif',\n '.webp',\n '.webm',\n '.ico',\n '.woff',\n '.eot',\n ];\n }\n\n if (!this.input.downloadCss) {\n this.blockedUrlPatterns.push('.css');\n }\n\n this.isDevRun = this.input.runMode === RunMode.Development;\n\n // Named storages\n this.datasetName = this.input.datasetName;\n this.keyValueStoreName = this.input.keyValueStoreName;\n this.requestQueueName = this.input.requestQueueName;\n\n // Initialize async operations.\n this.crawler = null!;\n this.requestQueue = null!;\n this.dataset = null!;\n this.keyValueStore = null!;\n this.initPromise = this._initializeAsync();\n }\n\n private async _initializeAsync() {\n // RequestList\n const startUrls = this.input.startUrls.map((req) => {\n req.useExtendedUniqueKey = true;\n req.keepUrlFragment = this.input.keepUrlFragments;\n return req;\n });\n\n // KeyValueStore\n this.keyValueStore = await KeyValueStore.open(this.keyValueStoreName);\n\n // RequestQueue\n this.requestQueue = await RequestQueueV2.open(this.requestQueueName);\n\n if (\n !(await this.keyValueStore.recordExists(\n REQUEST_QUEUE_INIT_FLAG_KEY,\n ))\n ) {\n const requests: Request[] = [];\n for await (const request of await RequestList.open(\n null,\n startUrls,\n )) {\n if (\n this.input.maxResultsPerCrawl > 0 &&\n requests.length >= 1.5 * this.input.maxResultsPerCrawl\n ) {\n break;\n }\n requests.push(request);\n }\n\n const { waitForAllRequestsToBeAdded } =\n await this.requestQueue.addRequestsBatched(requests);\n\n void waitForAllRequestsToBeAdded.then(async () => {\n await this.keyValueStore.setValue(\n REQUEST_QUEUE_INIT_FLAG_KEY,\n '1',\n );\n });\n }\n\n // Dataset\n this.dataset = await Dataset.open(this.datasetName);\n const info = await this.dataset.getInfo();\n this.pagesOutputted = info?.itemCount ?? 0;\n }\n\n /**\n * Resolves to a `PuppeteerCrawler` instance.\n * constructor.\n */\n async createCrawler() {\n await this.initPromise;\n\n const args = ['--disable-dev-shm-usage'];\n if (this.input.ignoreCorsAndCsp) args.push('--disable-web-security');\n if (this.isDevRun)\n args.push(`--remote-debugging-port=${CHROME_DEBUGGER_PORT}`);\n\n const options: PuppeteerCrawlerOptions = {\n requestHandler: this._requestHandler.bind(this),\n requestQueue: this.requestQueue,\n requestHandlerTimeoutSecs: this.isDevRun\n ? DEVTOOLS_TIMEOUT_SECS\n : this.input.pageFunctionTimeoutSecs,\n preNavigationHooks: [],\n postNavigationHooks: [],\n failedRequestHandler: this._failedRequestHandler.bind(this),\n respectRobotsTxtFile: this.input.respectRobotsTxtFile,\n maxConcurrency: this.isDevRun\n ? MAX_CONCURRENCY_IN_DEVELOPMENT\n : this.input.maxConcurrency,\n maxRequestRetries: this.input.maxRequestRetries,\n maxRequestsPerCrawl:\n this.input.maxPagesPerCrawl === 0\n ? undefined\n : this.input.maxPagesPerCrawl,\n proxyConfiguration: (await Actor.createProxyConfiguration(\n this.input.proxyConfiguration,\n )) as any as ProxyConfiguration,\n browserPoolOptions: {\n preLaunchHooks: [\n async () => {\n if (!this.isDevRun) {\n return;\n }\n\n const devToolsServer = new DevToolsServer({\n containerHost: new URL(\n process.env.ACTOR_WEB_SERVER_URL!,\n ).host,\n devToolsServerPort:\n process.env.ACTOR_WEB_SERVER_PORT,\n chromeRemoteDebuggingPort: CHROME_DEBUGGER_PORT,\n });\n await devToolsServer.start();\n },\n ],\n },\n launchContext: {\n useChrome: this.input.useChrome,\n launchOptions: {\n acceptInsecureCerts: this.input.ignoreSslErrors,\n defaultViewport: DEFAULT_VIEWPORT,\n args,\n headless: this.input.headless,\n },\n },\n useSessionPool: !this.isDevRun,\n persistCookiesPerSession: !this.isDevRun,\n sessionPoolOptions: {\n persistStateKeyValueStoreId: this.input.sessionPoolName\n ? SESSION_STORE_NAME\n : undefined,\n persistStateKey: this.input.sessionPoolName,\n sessionOptions: {\n maxUsageCount: this.maxSessionUsageCount,\n },\n },\n experiments: {\n requestLocking: true,\n },\n };\n\n this._createNavigationHooks(options);\n\n if (this.input.proxyRotation === ProxyRotation.UntilFailure) {\n options.sessionPoolOptions!.maxPoolSize = 1;\n }\n if (this.isDevRun) {\n options.browserPoolOptions!.retireBrowserAfterPageCount = Infinity;\n }\n\n this.crawler = new PuppeteerCrawler(options);\n\n if (this.isDevRun) logDevRunWarning();\n return this.crawler;\n }\n\n private _createNavigationHooks(options: PuppeteerCrawlerOptions) {\n options.preNavigationHooks!.push(\n async ({ request, page, session }, gotoOptions) => {\n const start = process.hrtime();\n\n // Create a new page context with a new random key for Apify namespace.\n const pageContext: PageContext = {\n apifyNamespace: await tools.createRandomHash(),\n skipLinks: false,\n timers: { start },\n };\n this.pageContexts.set(page, pageContext);\n\n // Attach a console listener to get all logs as soon as possible.\n if (this.input.browserLog) browserTools.dumpConsole(page);\n\n // Prevent download of stylesheets and media, unless selected otherwise\n if (this.blockedUrlPatterns.length) {\n await puppeteerUtils.blockRequests(page, {\n urlPatterns: this.blockedUrlPatterns,\n });\n }\n\n // Add initial cookies, if any.\n if (\n this.input.initialCookies &&\n this.input.initialCookies.length\n ) {\n const cookiesToSet = session\n ? tools.getMissingCookiesFromSession(\n session,\n this.input.initialCookies,\n request.url,\n )\n : this.input.initialCookies;\n if (cookiesToSet && cookiesToSet.length) {\n // setting initial cookies that are not already in the session and page\n // TODO: We can remove the condition when there is an option to define blocked status codes in sessionPool\n session?.setCookies(cookiesToSet, request.url);\n await page.setCookie(...cookiesToSet);\n }\n }\n\n // Disable content security policy.\n if (this.input.ignoreCorsAndCsp) await page.setBypassCSP(true);\n\n tools.logPerformance(request, 'gotoFunction INIT', start);\n const handleStart = process.hrtime();\n pageContext.browserHandles = await this._injectBrowserHandles(\n page,\n pageContext,\n );\n tools.logPerformance(\n request,\n 'gotoFunction INJECTION HANDLES',\n handleStart,\n );\n\n const evalStart = process.hrtime();\n await Promise.all([\n page.evaluateOnNewDocument(\n createBundle,\n pageContext.apifyNamespace,\n ),\n page.evaluateOnNewDocument(\n browserTools.wrapPageFunction(\n this.input.pageFunction,\n pageContext.apifyNamespace,\n ),\n ),\n ]);\n tools.logPerformance(\n request,\n 'gotoFunction INJECTION EVAL',\n evalStart,\n );\n\n if (this.isDevRun) {\n const cdpClient = await page.target().createCDPSession();\n await cdpClient.send('Debugger.enable');\n if (\n this.input.breakpointLocation ===\n BreakpointLocation.BeforeGoto\n ) {\n await cdpClient.send('Debugger.pause');\n }\n }\n\n pageContext.timers.navStart = process.hrtime();\n if (gotoOptions) {\n gotoOptions.timeout = this.input.pageLoadTimeoutSecs * 1000;\n gotoOptions.waitUntil = this.input.waitUntil;\n }\n },\n );\n\n options.preNavigationHooks!.push(\n ...this._runHookWithEnhancedContext(this.evaledPreNavigationHooks),\n );\n options.postNavigationHooks!.push(\n ...this._runHookWithEnhancedContext(this.evaledPostNavigationHooks),\n );\n\n options.postNavigationHooks!.push(\n async ({ request, page, response }) => {\n await this._waitForLoadEventWhenXml(page, response);\n const pageContext = this.pageContexts.get(page)!;\n tools.logPerformance(\n request,\n 'gotoFunction NAVIGATION',\n pageContext.timers.navStart!,\n );\n\n const delayStart = process.hrtime();\n await this._assertNamespace(page, pageContext.apifyNamespace);\n\n // Inject selected libraries\n if (this.input.injectJQuery)\n await puppeteerUtils.injectJQuery(page);\n\n tools.logPerformance(\n request,\n 'gotoFunction INJECTION DELAY',\n delayStart,\n );\n tools.logPerformance(\n request,\n 'gotoFunction EXECUTION',\n pageContext.timers.start,\n );\n },\n );\n }\n\n private _runHookWithEnhancedContext(\n hooks: ((...args: unknown[]) => Awaitable)[],\n ) {\n return hooks.map((hook) => (ctx: Dictionary, ...args: unknown[]) => {\n const { customData } = this.input;\n return hook({ ...ctx, customData }, ...args);\n });\n }\n\n private async _failedRequestHandler({ request }: PuppeteerCrawlingContext) {\n const lastError =\n request.errorMessages[request.errorMessages.length - 1];\n const errorMessage = lastError ? lastError.split('\\n')[0] : 'no error';\n log.error(\n `Request ${request.url} failed and will not be retried anymore. Marking as failed.\\nLast Error Message: ${errorMessage}`,\n );\n return this._handleResult(request, undefined, undefined, true);\n }\n\n /**\n * First of all, it initializes the state that is exposed to the user via\n * `pageFunction` context.\n *\n * Then it invokes the user provided `pageFunction` with the prescribed context\n * and saves its return value.\n *\n * Finally, it makes decisions based on the current state and post-processes\n * the data returned from the `pageFunction`.\n */\n private async _requestHandler(crawlingContext: PuppeteerCrawlingContext) {\n const { request, response, page, crawler, proxyInfo } = crawlingContext;\n const start = process.hrtime();\n const pageContext = this.pageContexts.get(page)!;\n\n /**\n * PRE-PROCESSING\n */\n // Make sure that an object containing internal metadata\n // is present on every request.\n tools.ensureMetaData(request);\n\n // Abort the crawler if the maximum number of results was reached.\n const aborted = await this._handleMaxResultsPerCrawl(\n crawler.autoscaledPool,\n );\n if (aborted) return;\n\n // Setup Context and pass the configuration down to Browser.\n const contextOptions = {\n crawlerSetup: {\n rawInput: this.rawInput,\n env: this.env,\n customData: this.input.customData,\n injectJQuery: this.input.injectJQuery,\n META_KEY,\n },\n browserHandles: pageContext.browserHandles,\n pageFunctionArguments: {\n request,\n proxyInfo,\n response: {\n status: response && response.status(),\n headers: response && response.headers(),\n },\n },\n };\n\n /**\n * USER FUNCTION EXECUTION\n */\n tools.logPerformance(request, 'requestHandler PREPROCESSING', start);\n\n if (\n this.isDevRun &&\n this.input.breakpointLocation ===\n BreakpointLocation.BeforePageFunction\n ) {\n await page.evaluate(async () => {\n // eslint-disable-next-line no-debugger -- Debugger is enabled in dev run\n debugger;\n });\n }\n\n if (this.input.closeCookieModals) {\n await setTimeout(500);\n await page.evaluate(getInjectableScript());\n await setTimeout(2000);\n }\n\n if (this.input.maxScrollHeightPixels > 0) {\n await crawlingContext.infiniteScroll({\n maxScrollHeight: this.input.maxScrollHeightPixels,\n });\n }\n\n const startUserFn = process.hrtime();\n\n const namespace = pageContext.apifyNamespace;\n const output = await page.evaluate(\n async (ctxOpts: typeof contextOptions, namespc: string) => {\n const context: ReturnType['context'] =\n window[namespc].createContext(ctxOpts);\n // eslint-disable-next-line @typescript-eslint/no-shadow\n const output = {} as Output;\n try {\n output.pageFunctionResult =\n await window[namespc].pageFunction(context);\n } catch (err) {\n const casted = err as Error;\n output.pageFunctionError = Object.getOwnPropertyNames(\n casted,\n ).reduce((memo, name) => {\n memo[name] = casted[name as keyof Error];\n return memo;\n }, {} as Dictionary);\n }\n\n // This needs to be added after pageFunction has run.\n output.requestFromBrowser = context.request as Request;\n\n /**\n * Since Dates cannot travel back to Node and Puppeteer does not use .toJSON\n * to stringify, they come back as empty objects. We could use JSON.stringify\n * ourselves, but that exposes us to overridden .toJSON in the target websites.\n * This hack is not ideal, but it avoids both problems.\n */\n function replaceAllDatesInObjectWithISOStrings(obj: Output) {\n for (const [key, value] of Object.entries(obj)) {\n if (\n value instanceof Date &&\n typeof value.toISOString === 'function'\n ) {\n Reflect.set(obj, key, value.toISOString());\n } else if (value && typeof value === 'object') {\n replaceAllDatesInObjectWithISOStrings(\n value as Output,\n );\n }\n }\n return obj;\n }\n\n return replaceAllDatesInObjectWithISOStrings(output);\n },\n contextOptions,\n namespace,\n );\n\n tools.logPerformance(\n request,\n 'requestHandler USER FUNCTION',\n startUserFn,\n );\n const finishUserFn = process.hrtime();\n\n /**\n * POST-PROCESSING\n */\n const { pageFunctionResult, requestFromBrowser, pageFunctionError } =\n output;\n // Merge requestFromBrowser into request to preserve modifications that\n // may have been made in browser context.\n Object.assign(request, requestFromBrowser);\n\n // Throw error from pageFunction, if any.\n if (pageFunctionError) throw tools.createError(pageFunctionError);\n\n // Enqueue more links if a link selector is available,\n // unless the user invoked the `skipLinks()` context function\n // or maxCrawlingDepth would be exceeded.\n if (!pageContext.skipLinks) {\n await this._handleLinks(crawlingContext);\n }\n\n // Save the `pageFunction`s result (or just metadata) to the default dataset.\n await this._handleResult(request, response, pageFunctionResult);\n\n tools.logPerformance(\n request,\n 'requestHandler POSTPROCESSING',\n finishUserFn,\n );\n tools.logPerformance(request, 'requestHandler EXECUTION', start);\n\n if (\n this.isDevRun &&\n this.input.breakpointLocation ===\n BreakpointLocation.AfterPageFunction\n ) {\n await page.evaluate(async () => {\n // eslint-disable-next-line no-debugger -- Debugger is enabled in dev run\n debugger;\n });\n }\n }\n\n private async _handleMaxResultsPerCrawl(autoscaledPool?: AutoscaledPool) {\n if (\n !this.input.maxResultsPerCrawl ||\n this.pagesOutputted < this.input.maxResultsPerCrawl\n )\n return false;\n if (!autoscaledPool) return false;\n log.info(\n `User set limit of ${this.input.maxResultsPerCrawl} results was reached. Finishing the crawl.`,\n );\n await autoscaledPool.abort();\n return true;\n }\n\n private async _handleLinks({\n request,\n enqueueLinks,\n }: PuppeteerCrawlingContext) {\n if (!(this.input.linkSelector && this.requestQueue)) return;\n const start = process.hrtime();\n\n const currentDepth = (\n request.userData![META_KEY] as tools.RequestMetadata\n ).depth;\n const hasReachedMaxDepth =\n this.input.maxCrawlingDepth &&\n currentDepth >= this.input.maxCrawlingDepth;\n if (hasReachedMaxDepth) {\n log.debug(\n `Request ${request.url} reached the maximum crawling depth of ${currentDepth}.`,\n );\n return;\n }\n\n await enqueueLinks({\n selector: this.input.linkSelector,\n globs: this.input.globs,\n pseudoUrls: this.input.pseudoUrls,\n exclude: this.input.excludes,\n transformRequestFunction: (requestOptions) => {\n requestOptions.userData ??= {};\n requestOptions.userData[META_KEY] = {\n parentRequestId: request.id || request.uniqueKey,\n depth: currentDepth + 1,\n };\n\n requestOptions.useExtendedUniqueKey = true;\n requestOptions.keepUrlFragment = this.input.keepUrlFragments;\n return requestOptions;\n },\n });\n\n tools.logPerformance(request, 'handleLinks EXECUTION', start);\n }\n\n private async _handleResult(\n request: Request,\n response?: HTTPResponse,\n pageFunctionResult?: Dictionary,\n isError?: boolean,\n ) {\n const start = process.hrtime();\n const payload = tools.createDatasetPayload(\n request,\n response,\n pageFunctionResult,\n isError,\n );\n await this.dataset.pushData(payload);\n this.pagesOutputted++;\n tools.logPerformance(request, 'handleResult EXECUTION', start);\n }\n\n private async _assertNamespace(page: Page, namespace: string) {\n try {\n await page.waitForFunction(\n (nmspc: string) => !!window[nmspc],\n { timeout: this.input.pageLoadTimeoutSecs * 1000 },\n namespace,\n );\n } catch (err) {\n const casted = err as Error;\n if (casted.stack!.startsWith('TimeoutError')) {\n throw new Error(\n 'Injection of environment into the browser context timed out. ' +\n 'If this persists even after retries, try increasing the Page load timeout input setting.',\n );\n } else {\n throw err;\n }\n }\n }\n\n private async _waitForLoadEventWhenXml(\n page: Page,\n response?: HTTPResponse,\n ) {\n // Response can sometimes be null.\n if (!response) return;\n\n const cTypeHeader = response.headers()['content-type'];\n try {\n const { type } = contentType.parse(cTypeHeader);\n if (!/^(text|application)\\/xml$|\\+xml$/.test(type)) return;\n } catch (err) {\n // Invalid type is not XML.\n return;\n }\n\n try {\n const timeout = this.input.pageLoadTimeoutSecs * 1000;\n await page.waitForFunction(\n () => document.readyState === 'complete',\n { timeout },\n );\n } catch (err) {\n const casted = err as Error;\n if (casted.stack!.startsWith('TimeoutError')) {\n throw new Error(\n \"Parsing of XML in the page timed out. If you're expecting a large XML file, \" +\n ' such as a site map, try increasing the Page load timeout input setting.',\n );\n } else {\n throw err;\n }\n }\n }\n\n private async _injectBrowserHandles(page: Page, pageContext: PageContext) {\n const saveSnapshotP = browserTools.createBrowserHandle(page, async () =>\n browserTools.saveSnapshot({ page }),\n );\n const skipLinksP = browserTools.createBrowserHandle(page, () => {\n pageContext.skipLinks = true;\n });\n const globalStoreP = browserTools.createBrowserHandlesForObject(\n page,\n this.globalStore,\n [\n 'size',\n 'clear',\n 'delete',\n 'entries',\n 'get',\n 'has',\n 'keys',\n 'set',\n 'values',\n ],\n ['size'],\n );\n const logP = browserTools.createBrowserHandlesForObject(page, log, [\n 'LEVELS',\n 'setLevel',\n 'getLevel',\n 'debug',\n 'info',\n 'warning',\n 'error',\n 'exception',\n ]);\n const requestQueueP = this.requestQueue\n ? browserTools.createBrowserHandlesForObject(\n page,\n this.requestQueue,\n ['addRequest'],\n )\n : null;\n const keyValueStoreP = this.keyValueStore\n ? browserTools.createBrowserHandlesForObject(\n page,\n this.keyValueStore,\n ['getValue', 'setValue'],\n )\n : null;\n\n const [\n saveSnapshot,\n skipLinks,\n globalStore,\n logHandle,\n requestQueue,\n keyValueStore,\n ] = await Promise.all([\n saveSnapshotP,\n skipLinksP,\n globalStoreP,\n logP,\n requestQueueP,\n keyValueStoreP,\n ]);\n\n return {\n saveSnapshot,\n skipLinks,\n globalStore,\n log: logHandle,\n keyValueStore,\n requestQueue,\n };\n }\n}\n\nfunction logDevRunWarning() {\n log.warning(`\n*****************************************************************\n* Web Scraper is running in DEVELOPMENT MODE! *\n* Concurrency is limited, sessionPool is not available, *\n* timeouts are increased and debugger is enabled. *\n* If you want full control and performance switch *\n* Run type to PRODUCTION! *\n*****************************************************************\n`);\n}" }, { "name": "src/internals/global_store.ts", "format": "TEXT", "content": "/**\n * GlobalStore is a trivial storage that resembles a Map to be used from Browser contexts\n * to retain data through page navigations and browser instances. It limits Map's functionality\n * because it's currently impossible for functions and object references to cross Node-Browser threshold.\n */\nexport class GlobalStore extends Map {\n override get(key: string) {\n if (typeof key !== 'string') {\n throw new Error(\n 'GlobalStore#get parameter \"key\" must be a string.',\n );\n }\n return super.get(key);\n }\n\n override set(key: string, value: V) {\n if (typeof key !== 'string') {\n throw new Error(\n 'GlobalStore#set parameter \"key\" must be a string.',\n );\n }\n return super.set(key, value);\n }\n\n override forEach(): never {\n throw new Error(\n 'GlobalStore#forEach function is not available due to underlying technology limitations.',\n );\n }\n\n override values() {\n return Array.from(super.values()) as any;\n }\n\n override keys() {\n return Array.from(super.keys()) as any;\n }\n\n override entries() {\n return Array.from(super.entries()) as any;\n }\n}" }, { "name": "src/main.ts", "format": "TEXT", "content": "import { runActor } from '@apify/scraper-tools';\n\nimport { CrawlerSetup } from './internals/crawler_setup.js';\n\nrunActor(CrawlerSetup);" }, { "name": "test/bundle.browser.test.ts", "format": "TEXT", "content": "import { launchPuppeteer } from '@crawlee/puppeteer';\nimport type { Browser, Page } from 'puppeteer';\n\nimport { createBundle } from '../src/internals/bundle.browser';\n\nconst NAMESPACE = 'Apify';\n\ndescribe('Bundle', () => {\n let browser: Browser;\n let page: Page;\n\n beforeAll(async () => {\n browser = await launchPuppeteer({ launchOptions: { headless: true } });\n });\n\n afterAll(async () => {\n await browser.close();\n });\n\n beforeEach(async () => {\n page = await browser.newPage();\n await page.evaluateOnNewDocument(createBundle, NAMESPACE);\n });\n\n afterEach(async () => {\n await page.close();\n });\n\n describe('Context', () => {\n const CONTEXT_OPTIONS = {\n crawlerSetup: {\n rawInput: '{}',\n },\n browserHandles: {\n apify: {},\n globalStore: {},\n log: {},\n },\n pageFunctionArguments: {\n request: {},\n },\n };\n\n beforeEach(async () => {\n await page.goto('about:chrome');\n await page.waitForFunction(\n (namespace: string) => !!window[namespace],\n {},\n NAMESPACE,\n );\n await page.evaluate(\n (namespace: string, contextOptions) => {\n window.contextInstance =\n window[namespace].createContext(contextOptions);\n },\n NAMESPACE,\n CONTEXT_OPTIONS,\n );\n });\n\n describe('waitFor', () => {\n it('should work with a number', async () => {\n const millis = await page.evaluate(async () => {\n const ctx = window.contextInstance;\n const start = Date.now();\n await ctx.waitFor(10);\n return Date.now() - start;\n });\n expect(millis).toBeGreaterThan(9);\n });\n\n it('should work with a selector', async () => {\n const millis = await page.evaluate(async () => {\n const ctx = window.contextInstance;\n const start = Date.now();\n setTimeout(() => {\n const el = document.createElement('div');\n el.id = 'very-unique-id';\n document.body.appendChild(el);\n }, 10);\n await ctx.waitFor('#very-unique-id');\n return Date.now() - start;\n });\n expect(millis).toBeGreaterThan(9);\n });\n\n it('should work with a function', async () => {\n const millis = await page.evaluate(async () => {\n const ctx = window.contextInstance;\n let done = false;\n const start = Date.now();\n setTimeout(() => {\n done = true;\n }, 10);\n await ctx.waitFor(() => done);\n return Date.now() - start;\n });\n expect(millis).toBeGreaterThan(9);\n });\n });\n });\n});" }, { "name": ".dockerignore", "format": "TEXT", "content": "# configurations\n.idea\n\n# crawlee and apify storage folders\napify_storage\ncrawlee_storage\nstorage\n\n# installed files\nnode_modules" }, { "name": "CHANGELOG.md", "format": "TEXT", "content": "# Change Log\n\n## 3.0.18 (2024-10-25)\n\n- Updated Crawlee version to v3.11.5 and SDK v3.2.6\n- Updated Node to v22\n\n## 3.0.17 (2024-04-09)\n\n- Updated Crawlee version to v3.8.0.\n- Updated to use new request queue in scraper\n\n## 3.0.14 (2023-08-22)\n\n- Updated Crawlee version to v3.5.2.\n- Updated Node.js version to v18.\n- Added new options:\n - **Dismiss cookie modals** (`closeCookieModals`): Using the [I don't care about cookies](https://addons.mozilla.org/en-US/firefox/addon/i-dont-care-about-cookies/) browser extension. When on, the crawler will automatically try to dismiss cookie consent modals. This can be useful when crawling European websites that show cookie consent modals.\n - **Maximum scrolling distance in pixels** (`maxScrollHeightPixels`): The crawler will scroll down the page until all content is loaded or the maximum scrolling distance is reached. Setting this to `0` disables scrolling altogether.\n - **Exclude Glob Patterns** (`excludes`): Glob patterns to match links in the page that you want to exclude from being enqueued.\n\n## 3.0 (`version-3`)\n\n- Rewrite from Apify SDK to Crawlee, see the [v3 migration guide](https://sdk.apify.com/docs/upgrading/upgrading-to-v3) for more details.\n- Proxy usage is now required.\n\n## 2.0 (`version-2`)\n\nMain difference between v1 and v2 of the scrapers is the upgrade of SDK to v2, which requires node v15.10+. SDK v2 uses http2 to do the requests with cheerio-scraper, and the http2 support in older node versions were too buggy, so we decided to drop support for those. If you need to run on older node version, use SDK v1.\n\nPlease refer to the SDK 1.0 migration guide for more details about functional changes in the SDK. SDK v2 basically only changes the required node version and has no other breaking changes.\n\n- deprecated `useRequestQueue` option has been removed\n - `RequestQueue` will be always used\n- deprecated `context.html` getter from the `cheerio-scraper` has been removed\n - use `context.body` instead\n- deprecated prepareRequestFunction input option\n - use `pre/postNavigationHooks` instead\n- removed `puppeteerPool/autoscaledPool` from the crawlingContext object\n - `puppeteerPool` was replaced by `browserPool`\n - `autoscaledPool` and `browserPool` and available on the `crawler` property of `crawlingContext` object\n- custom \"Key-value store name\" option in Advanced configuration is now fixed, previously the default store was always used" }, { "name": "Dockerfile", "format": "TEXT", "content": "FROM apify/actor-node-puppeteer-chrome:22 AS builder\n\nCOPY --chown=myuser package*.json ./\n\nRUN npm install --include=dev --audit=false\n\nCOPY --chown=myuser . ./\n\nRUN npm run build\n\nFROM apify/actor-node-puppeteer-chrome:22\n\nCOPY --from=builder --chown=myuser /home/myuser/dist ./dist\n\nCOPY --chown=myuser package*.json ./\n\nRUN npm --quiet set progress=false \\\n && npm install --omit=dev --omit=optional \\\n && echo \"Installed NPM packages:\" \\\n && (npm list --omit=dev --all || true) \\\n && echo \"Node.js version:\" \\\n && node --version \\\n && echo \"NPM version:\" \\\n && npm --version \\\n && rm -r ~/.npm\n\nCOPY --chown=myuser . ./\n\nENV APIFY_DISABLE_OUTDATED_WARNING=1\n\nCMD ./start_xvfb_and_run_cmd.sh && npm run start:prod --silent" }, { "name": "INPUT_SCHEMA.json", "format": "TEXT", "content": "{\n \"title\": \"Web Scraper Input\",\n \"type\": \"object\",\n \"description\": \"Web Scraper loads Start URLs in the Chrome browser and executes Page function on each page to extract data from it. To follow links and scrape additional pages, set Link selector with Pseudo-URLs and/or Glob patterns to specify which links to follow. Alternatively, you can manually enqueue new links in Page function. For details, see Actor's README or Web scraping tutorial in the Apify documentation.\",\n \"schemaVersion\": 1,\n \"properties\": {\n \"runMode\": {\n \"sectionCaption\": \"Basic configuration\",\n \"title\": \"Run mode\",\n \"type\": \"string\",\n \"description\": \"This property indicates the scraper's mode of operation. In DEVELOPMENT mode, the scraper ignores page timeouts, doesn't use sessionPool, opens pages one by one and enables debugging via Chrome DevTools. Open the live view tab or the container URL to access the debugger. Further debugging options can be configured in the Advanced configuration section. PRODUCTION mode disables debugging and enables timeouts and concurrency.

For details, see Run mode in README.\",\n \"default\": \"PRODUCTION\",\n \"prefill\": \"DEVELOPMENT\",\n \"editor\": \"select\",\n \"enum\": [\"PRODUCTION\", \"DEVELOPMENT\"]\n },\n \"startUrls\": {\n \"title\": \"Start URLs\",\n \"type\": \"array\",\n \"description\": \"A static list of URLs to scrape.

For details, see Start URLs in README.\",\n \"prefill\": [{ \"url\": \"https://crawlee.dev/js\" }],\n \"editor\": \"requestListSources\"\n },\n \"keepUrlFragments\": {\n \"title\": \"URL #fragments identify unique pages\",\n \"type\": \"boolean\",\n \"description\": \"Indicates that URL fragments (e.g. http://example.com#fragment) should be included when checking whether a URL has already been visited or not. Typically, URL fragments are used for page navigation only and therefore they should be ignored, as they don't identify separate pages. However, some single-page websites use URL fragments to display different pages; in such a case, this option should be enabled.\",\n \"default\": false,\n \"groupCaption\": \"Options\"\n },\n \"respectRobotsTxtFile\": {\n \"title\": \"Respect the robots.txt file\",\n \"type\": \"boolean\",\n \"description\": \"If enabled, the crawler will consult the robots.txt file for the target website before crawling each page. At the moment, the crawler does not use any specific user agent identifier. The crawl-delay directive is also not supported yet.\",\n \"default\": false,\n \"prefill\": true\n },\n \"linkSelector\": {\n \"title\": \"Link selector\",\n \"type\": \"string\",\n \"description\": \"A CSS selector saying which links on the page (<a> elements with href attribute) shall be followed and added to the request queue. To filter the links added to the queue, use the Pseudo-URLs and/or Glob patterns setting.

If Link selector is empty, the page links are ignored.

For details, see Link selector in README.\",\n \"editor\": \"textfield\",\n \"prefill\": \"a[href]\"\n },\n \"globs\": {\n \"title\": \"Glob Patterns\",\n \"type\": \"array\",\n \"description\": \"Glob patterns to match links in the page that you want to enqueue. Combine with Link selector to tell the scraper where to find links. Omitting the Glob patterns will cause the scraper to enqueue all links matched by the Link selector.\",\n \"editor\": \"globs\",\n \"default\": [],\n \"prefill\": [\n {\n \"glob\": \"https://crawlee.dev/js/*/*\"\n }\n ]\n },\n \"pseudoUrls\": {\n \"title\": \"Pseudo-URLs\",\n \"type\": \"array\",\n \"description\": \"Specifies what kind of URLs found by Link selector should be added to the request queue. A pseudo-URL is a URL with regular expressions enclosed in [] brackets, e.g. http://www.example.com/[.*].

If Pseudo-URLs are omitted, the Actor enqueues all links matched by the Link selector.

For details, see Pseudo-URLs in README.\",\n \"editor\": \"pseudoUrls\",\n \"default\": [],\n \"prefill\": []\n },\n \"excludes\": {\n \"title\": \"Exclude Glob Patterns\",\n \"type\": \"array\",\n \"description\": \"Glob patterns to match links in the page that you want to exclude from being enqueued.\",\n \"editor\": \"globs\",\n \"default\": [],\n \"prefill\": [\n {\n \"glob\": \"/**/*.{png,jpg,jpeg,pdf}\"\n }\n ]\n },\n \"pageFunction\": {\n \"title\": \"Page function\",\n \"type\": \"string\",\n \"description\": \"JavaScript (ES6) function that is executed in the context of every page loaded in the Chrome browser. Use it to scrape data from the page, perform actions or add new URLs to the request queue.

For details, see Page function in README.\",\n \"prefill\": \"// The function accepts a single argument: the \\\"context\\\" object.\\n// For a complete list of its properties and functions,\\n// see https://apify.com/apify/web-scraper#page-function \\nasync function pageFunction(context) {\\n // This statement works as a breakpoint when you're trying to debug your code. Works only with Run mode: DEVELOPMENT!\\n // debugger; \\n\\n // jQuery is handy for finding DOM elements and extracting data from them.\\n // To use it, make sure to enable the \\\"Inject jQuery\\\" option.\\n const $ = context.jQuery;\\n const pageTitle = $('title').first().text();\\n const h1 = $('h1').first().text();\\n const first_h2 = $('h2').first().text();\\n const random_text_from_the_page = $('p').first().text();\\n\\n\\n // Print some information to Actor log\\n context.log.info(`URL: ${context.request.url}, TITLE: ${pageTitle}`);\\n\\n // Manually add a new page to the queue for scraping.\\n await context.enqueueRequest({ url: 'http://www.example.com' });\\n\\n // Return an object with the data extracted from the page.\\n // It will be stored to the resulting dataset.\\n return {\\n url: context.request.url,\\n pageTitle,\\n h1,\\n first_h2,\\n random_text_from_the_page\\n };\\n}\",\n \"editor\": \"javascript\"\n },\n \"injectJQuery\": {\n \"title\": \"Inject jQuery\",\n \"type\": \"boolean\",\n \"description\": \"If enabled, the scraper will inject the jQuery library into every web page loaded, before Page function is invoked. Note that the jQuery object ($) will not be registered into global namespace in order to avoid conflicts with libraries used by the web page. It can only be accessed through context.jQuery in Page function.\",\n \"default\": true\n },\n \"proxyConfiguration\": {\n \"sectionCaption\": \"Proxy and browser configuration\",\n \"title\": \"Proxy configuration\",\n \"type\": \"object\",\n \"description\": \"Specifies proxy servers that will be used by the scraper in order to hide its origin.

For details, see Proxy configuration in README.\",\n \"prefill\": { \"useApifyProxy\": true },\n \"default\": { \"useApifyProxy\": true },\n \"editor\": \"proxy\"\n },\n \"proxyRotation\": {\n \"title\": \"Proxy rotation\",\n \"type\": \"string\",\n \"description\": \"This property indicates the strategy of proxy rotation and can only be used in conjunction with Apify Proxy. The recommended setting automatically picks the best proxies from your available pool and rotates them evenly, discarding proxies that become blocked or unresponsive. If this strategy does not work for you for any reason, you may configure the scraper to either use a new proxy for each request, or to use one proxy as long as possible, until the proxy fails. IMPORTANT: This setting will only use your available Apify Proxy pool, so if you don't have enough proxies for a given task, no rotation setting will produce satisfactory results.\",\n \"default\": \"RECOMMENDED\",\n \"editor\": \"select\",\n \"enum\": [\"RECOMMENDED\", \"PER_REQUEST\", \"UNTIL_FAILURE\"],\n \"enumTitles\": [\n \"Use recommended settings\",\n \"Rotate proxy after each request\",\n \"Use one proxy until failure\"\n ]\n },\n \"sessionPoolName\": {\n \"title\": \"Session pool name\",\n \"type\": \"string\",\n \"description\": \"Use only english alphanumeric characters dashes and underscores. A session is a representation of a user. It has it's own IP and cookies which are then used together to emulate a real user. Usage of the sessions is controlled by the Proxy rotation option. By providing a session pool name, you enable sharing of those sessions across multiple Actor runs. This is very useful when you need specific cookies for accessing the websites or when a lot of your proxies are already blocked. Instead of trying randomly, a list of working sessions will be saved and a new Actor run can reuse those sessions. Note that the IP lock on sessions expires after 24 hours, unless the session is used again in that window.\",\n \"editor\": \"textfield\",\n \"minLength\": 3,\n \"maxLength\": 200,\n \"pattern\": \"[0-9A-z-]\"\n },\n \"initialCookies\": {\n \"title\": \"Initial cookies\",\n \"type\": \"array\",\n \"description\": \"A JSON array with cookies that will be set to every Chrome browser tab opened before loading the page, in the format accepted by Puppeteer's Page.setCookie() function. This option is useful for transferring a logged-in session from an external web browser.\",\n \"default\": [],\n \"prefill\": [],\n \"editor\": \"json\"\n },\n \"useChrome\": {\n \"title\": \"Use Chrome\",\n \"type\": \"boolean\",\n \"description\": \"If enabled, the scraper will use a real Chrome browser instead of Chromium bundled with Puppeteer. This option may help bypass certain anti-scraping protections, but might make the scraper unstable. Use at your own risk \\uD83D\\uDE42\",\n \"default\": false,\n \"groupCaption\": \"Browser masking\"\n },\n \"headless\": {\n \"title\": \"Run browsers in headless mode\",\n \"type\": \"boolean\",\n \"description\": \"By default, browsers run in headless mode. You can toggle this off to run them in headful mode, which can help with certain rare anti-scraping protections but is slower and more costly.\",\n \"default\": true\n },\n \"ignoreSslErrors\": {\n \"title\": \"Ignore SSL errors\",\n \"type\": \"boolean\",\n \"description\": \"If enabled, the scraper will ignore SSL/TLS certificate errors. Use at your own risk.\",\n \"default\": false,\n \"groupCaption\": \"Security\"\n },\n \"ignoreCorsAndCsp\": {\n \"title\": \"Ignore CORS and CSP\",\n \"type\": \"boolean\",\n \"description\": \"If enabled, the scraper will ignore Content Security Policy (CSP) and Cross-Origin Resource Sharing (CORS) settings of visited pages and requested domains. This enables you to freely use XHR/Fetch to make HTTP requests from Page function.\",\n \"default\": false\n },\n \"downloadMedia\": {\n \"sectionCaption\": \"Performance and limits\",\n \"title\": \"Download media files\",\n \"type\": \"boolean\",\n \"description\": \"If enabled, the scraper will download media such as images, fonts, videos and sound files, as usual. Disabling this option might speed up the scrape, but certain websites could stop working correctly.\",\n \"default\": true,\n \"groupCaption\": \"Page resources\"\n },\n \"downloadCss\": {\n \"title\": \"Download CSS files\",\n \"type\": \"boolean\",\n \"description\": \"If enabled, the scraper will download CSS files with stylesheets, as usual. Disabling this option may speed up the scrape, but certain websites could stop working correctly, and the live view will not look as cool.\",\n \"default\": true\n },\n \"maxRequestRetries\": {\n \"title\": \"Max page retries\",\n \"type\": \"integer\",\n \"description\": \"The maximum number of times the scraper will retry to load each web page on error, in case of a page load error or an exception thrown by Page function.

If set to 0, the page will be considered failed right after the first error.\",\n \"minimum\": 0,\n \"default\": 3\n },\n \"maxPagesPerCrawl\": {\n \"title\": \"Max pages per run\",\n \"type\": \"integer\",\n \"description\": \"The maximum number of pages that the scraper will load. The scraper will stop when this limit is reached. It's always a good idea to set this limit in order to prevent excess platform usage for misconfigured scrapers. Note that the actual number of pages loaded might be slightly higher than this value.

If set to 0, there is no limit.\",\n \"minimum\": 0,\n \"default\": 0\n },\n \"maxResultsPerCrawl\": {\n \"title\": \"Max result records\",\n \"type\": \"integer\",\n \"description\": \"The maximum number of records that will be saved to the resulting dataset. The scraper will stop when this limit is reached.

If set to 0, there is no limit.\",\n \"minimum\": 0,\n \"default\": 0\n },\n \"maxCrawlingDepth\": {\n \"title\": \"Max crawling depth\",\n \"type\": \"integer\",\n \"description\": \"Specifies how many links away from Start URLs the scraper will descend. This value is a safeguard against infinite crawling depths for misconfigured scrapers. Note that pages added using context.enqueuePage() in Page function are not subject to the maximum depth constraint.

If set to 0, there is no limit. To crawl only the pages specified by the Start URLs, set linkSelector empty instead.\",\n \"minimum\": 0,\n \"default\": 0\n },\n \"maxConcurrency\": {\n \"title\": \"Max concurrency\",\n \"type\": \"integer\",\n \"description\": \"Specified the maximum number of pages that can be processed by the scraper in parallel. The scraper automatically increases and decreases concurrency based on available system resources. This option enables you to set an upper limit, for example to reduce the load on a target web server.\",\n \"minimum\": 1,\n \"default\": 50\n },\n \"pageLoadTimeoutSecs\": {\n \"title\": \"Page load timeout\",\n \"type\": \"integer\",\n \"description\": \"The maximum amount of time the scraper will wait for a web page to load, in seconds. If the web page does not load in this timeframe, it is considered to have failed and will be retried (subject to Max page retries), similarly as with other page load errors.\",\n \"minimum\": 1,\n \"default\": 60,\n \"unit\": \"seconds\"\n },\n \"pageFunctionTimeoutSecs\": {\n \"title\": \"Page function timeout\",\n \"type\": \"integer\",\n \"description\": \"The maximum amount of time the scraper will wait for Page function to execute, in seconds. It's a good idea to set this limit, to ensure that unexpected behavior in page function will not get the scraper stuck.\",\n \"minimum\": 1,\n \"default\": 60,\n \"unit\": \"seconds\"\n },\n \"waitUntil\": {\n \"title\": \"Navigation waits until\",\n \"type\": \"array\",\n \"description\": \"Contains a JSON array with names of page events to wait, before considering a web page fully loaded. The scraper will wait until all of the events are triggered in the web page before executing Page function. Available events are domcontentloaded, load, networkidle2 and networkidle0.

For details, see waitUntil option in Puppeteer's Page.goto() function documentation.\",\n \"default\": [\"networkidle2\"],\n \"prefill\": [\"networkidle2\"],\n \"editor\": \"json\"\n },\n \"preNavigationHooks\": {\n \"sectionCaption\": \"Advanced configuration\",\n \"title\": \"Pre-navigation hooks\",\n \"type\": \"string\",\n \"description\": \"Async functions that are sequentially evaluated before the navigation. Good for setting additional cookies or browser properties before navigation. The function accepts two parameters, `crawlingContext` and `gotoOptions`, which are passed to the `page.goto()` function the crawler calls to navigate.\",\n \"prefill\": \"// We need to return array of (possibly async) functions here.\\n// The functions accept two arguments: the \\\"crawlingContext\\\" object\\n// and \\\"gotoOptions\\\".\\n[\\n async (crawlingContext, gotoOptions) => {\\n // ...\\n },\\n]\\n\",\n \"editor\": \"javascript\"\n },\n \"postNavigationHooks\": {\n \"title\": \"Post-navigation hooks\",\n \"type\": \"string\",\n \"description\": \"Async functions that are sequentially evaluated after the navigation. Good for checking if the navigation was successful. The function accepts `crawlingContext` as the only parameter.\",\n \"prefill\": \"// We need to return array of (possibly async) functions here.\\n// The functions accept a single argument: the \\\"crawlingContext\\\" object.\\n[\\n async (crawlingContext) => {\\n // ...\\n },\\n]\",\n \"editor\": \"javascript\"\n },\n \"breakpointLocation\": {\n \"title\": \"Insert breakpoint\",\n \"type\": \"string\",\n \"description\": \"This property has no effect if Run mode is set to PRODUCTION. When set to DEVELOPMENT it inserts a breakpoint at the selected location in every page the scraper visits. Execution of code stops at the breakpoint until manually resumed in the DevTools window accessible via Live View tab or Container URL. Additional breakpoints can be added by adding debugger; statements within your Page function.

See Run mode in README for details.\",\n \"default\": \"NONE\",\n \"prefill\": \"NONE\",\n \"editor\": \"select\",\n \"enum\": [\n \"NONE\",\n \"BEFORE_GOTO\",\n \"BEFORE_PAGE_FUNCTION\",\n \"AFTER_PAGE_FUNCTION\"\n ],\n \"enumTitles\": [\n \"Nowhere. Break only on debugger; statements\",\n \"Before navigation to URL\",\n \"Before Page function invocation\",\n \"After Page function invocation\"\n ]\n },\n \"closeCookieModals\": {\n \"title\": \"Dismiss cookie modals\",\n \"type\": \"boolean\",\n \"description\": \"Using the [I don't care about cookies](https://addons.mozilla.org/en-US/firefox/addon/i-dont-care-about-cookies/) browser extension. When on, the crawler will automatically try to dismiss cookie consent modals. This can be useful when crawling European websites that show cookie consent modals.\",\n \"default\": false\n },\n \"maxScrollHeightPixels\": {\n \"title\": \"Maximum scrolling distance in pixels\",\n \"type\": \"integer\",\n \"description\": \"The crawler will scroll down the page until all content is loaded or the maximum scrolling distance is reached. Setting this to `0` disables scrolling altogether.\",\n \"default\": 5000\n },\n \"debugLog\": {\n \"title\": \"Debug log\",\n \"type\": \"boolean\",\n \"description\": \"If enabled, the Actor log will include debug messages. Beware that this can be quite verbose. Use context.log.debug('message') to log your own debug messages from Page function.\",\n \"default\": false,\n \"groupCaption\": \"Logging\"\n },\n \"browserLog\": {\n \"title\": \"Browser log\",\n \"type\": \"boolean\",\n \"description\": \"If enabled, the Actor log will include console messages produced by JavaScript executed by the web pages (e.g. using console.log()). Beware that this may result in the log being flooded by error messages, warnings and other messages of little value, especially with high concurrency.\",\n \"default\": false\n },\n \"customData\": {\n \"title\": \"Custom data\",\n \"type\": \"object\",\n \"description\": \"A custom JSON object that is passed to Page function as context.customData. This setting is useful when invoking the scraper via API, in order to pass some arbitrary parameters to your code.\",\n \"default\": {},\n \"prefill\": {},\n \"editor\": \"json\"\n },\n \"datasetName\": {\n \"title\": \"Dataset name\",\n \"type\": \"string\",\n \"description\": \"Name or ID of the dataset that will be used for storing results. If left empty, the default dataset of the run will be used.\",\n \"editor\": \"textfield\"\n },\n \"keyValueStoreName\": {\n \"title\": \"Key-value store name\",\n \"type\": \"string\",\n \"description\": \"Name or ID of the key-value store that will be used for storing records. If left empty, the default key-value store of the run will be used.\",\n \"editor\": \"textfield\"\n },\n \"requestQueueName\": {\n \"title\": \"Request queue name\",\n \"type\": \"string\",\n \"description\": \"Name of the request queue that will be used for storing requests. If left empty, the default request queue of the run will be used.\",\n \"editor\": \"textfield\"\n }\n },\n \"required\": [\"startUrls\", \"pageFunction\", \"proxyConfiguration\"]\n}" }, { "name": "README.md", "format": "TEXT", "content": "# Web Scraper\n\n## What is Web Scraper?\n\nWeb Scraper is a tool for extracting data from any website. It can navigate pages, render JavaScript, and extract structured data using a few simple commands. Whether you need to scrape product prices, real estate data, or social media profiles, this Actor turns any web page into an API.\n\n- Configurable with an **intuitive user interface**\n- Can handle almost **any website** and can scrape dynamic content\n- Scrape a list of **URLs or crawl an entire website** by following links\n- Runs entirely on the **Apify platform**; no need to manage servers or proxies\n- Set your scraper to **run on a schedule** and get data delivered automatically\n- Can be used as a template to **create your own scraper**\n\n## What can Web Scraper data be used for?\n\nWeb Scraper can extract almost any data from any site, effectively turning any site into a data source. All data can be exported into **JSON, CSV, HTML, and Excel** formats.\n\nHere are some examples:\n\n- **Extract reviews** from sites like Yelp or Amazon\n- Gather **real estate data** from Zillow or local realtor pages\n- Get **contact details** and social media accounts from local businesses\n- **Monitor mentions** of a brand or person on specific sites\n- **Collect and monitor product prices** on e-commerce websites\n\nAs a generic tool, Web Scraper can also serve as a template to **build your own scraper** which you can then market on Apify Store.\n\n## How much does the Web Scraper cost?\n\nWeb Scraper is free to use, but you do pay for Apify platform usage, which is calculated in [compute units](https://help.apify.com/en/articles/3490384-what-is-a-compute-unit?ref=apify) (CU). On the free plan, these are charged at $0.04 per CU. CUs get cheaper with higher subscription plans - [see our pricing page](https://apify.com/pricing) for more details.\n\nWith our free plan, you get **$5 in platform credits every month**, which is enough to scrape from 500 to 1,000 **web pages**. If you sign up to our Starter plan, you can expect to scrape thousands.\n\n## How to use Web Scraper\n\n1. [Create](https://console.apify.com/actors/moJRLRc85AitArpNN?addFromActorId=moJRLRc85AitArpNN) a free Apify account using your email and open [Web Scraper](https://apify.com/apify/web-scraper)\n2. Add one or more URLs you want to scrape\n3. Set paths that you’d like to include or exclude from crawling by configuring glob patterns or pseudo-URLs\n4. Configure the page function that determines the data that needs to be scraped\n5. Click the “Start” button and wait for the data to be extracted\n6. Download your data in JSON, XML, CSV, Excel, or HTML\n\nFor more in-depth instructions, please read our article on [scraping with Web Scraper](https://docs.apify.com/tutorials/apify-scrapers/web-scraper), which features step-by-step instructions on how to use Web Scraper on the basis of real-life examples. We also have a video tutorial you can follow along with:\n\nhttps://www.youtube.com/watch?v=5kcaHAuGxmY\n\n## Using Web Scraper with the Apify API\n\nThe Apify API gives you programmatic access to the Apify platform. The API is organized around RESTful HTTP endpoints that enable you to manage, schedule, and run Apify Actors. The API also lets you access any datasets, monitor actor performance, fetch results, create and update versions, and more.\n\nTo access the API using Node.js, use the `apify-client` [NPM package](https://apify.com/apify/web-scraper/api/javascript). To access the API using Python, use the `apify-client` [PyPI package](https://apify.com/apify/web-scraper/api/python).\n\nClick on the [API tab](https://apify.com/apify/web-scraper/api/python) for code examples, or check out the [Apify API reference](https://docs.apify.com/api/v2) docs for all the details.\n\n## Web Scraper and MCP Server\n\nWith Apify API, you can use almost any Actor in conjunction with an MCP server. You can connect to the MCP server using clients like ClaudeDesktop and LibreChat, or even build your own. Read all about how you can [set up Apify Actors with MCP](https://blog.apify.com/how-to-use-mcp/).\n\nFor Web Scraper, go to the [MCP tab](https://apify.com/apify/web-scraper/api/mcp) and then go through the following steps:\n\n1. Start a Server-Sent Events (SSE) session to receive a `sessionId`\n2. Send API messages using that `sessionId` to trigger the scraper\n3. The message starts the Web Scraper with the provided input\n4. The response should be: `Accepted`\n\n## Integrating Web Scraper into your workflows\n\nYou can integrate Web Scraper with almost any cloud service or web app. We offer integrations with **Make, Zapier, Slack, Airbyte, GitHub, Google Sheets, Google Drive**, [and plenty more](https://docs.apify.com/integrations).\n\nAlternatively, you could use [webhooks](https://docs.apify.com/integrations/webhooks) to carry out an action whenever an event occurs, such as getting a notification whenever Web Scraper successfully finishes a run.\n\n## Advanced configuration settings\n\nBelow you’ll find detailed instructions on more advanced configuration settings for Web Scraper.\n\n## Input configurations\n\nOn input, the Web Scraper Actor accepts a number of configuration settings. These can be entered either manually in the user interface in [Apify Console](https://console.apify.com/), or programmatically in a JSON object using the [Apify API](https://docs.apify.com/api/v2#/reference/actors/run-collection/run-actor).\n\nFor a complete list of input fields and their type, please see the [input tab](https://apify.com/apify/web-scraper/input-schema).\n\n### Run mode\n\nRun mode allows you to switch between two modes of operation for Web Scraper.\n\n**PRODUCTION** mode gives you full control and full performance. You should always switch Web Scraper to production mode once you're done making changes to your scraper.\n\nWhen starting to develop your Scraper, you want to be able to inspect what's happening in the browser and debug your code. You can do that with the scraper's **DEVELOPMENT** mode. It allows you to directly control the browser using Chrome DevTools. Open the Live View tab to access the DevTools. It will also limit concurrency and prevent timeouts to improve your DevTools experience. Other debugging related options can be configured in the **Advanced configuration** section.\n\n### Start URLs\n\nThe **Start URLs** (`startUrls`) field represent the initial list of URLs of pages that the scraper will visit. You can either enter these URLs manually one by one, upload them in a CSV file or\n[link URLs from the Google Sheets](https://help.apify.com/en/articles/2906022-scraping-a-list-of-urls-from-a-google-sheets-document) document. Each URL must start with either a `http://` or `https://` protocol prefix.\n\nThe scraper supports adding new URLs to scrape on the fly, either using the [**Link selector**](#link-selector) and [**Glob Patterns**](#glob-patterns)/[**Pseudo-URLs**](#pseudo-urls) options or by calling `await context.enqueueRequest()` inside [**Page function**](#page-function).\n\nOptionally, each URL can be associated with custom user data - a JSON object that can be referenced from your JavaScript code in [**Page function**](#page-function) under `context.request.userData`. This is useful for determining which start URL is currently loaded, in order to perform some page-specific actions. For example, when crawling an online store, you might want to perform different\nactions on a page listing the products vs. a product detail page. For details, see our [web scraping tutorial](https://docs.apify.com/tutorials/apify-scrapers/getting-started#the-start-url).\n\n\n\n### Link selector\n\nThe **Link selector** (`linkSelector`) field contains a CSS selector that is used to find links to other web pages, i.e. `` elements with the `href` attribute.\n\nOn every page loaded, the scraper looks for all links matching **Link selector**, checks that the target URL matches one of the [**Glob Patterns**](#glob-patterns)/[**Pseudo-URLs**](#pseudo-urls), and if so then adds the URL to the request queue, so that it's loaded by the scraper later.\n\nBy default, new scrapers are created with the following selector that matches all links:\n\n```\na[href]\n```\n\nIf **Link selector** is empty, the page links are ignored, and the scraper only loads pages that were specified in [**Start URLs**](#start-urls) or that were manually added to the request queue by calling `await context.enqueueRequest()` in [**Page function**](#page-function).\n\n### Glob Patterns\n\nThe **Glob Patterns** (`globs`) field specifies which types of URLs found by [**Link selector**](#link-selector) should be added to the request queue.\n\nA glob pattern is simply a string with wildcard characters.\n\nFor example, a glob pattern `http://www.example.com/pages/**/*` will match all the\nfollowing URLs:\n\n- `http://www.example.com/pages/deeper-level/page`\n- `http://www.example.com/pages/my-awesome-page`\n- `http://www.example.com/pages/something`\n\nNote that you don't need to use the **Glob Patterns** setting at all, because you can completely control which pages the scraper will access by calling `await context.enqueueRequest()` from the [**Page function**](#page-function).\n\n### Pseudo-URLs\n\nThe **Pseudo-URLs** (`pseudoUrls`) field specifies what kind of URLs found by [**Link selector**](#link-selector) should be added to the request queue.\n\nA pseudo-URL is simply a URL with special directives enclosed in `[]` brackets. Currently, the only supported directive is `[regexp]`, which defines a JavaScript-style regular expression to match against the URL.\n\nFor example, a pseudo-URL `http://www.example.com/pages/[(\\w|-)*]` will match all the\nfollowing URLs:\n\n- `http://www.example.com/pages/`\n- `http://www.example.com/pages/my-awesome-page`\n- `http://www.example.com/pages/something`\n\nIf either `[` or `]` is part of the normal query string, it must be encoded as `[\\x5B]` or `[\\x5D]`, respectively. For example, the following pseudo-URL:\n\n```\nhttp://www.example.com/search?do[\\x5B]load[\\x5D]=1\n```\n\nwill match the URL:\n\n```\nhttp://www.example.com/search?do[load]=1\n```\n\nOptionally, each pseudo-URL can be associated with user data that can be referenced from\nyour [**Page function**](#page-function) using `context.request.label` to determine which kind of page is currently loaded in the browser.\n\nNote that you don't need to use the **Pseudo-URLs** setting at all, because you can completely control which pages the scraper will access by calling `await context.enqueueRequest()` from [**Page function**](#page-function).\n\n### Page function\n\nThe **Page function** (`pageFunction`) field contains a JavaScript function that is executed in the context of every page loaded in the Chromium browser. The purpose of this function is to extract\ndata from the web page, manipulate the DOM by clicking elements, add new URLs to the request queue and otherwise control Web Scraper's operation.\n\nExample:\n\n```javascript\nasync function pageFunction(context) {\n // jQuery is handy for finding DOM elements and extracting data from them.\n // To use it, make sure to enable the \"Inject jQuery\" option.\n const $ = context.jQuery;\n const pageTitle = $('title').first().text();\n\n // Print some information to Actor log\n context.log.info(`URL: ${context.request.url}, TITLE: ${pageTitle}`);\n\n // Manually add a new page to the scraping queue.\n await context.enqueueRequest({ url: 'http://www.example.com' });\n\n // Return an object with the data extracted from the page.\n // It will be stored to the resulting dataset.\n return {\n url: context.request.url,\n pageTitle,\n };\n}\n```\n\nThe page function accepts a single argument, the `context` object, whose properties are listed in the table below. Since the function is executed in the context of the web page, it can access the DOM, e.g. using the `window` or `document` global variables.\n\nThe return value of the page function is an object (or an array of objects) representing the data extracted from the web page. The return value must be stringify-able to JSON, i.e. it can only contain basic types and no circular references. If you don't want to extract any data from the page and skip it in the clean results, simply return `null` or `undefined`.\n\nThe page function supports the JavaScript ES6 syntax and is asynchronous, which means you can use the `await` keyword to wait for background operations to finish. To learn more about `async` functions, see Mozilla documentation.\n\n**Properties of the `context` object:**\n\n- **`customData: Object`**\n Contains the object provided in the **Custom data** (`customData`) input setting. This is useful for passing dynamic parameters to your Web Scraper using API.\n- **`enqueueRequest(request, [options]): AsyncFunction`**\n Adds a new URL to the request queue, if it wasn't already there. The `request` parameter is an object containing details of the request, with properties such as `url`, `label`, `userData`, `headers` etc. For the full list of the supported properties, see the `Request` object's constructor in Crawlee documentation.\n The optional `options` parameter is an object with additional options. Currently, it only supports the `forefront` boolean flag. If it's `true`, the request is added to the beginning of the queue. By default, requests are added to the end.\n Example:\n ```javascript\n await context.enqueueRequest({ url: 'https://www.example.com' });\n await context.enqueueRequest(\n { url: 'https://www.example.com/first' },\n { forefront: true },\n );\n ```\n- **`env: Object`**\n A map of all relevant values set by the Apify platform to the Actor run via the `APIFY_` environment variables. For example, you can find here information such as Actor run ID, timeouts, Actor run memory, etc.\n For the full list of available values, see\n `ApifyEnv` interface in Apify SDK.\n Example:\n ```javascript\n console.log(`Actor run ID: ${context.env.actorRunId}`);\n ```\n- **`getValue(key): AsyncFunction`**\n Gets a value from the default key-value store associated with the Actor run. The key-value store is useful for persisting named data records, such as state objects, files, etc. The function is very similar to `Actor.getValue()` function in Apify SDK.\n To set the value, use the dual function `context.setValue(key, value)`.\n Example:\n ```javascript\n const value = await context.getValue('my-key');\n console.dir(value);\n ```\n- **`globalStore: Object`**\n Represents an in-memory store that can be used to share data across page function invocations, e.g. state variables, API responses or other data. The `globalStore` object has an equivalent interface as JavaScript's `Map` object, with a few important differences:\n - All functions of `globalStore` are `async`; use `await` when calling them.\n - Keys must be strings and values need to be JSON stringify-able.\n - `forEach()` function is not supported.\n Note that the stored data is not persisted. If the Actor run is restarted or migrated to another worker server, the content of `globalStore` is reset. Therefore, never depend on a specific value to be present in the store.\n Example:\n ```javascript\n let movies = await context.globalStore.get('cached-movies');\n if (!movies) {\n movies = await fetch('http://example.com/movies.json');\n await context.globalStore.set('cached-movies', movies);\n }\n console.dir(movies);\n ```\n- **`input: Object`**\n An object containing the Actor run input, i.e. the Web Scraper's configuration. Each page function invocation gets a fresh copy of the `input` object, so changing its properties has no effect.\n- **`jQuery: Function`**\n A reference to the `jQuery` library, which is extremely useful for DOM traversing, manipulation, querying and data extraction. This field is only available if the **Inject jQuery** option is enabled.\n Typically, the jQuery function is registered under a global variable called $.\n However, the web page might use this global variable for something else. To avoid conflicts, the jQuery object is not registered globally and is only available through the `context.jQuery` property.\n Example:\n ```javascript\n const $ = context.jQuery;\n const pageTitle = $('title').first().text();\n ```\n- **`log: Object`**\n An object containing logging functions, with the same interface as provided by the `Crawlee.utils.log` object in Crawlee. The log messages are written directly to the Actor run log, which is useful for monitoring and debugging. Note that `log.debug()` only prints messages to the log if the **Enable debug log** input setting is set.\n Example:\n ```javascript\n const log = context.log;\n log.debug('Debug message', { hello: 'world!' });\n log.info('Information message', { all: 'good' });\n log.warning('Warning message');\n log.error('Error message', { details: 'This is bad!' });\n try {\n throw new Error('Not good!');\n } catch (e) {\n log.exception(e, 'Exception occurred', {\n details: 'This is really bad!',\n });\n }\n ```\n- **`request: Object`**\n An object containing information about the currently loaded web page, such as the URL, number of retries, a unique key, etc. Its properties are equivalent to the `Request` object in Crawlee.\n- **`response: Object`**\n An object containing information about the HTTP response from the web server. Currently, it only contains the `status` and `headers` properties. For example:\n\n ```javascript\n {\n // HTTP status code\n status: 200,\n\n // HTTP headers\n headers: {\n 'content-type': 'text/html; charset=utf-8',\n 'date': 'Wed, 06 Nov 2019 16:01:53 GMT',\n 'cache-control': 'no-cache',\n 'content-encoding': 'gzip',\n },\n }\n\n ```\n\n- **`saveSnapshot(): AsyncFunction`**\n Saves a screenshot and full HTML of the current page to the key-value store\n associated with the Actor run, under the `SNAPSHOT-SCREENSHOT` and `SNAPSHOT-HTML` keys, respectively. This feature is useful when debugging your scraper.\n Note that each snapshot overwrites the previous one and the `saveSnapshot()` calls are throttled to at most one call in two seconds, in order to avoid excess consumption of resources and slowdown of the Actor.\n- **`setValue(key, data, options): AsyncFunction`**\n Sets a value to the default key-value store associated with the Actor run. The key-value store is useful for persisting named data records, such as state objects, files, etc. The function is very similar to `KeyValueStore.setValue()` function in Crawlee.\n To get the value, use the dual function `await context.getValue(key)`.\n Example:\n ```javascript\n await context.setValue('my-key', { hello: 'world' });\n ```\n- **`skipLinks(): AsyncFunction`**\n Calling this function ensures that page links from the current page will not be added to the request queue, even if they match the [**Link selector**](#link-selector) and/or [**Glob Patterns**](#glob-patterns)/[**Pseudo-URLs**](#pseudo-urls) settings. This is useful to programmatically stop recursive crawling, e.g. if you know there are no more interesting links on the current page to follow.\n- **`waitFor(task, options): AsyncFunction`**\n A helper function that waits either a specific amount of time (in milliseconds), for an element specified using a CSS selector to appear in the DOM or for a provided function to return `true`.\n This is useful for extracting data from web pages with dynamic content, where the content might not be available at the time when the page function is called.\n The `options` parameter is an object with the following properties and default values:\n\n ```javascript\n {\n // Maximum time to wait\n timeoutMillis: 20000,\n\n // How often to check if the condition changes\n pollingIntervalMillis: 50,\n }\n ```\n\n Example:\n\n ```javascript\n // Wait for selector\n await context.waitFor('.foo');\n // Wait for 1 second\n await context.waitFor(1000);\n // Wait for predicate\n await context.waitFor(() => !!document.querySelector('.foo'), {\n timeoutMillis: 5000,\n });\n ```\n\n## Proxy configuration\n\nThe **Proxy configuration** (`proxyConfiguration`) option enables you to set proxies that will be used by the scraper in order to prevent its detection by target websites. You can use both [Apify Proxy](https://apify.com/proxy) and custom HTTP or SOCKS5 proxy servers.\n\nProxy is required to run the scraper. The following table lists the available options of the proxy configuration setting:\n\n\n \n \n \n \n \n \n \n \n \n \n \n
Apify Proxy (automatic)\n \n The scraper will load all web pages using Apify Proxy in the automatic mode. In this mode, the proxy uses all proxy groups that are available to the user, and for each new web page it automatically selects the proxy that hasn't been used in the longest time for the specific hostname, in order to reduce the chance of detection by the website. You can view the list of available proxy groups on the Proxy page in Apify Console.\n
Apify Proxy (selected groups)\n \n The scraper will load all web pages using Apify Proxy with specific groups of target proxy servers.\n
Custom proxies\n \n

\n The scraper will use a custom list of proxy servers. The proxies must be specified in the `scheme://user:password@host:port` format, multiple proxies should be separated by a space or new line. The URL scheme can be either `HTTP` or `SOCKS5`. User and password might be omitted, but the port must always be present.\n

\n

\n Example:\n

\n 
http://bob:password@proxy1.example.com:8000\nhttp://bob:password@proxy2.example.com:8000
\n
\n\nThe proxy configuration can be set programmatically when calling the Actor using the API\nby setting the `proxyConfiguration` field. It accepts a JSON object with the following structure:\n\n```javascript\n{\n // Indicates whether to use Apify Proxy or not.\n \"useApifyProxy\": Boolean,\n\n // Array of Apify Proxy groups, only used if \"useApifyProxy\" is true.\n // If missing or null, Apify Proxy will use the automatic mode.\n \"apifyProxyGroups\": String[],\n\n // Array of custom proxy URLs, in \"scheme://user:password@host:port\" format.\n // If missing or null, custom proxies are not used.\n \"proxyUrls\": String[],\n}\n```\n\n### Logging into websites with Web Scraper\n\nThe **Initial cookies** field allows you to set cookies that will be used by the scraper to log into websites. Cookies are small text files that are stored on your computer by your web browser. Various websites use cookies to store information about your current session. By transferring this information to the scraper, it will be able to log into websites using your credentials. To learn more about logging into websites by transferring cookies, check out our [tutorial](https://docs.apify.com/tutorials/log-in-by-transferring-cookies).\n\nBe aware that cookies usually have a limited lifespan and will expire after a certain period of time. This means that you will have to update the cookies periodically in order to keep the scraper logged in. Alternative approach is to make the scraper actively log in to the website in the Page function. For more info about this approach, check out our [tutorial](https://docs.apify.com/tutorials/log-into-a-website-using-puppeteer) on logging into websites using Puppeteer.\n\nThe scraper expects the cookies in the **Initial cookies** field to be stored as separate JSON objects in a JSON array, see example below:\n\n```json\n[\n {\n \"name\": \" ga\",\n \"value\": \"GA1.1.689972112. 1627459041\",\n \"domain\": \".apify.com\",\n \"hostOnly\": false,\n \"path\": \"/\",\n \"secure\": false,\n \"httpOnly\": false,\n \"sameSite\": \"no_restriction\",\n \"session\": false,\n \"firstPartyDomain\": \"\",\n \"expirationDate\": 1695304183,\n \"storelId\": \"firefox-default\",\n \"id\": 1\n }\n]\n```\n\n## Advanced configuration\n\n### Pre-navigation hooks\n\nThis is an array of functions that will be executed **BEFORE** the main `pageFunction` is run. A similar `context` object is passed into each of these functions as is passed into the `pageFunction`; however, a second \"[DirectNavigationOptions](https://crawlee.dev/api/puppeteer-crawler/namespace/puppeteerUtils#DirectNavigationOptions)\" object is also passed in.\n\nThe available options can be seen here:\n\n```javascript\npreNavigationHooks: [\n async (\n { id, request, session, proxyInfo },\n { timeout, waitUntil, referer },\n ) => {},\n];\n```\n\n> Unlike with playwright, puppeteer and cheerio scrapers, in web scraper we don't have the Actor object available in the hook parameters, as the hook is executed inside the browser.\n\nCheck out the docs for [Pre-navigation hooks](https://crawlee.dev/api/puppeteer-crawler/interface/PuppeteerCrawlerOptions#preNavigationHooks) and the [PuppeteerHook type](https://crawlee.dev/api/puppeteer-crawler/interface/PuppeteerHook) for more info regarding the objects passed into these functions.\n\n### Post-navigation hooks\n\nAn array of functions that will be executed **AFTER** the main `pageFunction` is run. The only available parameter is the `CrawlingContext` object.\n\n```javascript\npostNavigationHooks: [\n async ({ id, request, session, proxyInfo, response }) => {},\n],\n```\n\n> Unlike with playwright, puppeteer and cheerio scrapers, in web scraper we don't have the Actor object available in the hook parameters, as the hook is executed inside the browser.\n\nCheck out the docs for [Post-navigation hooks](https://crawlee.dev/api/puppeteer-crawler/interface/PuppeteerCrawlerOptions#postNavigationHooks) and the [PuppeteerHook type](https://crawlee.dev/api/puppeteer-crawler/interface/PuppeteerHook) for more info regarding the objects passed into these functions.\n\n### Insert breakpoint\n\nThis property has no effect if [Run mode](#run-mode) is set to **PRODUCTION**. When set to **DEVELOPMENT** it inserts a breakpoint at the selected location in every page the scraper visits. Execution of code stops at the breakpoint until manually resumed in the DevTools window accessible via Live View tab or Container URL. Additional breakpoints can be added by adding debugger; statements within your Page function.\n\n### Debug log\n\nWhen set to true, debug messages will be included in the log. Use `context.log.debug('message')` to log your own debug messages.\n\n### Browser log\n\nWhen set to true, console messages from the browser will be included in the Actor's log. This may result in the log being flooded by error messages, warnings and other messages of little value (especially with a high concurrency).\n\n### Custom data\n\nSince the input UI is fixed, it does not support adding of other fields that may be needed for all specific use cases. If you need to pass arbitrary data to the scraper, use the [Custom data](#custom-data) input field within [Advanced configuration](#advanced-configuration) and its contents will be available under the `customData` context key as an object within the [pageFunction](#page-function).\n\n### Custom names\n\nWith the final three options in the **Advanced configuration**, you can set custom names for the following:\n\n- Dataset\n- Key-value store\n- Request queue\n\nLeave the storage unnamed if you only want the data within it to be persisted on the Apify platform for a number of days corresponding to your [plan](https://apify.com/pricing) (after which it will expire). Named storages are retained indefinitely. Additionally, using a named storage allows you to share it across multiple runs (e.g. instead of having 10 different unnamed datasets for 10 different runs, all the data from all 10 runs can be accumulated into a single named dataset). Learn more [here](https://docs.apify.com/storage#named-and-unnamed-storages).\n\n## Results\n\nAll scraping results returned by [**Page function**](#page-function) are stored in the default dataset associated with the Actor run, and can be saved in several different formats, such as JSON, XML, CSV or Excel. For each object returned by [**Page function**](#page-function), Web Scraper pushes one record into the dataset, and extends it with metadata such as the URL of the web page where the results come from.\n\nFor example, if your page function returned the following object:\n\n```javascript\n{\n message: 'Hello world!',\n}\n```\n\nThe full object stored in the dataset will look as follows\n(in JSON format, including the metadata fields `#error` and `#debug`):\n\n```json\n{\n \"message\": \"Hello world!\",\n \"#error\": false,\n \"#debug\": {\n \"requestId\": \"fvwscO2UJLdr10B\",\n \"url\": \"https://www.example.com/\",\n \"loadedUrl\": \"https://www.example.com/\",\n \"method\": \"GET\",\n \"retryCount\": 0,\n \"errorMessages\": null,\n \"statusCode\": 200\n }\n}\n```\n\nTo download the results, call the [Get dataset items](https://docs.apify.com/api/v2#/reference/datasets/item-collection) API endpoint:\n\n```\nhttps://api.apify.com/v2/datasets/[DATASET_ID]/items?format=json\n```\n\nwhere `[DATASET_ID]` is the ID of the Actor's run dataset, in which you can find the Run object returned when starting the Actor. Alternatively, you'll find the download links for the results in Apify Console.\n\nTo skip the `#error` and `#debug` metadata fields from the results and not include empty result records, simply add the `clean=true` query parameter to the API URL, or select the **Clean items** option when downloading the dataset in Apify Console.\n\nTo get the results in other formats, set the `format` query parameter to `xml`, `xlsx`, `csv`, `html`, etc. For more information, see [Datasets](https://apify.com/docs/storage#dataset) in documentation or the [Get dataset items](https://docs.apify.com/api/v2#/reference/datasets/item-collection) endpoint in Apify API reference.\n\n## Additional resources\n\nIf you’d like to learn more about Web Scraper or Apify’s other Actors and tools, check out these resources:\n\n- [Cheerio Scraper](https://apify.com/apify/cheerio-scraper), another web scraping Actor that downloads and processes pages in raw HTML for much higher performance.\n- [Playwright Scraper](https://apify.com/apify/playwright-scraper), a similar web scraping Actor to Web Scraper, which provides lower-level control of the underlying [Playwright](https://github.com/microsoft/playwright) library and the ability to use server-side libraries.\n- [Puppeteer Scraper](https://apify.com/apify/puppeteer-scraper), an Actor similar to Web Scraper, which provides lower-level control of the underlying [Puppeteer](https://github.com/puppeteer/puppeteer) library and the ability to use server-side libraries.\n- [Actors documentation](https://apify.com/docs/actor) for the Apify cloud computing platform.\n- [Apify SDK documentation](https://sdk.apify.com/), where you can learn more about the tools required to run your own Apify Actors.\n- [Crawlee documentation](https://crawlee.dev/?__hstc=160404322.4ff1f55e48512a0b19aa0955767abc98.1753772621636.1753781308751.1753783813114.4&__hssc=160404322.2.1753783813114&__hsfp=3081399490), how to build a new web scraping project from scratch using the world's most popular web crawling and scraping library for Node.js.\n\n## Frequently asked questions\n\n### Are there any limitations to using Web Scraper?\n\nWeb Scraper is designed to be user-friendly and generic, which may affect its performance and flexibility compared to more specialized solutions. It uses a resource-intensive Chromium browser to supports client-side JavaScript code.\n\n### Is web scraping legal?\n\nIt is legal to scrape any non-personal data. Personal data is protected by the [GDPR](https://en.wikipedia.org/wiki/General_Data_Protection_Regulation) in the European Union and by other regulations around the world. You should not scrape personal data unless you have a legitimate reason to do so. If you're unsure whether your reason is legitimate, consult your lawyers. You can also read our blog post on the [legality of web scraping](https://blog.apify.com/is-web-scraping-legal/).\n\n### Can I control the crawling behavior of Web Scraper?\n\nYes, you can control the crawling behavior of Web Scraper. You can specify start URLs, define link selectors, enter pseudo-URLs to guide the scraper in following specific page links, and plenty of other configurations options. This allows recursive crawling of websites or targeted extraction of data.\n\n### Is it possible to use proxies with Web Scraper?\n\nYes, you can configure proxies for Web Scraper. You have the option to use [Apify Proxy](https://apify.com/proxy), which under the free plan is set up for you. On paid plans, you can configure them yourself, or even set up your own.\n\n### How can I access and export the data scraped by Web Scraper?\n\nThe data scraped by Web Scraper is stored in a dataset which you can access and export in various formats such as JSON, XML, CSV, or as an Excel spreadsheet. The results can be downloaded using the Apify API or through the Apify Console.\n\n### Your feedback\n\nWe’re always working on improving the performance of our Actors. If you have any technical feedback for Web Scraper or found a bug, please create an issue in the [Issues tab](https://apify.com/apify/web-scraper/issues/open)." }, { "name": "package.json", "format": "TEXT", "content": "{\n \"name\": \"actor-web-scraper\",\n \"version\": \"3.1.0\",\n \"private\": true,\n \"description\": \"Crawl web pages using Apify, headless Chrome and Puppeteer\",\n \"main\": \"dist/main.js\",\n \"type\": \"module\",\n \"dependencies\": {\n \"@apify/scraper-tools\": \"^1.1.4\",\n \"@crawlee/puppeteer\": \"^3.14.1\",\n \"apify\": \"^3.2.6\",\n \"content-type\": \"^1.0.5\",\n \"crawlee\": \"^3.13.2\",\n \"devtools-server\": \"^0.0.2\",\n \"idcac-playwright\": \"^0.1.3\",\n \"puppeteer\": \"*\"\n },\n \"devDependencies\": {\n \"@apify/tsconfig\": \"^0.1.0\",\n \"@types/content-type\": \"^1.1.8\",\n \"@types/node\": \"^22.7.4\",\n \"tsx\": \"^4.19.1\",\n \"typescript\": \"~5.9.0\"\n },\n \"scripts\": {\n \"start\": \"npm run start:dev\",\n \"start:prod\": \"node dist/main.js\",\n \"start:dev\": \"tsx src/main.ts\",\n \"build\": \"tsc\"\n },\n \"repository\": {\n \"type\": \"git\",\n \"url\": \"https://github.com/apify/apify-sdk-js\"\n },\n \"author\": {\n \"name\": \"Apify Technologies\",\n \"email\": \"support@apify.com\",\n \"url\": \"https://apify.com\"\n },\n \"contributors\": [\n \"Marek Trunkat \",\n \"Ondra Urban \"\n ],\n \"license\": \"Apache-2.0\",\n \"homepage\": \"https://github.com/apify/apify-sdk-js\"\n}" }, { "name": "tsconfig.json", "format": "TEXT", "content": "{\n \"extends\": \"@apify/tsconfig\",\n \"compilerOptions\": {\n \"outDir\": \"dist\",\n \"module\": \"ESNext\",\n \"allowJs\": true,\n \"skipLibCheck\": true,\n \"lib\": [\"dom\"]\n },\n \"include\": [\"src\"]\n}" } ] } ], "defaultRunOptions": { "build": "version-3", "timeoutSecs": 3600, "memoryMbytes": 4096, "restartOnError": true }, "exampleRunInput": { "body": "{ \"helloWorld\": 123 }", "contentType": "application/json; charset=utf-8" }, "categories": [ "DEVELOPER_TOOLS", "OPEN_SOURCE" ], "isDeprecated": false, "title": "Web Scraper", "pictureUrl": "https://apify-image-uploads-prod.s3.amazonaws.com/moJRLRc85AitArpNN/Zn8vbWTika7anCQMn-SD-02-02.png", "seoTitle": "Web Scraper", "seoDescription": "Crawls websites using Chrome and extracts data from pages using JavaScript. Supports recursive crawling and URL lists and automatically manages concurrency.", "notice": "NONE", "isCritical": true, "isGeneric": true, "hasNoDataset": false, "isSourceCodeHidden": false, "standbyUrl": null, "actorPermissionLevel": "FULL_PERMISSIONS", "readmeSummary": "## Web Scraper\n\nA browser-based web scraping Actor that navigates web pages with a headless Chromium browser, renders client-side JavaScript, follows links and link patterns, and executes a per-page JavaScript extractor to produce structured data. It performs site crawling or targeted URL extraction, supports link filtering using pattern/regex-style rules, injection of DOM utilities, programmable page interactions (clicks, waits, form handling), session/cookie-driven logins, proxy usage (HTTP/SOCKS5 or managed proxy groups), runtime hooks (pre- and post-navigation), debugging with live DevTools and breakpoints, and can capture page snapshots (screenshot + full HTML). Outputs are structured records in common data formats (JSON, CSV, XML, XLSX) enriched with per-record source metadata such as page URL and HTTP response details.\n\n## Use cases\n\n- Extract product prices and monitor price changes on e-commerce sites \n- Collect reviews and ratings from marketplace or review platforms \n- Scrape real estate listings and property details from listing sites \n- Harvest contact details and social media profiles from business pages \n- Monitor mentions of brands or individuals on specific websites \n- Crawl a list of URLs or perform recursive site crawls to turn pages into structured data \n- Use as a template to build custom scrapers for domain-specific data extraction", "deploymentKey": "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQDBdK6hmM7rlDHGimuBL41zbMjEaU6Nt2oWzlD/LKji/5+0jRd8IKD9mZOQ5OkciPdscFpcNibj7nDkZI7qGl8Rb8gccO8YcvHB5L285IbRjqPYAewgB0CZjyMdZWdgZOE9x/29165NBzWw62jaDtkID8pCF5CG68zVm8LD1jfxT+1B6AeiabovmSGfYB6dRxKyhuQCw8xr1AQDM+eop8sdwl3VpHneA0Jun8RSpM1li+iGU7oaNasffMMrD6CRRv43AFxZUaqrv8fMW8ZxIOZVILRmpE6NJqMmXdrqMKNK9KKPtk2R3C30sZw5yfwi5vhIkow9JXV3ZxrTrNbIvH19 \n" } }