{ "openapi": "3.0.1", "info": { "title": "Website Content Crawler (Advanced Feature)", "description": "Website Content Crawler extracts clean text, Markdown, HTML, metadata, and links from websites. It checks broken links, redirects, and external URLs while supporting SEO audits, content analysis, web crawling, LLM/RAG pipelines, and search indexing.", "version": "0.1", "x-build-id": "Qa5NECJFicquZ4uDL" }, "servers": [ { "url": "https://api.apify.com/v2" } ], "paths": { "/acts/scrapevanta~website-content-crawler/run-sync-get-dataset-items": { "post": { "operationId": "run-sync-get-dataset-items-scrapevanta-website-content-crawler", "x-openai-isConsequential": false, "summary": "Executes an Actor, waits for its completion, and returns Actor's dataset items in response.", "tags": [ "Run Actor" ], "requestBody": { "required": true, "content": { "application/json": { "schema": { "$ref": "#/components/schemas/inputSchema" } } } }, "parameters": [ { "name": "token", "in": "query", "required": true, "schema": { "type": "string" }, "description": "Enter your Apify token here" } ], "responses": { "200": { "description": "OK" } } } }, "/acts/scrapevanta~website-content-crawler/runs": { "post": { "operationId": "runs-sync-scrapevanta-website-content-crawler", "x-openai-isConsequential": false, "summary": "Executes an Actor and returns information about the initiated run in response.", "tags": [ "Run Actor" ], "requestBody": { "required": true, "content": { "application/json": { "schema": { "$ref": "#/components/schemas/inputSchema" } } } }, "parameters": [ { "name": "token", "in": "query", "required": true, "schema": { "type": "string" }, "description": "Enter your Apify token here" } ], "responses": { "200": { "description": "OK", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/runsResponseSchema" } } } } } } }, "/acts/scrapevanta~website-content-crawler/run-sync": { "post": { "operationId": "run-sync-scrapevanta-website-content-crawler", "x-openai-isConsequential": false, "summary": "Executes an Actor, waits for completion, and returns the OUTPUT from Key-value store in response.", "tags": [ "Run Actor" ], "requestBody": { "required": true, "content": { "application/json": { "schema": { "$ref": "#/components/schemas/inputSchema" } } } }, "parameters": [ { "name": "token", "in": "query", "required": true, "schema": { "type": "string" }, "description": "Enter your Apify token here" } ], "responses": { "200": { "description": "OK" } } } } }, "components": { "schemas": { "inputSchema": { "type": "object", "required": [ "startUrls" ], "properties": { "startUrls": { "title": "๐Ÿ”— Start URLs", "type": "array", "description": "One or more URLs to start crawling from. Add as many rows as you like โ€” bulk input is fully supported. By default the crawler also follows internal links found on these pages (see **Max crawling depth** below).", "items": { "type": "string" } }, "maxResults": { "title": "๐ŸŽฏ Max results", "minimum": 1, "maximum": 100000, "type": "integer", "description": "Stop once this many pages have been successfully collected. This is your main \"how much do I get\" dial โ€” the crawl stops the moment it's reached, even if more pages are still queued.", "default": 20 }, "includeUrlGlobs": { "title": "โœ… Include URL patterns (globs)", "type": "array", "description": "Only crawl links matching one of these glob patterns, e.g. `https://example.com/blog/**`. Leave empty to follow all same-domain sub-pages of your Start URLs.", "default": [], "items": { "type": "object", "required": [ "glob" ], "properties": { "glob": { "type": "string", "title": "Glob of a web page" } } } }, "excludeUrlGlobs": { "title": "๐Ÿšซ Exclude URL patterns (globs)", "type": "array", "description": "Skip any discovered link matching one of these glob patterns, e.g. `https://example.com/login/**`.", "default": [], "items": { "type": "object", "required": [ "glob" ], "properties": { "glob": { "type": "string", "title": "Glob of a web page" } } } }, "maxCrawlDepth": { "title": "๐Ÿชœ Max crawling depth", "minimum": 0, "maximum": 100, "type": "integer", "description": "How many link-hops away from your Start URLs to follow. `0` = only the Start URLs themselves; `1` = also pages directly linked from them; and so on.", "default": 3 }, "maxCrawlPages": { "title": "๐Ÿ“„ Max pages to visit (optional)", "minimum": 0, "type": "integer", "description": "A hard ceiling on how many pages the crawler is allowed to visit in total, including ones that turn out empty or duplicate. Leave empty to let **Max results** alone decide when to stop." }, "keepUrlFragments": { "title": "#๏ธโƒฃ Treat URL fragments as distinct pages", "type": "boolean", "description": "If the target site uses `#fragment` URLs to represent different pages (common on some single-page apps), enable this so `page#a` and `page#b` are crawled as separate pages instead of being deduplicated together.", "default": false }, "ignoreCanonicalUrl": { "title": "๐Ÿ™ˆ Ignore canonical URLs", "type": "boolean", "description": "By default, a page's own `` tag decides its final URL and dedup identity. Enable this to always use the actually-visited URL instead โ€” useful for sites that report incorrect canonical URLs.", "default": false }, "removeElementsCssSelector": { "title": "โœ‚๏ธ Remove HTML elements (CSS selector)", "type": "string", "description": "Comma-separated CSS selectors removed from a page before text/HTML extraction. The default strips navigation, footers, scripts, and common cookie/alert dialogs.", "default": "nav, footer, script, style, noscript, svg, img[src^=\"data:\"], [role=\"alert\"], [role=\"banner\"], [role=\"dialog\"], [role=\"alertdialog\"], [role=\"region\"][aria-label*=\"skip\" i], [aria-modal=\"true\"]" }, "readableTextCharThreshold": { "title": "๐Ÿ“ Minimum readable text length", "minimum": 0, "maximum": 10000, "type": "integer", "description": "A page whose extracted text falls below this many characters is treated as empty/thin and dropped instead of being saved as a result.", "default": 100 }, "saveMarkdown": { "title": "๐Ÿ“ฐ Save Markdown", "type": "boolean", "description": "Convert each page's HTML to Markdown and include it in the `markdown` output field โ€” handy for feeding LLMs directly.", "default": true }, "saveHtmlAsFile": { "title": "๐Ÿ—‚๏ธ Save HTML snapshots", "type": "boolean", "description": "Save the original, cleaned, and readable HTML for each page to the key-value store, linked from the `htmlUrl` / `debug.snapshots` output fields. Off by default to keep storage usage down.", "default": false }, "saveScreenshots": { "title": "๐Ÿ“ธ Save screenshots", "type": "boolean", "description": "Save a full-page PNG screenshot for every page rendered by the browser tier, linked from the `screenshotUrl` output field. Useful for debugging; adds storage cost.", "default": false }, "debugLog": { "title": "๐Ÿž Verbose debug logging", "type": "boolean", "description": "Log low-level crawl internals (tier escalations, vendor detection, session bookkeeping) at DEBUG level. Off by default โ€” normal runs only show clean, high-level progress in the log.", "default": false }, "browserFallback": { "title": "๐Ÿ–ฅ๏ธ Enable headless browser fallback", "type": "boolean", "description": "When a fast HTTP request is blocked or a page needs JavaScript to render, escalate to a real, fingerprint-resistant headless browser. Turning this off makes the crawler HTTP-only โ€” faster, but it will fail on JS-heavy or heavily-protected sites.", "default": true }, "browserHeadless": { "title": "๐Ÿ‘ป Run browser headless", "type": "boolean", "description": "Keep this on for normal runs. Disabling only makes sense when debugging locally with a visible browser window, and has no effect on the Apify platform.", "default": true }, "browserHumanize": { "title": "๐Ÿ–ฑ๏ธ Human-like cursor movement", "type": "boolean", "description": "Move the mouse along realistic paths instead of teleporting the cursor. Adds roughly 1-2 seconds per browser page but meaningfully helps against behavior-based anti-bot systems.", "default": true }, "challengeWaitSecs": { "title": "โฑ๏ธ Challenge clearance timeout", "minimum": 5, "maximum": 120, "type": "integer", "description": "How many seconds to let the browser wait for a bot-detection interstitial to clear on its own before giving up on that page.", "default": 30 }, "waitForSelector": { "title": "๐ŸŽฏ Wait for CSS selector (optional)", "type": "string", "description": "When using the browser tier, wait for this CSS selector to appear before extracting content โ€” useful for pages whose content loads asynchronously.", "default": "" }, "capsolverEnabled": { "title": "๐Ÿงฉ Enable captcha solving", "type": "boolean", "description": "If enabled and an API key is supplied below, the crawler pays a solver service (CapSolver.com) to clear interactive captchas it can't get past on its own.", "default": false }, "capsolverApiKey": { "title": "๐Ÿ”‘ CapSolver API key", "type": "string", "description": "Your CapSolver.com API key. Required only when **Enable captcha solving** is on. Stored encrypted and never shown in logs." }, "capsolverMaxSolves": { "title": "๐Ÿ’ฐ Max captcha solves per run", "minimum": 0, "maximum": 1000, "type": "integer", "description": "Hard ceiling on how many paid captcha solves this run is allowed to spend, regardless of how many pages remain.", "default": 50 }, "maxConcurrency": { "title": "โšก Max concurrency", "minimum": 1, "maximum": 200, "type": "integer", "description": "The overall number of pages the crawler works on in parallel, across all domains combined.", "default": 10 }, "perDomainConcurrency": { "title": "๐Ÿ—๏ธ Max concurrency per domain", "minimum": 1, "maximum": 50, "type": "integer", "description": "How many requests to the *same* domain run in parallel. This automatically halves after a block and slowly ramps back up after a run of clean responses.", "default": 4 }, "perDomainDelaySecs": { "title": "โณ Delay between requests per domain", "minimum": 0, "maximum": 60, "type": "number", "description": "Starting spacing (seconds, with jitter) between requests to the same domain. Doubles automatically after a block, eases back down after a run of clean responses.", "default": 1 }, "maxRequestRetries": { "title": "๐Ÿ” Max retries per page", "minimum": 1, "maximum": 20, "type": "integer", "description": "How many times to retry a single page over the fast HTTP tier before escalating it to the browser tier. Also acts as the floor for the proxy ladder's own no-proxy โ†’ datacenter โ†’ residential retry sequence.", "default": 3 }, "requestTimeoutSecs": { "title": "โฐ Request timeout", "minimum": 5, "maximum": 300, "type": "integer", "description": "Timeout in seconds for a single HTTP request or browser page navigation.", "default": 60 }, "proxyConfiguration": { "title": "๐Ÿ›ก๏ธ Proxy configuration", "type": "object", "description": "**Recommended: leave this on \"No proxy\".** Every request starts as a direct, no-proxy connection and *automatically* escalates to Apify's Datacenter proxy, then Residential proxy, only when a target site actually blocks it โ€” each escalation is logged clearly so you can see exactly when and why it happened, and once Residential is reached it's used for the rest of the run. Pick a specific proxy group here only if you want to force one from the very first request.", "default": { "useApifyProxy": false } }, "fingerprintProfile": { "title": "๐Ÿงฌ Browser fingerprint", "enum": [ "chrome_windows", "chrome_macos", "firefox_windows" ], "type": "string", "description": "The TLS/HTTP2 fingerprint and User-Agent identity used for fast HTTP requests (tier 0 of the anti-bot ladder). All three are kept internally self-consistent โ€” mixing them is a bigger giveaway than sending no headers at all.", "default": "chrome_windows" }, "customHttpHeaders": { "title": "๐Ÿ“‹ Custom HTTP headers", "type": "object", "description": "Extra HTTP headers merged into every request (applied last, so they override the fingerprint profile's own values). Useful for authentication headers required by the target site. Example: `{\"Authorization\": \"Bearer ...\"}`.", "default": {} }, "checkLinkStatus": { "title": "๐Ÿšฆ Check link status", "type": "boolean", "description": "For every link found on a crawled page, send a live request to it and record its HTTP status, whether it's broken (4xx/5xx or unreachable), and whether it redirects. This is what turns the crawl into a broken-link/redirect audit โ€” turn it off to crawl exactly like the base content crawler.", "default": true }, "checkExternalLinks": { "title": "๐ŸŒ Check external links too", "type": "boolean", "description": "Also live-check links that point to other domains, not just links that stay on the crawled site. Turn off to only audit the site's own internal links (faster, no requests leave the domain being crawled).", "default": true }, "maxLinksPerPage": { "title": "๐Ÿ”ข Max links checked per page", "minimum": 1, "maximum": 1000, "type": "integer", "description": "Cap on how many of a page's links get a live status check. Pages with more links than this only have their first N checked, in the order found on the page. Keeps a link-heavy page from ballooning the run's request count.", "default": 200 }, "linkCheckTimeoutSecs": { "title": "โŒ› Link check timeout", "minimum": 1, "maximum": 60, "type": "integer", "description": "Timeout in seconds for a single link's status check request, independent of the page-load timeout above.", "default": 10 }, "linkCheckConcurrency": { "title": "๐Ÿงต Link check concurrency", "minimum": 1, "maximum": 50, "type": "integer", "description": "How many link status checks run in parallel, across all pages combined.", "default": 10 } } }, "runsResponseSchema": { "type": "object", "properties": { "data": { "type": "object", "properties": { "id": { "type": "string" }, "actId": { "type": "string" }, "userId": { "type": "string" }, "startedAt": { "type": "string", "format": "date-time", "example": "2025-01-08T00:00:00.000Z" }, "finishedAt": { "type": "string", "format": "date-time", "example": "2025-01-08T00:00:00.000Z" }, "status": { "type": "string", "example": "READY" }, "meta": { "type": "object", "properties": { "origin": { "type": "string", "example": "API" }, "userAgent": { "type": "string" } } }, "stats": { "type": "object", "properties": { "inputBodyLen": { "type": "integer", "example": 2000 }, "rebootCount": { "type": "integer", "example": 0 }, "restartCount": { "type": "integer", "example": 0 }, "resurrectCount": { "type": "integer", "example": 0 }, "computeUnits": { "type": "integer", "example": 0 } } }, "options": { "type": "object", "properties": { "build": { "type": "string", "example": "latest" }, "timeoutSecs": { "type": "integer", "example": 300 }, "memoryMbytes": { "type": "integer", "example": 1024 }, "diskMbytes": { "type": "integer", "example": 2048 } } }, "buildId": { "type": "string" }, "defaultKeyValueStoreId": { "type": "string" }, "defaultDatasetId": { "type": "string" }, "defaultRequestQueueId": { "type": "string" }, "buildNumber": { "type": "string", "example": "1.0.0" }, "containerUrl": { "type": "string" }, "usage": { "type": "object", "properties": { "ACTOR_COMPUTE_UNITS": { "type": "integer", "example": 0 }, "DATASET_READS": { "type": "integer", "example": 0 }, "DATASET_WRITES": { "type": "integer", "example": 0 }, "KEY_VALUE_STORE_READS": { "type": "integer", "example": 0 }, "KEY_VALUE_STORE_WRITES": { "type": "integer", "example": 1 }, "KEY_VALUE_STORE_LISTS": { "type": "integer", "example": 0 }, "REQUEST_QUEUE_READS": { "type": "integer", "example": 0 }, "REQUEST_QUEUE_WRITES": { "type": "integer", "example": 0 }, "DATA_TRANSFER_INTERNAL_GBYTES": { "type": "integer", "example": 0 }, "DATA_TRANSFER_EXTERNAL_GBYTES": { "type": "integer", "example": 0 }, "PROXY_RESIDENTIAL_TRANSFER_GBYTES": { "type": "integer", "example": 0 }, "PROXY_SERPS": { "type": "integer", "example": 0 } } }, "usageTotalUsd": { "type": "number", "example": 0.00005 }, "usageUsd": { "type": "object", "properties": { "ACTOR_COMPUTE_UNITS": { "type": "integer", "example": 0 }, "DATASET_READS": { "type": "integer", "example": 0 }, "DATASET_WRITES": { "type": "integer", "example": 0 }, "KEY_VALUE_STORE_READS": { "type": "integer", "example": 0 }, "KEY_VALUE_STORE_WRITES": { "type": "number", "example": 0.00005 }, "KEY_VALUE_STORE_LISTS": { "type": "integer", "example": 0 }, "REQUEST_QUEUE_READS": { "type": "integer", "example": 0 }, "REQUEST_QUEUE_WRITES": { "type": "integer", "example": 0 }, "DATA_TRANSFER_INTERNAL_GBYTES": { "type": "integer", "example": 0 }, "DATA_TRANSFER_EXTERNAL_GBYTES": { "type": "integer", "example": 0 }, "PROXY_RESIDENTIAL_TRANSFER_GBYTES": { "type": "integer", "example": 0 }, "PROXY_SERPS": { "type": "integer", "example": 0 } } } } } } } } } }