{ "openapi": "3.0.1", "info": { "title": "Website Content Crawler - Clean Text, Markdown & RAG Chunks", "description": "Crawl any website or URL list for $1.20 per 1,000 pages: one row per page with clean text, Markdown, HTML, metadata, links, headings, tables and RAG chunks. Plain HTTP first, headless Chrome only for JavaScript pages. Sitemaps, llms.txt, PDF/DOCX text, Bing/DuckDuckGo search, change monitor.", "version": "0.1", "x-build-id": "eKOnOWMnbWx2XqPns" }, "servers": [ { "url": "https://api.apify.com/v2" } ], "paths": { "/acts/topapi~website-content-crawler/run-sync-get-dataset-items": { "post": { "operationId": "run-sync-get-dataset-items-topapi-website-content-crawler", "x-openai-isConsequential": false, "summary": "Executes an Actor, waits for its completion, and returns Actor's dataset items in response.", "tags": [ "Run Actor" ], "requestBody": { "required": true, "content": { "application/json": { "schema": { "$ref": "#/components/schemas/inputSchema" } } } }, "parameters": [ { "name": "token", "in": "query", "required": true, "schema": { "type": "string" }, "description": "Enter your Apify token here" } ], "responses": { "200": { "description": "OK" } } } }, "/acts/topapi~website-content-crawler/runs": { "post": { "operationId": "runs-sync-topapi-website-content-crawler", "x-openai-isConsequential": false, "summary": "Executes an Actor and returns information about the initiated run in response.", "tags": [ "Run Actor" ], "requestBody": { "required": true, "content": { "application/json": { "schema": { "$ref": "#/components/schemas/inputSchema" } } } }, "parameters": [ { "name": "token", "in": "query", "required": true, "schema": { "type": "string" }, "description": "Enter your Apify token here" } ], "responses": { "200": { "description": "OK", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/runsResponseSchema" } } } } } } }, "/acts/topapi~website-content-crawler/run-sync": { "post": { "operationId": "run-sync-topapi-website-content-crawler", "x-openai-isConsequential": false, "summary": "Executes an Actor, waits for completion, and returns the OUTPUT from Key-value store in response.", "tags": [ "Run Actor" ], "requestBody": { "required": true, "content": { "application/json": { "schema": { "$ref": "#/components/schemas/inputSchema" } } } }, "parameters": [ { "name": "token", "in": "query", "required": true, "schema": { "type": "string" }, "description": "Enter your Apify token here" } ], "responses": { "200": { "description": "OK" } } } } }, "components": { "schemas": { "inputSchema": { "type": "object", "properties": { "startUrls": { "title": "Start URLs", "type": "array", "description": "Pages where the crawl starts, e.g. https://docs.example.com/guide. Sub-pages under that path are crawled too. A linked text file of URLs also works.", "items": { "type": "object", "required": [ "url" ], "properties": { "url": { "type": "string", "title": "URL of a web page", "format": "uri" } } } }, "searchQueries": { "title": "Search queries", "type": "array", "description": "Optional. Each query is searched and its top results are crawled, e.g. \"web scraping python tutorial\". Charged per query that returns results.", "items": { "type": "string" } }, "searchEngine": { "title": "Search engine", "enum": [ "bing", "duckduckgo" ], "type": "string", "description": "Engine for the search queries. When it returns fewer results than asked (or nothing), the other engine tops up the list.", "default": "bing" }, "maxResultsPerSearch": { "title": "Results per search query", "minimum": 1, "maximum": 50, "type": "integer", "description": "How many top results of each search query are crawled (1-50).", "default": 3 }, "useSitemaps": { "title": "Use sitemaps", "type": "boolean", "description": "Also find pages in the site's sitemaps (robots.txt Sitemap lines, /sitemap.xml, sitemap indexes, .gz files). Only in-scope URLs are added.", "default": false }, "sitemapUrls": { "title": "Sitemap URLs", "type": "array", "description": "Optional sitemap files to read, e.g. https://example.com/sitemap-posts.xml. Only in-scope URLs are added.", "items": { "type": "string" } }, "useLlmsTxt": { "title": "Crawl /llms.txt and Markdown files", "type": "boolean", "description": "Also crawl /llms.txt and /llms-full.txt at the root of each start URL's site, and the Markdown links listed in them.", "default": false }, "crawlMode": { "title": "Which links to follow", "enum": [ "startUrlPath", "sameHostname", "sameDomain", "allLinks", "urlsOnly" ], "type": "string", "description": "Default: pages under each start URL's path, e.g. start /blog crawls /blog/post-1. Pick a wider scope or only the start URLs.", "default": "startUrlPath" }, "includeUrlGlobs": { "title": "Include URL patterns (globs)", "type": "array", "description": "Links matching any pattern are crawled even outside the scope above, e.g. https://example.com/blog/**", "default": [], "items": { "type": "object", "required": [ "glob" ], "properties": { "glob": { "type": "string", "title": "Glob of a web page" } } } }, "excludeUrlGlobs": { "title": "Exclude URL patterns (globs)", "type": "array", "description": "Links matching any pattern are never crawled, e.g. https://example.com/**/tag/**", "default": [], "items": { "type": "object", "required": [ "glob" ], "properties": { "glob": { "type": "string", "title": "Glob of a web page" } } } }, "maxCrawlDepth": { "title": "Max crawling depth", "minimum": 0, "maximum": 1000, "type": "integer", "description": "Links followed from the start URLs (depth 0). 1 = start pages plus the pages they link to. 0 = start URLs only.", "default": 20 }, "maxCrawlPages": { "title": "Max pages", "minimum": 0, "type": "integer", "description": "Stop after this many pages are crawled. 0 = no limit. Each delivered page is billed, so this also caps the cost.", "default": 200 }, "maxPagesPerStartUrl": { "title": "Max pages per start URL", "minimum": 0, "type": "integer", "description": "Optional cap per start URL, so one big site cannot use the whole page budget. 0 = no cap.", "default": 0 }, "linkSelector": { "title": "Link selector", "type": "string", "description": "CSS selector of the links to follow. Default a[href] (every link).", "default": "a[href]" }, "onlyCrawlPagesModifiedSince": { "title": "Only pages changed since", "type": "string", "description": "ISO date (2026-09-01) or relative (\"7 days\"). Uses the server's 304 answer, sitemap lastmod, Last-Modified and the page's own dates." }, "minWords": { "title": "Minimum words", "minimum": 0, "type": "integer", "description": "Skip pages whose extracted text has fewer words, e.g. 150 for articles only. 0 = keep all.", "default": 0 }, "articlesOnly": { "title": "Articles only", "type": "boolean", "description": "Keep only pages marked as articles (og:type article, article:published_time or Article / BlogPosting JSON-LD). Links on other pages are still followed.", "default": false }, "mustHaveDate": { "title": "Must have a publish date", "type": "boolean", "description": "Keep only pages that state a publish date.", "default": false }, "respectRobotsTxtFile": { "title": "Respect robots.txt", "type": "boolean", "description": "Skip URLs the site's robots.txt disallows.", "default": false }, "keepUrlFragments": { "title": "URL #fragments identify unique pages", "type": "boolean", "description": "Treat example.com/#a and example.com/#b as different pages (some single-page sites route by fragment).", "default": false }, "ignoreCanonicalUrl": { "title": "Ignore canonical URLs", "type": "boolean", "description": "By default a page whose canonical URL was already delivered is skipped as a duplicate. Turn on for sites with wrong canonical tags.", "default": false }, "renderJavaScript": { "title": "Render JavaScript", "enum": [ "auto", "never", "always" ], "type": "string", "description": "Auto renders only pages that are app shells over HTTP. Always renders every page (slower). Never uses plain HTTP only.", "default": "auto" }, "maxBrowserPages": { "title": "Max pages rendered in the browser", "minimum": 0, "type": "integer", "description": "Cap on browser-rendered pages per run. 0 = automatic: 20% of Max pages (at least 5, at most 500), or every page in Always mode.", "default": 0 }, "dynamicContentWaitSecs": { "title": "Wait for dynamic content (s)", "minimum": 0, "maximum": 60, "type": "integer", "description": "Browser only: wait up to this long for the network to go quiet before reading the page.", "default": 10 }, "waitForSelector": { "title": "Wait for selector", "type": "string", "description": "Browser only: wait until this CSS selector exists; the page fails if it never appears." }, "softWaitForSelector": { "title": "Soft wait for selector", "type": "string", "description": "Browser only: wait for this CSS selector, but continue if it never appears." }, "maxScrollHeightPixels": { "title": "Maximum scroll height (px)", "minimum": 0, "type": "integer", "description": "Browser only: scroll down to load lazy content, up to this height. 0 = no scrolling.", "default": 5000 }, "clickElementsCssSelector": { "title": "Expand clickable elements", "type": "string", "description": "Browser only: click elements matching this selector to expand collapsed sections.", "default": "[aria-expanded=\"false\"]" }, "removeCookieWarnings": { "title": "Remove cookie warnings", "type": "boolean", "description": "Remove cookie-consent banners (OneTrust, Cookiebot, Didomi, Quantcast and others) from the content.", "default": true }, "blockMedia": { "title": "Block images and media", "type": "boolean", "description": "Browser only: do not load images, video and fonts (faster, cheaper). Turn off for screenshots with images.", "default": true }, "expandIframes": { "title": "Expand iframes", "type": "boolean", "description": "Browser only: include the content of iframes in the page.", "default": true }, "htmlTransformer": { "title": "HTML transformer", "enum": [ "readableTextIfPossible", "readableText", "extractus", "defuddle", "none" ], "type": "string", "description": "Readability with fallback keeps the whole cleaned page when Readability finds no clear article. Extractus and Defuddle are alternative extractors.", "default": "readableTextIfPossible" }, "readableTextCharThreshold": { "title": "Readable text threshold (chars)", "minimum": 0, "type": "integer", "description": "Readability's minimum article length; also the text length under which an HTTP page counts as an app shell.", "default": 100 }, "removeElementsCssSelector": { "title": "Remove HTML elements (CSS selector)", "type": "string", "description": "Elements removed before extraction: navigation, footers, scripts, dialogs and banners by default.", "default": "nav, footer, script, style, noscript, svg, img[src^='data:'],\n[role=\"alert\"],\n[role=\"banner\"],\n[role=\"dialog\"],\n[role=\"alertdialog\"],\n[role=\"region\"][aria-label*=\"skip\" i],\n[aria-modal=\"true\"]" }, "keepElementsCssSelector": { "title": "Keep only HTML elements (CSS selector)", "type": "string", "description": "Optional: keep only elements matching this selector, e.g. article, main. Everything else is dropped first." }, "aggressivePrune": { "title": "Remove duplicate text lines", "type": "boolean", "description": "Drop text lines already seen on earlier pages of the run (menus, footers, repeated banners). May remove a repeated line that matters.", "default": false }, "outputFormats": { "title": "Output formats", "type": "array", "description": "Content formats on each row. Markdown and text by default; HTML = the cleaned content HTML; Raw = the page's original HTML.", "items": { "type": "string", "enum": [ "markdown", "text", "html", "raw" ], "enumTitles": [ "Markdown", "Plain text", "Cleaned HTML", "Raw HTML (original page)" ] }, "default": [ "markdown", "text" ] }, "chunkSize": { "title": "RAG chunk size (chars)", "minimum": 0, "maximum": 100000, "type": "integer", "description": "Split each page's Markdown into chunks of up to this many characters, with their heading path. 0 = no chunks. Typical: 1000-2000.", "default": 0 }, "chunkOverlap": { "title": "Chunk overlap (chars)", "minimum": 0, "maximum": 10000, "type": "integer", "description": "Characters repeated from the end of one chunk at the start of the next.", "default": 200 }, "saveHtmlAsFile": { "title": "Save HTML to key-value store", "type": "boolean", "description": "Store each page's cleaned HTML as a file and put its link in htmlUrl (no size limit).", "default": false }, "saveFiles": { "title": "Crawl linked documents (PDF, DOCX…)", "type": "boolean", "description": "Also crawl in-scope PDF, DOCX and other document links: text is extracted from PDF and DOCX, the file is stored and linked in file.kvUrl.", "default": false }, "saveScreenshots": { "title": "Save screenshots", "type": "boolean", "description": "Render every page in the browser and store a screenshot, linked in screenshotUrl. Charged as screenshot per stored screenshot.", "default": false }, "maxResults": { "title": "Max results", "minimum": 0, "type": "integer", "description": "Stop after this many pages are delivered. 0 = no extra limit (Max pages still applies).", "default": 0 }, "compact": { "title": "Compact output", "type": "boolean", "description": "Core fields only (URL, title, description, date, Markdown, word count, hash) - sized for AI agents and MCP.", "default": false }, "customMapFunction": { "title": "Custom map function", "type": "string", "description": "Optional JavaScript (item, { $, html }) => item to reshape each row; $ is Cheerio on the page HTML. Return null to drop a page (not charged)." }, "monitorMode": { "title": "Monitor mode (new and changed pages only)", "type": "boolean", "description": "Deliver only pages not delivered before or whose text changed (isNew / changed / previousHash). Unchanged pages are not delivered.", "default": false }, "monitorStoreName": { "title": "Monitor store name", "type": "string", "description": "Named key-value store that remembers delivered pages. Use one name per monitored site or task.", "default": "website-content-crawler-monitor" }, "proxyConfiguration": { "title": "Proxy configuration", "type": "object", "description": "Default: each site is fetched directly and moved to Apify Proxy only if it refuses. Pick a group (e.g. RESIDENTIAL) to send every request through it.", "default": { "useApifyProxy": true } }, "customHttpHeaders": { "title": "Custom HTTP headers", "type": "object", "description": "Extra headers sent with every request, e.g. {\"Accept-Language\": \"de-DE\"}.", "default": {} }, "device": { "title": "Device", "enum": [ "desktop", "mobile" ], "type": "string", "description": "Desktop or mobile browser headers (and viewport in the browser).", "default": "desktop" }, "maxConcurrency": { "title": "Max concurrency", "minimum": 1, "maximum": 100, "type": "integer", "description": "Pages fetched in parallel. Capped at 8 per GB of run memory (more starves the CPU).", "default": 10 }, "requestTimeoutSecs": { "title": "Request timeout (s)", "minimum": 5, "maximum": 180, "type": "integer", "description": "Timeout for one page request.", "default": 30 }, "maxRequestRetries": { "title": "Max request retries", "minimum": 0, "maximum": 6, "type": "integer", "description": "Retries after a network error, 5xx or 429 (other HTTP version, then a fresh proxy session).", "default": 3 }, "minFileDownloadSpeedKBps": { "title": "Minimum file download speed (KB/s)", "minimum": 0, "type": "integer", "description": "Abort a download that stays slower than this after 10 seconds. 0 = no limit.", "default": 128 }, "ignoreHttpsErrors": { "title": "Ignore HTTPS errors", "type": "boolean", "description": "Accept invalid or self-signed TLS certificates.", "default": false }, "storeSkippedUrls": { "title": "Store skipped URLs", "type": "boolean", "description": "Save every URL that was not delivered, grouped by reason, to the SKIPPED_URLS record of the run's key-value store.", "default": false }, "debugMode": { "title": "Debug mode", "type": "boolean", "description": "Add a debug object to each row (app-shell verdict, transformer, timings) and save SKIPPED_URLS.", "default": false } } }, "runsResponseSchema": { "type": "object", "properties": { "data": { "type": "object", "properties": { "id": { "type": "string" }, "actId": { "type": "string" }, "userId": { "type": "string" }, "startedAt": { "type": "string", "format": "date-time", "example": "2025-01-08T00:00:00.000Z" }, "finishedAt": { "type": "string", "format": "date-time", "example": "2025-01-08T00:00:00.000Z" }, "status": { "type": "string", "example": "READY" }, "meta": { "type": "object", "properties": { "origin": { "type": "string", "example": "API" }, "userAgent": { "type": "string" } } }, "stats": { "type": "object", "properties": { "inputBodyLen": { "type": "integer", "example": 2000 }, "rebootCount": { "type": "integer", "example": 0 }, "restartCount": { "type": "integer", "example": 0 }, "resurrectCount": { "type": "integer", "example": 0 }, "computeUnits": { "type": "integer", "example": 0 } } }, "options": { "type": "object", "properties": { "build": { "type": "string", "example": "latest" }, "timeoutSecs": { "type": "integer", "example": 300 }, "memoryMbytes": { "type": "integer", "example": 1024 }, "diskMbytes": { "type": "integer", "example": 2048 } } }, "buildId": { "type": "string" }, "defaultKeyValueStoreId": { "type": "string" }, "defaultDatasetId": { "type": "string" }, "defaultRequestQueueId": { "type": "string" }, "buildNumber": { "type": "string", "example": "1.0.0" }, "containerUrl": { "type": "string" }, "usage": { "type": "object", "properties": { "ACTOR_COMPUTE_UNITS": { "type": "integer", "example": 0 }, "DATASET_READS": { "type": "integer", "example": 0 }, "DATASET_WRITES": { "type": "integer", "example": 0 }, "KEY_VALUE_STORE_READS": { "type": "integer", "example": 0 }, "KEY_VALUE_STORE_WRITES": { "type": "integer", "example": 1 }, "KEY_VALUE_STORE_LISTS": { "type": "integer", "example": 0 }, "REQUEST_QUEUE_READS": { "type": "integer", "example": 0 }, "REQUEST_QUEUE_WRITES": { "type": "integer", "example": 0 }, "DATA_TRANSFER_INTERNAL_GBYTES": { "type": "integer", "example": 0 }, "DATA_TRANSFER_EXTERNAL_GBYTES": { "type": "integer", "example": 0 }, "PROXY_RESIDENTIAL_TRANSFER_GBYTES": { "type": "integer", "example": 0 }, "PROXY_SERPS": { "type": "integer", "example": 0 }, "PROXY_UNBLOCKER_UNITS": { "type": "integer", "example": 0 } } }, "usageTotalUsd": { "type": "number", "example": 0.00005 }, "usageUsd": { "type": "object", "properties": { "ACTOR_COMPUTE_UNITS": { "type": "integer", "example": 0 }, "DATASET_READS": { "type": "integer", "example": 0 }, "DATASET_WRITES": { "type": "integer", "example": 0 }, "KEY_VALUE_STORE_READS": { "type": "integer", "example": 0 }, "KEY_VALUE_STORE_WRITES": { "type": "number", "example": 0.00005 }, "KEY_VALUE_STORE_LISTS": { "type": "integer", "example": 0 }, "REQUEST_QUEUE_READS": { "type": "integer", "example": 0 }, "REQUEST_QUEUE_WRITES": { "type": "integer", "example": 0 }, "DATA_TRANSFER_INTERNAL_GBYTES": { "type": "integer", "example": 0 }, "DATA_TRANSFER_EXTERNAL_GBYTES": { "type": "integer", "example": 0 }, "PROXY_RESIDENTIAL_TRANSFER_GBYTES": { "type": "integer", "example": 0 }, "PROXY_SERPS": { "type": "integer", "example": 0 }, "PROXY_UNBLOCKER_UNITS": { "type": "integer", "example": 0 } } } } } } } } } }