{ "openapi": "3.0.1", "info": { "title": "๐ŸŒ Website Content Crawler", "description": "Crawl any website into clean text, Markdown & rich metadata โ€” AI-ready output for RAG, LLMs & vector databases. Streams to your dataset in real time.", "version": "1.1", "x-build-id": "uPZ6EcAT89iYOKx10" }, "servers": [ { "url": "https://api.apify.com/v2" } ], "paths": { "/acts/citrine_venus~website-content-crawler/run-sync-get-dataset-items": { "post": { "operationId": "run-sync-get-dataset-items-citrine_venus-website-content-crawler", "x-openai-isConsequential": false, "summary": "Executes an Actor, waits for its completion, and returns Actor's dataset items in response.", "tags": [ "Run Actor" ], "requestBody": { "required": true, "content": { "application/json": { "schema": { "$ref": "#/components/schemas/inputSchema" } } } }, "parameters": [ { "name": "token", "in": "query", "required": true, "schema": { "type": "string" }, "description": "Enter your Apify token here" } ], "responses": { "200": { "description": "OK" } } } }, "/acts/citrine_venus~website-content-crawler/runs": { "post": { "operationId": "runs-sync-citrine_venus-website-content-crawler", "x-openai-isConsequential": false, "summary": "Executes an Actor and returns information about the initiated run in response.", "tags": [ "Run Actor" ], "requestBody": { "required": true, "content": { "application/json": { "schema": { "$ref": "#/components/schemas/inputSchema" } } } }, "parameters": [ { "name": "token", "in": "query", "required": true, "schema": { "type": "string" }, "description": "Enter your Apify token here" } ], "responses": { "200": { "description": "OK", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/runsResponseSchema" } } } } } } }, "/acts/citrine_venus~website-content-crawler/run-sync": { "post": { "operationId": "run-sync-citrine_venus-website-content-crawler", "x-openai-isConsequential": false, "summary": "Executes an Actor, waits for completion, and returns the OUTPUT from Key-value store in response.", "tags": [ "Run Actor" ], "requestBody": { "required": true, "content": { "application/json": { "schema": { "$ref": "#/components/schemas/inputSchema" } } } }, "parameters": [ { "name": "token", "in": "query", "required": true, "schema": { "type": "string" }, "description": "Enter your Apify token here" } ], "responses": { "200": { "description": "OK" } } } } }, "components": { "schemas": { "inputSchema": { "type": "object", "required": [ "startUrls" ], "properties": { "startUrls": { "title": "๐ŸŒฑ Start URLs", "type": "array", "description": "๐Ÿ“ **Required.** Paste the page(s) where crawling should begin. Example: your docs home, a blog index, or a product category. URL filters below are applied to *discovered* links only โ€” your seeds are always crawled.", "items": { "type": "string" } }, "referrerUrl": { "title": "๐Ÿ”— Default referrer (optional)", "type": "string", "description": "๐Ÿท๏ธ Recorded as `referrerUrl` on rows produced from your start URLs (they have no natural โ€œcame fromโ€ page). Discovered pages always record the page that linked to them. Leave blank for an empty referrer." }, "blockAds": { "title": "๐Ÿšซ Block ad & tracker domains (rendered pages)", "type": "boolean", "description": "๐Ÿ›ก๏ธ Abort requests to known ad/tracker hosts (doubleclick, Google Ads, Taboola, Criteo, and more) while rendering โ€” cleaner, faster pages. Only affects rendered pages.", "default": false }, "mobileView": { "title": "๐Ÿ“ฑ Emulate a mobile browser (rendered pages)", "type": "boolean", "description": "๐Ÿ“ฒ Render pages with a mobile viewport and device profile instead of desktop. Useful for sites that serve different content to mobile visitors. A custom User-Agent above still overrides the mobile one.", "default": false }, "redactPii": { "title": "๐Ÿ•ต๏ธ Redact personal info (best-effort)", "type": "boolean", "description": "๐Ÿ”’ Mask emails, phone numbers, credit-card-like numbers, and SSNs in `text` and `markdown` before saving. Regex-based heuristics โ€” not a compliance guarantee.", "default": false }, "screenshotFullPage": { "title": "๐Ÿ“ Full-page screenshots", "type": "boolean", "description": "๐Ÿ–ผ๏ธ When **on** (default), screenshots capture the entire scrolled page. Turn **off** to capture just the visible viewport โ€” faster and smaller. Only applies when `saveScreenshots` is on.", "default": true }, "extractPdfText": { "title": "๐Ÿ“• Extract text from linked PDFs", "type": "boolean", "description": "๐Ÿ“„ When **on**, discovered PDF links are opened and their text is extracted into `text` (counted toward your page/result limits like any other page) instead of only being listed as a file link.", "default": false }, "includeUrlGlobs": { "title": "โœ… Only crawl URLs matchingโ€ฆ", "type": "array", "description": "๐Ÿงฉ **Include patterns.** If you add any, *only* discovered URLs matching at least one pattern are queued. Examples: `/blog/*` ยท `/docs/**` ยท `https://site.com/product/**`. Empty = no extra filter.", "items": { "type": "string" } }, "excludeUrlGlobs": { "title": "๐Ÿšซ Skip URLs matchingโ€ฆ", "type": "array", "description": "๐Ÿ›‘ URLs matching **any** pattern here are skipped โ€” handy for login, admin, or file folders. Examples: `/admin/**` ยท `/login` ยท `**/*.pdf`.", "items": { "type": "string" } }, "useSitemaps": { "title": "๐Ÿ—บ๏ธ Seed from sitemap.xml", "type": "boolean", "description": "When **on**, the crawler reads `robots.txt`-advertised sitemaps and `/sitemap.xml` (following sitemap indexes) and queues those URLs โ€” great for reaching pages that aren't linked from your start page. Your include/exclude filters still apply.", "default": false }, "useLlmsTxt": { "title": "๐Ÿ“„ Seed from llms.txt", "type": "boolean", "description": "When **on**, the crawler reads `/llms.txt` at each start domain and queues the URLs it lists. Your include/exclude filters still apply.", "default": false }, "respectRobotsTxtFile": { "title": "๐Ÿค– Respect robots.txt", "type": "boolean", "description": "When **on**, discovered URLs disallowed by the site's `robots.txt` are skipped. Your explicit start URLs are always crawled.", "default": false }, "keepUrlFragments": { "title": "๐Ÿ”– Treat `#anchors` as different pages", "type": "boolean", "description": "๐Ÿ“Œ Normally `#section` is ignored. Turn **on** if you need each fragment to count as its own URL.", "default": false }, "ignoreCanonicalUrl": { "title": "๐Ÿงญ Ignore canonical `` tags", "type": "boolean", "description": "๐Ÿ”€ When **off**, the page's canonical URL becomes the row's `url` and drives deduplication. When **on**, the loaded URL wins.", "default": false }, "maxCrawlDepth": { "title": "๐Ÿ“‚ Max crawl depth", "minimum": 0, "maximum": 50, "type": "integer", "description": "0๏ธโƒฃ = only your start URLs ยท 1๏ธโƒฃ = their direct links ยท 2๏ธโƒฃ = one more hopโ€ฆ and so on. Great for staying on-topic!", "default": 2 }, "maxCrawlPages": { "title": "๐Ÿ”ข Max pages to visit (0 = unlimited)", "minimum": 0, "maximum": 1000000, "type": "integer", "description": "๐Ÿ›ก๏ธ Hard cap on how many **unique pages** may be opened, including pages that turn out to be empty or blocked. **0** means โ€œno extra capโ€ (max rows below still applies).", "default": 0 }, "maxResults": { "title": "๐Ÿ“ฌ Max rows in your dataset", "minimum": 1, "maximum": 100000, "type": "integer", "description": "๐ŸŽ Hard stop on how many **successful page records** are saved โ€” perfect for sampling or quick tests.", "default": 10 }, "maxConcurrency": { "title": "โšก Parallel pages", "minimum": 1, "maximum": 50, "type": "integer", "description": "๐Ÿ”„ How many pages load **at the same time**. Higher = faster, but be kind to small sites โ€” start moderate.", "default": 10 }, "rateLimitDelaySecs": { "title": "โณ Pause between requests (seconds)", "type": "string", "description": "๐ŸŒŠ Throttle applied before each fetch, e.g. `0.1` or `1.5`. Use `0` to go as fast as your other limits allow. Non-numeric values fall back to `0`." }, "maxRequestRetries": { "title": "๐Ÿ” Attempts per page", "minimum": 1, "maximum": 20, "type": "integer", "description": "โ™ป๏ธ Total fetch **attempts** for a URL before giving up (`3` = one try plus two retries, with exponential backoff). `429` responses honour `Retry-After`.", "default": 3 }, "requestTimeoutSecs": { "title": "โŒ› Page load timeout (seconds)", "minimum": 5, "maximum": 300, "type": "integer", "description": "โฒ๏ธ Max time to wait for a page to respond โ€” useful on slow or heavy sites.", "default": 60 }, "dynamicContentWaitSecs": { "title": "๐ŸŽฌ Extra wait for dynamic sites (seconds)", "minimum": 0, "maximum": 120, "type": "integer", "description": "๐Ÿ“บ Real settle time **after** a rendered page finishes loading, so late-arriving content is captured. Only applies to rendered pages. Higher values cost more compute.", "default": 3 }, "waitForSelector": { "title": "๐ŸŽฏ Wait for CSS selector (optional)", "type": "string", "description": "โœจ Wait until this **CSS selector** appears before extracting โ€” ideal for key content that pops in late. A page missing the selector is treated as incomplete and collected again more thoroughly." }, "renderingMode": { "title": "๐ŸŽฎ Rendering mode", "enum": [ "auto", "always", "never" ], "type": "string", "description": "**Auto** is recommended: most pages are collected quickly, and only the ones that need it are rendered. **Always** renders every page (most reliable for JavaScript-heavy sites, slowest). **Never** is the cheapest and fastest, but cannot capture screenshots or JavaScript-built content.", "default": "auto" }, "retryFailedWithRendering": { "title": "๐Ÿ†˜ Second chance if a page won't load", "type": "boolean", "description": "๐Ÿ’ซ If a page cannot be collected on the first attempt, try it once more with rendering. Great for flaky or protected pages. Ignored when the rendering mode is *Never*.", "default": true }, "blockMedia": { "title": "๐Ÿšซ Block images, fonts & media (rendered pages)", "type": "boolean", "description": "โšก Skip downloading images, fonts and video while rendering โ€” faster and cheaper on heavy pages. Only affects rendered pages; text and links are unaffected.", "default": false }, "maxScrollHeightPixels": { "title": "๐Ÿ“œ Auto-scroll height (pixels, 0 = off)", "minimum": 0, "maximum": 200000, "type": "integer", "description": "๐Ÿ–ฑ๏ธ On rendered pages, scroll down up to this many pixels to trigger lazy-loaded / infinite-scroll content before extracting. `0` disables scrolling. Only applies to rendered pages.", "default": 0 }, "clickElementsCssSelector": { "title": "๐Ÿ‘† Click elements before extracting (CSS selector)", "type": "string", "description": "๐Ÿ”ฝ On rendered pages, click elements matching this selector (accordions, โ€œshow moreโ€, expanders) before extracting. Best-effort โ€” unclickable elements are skipped. Example: `[aria-expanded=\"false\"]`." }, "ignoreHttpsErrors": { "title": "โš ๏ธ Allow insecure / bad certs (dev only)", "type": "boolean", "description": "๐Ÿ”“ **Off** by default (secure). Turn **on** only for local/dev sites with certificate issues โ€” not for production.", "default": false }, "keepElementsCssSelector": { "title": "๐ŸŽฏ Keep only elements (CSS selectors)", "type": "string", "description": "๐Ÿงฒ If set, **only** the elements matching this selector are kept โ€” everything else is discarded before extraction. Ideal for grabbing just an article body, e.g. `article, main`. Leave empty to keep the whole page. Applied before the remove list below." }, "removeElementsCssSelector": { "title": "๐Ÿ—‘๏ธ Remove elements (CSS selectors)", "type": "string", "description": "Comma-separated selectors. Commas inside `[attr=\"a,b\"]` or `:is(a,b)` are handled correctly, and an invalid selector is skipped with a warning instead of failing the page.", "default": "nav, footer, script, style, noscript, svg, img[src^='data:'], [role=\"alert\"], [role=\"banner\"], [role=\"dialog\"], [role=\"alertdialog\"], [role=\"region\"][aria-label*=\"skip\" i], [aria-modal=\"true\"]" }, "readableTextCharThreshold": { "title": "๐Ÿ“ โ€œEnough text?โ€ sensitivity", "minimum": 0, "maximum": 100000, "type": "integer", "description": "๐Ÿ“Š A page whose cleaned text is shorter than this counts as **too thin** and, in *Auto* rendering mode, is collected again more thoroughly.", "default": 100 }, "kvDir": { "title": "๐Ÿ“ Storage prefix", "type": "string", "description": "Prefix for saved records (e.g. `kv`). Characters that are illegal in key-value store keys are replaced automatically. Leave **empty** to skip writing files entirely." }, "saveScreenshots": { "title": "๐Ÿ“ธ Save full-page screenshots", "type": "boolean", "description": "๐Ÿ–ผ๏ธ Captures a full-page **PNG** for every saved row. โš ๏ธ Screenshots require rendering every page, so runs are slower and cost more. Needs a storage prefix and a rendering mode other than *Never*.", "default": false }, "saveMarkdown": { "title": "๐Ÿ“ Include Markdown in each row", "type": "boolean", "description": "๐Ÿ“’ Adds a real **Markdown** rendering (headings, links, lists, tables, code fences) alongside plain text โ€” turn off if you only need raw text.", "default": true }, "saveHtml": { "title": "๐Ÿ“„ Include cleaned HTML in each row", "type": "boolean", "description": "๐Ÿงฑ Embeds the cleaned readable HTML in the `html` field. Turn **off** for much smaller dataset rows โ€” the snapshot links in `htmlUrl` still work.", "default": true }, "proxyConfiguration": { "title": "๐ŸŒ Proxy configuration", "type": "object", "description": "Pick Apify Proxy (datacenter or residential) or supply custom proxy URLs. If the selected proxy is unavailable, the run continues rather than failing.", "default": {} }, "headerAcceptLanguage": { "title": "๐Ÿ—ฃ๏ธ Accept-Language", "type": "string", "description": "Preferred content language, e.g. `en-US,en;q=0.9` or `de-DE,de;q=0.9`." }, "headerDnt": { "title": "๐Ÿ›ก๏ธ DNT (Do Not Track)", "type": "string", "description": "Typical value `1`. Leave empty to omit the header." }, "headerUpgradeInsecureRequests": { "title": "โฌ†๏ธ Upgrade insecure requests", "type": "string", "description": "Standard browser hint โ€” usually `1`. Leave empty to omit." }, "headerUserAgent": { "title": "๐Ÿงฉ User-Agent string", "type": "string", "description": "๐ŸŽญ Leave at the default for the best success rate. Change it only if a site needs a specific User-Agent โ€” a custom value is then used verbatim." }, "headerSecChUa": { "title": "๐Ÿ”’ sec-ch-ua", "type": "string", "description": "Client hint header โ€” keep the default unless you also customise the User-Agent." }, "headerSecChUaMobile": { "title": "๐Ÿ“ฑ sec-ch-ua-mobile", "type": "string", "description": "Usually `?0` for desktop mode." }, "headerSecChUaPlatform": { "title": "๐ŸชŸ sec-ch-ua-platform", "type": "string", "description": "e.g. `\"Windows\"` for desktop." }, "customHttpHeaders": { "title": "โž• Extra HTTP headers (JSON)", "type": "object", "description": "๐Ÿงฉ Merge **custom headers** (cookies, auth, language). These win over every header above. Invalid JSON fails validation." }, "initialCookies": { "title": "๐Ÿช Initial cookies (JSON)", "type": "array", "description": "๐Ÿ” Cookies sent with every request and loaded into the browser โ€” useful for session or light login. Each item: `{ \"name\": \"...\", \"value\": \"...\", \"domain\": \"...\", \"path\": \"/\" }` (`domain`/`path` optional). Example: `[{\"name\":\"sessionid\",\"value\":\"abc123\"}]`.", "items": { "type": "object" } }, "debugLog": { "title": "๐Ÿ—ฃ๏ธ Verbose logging", "type": "boolean", "description": "๐Ÿ“ฃ When **on**, the run log drops to DEBUG level and prints queue decisions, fallbacks, and parse notes. Noisier, but great for troubleshooting.", "default": true }, "storeSkippedUrls": { "title": "๐Ÿงพ Save skipped URLs", "type": "boolean", "description": "๐Ÿ“„ When **on**, URLs that were discovered but not crawled (off-domain, filtered out, robots-disallowed) are written to `skipped-urls.json` in the key-value store, with a reason for each.", "default": false } } }, "runsResponseSchema": { "type": "object", "properties": { "data": { "type": "object", "properties": { "id": { "type": "string" }, "actId": { "type": "string" }, "userId": { "type": "string" }, "startedAt": { "type": "string", "format": "date-time", "example": "2025-01-08T00:00:00.000Z" }, "finishedAt": { "type": "string", "format": "date-time", "example": "2025-01-08T00:00:00.000Z" }, "status": { "type": "string", "example": "READY" }, "meta": { "type": "object", "properties": { "origin": { "type": "string", "example": "API" }, "userAgent": { "type": "string" } } }, "stats": { "type": "object", "properties": { "inputBodyLen": { "type": "integer", "example": 2000 }, "rebootCount": { "type": "integer", "example": 0 }, "restartCount": { "type": "integer", "example": 0 }, "resurrectCount": { "type": "integer", "example": 0 }, "computeUnits": { "type": "integer", "example": 0 } } }, "options": { "type": "object", "properties": { "build": { "type": "string", "example": "latest" }, "timeoutSecs": { "type": "integer", "example": 300 }, "memoryMbytes": { "type": "integer", "example": 1024 }, "diskMbytes": { "type": "integer", "example": 2048 } } }, "buildId": { "type": "string" }, "defaultKeyValueStoreId": { "type": "string" }, "defaultDatasetId": { "type": "string" }, "defaultRequestQueueId": { "type": "string" }, "buildNumber": { "type": "string", "example": "1.0.0" }, "containerUrl": { "type": "string" }, "usage": { "type": "object", "properties": { "ACTOR_COMPUTE_UNITS": { "type": "integer", "example": 0 }, "DATASET_READS": { "type": "integer", "example": 0 }, "DATASET_WRITES": { "type": "integer", "example": 0 }, "KEY_VALUE_STORE_READS": { "type": "integer", "example": 0 }, "KEY_VALUE_STORE_WRITES": { "type": "integer", "example": 1 }, "KEY_VALUE_STORE_LISTS": { "type": "integer", "example": 0 }, "REQUEST_QUEUE_READS": { "type": "integer", "example": 0 }, "REQUEST_QUEUE_WRITES": { "type": "integer", "example": 0 }, "DATA_TRANSFER_INTERNAL_GBYTES": { "type": "integer", "example": 0 }, "DATA_TRANSFER_EXTERNAL_GBYTES": { "type": "integer", "example": 0 }, "PROXY_RESIDENTIAL_TRANSFER_GBYTES": { "type": "integer", "example": 0 }, "PROXY_SERPS": { "type": "integer", "example": 0 } } }, "usageTotalUsd": { "type": "number", "example": 0.00005 }, "usageUsd": { "type": "object", "properties": { "ACTOR_COMPUTE_UNITS": { "type": "integer", "example": 0 }, "DATASET_READS": { "type": "integer", "example": 0 }, "DATASET_WRITES": { "type": "integer", "example": 0 }, "KEY_VALUE_STORE_READS": { "type": "integer", "example": 0 }, "KEY_VALUE_STORE_WRITES": { "type": "number", "example": 0.00005 }, "KEY_VALUE_STORE_LISTS": { "type": "integer", "example": 0 }, "REQUEST_QUEUE_READS": { "type": "integer", "example": 0 }, "REQUEST_QUEUE_WRITES": { "type": "integer", "example": 0 }, "DATA_TRANSFER_INTERNAL_GBYTES": { "type": "integer", "example": 0 }, "DATA_TRANSFER_EXTERNAL_GBYTES": { "type": "integer", "example": 0 }, "PROXY_RESIDENTIAL_TRANSFER_GBYTES": { "type": "integer", "example": 0 }, "PROXY_SERPS": { "type": "integer", "example": 0 } } } } } } } } } }