{ "openapi": "3.0.1", "info": { "title": "Cloudflare Crawl Router", "description": "An intelligent crawl orchestration Actor that analyzes websites and routes each target to the most suitable compliant crawl strategy for efficient extraction and transparent diagnostics.", "version": "0.1", "x-build-id": "6GhQfKWIpDNn7LOec" }, "servers": [ { "url": "https://api.apify.com/v2" } ], "paths": { "/acts/solutionssmart~cloudflare-crawl-router/run-sync-get-dataset-items": { "post": { "operationId": "run-sync-get-dataset-items-solutionssmart-cloudflare-crawl-router", "x-openai-isConsequential": false, "summary": "Executes an Actor, waits for its completion, and returns Actor's dataset items in response.", "tags": [ "Run Actor" ], "requestBody": { "required": true, "content": { "application/json": { "schema": { "$ref": "#/components/schemas/inputSchema" } } } }, "parameters": [ { "name": "token", "in": "query", "required": true, "schema": { "type": "string" }, "description": "Enter your Apify token here" } ], "responses": { "200": { "description": "OK" } } } }, "/acts/solutionssmart~cloudflare-crawl-router/runs": { "post": { "operationId": "runs-sync-solutionssmart-cloudflare-crawl-router", "x-openai-isConsequential": false, "summary": "Executes an Actor and returns information about the initiated run in response.", "tags": [ "Run Actor" ], "requestBody": { "required": true, "content": { "application/json": { "schema": { "$ref": "#/components/schemas/inputSchema" } } } }, "parameters": [ { "name": "token", "in": "query", "required": true, "schema": { "type": "string" }, "description": "Enter your Apify token here" } ], "responses": { "200": { "description": "OK", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/runsResponseSchema" } } } } } } }, "/acts/solutionssmart~cloudflare-crawl-router/run-sync": { "post": { "operationId": "run-sync-solutionssmart-cloudflare-crawl-router", "x-openai-isConsequential": false, "summary": "Executes an Actor, waits for completion, and returns the OUTPUT from Key-value store in response.", "tags": [ "Run Actor" ], "requestBody": { "required": true, "content": { "application/json": { "schema": { "$ref": "#/components/schemas/inputSchema" } } } }, "parameters": [ { "name": "token", "in": "query", "required": true, "schema": { "type": "string" }, "description": "Enter your Apify token here" } ], "responses": { "200": { "description": "OK" } } } } }, "components": { "schemas": { "inputSchema": { "type": "object", "required": [ "startUrls" ], "properties": { "startUrls": { "title": "🌐 Start URLs", "type": "array", "description": "Add one or more seed URLs to inspect and process. For best results, use direct content URLs such as product pages, article pages, category pages, or homepages depending on your crawl goal.", "items": { "type": "object", "required": [ "url" ], "properties": { "url": { "type": "string", "title": "URL of a web page", "format": "uri" } } } }, "routing": { "title": "🧠 Routing Strategy", "required": [ "mode", "renderJavaScript", "fallbackToBrowser" ], "type": "object", "description": "Control how the Actor decides between focused extraction and broader discovery crawling. Auto mode is recommended for most users.", "properties": { "mode": { "title": "🧭 Crawl mode", "type": "string", "description": "Choose whether to extract a single page, crawl multiple pages on a site, or let the router automatically decide the best mode.", "default": "auto", "enum": [ "single-page", "site-crawl", "auto" ], "enumTitles": [ "Single page", "Site crawl", "Auto" ], "editor": "select" }, "renderJavaScript": { "title": "πŸ–₯️ Render JavaScript", "type": "string", "description": "Force browser rendering, disable it, or let the router automatically decide based on page complexity and frontend signals.", "default": "auto", "enum": [ "auto", "true", "false" ], "enumTitles": [ "Auto", "Always render", "Never render" ], "editor": "select" }, "followSitemaps": { "title": "πŸ—ΊοΈ Follow sitemaps", "type": "boolean", "description": "Use sitemap URLs discovered from robots.txt to seed site crawls. Helpful for structured site discovery and broader coverage.", "default": false, "editor": "checkbox" }, "fallbackToBrowser": { "title": "πŸ” Fallback to browser", "type": "boolean", "description": "Allow the router to escalate to Playwright when HTTP-based extraction looks unreliable or JavaScript rendering is needed.", "default": true, "editor": "checkbox" } }, "additionalProperties": false }, "limits": { "title": "βš™οΈ Crawl Limits", "required": [ "maxDepth", "maxPages", "timeoutSecs" ], "type": "object", "description": "Set safety and budget controls to balance crawl breadth, runtime, and output quality.", "properties": { "maxDepth": { "title": "πŸ“ Max depth", "type": "integer", "description": "Maximum link depth when site crawling is enabled. Use lower values for tighter, more relevant crawls and higher values for broader discovery.", "default": 1, "minimum": 0, "maximum": 10, "editor": "number" }, "maxPages": { "title": "πŸ“„ Max pages", "type": "integer", "description": "Maximum number of pages to process for each start URL. Lower values help keep runs focused and cost efficient.", "default": 20, "minimum": 1, "maximum": 1000, "editor": "number" }, "timeoutSecs": { "title": "⏱️ Timeout (seconds)", "type": "integer", "description": "Per-request timeout budget for HTTP fetches and browser navigations. Increase for slow sites or heavy browser rendering.", "default": 45, "minimum": 5, "maximum": 180, "editor": "number" } }, "additionalProperties": false }, "extraction": { "title": "πŸ“¦ Content Extraction", "required": [ "extractionType" ], "type": "object", "description": "Tune how the Actor cleans and structures page content for datasets, AI pipelines, and downstream processing.", "properties": { "extractionType": { "title": "🧾 Extraction type", "type": "string", "description": "Bias the extraction toward a specific content type such as article, product, job, or profile. Use Auto if you want the Actor to infer the best fit.", "default": "auto", "enum": [ "auto", "article", "product", "job", "profile", "page" ], "enumTitles": [ "Auto", "Article", "Product", "Job", "Profile", "Page" ], "editor": "select" } }, "additionalProperties": false }, "output": { "title": "πŸ“Š Output & Data", "required": [ "outputFormat", "includeMetadata", "includeDiagnostics" ], "type": "object", "description": "Configure the format and level of detail you want in the final dataset and supporting records.", "properties": { "outputFormat": { "title": "πŸ“€ Output format", "type": "string", "description": "Choose the primary content representation for dataset output. JSON is best for structured workflows, while markdown and text are ideal for LLM and RAG use cases.", "default": "json", "enum": [ "json", "markdown", "text", "html" ], "enumTitles": [ "JSON", "Markdown", "Text", "HTML" ], "editor": "select" }, "includeMetadata": { "title": "🏷️ Include metadata", "type": "boolean", "description": "Include metadata such as canonical URL, headings, language, and response headers for better traceability and downstream analysis.", "default": true, "editor": "checkbox" }, "includeDiagnostics": { "title": "πŸ”Ž Include diagnostics", "type": "boolean", "description": "Include detection signals, routing rationale, and crawl diagnostics in each dataset item. Recommended for debugging, compliance review, and crawl analysis.", "default": true, "editor": "checkbox" } }, "additionalProperties": false }, "networking": { "title": "🌐 Networking & Access", "required": [ "useProxy" ], "type": "object", "description": "Configure network behavior, proxy support, and request resilience for difficult or protected targets.", "properties": { "useProxy": { "title": "🌍 Use proxy", "type": "string", "description": "Force proxies, disable proxies, or let the router decide when proxy usage is worth the added cost and reliability tradeoff.", "default": "auto", "enum": [ "auto", "true", "false" ], "enumTitles": [ "Auto", "Always use proxy", "Never use proxy" ], "editor": "select" }, "proxyConfiguration": { "title": "πŸ›‘οΈ Proxy configuration", "type": "object", "description": "Optional Apify Proxy or custom proxy settings for higher reliability on protected, geo-sensitive, or rate-limited targets.", "editor": "proxy" } }, "additionalProperties": false }, "compliance": { "title": "βœ… Compliance & Scope", "required": [ "obeyRobotsTxt", "sameDomainOnly" ], "type": "object", "description": "Keep crawls controlled, compliant, and relevant by limiting scope and respecting site directives.", "properties": { "obeyRobotsTxt": { "title": "🀝 Obey robots.txt", "type": "boolean", "description": "Respect robots.txt when deciding whether to crawl or enqueue pages. Recommended for compliant and production-safe workflows.", "default": true, "editor": "checkbox" }, "sameDomainOnly": { "title": "🏠 Same domain only", "type": "boolean", "description": "Restrict discovered links to the same hostname as the seed URL. Recommended to prevent noisy cross-domain crawling.", "default": true, "editor": "checkbox" } }, "additionalProperties": false }, "artifacts": { "title": "πŸ—‚οΈ Artifacts & Debug Assets", "required": [ "saveHtml", "saveMarkdown", "saveScreenshot" ], "type": "object", "description": "Optionally store raw crawl artifacts for QA, auditing, model ingestion, or custom post-processing.", "properties": { "saveHtml": { "title": "πŸ’Ύ Save HTML", "type": "boolean", "description": "Store raw HTML for each processed page in KeyValueStore. Useful for debugging, replay, or downstream custom parsers.", "default": false, "editor": "checkbox" }, "saveMarkdown": { "title": "πŸ“ Save markdown", "type": "boolean", "description": "Store generated markdown artifacts for each processed page in KeyValueStore. Recommended for AI, summarization, and RAG workflows.", "default": true, "editor": "checkbox" }, "saveScreenshot": { "title": "πŸ“Έ Save screenshot", "type": "boolean", "description": "Store a screenshot when a page is processed through a browser route. Useful for visual QA and troubleshooting dynamic pages.", "default": false, "editor": "checkbox" } }, "additionalProperties": false }, "diagnostics": { "title": "πŸ§ͺ Detection & Diagnostics", "required": [ "detectCloudflare" ], "type": "object", "description": "Enable richer site analysis so the Actor can explain crawl decisions, protection signals, and content-extraction confidence.", "properties": { "detectCloudflare": { "title": "☁️ Detect Cloudflare signals", "type": "boolean", "description": "Enable Cloudflare-aware header, challenge, and protection heuristics in the diagnostics layer for more transparent routing decisions.", "default": true, "editor": "checkbox" } }, "additionalProperties": false } } }, "runsResponseSchema": { "type": "object", "properties": { "data": { "type": "object", "properties": { "id": { "type": "string" }, "actId": { "type": "string" }, "userId": { "type": "string" }, "startedAt": { "type": "string", "format": "date-time", "example": "2025-01-08T00:00:00.000Z" }, "finishedAt": { "type": "string", "format": "date-time", "example": "2025-01-08T00:00:00.000Z" }, "status": { "type": "string", "example": "READY" }, "meta": { "type": "object", "properties": { "origin": { "type": "string", "example": "API" }, "userAgent": { "type": "string" } } }, "stats": { "type": "object", "properties": { "inputBodyLen": { "type": "integer", "example": 2000 }, "rebootCount": { "type": "integer", "example": 0 }, "restartCount": { "type": "integer", "example": 0 }, "resurrectCount": { "type": "integer", "example": 0 }, "computeUnits": { "type": "integer", "example": 0 } } }, "options": { "type": "object", "properties": { "build": { "type": "string", "example": "latest" }, "timeoutSecs": { "type": "integer", "example": 300 }, "memoryMbytes": { "type": "integer", "example": 1024 }, "diskMbytes": { "type": "integer", "example": 2048 } } }, "buildId": { "type": "string" }, "defaultKeyValueStoreId": { "type": "string" }, "defaultDatasetId": { "type": "string" }, "defaultRequestQueueId": { "type": "string" }, "buildNumber": { "type": "string", "example": "1.0.0" }, "containerUrl": { "type": "string" }, "usage": { "type": "object", "properties": { "ACTOR_COMPUTE_UNITS": { "type": "integer", "example": 0 }, "DATASET_READS": { "type": "integer", "example": 0 }, "DATASET_WRITES": { "type": "integer", "example": 0 }, "KEY_VALUE_STORE_READS": { "type": "integer", "example": 0 }, "KEY_VALUE_STORE_WRITES": { "type": "integer", "example": 1 }, "KEY_VALUE_STORE_LISTS": { "type": "integer", "example": 0 }, "REQUEST_QUEUE_READS": { "type": "integer", "example": 0 }, "REQUEST_QUEUE_WRITES": { "type": "integer", "example": 0 }, "DATA_TRANSFER_INTERNAL_GBYTES": { "type": "integer", "example": 0 }, "DATA_TRANSFER_EXTERNAL_GBYTES": { "type": "integer", "example": 0 }, "PROXY_RESIDENTIAL_TRANSFER_GBYTES": { "type": "integer", "example": 0 }, "PROXY_SERPS": { "type": "integer", "example": 0 } } }, "usageTotalUsd": { "type": "number", "example": 0.00005 }, "usageUsd": { "type": "object", "properties": { "ACTOR_COMPUTE_UNITS": { "type": "integer", "example": 0 }, "DATASET_READS": { "type": "integer", "example": 0 }, "DATASET_WRITES": { "type": "integer", "example": 0 }, "KEY_VALUE_STORE_READS": { "type": "integer", "example": 0 }, "KEY_VALUE_STORE_WRITES": { "type": "number", "example": 0.00005 }, "KEY_VALUE_STORE_LISTS": { "type": "integer", "example": 0 }, "REQUEST_QUEUE_READS": { "type": "integer", "example": 0 }, "REQUEST_QUEUE_WRITES": { "type": "integer", "example": 0 }, "DATA_TRANSFER_INTERNAL_GBYTES": { "type": "integer", "example": 0 }, "DATA_TRANSFER_EXTERNAL_GBYTES": { "type": "integer", "example": 0 }, "PROXY_RESIDENTIAL_TRANSFER_GBYTES": { "type": "integer", "example": 0 }, "PROXY_SERPS": { "type": "integer", "example": 0 } } } } } } } } } }