{ "openapi": "3.0.1", "info": { "title": "RAG Web Crawler: Website to Markdown & Text Chunks", "description": "Crawl public web pages into clean Markdown, plain text, and ready-to-embed text chunks for RAG pipelines and LLM apps. Every chunk keeps its source URL, title, and character offsets. Fast HTTP crawling that respects robots.txt and caps pages, requests, and output size.", "version": "0.1", "x-build-id": "xza1YEKoWQDxpDCLp" }, "servers": [ { "url": "https://api.apify.com/v2" } ], "paths": { "/acts/autofacts~rag-web-crawler/run-sync-get-dataset-items": { "post": { "operationId": "run-sync-get-dataset-items-autofacts-rag-web-crawler", "x-openai-isConsequential": false, "summary": "Executes an Actor, waits for its completion, and returns Actor's dataset items in response.", "tags": [ "Run Actor" ], "requestBody": { "required": true, "content": { "application/json": { "schema": { "$ref": "#/components/schemas/inputSchema" } } } }, "parameters": [ { "name": "token", "in": "query", "required": true, "schema": { "type": "string" }, "description": "Enter your Apify token here" } ], "responses": { "200": { "description": "OK" } } } }, "/acts/autofacts~rag-web-crawler/runs": { "post": { "operationId": "runs-sync-autofacts-rag-web-crawler", "x-openai-isConsequential": false, "summary": "Executes an Actor and returns information about the initiated run in response.", "tags": [ "Run Actor" ], "requestBody": { "required": true, "content": { "application/json": { "schema": { "$ref": "#/components/schemas/inputSchema" } } } }, "parameters": [ { "name": "token", "in": "query", "required": true, "schema": { "type": "string" }, "description": "Enter your Apify token here" } ], "responses": { "200": { "description": "OK", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/runsResponseSchema" } } } } } } }, "/acts/autofacts~rag-web-crawler/run-sync": { "post": { "operationId": "run-sync-autofacts-rag-web-crawler", "x-openai-isConsequential": false, "summary": "Executes an Actor, waits for completion, and returns the OUTPUT from Key-value store in response.", "tags": [ "Run Actor" ], "requestBody": { "required": true, "content": { "application/json": { "schema": { "$ref": "#/components/schemas/inputSchema" } } } }, "parameters": [ { "name": "token", "in": "query", "required": true, "schema": { "type": "string" }, "description": "Enter your Apify token here" } ], "responses": { "200": { "description": "OK" } } } } }, "components": { "schemas": { "inputSchema": { "type": "object", "properties": { "startUrls": { "title": "Start URLs", "maxItems": 1000, "type": "array", "description": "Public page URLs to crawl, up to 1,000. Through the API, each one can be a plain string or a `{ \"url\": \"…\" }` object.", "items": { "type": "string" } }, "query": { "title": "Search query", "pattern": "\\S", "minLength": 1, "maxLength": 128, "type": "string", "description": "One search query, merged with `queries` and deduplicated. Each result page is crawled like a start URL. Search needs a paid Apify plan; on the free plan, a run with a query fails before crawling." }, "queries": { "title": "Search queries", "maxItems": 1000, "type": "array", "description": "Several search queries, each at most 128 Unicode code points, merged with `query`. No search runs unless you add a query. Search needs a paid Apify plan.", "items": { "type": "string", "minLength": 1, "maxLength": 128, "pattern": "\\S" } }, "maxResultsPerQuery": { "title": "Results per query", "minimum": 1, "maximum": 50, "type": "integer", "description": "Maximum candidate results for each query; fewer may come back. Search is charged once per query that returns results, not per result.", "default": 5 }, "maxPages": { "title": "Max pages", "minimum": 1, "maximum": 1000, "type": "integer", "description": "Maximum distinct pages scheduled, including pages that fail or that robots.txt excludes. No extra pages are scheduled to make up for them.", "default": 20 }, "maxDepth": { "title": "Max link depth", "minimum": 0, "maximum": 5, "type": "integer", "description": "How many links deep to follow within each start URL's site, meaning the same scheme, host, and port. `0` fetches only the start URLs.", "default": 0 }, "maxConcurrency": { "title": "Max concurrency", "minimum": 1, "maximum": 10, "type": "integer", "description": "Pages processed at the same time in this run. It does not raise the shared search rate limit.", "default": 3 }, "maxSearchRequests": { "title": "Max search requests", "minimum": 0, "maximum": 1000, "type": "integer", "description": "Maximum search attempts in this run, retries included. `0` turns search off; start URLs are still crawled.", "default": 10 }, "maxFetchRequests": { "title": "Max fetch requests", "minimum": 0, "maximum": 10000, "type": "integer", "description": "Maximum network attempts for pages in this run, including robots.txt, redirects, retries, and resources loaded in browser mode. This is not the number of pages saved.", "default": 100 }, "requestTimeoutSecs": { "title": "Request timeout (seconds)", "minimum": 5, "maximum": 120, "type": "integer", "description": "Timeout for each network request or browser navigation. A page can take longer in total, for example when it is retried.", "default": 30 }, "maxResponseBytes": { "title": "Max response size (bytes)", "minimum": 1024, "maximum": 10485760, "type": "integer", "description": "Largest HTTP response accepted. A larger response fails the page instead of being saved incomplete.", "default": 2097152 }, "maxContentChars": { "title": "Max content length", "minimum": 1000, "maximum": 100000, "type": "integer", "description": "Maximum text and Markdown length per page, in Unicode code points, not tokens. Truncated pages have `contentTruncated` set to `true`.", "default": 50000 }, "maxOutputBytes": { "title": "Max output size (bytes)", "minimum": 1048576, "maximum": 104857600, "type": "integer", "description": "Total page-record and search-result JSON for the run; the run stops when it is reached. Logs, the `REQUESTS` record, and platform storage are not counted. Each page record is also limited to 1 MiB; a larger page is not saved, and the run continues.", "default": 20971520 }, "renderingMode": { "title": "Rendering mode", "enum": [ "http", "auto", "browser" ], "type": "string", "description": "`http` fetches pages without running JavaScript. `auto` and `browser` need a browser, which is not available in hosted runs, so keep `http`.", "default": "http" }, "respectRobotsTxt": { "title": "Respect robots.txt", "type": "boolean", "description": "Check each site's robots.txt before fetching its pages. This does not mean the site authorizes your use, or that its pages can be fetched and extracted.", "default": true }, "onlyMainContent": { "title": "Main content only", "type": "boolean", "description": "Extract the page's main content. When `false`, the whole page body is used, still without scripts and navigation. Content is never replaced by a summary.", "default": true }, "removeSelectors": { "title": "Remove elements (CSS selectors)", "maxItems": 30, "type": "array", "description": "Up to 30 CSS selectors, each at most 300 characters, for elements to remove before extraction. Selectors are checked during the run: an invalid one fails pages with `INVALID_SELECTOR`.", "items": { "type": "string", "minLength": 1, "maxLength": 300 }, "default": [] }, "chunkSize": { "title": "Chunk size", "minimum": 200, "maximum": 10000, "type": "integer", "description": "Maximum chunk length in Unicode code points of the cleaned page text, not tokens.", "default": 1000 }, "chunkOverlap": { "title": "Chunk overlap", "minimum": 0, "maximum": 9999, "type": "integer", "description": "Code points shared by consecutive chunks. Must be smaller than `chunkSize`. The form cannot check this, so a larger value fails the run when it starts.", "default": 150 } } }, "runsResponseSchema": { "type": "object", "properties": { "data": { "type": "object", "properties": { "id": { "type": "string" }, "actId": { "type": "string" }, "userId": { "type": "string" }, "startedAt": { "type": "string", "format": "date-time", "example": "2025-01-08T00:00:00.000Z" }, "finishedAt": { "type": "string", "format": "date-time", "example": "2025-01-08T00:00:00.000Z" }, "status": { "type": "string", "example": "READY" }, "meta": { "type": "object", "properties": { "origin": { "type": "string", "example": "API" }, "userAgent": { "type": "string" } } }, "stats": { "type": "object", "properties": { "inputBodyLen": { "type": "integer", "example": 2000 }, "rebootCount": { "type": "integer", "example": 0 }, "restartCount": { "type": "integer", "example": 0 }, "resurrectCount": { "type": "integer", "example": 0 }, "computeUnits": { "type": "integer", "example": 0 } } }, "options": { "type": "object", "properties": { "build": { "type": "string", "example": "latest" }, "timeoutSecs": { "type": "integer", "example": 300 }, "memoryMbytes": { "type": "integer", "example": 1024 }, "diskMbytes": { "type": "integer", "example": 2048 } } }, "buildId": { "type": "string" }, "defaultKeyValueStoreId": { "type": "string" }, "defaultDatasetId": { "type": "string" }, "defaultRequestQueueId": { "type": "string" }, "buildNumber": { "type": "string", "example": "1.0.0" }, "containerUrl": { "type": "string" }, "usage": { "type": "object", "properties": { "ACTOR_COMPUTE_UNITS": { "type": "integer", "example": 0 }, "DATASET_READS": { "type": "integer", "example": 0 }, "DATASET_WRITES": { "type": "integer", "example": 0 }, "KEY_VALUE_STORE_READS": { "type": "integer", "example": 0 }, "KEY_VALUE_STORE_WRITES": { "type": "integer", "example": 1 }, "KEY_VALUE_STORE_LISTS": { "type": "integer", "example": 0 }, "REQUEST_QUEUE_READS": { "type": "integer", "example": 0 }, "REQUEST_QUEUE_WRITES": { "type": "integer", "example": 0 }, "DATA_TRANSFER_INTERNAL_GBYTES": { "type": "integer", "example": 0 }, "DATA_TRANSFER_EXTERNAL_GBYTES": { "type": "integer", "example": 0 }, "PROXY_RESIDENTIAL_TRANSFER_GBYTES": { "type": "integer", "example": 0 }, "PROXY_SERPS": { "type": "integer", "example": 0 }, "PROXY_UNBLOCKER_UNITS": { "type": "integer", "example": 0 } } }, "usageTotalUsd": { "type": "number", "example": 0.00005 }, "usageUsd": { "type": "object", "properties": { "ACTOR_COMPUTE_UNITS": { "type": "integer", "example": 0 }, "DATASET_READS": { "type": "integer", "example": 0 }, "DATASET_WRITES": { "type": "integer", "example": 0 }, "KEY_VALUE_STORE_READS": { "type": "integer", "example": 0 }, "KEY_VALUE_STORE_WRITES": { "type": "number", "example": 0.00005 }, "KEY_VALUE_STORE_LISTS": { "type": "integer", "example": 0 }, "REQUEST_QUEUE_READS": { "type": "integer", "example": 0 }, "REQUEST_QUEUE_WRITES": { "type": "integer", "example": 0 }, "DATA_TRANSFER_INTERNAL_GBYTES": { "type": "integer", "example": 0 }, "DATA_TRANSFER_EXTERNAL_GBYTES": { "type": "integer", "example": 0 }, "PROXY_RESIDENTIAL_TRANSFER_GBYTES": { "type": "integer", "example": 0 }, "PROXY_SERPS": { "type": "integer", "example": 0 }, "PROXY_UNBLOCKER_UNITS": { "type": "integer", "example": 0 } } } } } } } } } }