{ "openapi": "3.0.1", "info": { "title": "Website Crawler - Markdown Scraper for AI & RAG", "description": "Crawl public websites, docs, blogs, and help centers into clean Markdown. Extract content with source URLs and page titles, or split it into chunks for RAG, chatbots, and AI agents. Export JSON or CSV. HTTP crawler for server-rendered pages; no AI API key needed.", "version": "1.0", "x-build-id": "WMk64KZz0Xwb9IS97" }, "servers": [ { "url": "https://api.apify.com/v2" } ], "paths": { "/acts/groupoject~ai-rag-web-crawler/run-sync-get-dataset-items": { "post": { "operationId": "run-sync-get-dataset-items-groupoject-ai-rag-web-crawler", "x-openai-isConsequential": false, "summary": "Executes an Actor, waits for its completion, and returns Actor's dataset items in response.", "tags": [ "Run Actor" ], "requestBody": { "required": true, "content": { "application/json": { "schema": { "$ref": "#/components/schemas/inputSchema" } } } }, "parameters": [ { "name": "token", "in": "query", "required": true, "schema": { "type": "string" }, "description": "Enter your Apify token here" } ], "responses": { "200": { "description": "OK" } } } }, "/acts/groupoject~ai-rag-web-crawler/runs": { "post": { "operationId": "runs-sync-groupoject-ai-rag-web-crawler", "x-openai-isConsequential": false, "summary": "Executes an Actor and returns information about the initiated run in response.", "tags": [ "Run Actor" ], "requestBody": { "required": true, "content": { "application/json": { "schema": { "$ref": "#/components/schemas/inputSchema" } } } }, "parameters": [ { "name": "token", "in": "query", "required": true, "schema": { "type": "string" }, "description": "Enter your Apify token here" } ], "responses": { "200": { "description": "OK", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/runsResponseSchema" } } } } } } }, "/acts/groupoject~ai-rag-web-crawler/run-sync": { "post": { "operationId": "run-sync-groupoject-ai-rag-web-crawler", "x-openai-isConsequential": false, "summary": "Executes an Actor, waits for completion, and returns the OUTPUT from Key-value store in response.", "tags": [ "Run Actor" ], "requestBody": { "required": true, "content": { "application/json": { "schema": { "$ref": "#/components/schemas/inputSchema" } } } }, "parameters": [ { "name": "token", "in": "query", "required": true, "schema": { "type": "string" }, "description": "Enter your Apify token here" } ], "responses": { "200": { "description": "OK" } } } } }, "components": { "schemas": { "inputSchema": { "type": "object", "required": [ "startUrls" ], "properties": { "startUrls": { "title": "Start URLs", "type": "array", "description": "URLs to start crawling from. Accepts plain URLs or { \"url\": \"...\" } objects.", "items": { "type": "object", "required": [ "url" ], "properties": { "url": { "type": "string", "title": "URL of a web page", "format": "uri" } } } }, "maxCrawlPages": { "title": "Max pages", "minimum": 1, "maximum": 10000, "type": "integer", "description": "Maximum number of pages to crawl in total.", "default": 50 }, "maxCrawlDepth": { "title": "Max crawl depth", "minimum": 0, "maximum": 20, "type": "integer", "description": "How many link-hops to follow from the start URLs. 0 = only the start URLs.", "default": 1 }, "sameDomainOnly": { "title": "Stay on same domain", "type": "boolean", "description": "Only follow links on the same domain(s) as the start URLs.", "default": true }, "includeUrlGlobs": { "title": "Include URL globs", "type": "array", "description": "Only crawl URLs matching these glob patterns (e.g. https://site.com/docs/*). Empty = no include filter.", "items": { "type": "string" } }, "excludeUrlGlobs": { "title": "Exclude URL globs", "type": "array", "description": "Skip URLs matching these glob patterns (e.g. *.pdf, https://site.com/blog/*).", "items": { "type": "string" } }, "chunkContent": { "title": "Chunk content for RAG", "type": "boolean", "description": "Split each page into overlapping chunks (one dataset row per chunk) — ready for embeddings. Off = one row per page.", "default": true }, "chunkSize": { "title": "Chunk size (characters)", "minimum": 200, "maximum": 20000, "type": "integer", "description": "Target characters per chunk.", "default": 1000 }, "chunkOverlap": { "title": "Chunk overlap (characters)", "minimum": 0, "maximum": 2000, "type": "integer", "description": "Characters of context carried between consecutive chunks.", "default": 100 }, "minChunkChars": { "title": "Minimum chunk size", "minimum": 0, "maximum": 5000, "type": "integer", "description": "Drop chunks shorter than this many characters.", "default": 50 }, "saveHtml": { "title": "Include cleaned HTML", "type": "boolean", "description": "Also include the cleaned HTML of the main content in each row.", "default": false }, "maxConcurrency": { "title": "Max concurrency", "minimum": 1, "maximum": 50, "type": "integer", "description": "How many pages to crawl in parallel.", "default": 10 }, "proxyConfiguration": { "title": "Proxy configuration", "type": "object", "description": "Optional. Use Apify Proxy to crawl sites that block datacenter IPs." }, "debugMode": { "title": "Debug mode", "type": "boolean", "description": "Verbose logging.", "default": false } } }, "runsResponseSchema": { "type": "object", "properties": { "data": { "type": "object", "properties": { "id": { "type": "string" }, "actId": { "type": "string" }, "userId": { "type": "string" }, "startedAt": { "type": "string", "format": "date-time", "example": "2025-01-08T00:00:00.000Z" }, "finishedAt": { "type": "string", "format": "date-time", "example": "2025-01-08T00:00:00.000Z" }, "status": { "type": "string", "example": "READY" }, "meta": { "type": "object", "properties": { "origin": { "type": "string", "example": "API" }, "userAgent": { "type": "string" } } }, "stats": { "type": "object", "properties": { "inputBodyLen": { "type": "integer", "example": 2000 }, "rebootCount": { "type": "integer", "example": 0 }, "restartCount": { "type": "integer", "example": 0 }, "resurrectCount": { "type": "integer", "example": 0 }, "computeUnits": { "type": "integer", "example": 0 } } }, "options": { "type": "object", "properties": { "build": { "type": "string", "example": "latest" }, "timeoutSecs": { "type": "integer", "example": 300 }, "memoryMbytes": { "type": "integer", "example": 1024 }, "diskMbytes": { "type": "integer", "example": 2048 } } }, "buildId": { "type": "string" }, "defaultKeyValueStoreId": { "type": "string" }, "defaultDatasetId": { "type": "string" }, "defaultRequestQueueId": { "type": "string" }, "buildNumber": { "type": "string", "example": "1.0.0" }, "containerUrl": { "type": "string" }, "usage": { "type": "object", "properties": { "ACTOR_COMPUTE_UNITS": { "type": "integer", "example": 0 }, "DATASET_READS": { "type": "integer", "example": 0 }, "DATASET_WRITES": { "type": "integer", "example": 0 }, "KEY_VALUE_STORE_READS": { "type": "integer", "example": 0 }, "KEY_VALUE_STORE_WRITES": { "type": "integer", "example": 1 }, "KEY_VALUE_STORE_LISTS": { "type": "integer", "example": 0 }, "REQUEST_QUEUE_READS": { "type": "integer", "example": 0 }, "REQUEST_QUEUE_WRITES": { "type": "integer", "example": 0 }, "DATA_TRANSFER_INTERNAL_GBYTES": { "type": "integer", "example": 0 }, "DATA_TRANSFER_EXTERNAL_GBYTES": { "type": "integer", "example": 0 }, "PROXY_RESIDENTIAL_TRANSFER_GBYTES": { "type": "integer", "example": 0 }, "PROXY_SERPS": { "type": "integer", "example": 0 }, "PROXY_UNBLOCKER_UNITS": { "type": "integer", "example": 0 } } }, "usageTotalUsd": { "type": "number", "example": 0.00005 }, "usageUsd": { "type": "object", "properties": { "ACTOR_COMPUTE_UNITS": { "type": "integer", "example": 0 }, "DATASET_READS": { "type": "integer", "example": 0 }, "DATASET_WRITES": { "type": "integer", "example": 0 }, "KEY_VALUE_STORE_READS": { "type": "integer", "example": 0 }, "KEY_VALUE_STORE_WRITES": { "type": "number", "example": 0.00005 }, "KEY_VALUE_STORE_LISTS": { "type": "integer", "example": 0 }, "REQUEST_QUEUE_READS": { "type": "integer", "example": 0 }, "REQUEST_QUEUE_WRITES": { "type": "integer", "example": 0 }, "DATA_TRANSFER_INTERNAL_GBYTES": { "type": "integer", "example": 0 }, "DATA_TRANSFER_EXTERNAL_GBYTES": { "type": "integer", "example": 0 }, "PROXY_RESIDENTIAL_TRANSFER_GBYTES": { "type": "integer", "example": 0 }, "PROXY_SERPS": { "type": "integer", "example": 0 }, "PROXY_UNBLOCKER_UNITS": { "type": "integer", "example": 0 } } } } } } } } } }