{ "openapi": "3.0.1", "info": { "title": "LLM Website Crawler: Clean Markdown & RAG Chunks", "description": "Crawl any website and get every page's main content as clean Markdown for ChatGPT, RAG and AI agents: no menus, footers or cookie banners, with title, description, headings and dates, plus ready-to-embed chunks. Sitemaps, JavaScript pages, include and exclude patterns. $1 per 1,000 pages.", "version": "0.1", "x-build-id": "E6ny7MDmmNZHa7Pdo" }, "servers": [ { "url": "https://api.apify.com/v2" } ], "paths": { "/acts/zaiq~llm-website-crawler/run-sync-get-dataset-items": { "post": { "operationId": "run-sync-get-dataset-items-zaiq-llm-website-crawler", "x-openai-isConsequential": false, "summary": "Executes an Actor, waits for its completion, and returns Actor's dataset items in response.", "tags": [ "Run Actor" ], "requestBody": { "required": true, "content": { "application/json": { "schema": { "$ref": "#/components/schemas/inputSchema" } } } }, "parameters": [ { "name": "token", "in": "query", "required": true, "schema": { "type": "string" }, "description": "Enter your Apify token here" } ], "responses": { "200": { "description": "OK" } } } }, "/acts/zaiq~llm-website-crawler/runs": { "post": { "operationId": "runs-sync-zaiq-llm-website-crawler", "x-openai-isConsequential": false, "summary": "Executes an Actor and returns information about the initiated run in response.", "tags": [ "Run Actor" ], "requestBody": { "required": true, "content": { "application/json": { "schema": { "$ref": "#/components/schemas/inputSchema" } } } }, "parameters": [ { "name": "token", "in": "query", "required": true, "schema": { "type": "string" }, "description": "Enter your Apify token here" } ], "responses": { "200": { "description": "OK", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/runsResponseSchema" } } } } } } }, "/acts/zaiq~llm-website-crawler/run-sync": { "post": { "operationId": "run-sync-zaiq-llm-website-crawler", "x-openai-isConsequential": false, "summary": "Executes an Actor, waits for completion, and returns the OUTPUT from Key-value store in response.", "tags": [ "Run Actor" ], "requestBody": { "required": true, "content": { "application/json": { "schema": { "$ref": "#/components/schemas/inputSchema" } } } }, "parameters": [ { "name": "token", "in": "query", "required": true, "schema": { "type": "string" }, "description": "Enter your Apify token here" } ], "responses": { "200": { "description": "OK" } } } } }, "components": { "schemas": { "inputSchema": { "type": "object", "required": [ "startUrls" ], "properties": { "startUrls": { "title": "Start URLs", "type": "array", "description": "Where the crawl starts. It follows links on the same site and under the same path (start at https://example.com/docs/ to crawl only the docs).", "items": { "type": "object", "required": [ "url" ], "properties": { "url": { "type": "string", "title": "URL of a web page", "format": "uri" } } } }, "maxPages": { "title": "Most pages", "minimum": 1, "maximum": 100000, "type": "integer", "description": "The crawl stops after this many pages delivered (in all start URLs).", "default": 100 }, "maxDepth": { "title": "Most link clicks from the start", "minimum": 0, "maximum": 100, "type": "integer", "description": "0 reads only the start pages (and their sitemaps when on).", "default": 20 }, "useSitemaps": { "title": "Use the site's sitemaps", "type": "boolean", "description": "Also find pages in the sitemaps named in robots.txt or at /sitemap.xml (pages no link points to as well).", "default": true }, "renderJs": { "title": "JavaScript pages", "enum": [ "auto", "always", "never" ], "type": "string", "description": "Most pages carry their content in the HTML; a page built by JavaScript is opened in Chrome to read it.", "default": "auto" }, "chunking": { "title": "Chunks for RAG", "enum": [ "none", "per-page", "rows" ], "type": "string", "description": "Split each page's Markdown on its headings into chunks of the size below, each with its heading path. A page is charged once however many chunks it has.", "default": "none" }, "chunkSize": { "title": "Chunk size (characters)", "minimum": 300, "maximum": 20000, "type": "integer", "description": "The longest a chunk may be, in characters (2,000 is about 400 words).", "default": 2000 }, "chunkOverlap": { "title": "Chunk overlap (characters)", "minimum": 0, "maximum": 10000, "type": "integer", "description": "Repeated at the cut when a section longer than the chunk size is split.", "default": 200 }, "includeUrlGlobs": { "title": "Only URLs matching (optional)", "type": "array", "description": "Patterns with * wildcards, e.g. https://example.com/blog/*. Empty: every page in scope.", "items": { "type": "string" } }, "excludeUrlGlobs": { "title": "Skip URLs matching (optional)", "type": "array", "description": "e.g. *?page=*, */tag/*, */login*", "items": { "type": "string" } }, "includeSubdomains": { "title": "Include subdomains", "type": "boolean", "description": "Also follow links to blog.example.com, docs.example.com and so on.", "default": false }, "stayUnderStartPath": { "title": "Stay under the start URL's path", "type": "boolean", "description": "Start at /docs/ and only /docs/ pages are read. Off: the whole site.", "default": true }, "respectRobotsTxt": { "title": "Respect robots.txt", "type": "boolean", "description": "Leave a page out when the site's robots.txt asks crawlers to. Switch off only for your own sites.", "default": true }, "contentSelector": { "title": "Content selector (optional)", "type": "string", "description": "A CSS selector for the part of the page to keep, e.g. article or main .content. Empty: found automatically." }, "removeSelectors": { "title": "Remove these elements (optional)", "type": "array", "description": "CSS selectors to drop before conversion, e.g. .newsletter-signup, #comments.", "items": { "type": "string" } }, "includeText": { "title": "Also plain text", "type": "boolean", "description": "Add each page's content as plain text (no Markdown) too.", "default": false }, "includeHtml": { "title": "Also the page's HTML", "type": "boolean", "description": "Add each page's full HTML too (larger rows).", "default": false }, "maxConcurrency": { "title": "Pages at once", "minimum": 1, "maximum": 20, "type": "integer", "description": "How many pages are read at the same time.", "default": 8 } } }, "runsResponseSchema": { "type": "object", "properties": { "data": { "type": "object", "properties": { "id": { "type": "string" }, "actId": { "type": "string" }, "userId": { "type": "string" }, "startedAt": { "type": "string", "format": "date-time", "example": "2025-01-08T00:00:00.000Z" }, "finishedAt": { "type": "string", "format": "date-time", "example": "2025-01-08T00:00:00.000Z" }, "status": { "type": "string", "example": "READY" }, "meta": { "type": "object", "properties": { "origin": { "type": "string", "example": "API" }, "userAgent": { "type": "string" } } }, "stats": { "type": "object", "properties": { "inputBodyLen": { "type": "integer", "example": 2000 }, "rebootCount": { "type": "integer", "example": 0 }, "restartCount": { "type": "integer", "example": 0 }, "resurrectCount": { "type": "integer", "example": 0 }, "computeUnits": { "type": "integer", "example": 0 } } }, "options": { "type": "object", "properties": { "build": { "type": "string", "example": "latest" }, "timeoutSecs": { "type": "integer", "example": 300 }, "memoryMbytes": { "type": "integer", "example": 1024 }, "diskMbytes": { "type": "integer", "example": 2048 } } }, "buildId": { "type": "string" }, "defaultKeyValueStoreId": { "type": "string" }, "defaultDatasetId": { "type": "string" }, "defaultRequestQueueId": { "type": "string" }, "buildNumber": { "type": "string", "example": "1.0.0" }, "containerUrl": { "type": "string" }, "usage": { "type": "object", "properties": { "ACTOR_COMPUTE_UNITS": { "type": "integer", "example": 0 }, "DATASET_READS": { "type": "integer", "example": 0 }, "DATASET_WRITES": { "type": "integer", "example": 0 }, "KEY_VALUE_STORE_READS": { "type": "integer", "example": 0 }, "KEY_VALUE_STORE_WRITES": { "type": "integer", "example": 1 }, "KEY_VALUE_STORE_LISTS": { "type": "integer", "example": 0 }, "REQUEST_QUEUE_READS": { "type": "integer", "example": 0 }, "REQUEST_QUEUE_WRITES": { "type": "integer", "example": 0 }, "DATA_TRANSFER_INTERNAL_GBYTES": { "type": "integer", "example": 0 }, "DATA_TRANSFER_EXTERNAL_GBYTES": { "type": "integer", "example": 0 }, "PROXY_RESIDENTIAL_TRANSFER_GBYTES": { "type": "integer", "example": 0 }, "PROXY_SERPS": { "type": "integer", "example": 0 }, "PROXY_UNBLOCKER_UNITS": { "type": "integer", "example": 0 } } }, "usageTotalUsd": { "type": "number", "example": 0.00005 }, "usageUsd": { "type": "object", "properties": { "ACTOR_COMPUTE_UNITS": { "type": "integer", "example": 0 }, "DATASET_READS": { "type": "integer", "example": 0 }, "DATASET_WRITES": { "type": "integer", "example": 0 }, "KEY_VALUE_STORE_READS": { "type": "integer", "example": 0 }, "KEY_VALUE_STORE_WRITES": { "type": "number", "example": 0.00005 }, "KEY_VALUE_STORE_LISTS": { "type": "integer", "example": 0 }, "REQUEST_QUEUE_READS": { "type": "integer", "example": 0 }, "REQUEST_QUEUE_WRITES": { "type": "integer", "example": 0 }, "DATA_TRANSFER_INTERNAL_GBYTES": { "type": "integer", "example": 0 }, "DATA_TRANSFER_EXTERNAL_GBYTES": { "type": "integer", "example": 0 }, "PROXY_RESIDENTIAL_TRANSFER_GBYTES": { "type": "integer", "example": 0 }, "PROXY_SERPS": { "type": "integer", "example": 0 }, "PROXY_UNBLOCKER_UNITS": { "type": "integer", "example": 0 } } } } } } } } } }