{ "openapi": "3.0.1", "info": { "title": "Website Content Crawler for LLMs & RAG", "description": "Crawl any site to clean Markdown for RAG, with no browser. Learns each site's navigation and footer from the crawl itself and strips them, counts tokens per page, and emits ready-to-embed chunks. Says NEEDS_JS instead of returning a blank page.", "version": "0.0", "x-build-id": "MwVIQZpxohWGsMf4L" }, "servers": [ { "url": "https://api.apify.com/v2" } ], "paths": { "/acts/eiv~llm-content-crawler/run-sync-get-dataset-items": { "post": { "operationId": "run-sync-get-dataset-items-eiv-llm-content-crawler", "x-openai-isConsequential": false, "summary": "Executes an Actor, waits for its completion, and returns Actor's dataset items in response.", "tags": [ "Run Actor" ], "requestBody": { "required": true, "content": { "application/json": { "schema": { "$ref": "#/components/schemas/inputSchema" } } } }, "parameters": [ { "name": "token", "in": "query", "required": true, "schema": { "type": "string" }, "description": "Enter your Apify token here" } ], "responses": { "200": { "description": "OK" } } } }, "/acts/eiv~llm-content-crawler/runs": { "post": { "operationId": "runs-sync-eiv-llm-content-crawler", "x-openai-isConsequential": false, "summary": "Executes an Actor and returns information about the initiated run in response.", "tags": [ "Run Actor" ], "requestBody": { "required": true, "content": { "application/json": { "schema": { "$ref": "#/components/schemas/inputSchema" } } } }, "parameters": [ { "name": "token", "in": "query", "required": true, "schema": { "type": "string" }, "description": "Enter your Apify token here" } ], "responses": { "200": { "description": "OK", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/runsResponseSchema" } } } } } } }, "/acts/eiv~llm-content-crawler/run-sync": { "post": { "operationId": "run-sync-eiv-llm-content-crawler", "x-openai-isConsequential": false, "summary": "Executes an Actor, waits for completion, and returns the OUTPUT from Key-value store in response.", "tags": [ "Run Actor" ], "requestBody": { "required": true, "content": { "application/json": { "schema": { "$ref": "#/components/schemas/inputSchema" } } } }, "parameters": [ { "name": "token", "in": "query", "required": true, "schema": { "type": "string" }, "description": "Enter your Apify token here" } ], "responses": { "200": { "description": "OK" } } } } }, "components": { "schemas": { "inputSchema": { "type": "object", "required": [ "startUrls" ], "properties": { "startUrls": { "title": "Start URLs", "type": "array", "description": "Where to begin. A bare domain works — example.com becomes https://example.com. Everything crawled stays on these hosts unless you turn on Include subdomains.", "items": { "type": "string" } }, "maxPages": { "title": "Max pages", "minimum": 1, "maximum": 10000, "type": "integer", "description": "How many pages to read at most. This is the number you are charged for, and the crawl stops the moment it is reached.", "default": 50 }, "maxDepth": { "title": "Max depth", "minimum": 0, "maximum": 20, "type": "integer", "description": "How many links deep to follow from a start URL. 0 reads only the start URLs themselves. Pages found in the sitemap enter at depth 1.", "default": 3 }, "followLinks": { "title": "Follow links", "type": "boolean", "description": "Discover more pages by following links found on each page. Turn this off to read only the start URLs and whatever the sitemap lists.", "default": true }, "useSitemap": { "title": "Use sitemap", "type": "boolean", "description": "Read sitemap.xml first, including any sitemaps robots.txt points at and any nested sitemap index. It is the fastest way to find every page on a site and costs one request.", "default": true }, "respectRobotsTxt": { "title": "Respect robots.txt", "type": "boolean", "description": "Honour the site's crawl rules and its Crawl-delay. Turn this off only for sites you own or have permission to crawl.", "default": true }, "includeSubdomains": { "title": "Include subdomains", "type": "boolean", "description": "Also crawl other hosts on the same root domain, so a start URL on example.com will also read docs.example.com and blog.example.com.", "default": false }, "includeUrlPatterns": { "title": "Include URL patterns", "type": "array", "description": "Only crawl URLs matching one of these. Globs by default (*/blog/*), or write /regex/ for a regular expression. Leave empty to allow everything in scope.", "items": { "type": "string" } }, "excludeUrlPatterns": { "title": "Exclude URL patterns", "type": "array", "description": "Skip URLs matching any of these. Same glob or /regex/ syntax. Useful for /tag/, /author/ and other pages that hold no content.", "items": { "type": "string" } }, "outputFormat": { "title": "Output format", "enum": [ "markdown", "text", "html" ], "type": "string", "description": "markdown keeps headings, lists, tables and code fences, which is what an LLM reads best. text is the same content with the markup stripped. html returns the original source untouched.", "default": "markdown" }, "removeBoilerplate": { "title": "Remove boilerplate", "type": "boolean", "description": "Drop the navigation, footer, cookie bar and sidebar. Blocks repeating across the crawl are learned and removed, and link-only menus are removed on sight even when the crawl is too small to learn from.", "default": true }, "keepLinks": { "title": "Keep links", "type": "boolean", "description": "Keep links as Markdown [text](url). Turn off for prose only — smaller, but the model can no longer cite or follow anything.", "default": true }, "keepImages": { "title": "Keep images", "type": "boolean", "description": "Include images as ![alt](url). Off by default: alt text is rarely worth the tokens unless you are indexing figures.", "default": false }, "keepTables": { "title": "Keep tables", "type": "boolean", "description": "Render tables as Markdown rows. Off means the cells still appear, but the row structure is lost.", "default": true }, "minWords": { "title": "Minimum words", "minimum": 0, "maximum": 5000, "type": "integer", "description": "Skip pages with fewer words than this once boilerplate is gone — redirect stubs, empty tag pages, bare listings. Skipped pages are not charged. A page that needs JavaScript is always kept, because its emptiness is the finding.", "default": 25 }, "chunk": { "title": "Split into chunks", "type": "boolean", "description": "Also emit ready-to-embed chunks as their own records, each with a stable id, its token count and the heading it sits under.", "default": false }, "chunkTokens": { "title": "Chunk size in tokens", "minimum": 64, "maximum": 8192, "type": "integer", "description": "Target size per chunk. Chunks break on block boundaries, so a chunk never starts mid-sentence and a code fence is never cut in half.", "default": 800 }, "chunkOverlapTokens": { "title": "Chunk overlap in tokens", "minimum": 0, "maximum": 4096, "type": "integer", "description": "Tokens carried from the end of one chunk into the start of the next, so a fact spanning a boundary survives. Must be below the chunk size.", "default": 100 }, "flattenOutput": { "title": "Flatten output", "type": "boolean", "description": "Turn arrays into pipe-delimited strings so the CSV and Excel exports have a fixed, readable column set.", "default": false }, "maxConcurrency": { "title": "Max concurrency", "minimum": 1, "maximum": 20, "type": "integer", "description": "Pages fetched at once. Raise it for large crawls of sites that can take it; lower it if you get rate limited.", "default": 5 }, "requestDelayMs": { "title": "Delay between requests", "minimum": 0, "maximum": 10000, "type": "integer", "description": "Milliseconds to wait between requests. Raised automatically if robots.txt asks for a longer Crawl-delay.", "default": 200 }, "requestTimeoutSecs": { "title": "Request timeout", "minimum": 5, "maximum": 180, "type": "integer", "description": "Seconds to wait for one page before giving up on it.", "default": 45 }, "maxRetries": { "title": "Max retries", "minimum": 0, "maximum": 5, "type": "integer", "description": "Retries for a page that times out or answers with a retryable status.", "default": 2 }, "proxyConfiguration": { "title": "Proxy", "type": "object", "description": "Optional. Not needed for most sites — this actor sends plain HTTP requests and does not run a browser." } } }, "runsResponseSchema": { "type": "object", "properties": { "data": { "type": "object", "properties": { "id": { "type": "string" }, "actId": { "type": "string" }, "userId": { "type": "string" }, "startedAt": { "type": "string", "format": "date-time", "example": "2025-01-08T00:00:00.000Z" }, "finishedAt": { "type": "string", "format": "date-time", "example": "2025-01-08T00:00:00.000Z" }, "status": { "type": "string", "example": "READY" }, "meta": { "type": "object", "properties": { "origin": { "type": "string", "example": "API" }, "userAgent": { "type": "string" } } }, "stats": { "type": "object", "properties": { "inputBodyLen": { "type": "integer", "example": 2000 }, "rebootCount": { "type": "integer", "example": 0 }, "restartCount": { "type": "integer", "example": 0 }, "resurrectCount": { "type": "integer", "example": 0 }, "computeUnits": { "type": "integer", "example": 0 } } }, "options": { "type": "object", "properties": { "build": { "type": "string", "example": "latest" }, "timeoutSecs": { "type": "integer", "example": 300 }, "memoryMbytes": { "type": "integer", "example": 1024 }, "diskMbytes": { "type": "integer", "example": 2048 } } }, "buildId": { "type": "string" }, "defaultKeyValueStoreId": { "type": "string" }, "defaultDatasetId": { "type": "string" }, "defaultRequestQueueId": { "type": "string" }, "buildNumber": { "type": "string", "example": "1.0.0" }, "containerUrl": { "type": "string" }, "usage": { "type": "object", "properties": { "ACTOR_COMPUTE_UNITS": { "type": "integer", "example": 0 }, "DATASET_READS": { "type": "integer", "example": 0 }, "DATASET_WRITES": { "type": "integer", "example": 0 }, "KEY_VALUE_STORE_READS": { "type": "integer", "example": 0 }, "KEY_VALUE_STORE_WRITES": { "type": "integer", "example": 1 }, "KEY_VALUE_STORE_LISTS": { "type": "integer", "example": 0 }, "REQUEST_QUEUE_READS": { "type": "integer", "example": 0 }, "REQUEST_QUEUE_WRITES": { "type": "integer", "example": 0 }, "DATA_TRANSFER_INTERNAL_GBYTES": { "type": "integer", "example": 0 }, "DATA_TRANSFER_EXTERNAL_GBYTES": { "type": "integer", "example": 0 }, "PROXY_RESIDENTIAL_TRANSFER_GBYTES": { "type": "integer", "example": 0 }, "PROXY_SERPS": { "type": "integer", "example": 0 } } }, "usageTotalUsd": { "type": "number", "example": 0.00005 }, "usageUsd": { "type": "object", "properties": { "ACTOR_COMPUTE_UNITS": { "type": "integer", "example": 0 }, "DATASET_READS": { "type": "integer", "example": 0 }, "DATASET_WRITES": { "type": "integer", "example": 0 }, "KEY_VALUE_STORE_READS": { "type": "integer", "example": 0 }, "KEY_VALUE_STORE_WRITES": { "type": "number", "example": 0.00005 }, "KEY_VALUE_STORE_LISTS": { "type": "integer", "example": 0 }, "REQUEST_QUEUE_READS": { "type": "integer", "example": 0 }, "REQUEST_QUEUE_WRITES": { "type": "integer", "example": 0 }, "DATA_TRANSFER_INTERNAL_GBYTES": { "type": "integer", "example": 0 }, "DATA_TRANSFER_EXTERNAL_GBYTES": { "type": "integer", "example": 0 }, "PROXY_RESIDENTIAL_TRANSFER_GBYTES": { "type": "integer", "example": 0 }, "PROXY_SERPS": { "type": "integer", "example": 0 } } } } } } } } } }