{ "openapi": "3.0.1", "info": { "title": "Website to LLM-Ready Dataset - Markdown & Chunks for RAG", "description": "Crawl a website or docs site and get clean Markdown, token-sized semantic chunks and metadata for every page. Ready for vector databases and RAG pipelines. $1.50 per 1,000 pages.", "version": "1.1", "x-build-id": "mADK6mcoaNt26u15m" }, "servers": [ { "url": "https://api.apify.com/v2" } ], "paths": { "/acts/atesen-software~llm-dataset-builder/run-sync-get-dataset-items": { "post": { "operationId": "run-sync-get-dataset-items-atesen-software-llm-dataset-builder", "x-openai-isConsequential": false, "summary": "Executes an Actor, waits for its completion, and returns Actor's dataset items in response.", "tags": [ "Run Actor" ], "requestBody": { "required": true, "content": { "application/json": { "schema": { "$ref": "#/components/schemas/inputSchema" } } } }, "parameters": [ { "name": "token", "in": "query", "required": true, "schema": { "type": "string" }, "description": "Enter your Apify token here" } ], "responses": { "200": { "description": "OK" } } } }, "/acts/atesen-software~llm-dataset-builder/runs": { "post": { "operationId": "runs-sync-atesen-software-llm-dataset-builder", "x-openai-isConsequential": false, "summary": "Executes an Actor and returns information about the initiated run in response.", "tags": [ "Run Actor" ], "requestBody": { "required": true, "content": { "application/json": { "schema": { "$ref": "#/components/schemas/inputSchema" } } } }, "parameters": [ { "name": "token", "in": "query", "required": true, "schema": { "type": "string" }, "description": "Enter your Apify token here" } ], "responses": { "200": { "description": "OK", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/runsResponseSchema" } } } } } } }, "/acts/atesen-software~llm-dataset-builder/run-sync": { "post": { "operationId": "run-sync-atesen-software-llm-dataset-builder", "x-openai-isConsequential": false, "summary": "Executes an Actor, waits for completion, and returns the OUTPUT from Key-value store in response.", "tags": [ "Run Actor" ], "requestBody": { "required": true, "content": { "application/json": { "schema": { "$ref": "#/components/schemas/inputSchema" } } } }, "parameters": [ { "name": "token", "in": "query", "required": true, "schema": { "type": "string" }, "description": "Enter your Apify token here" } ], "responses": { "200": { "description": "OK" } } } } }, "components": { "schemas": { "inputSchema": { "type": "object", "required": [ "startUrls" ], "properties": { "startUrls": { "title": "Start URLs", "type": "array", "description": "Pages where the crawl starts. By default the crawl stays under the path of each start URL, so https://docs.example.com/guide/ will not leave /guide/. Only public pages are crawled and robots.txt is always respected.", "items": { "type": "object", "required": [ "url" ], "properties": { "url": { "type": "string", "title": "URL of a web page", "format": "uri" } } } }, "maxPages": { "title": "Max pages", "minimum": 1, "maximum": 1000000, "type": "integer", "description": "Maximum number of pages to push to the dataset. The crawl stops as soon as this many pages have been extracted. You are charged per page pushed, so this is also your cost cap.", "default": 100 }, "crawlScope": { "title": "Crawl scope", "enum": [ "start-path-prefix", "same-hostname", "same-domain" ], "type": "string", "description": "Which links are followed. 'Under the start URL path' keeps the crawl inside the directory of each start URL (best for docs sites with several versions or languages). 'Same hostname' follows every link on the same host. 'Same domain' also follows subdomains.", "default": "start-path-prefix" }, "includeGlobs": { "title": "Include URLs (globs)", "type": "array", "description": "Optional. Only links matching at least one of these glob patterns are followed, for example https://docs.example.com/guide/**. Start URLs are always crawled.", "items": { "type": "object", "required": [ "glob" ], "properties": { "glob": { "type": "string", "title": "Glob of a web page" } } } }, "excludeGlobs": { "title": "Exclude URLs (globs)", "type": "array", "description": "Optional. Links matching any of these glob patterns are never followed, for example https://docs.example.com/**/changelog/**.", "items": { "type": "object", "required": [ "glob" ], "properties": { "glob": { "type": "string", "title": "Glob of a web page" } } } }, "maxCrawlDepth": { "title": "Max crawl depth", "minimum": 0, "maximum": 1000, "type": "integer", "description": "How many link hops away from the start URLs the crawl may go. 0 crawls only the start URLs.", "default": 20 }, "chunkSize": { "title": "Chunk size (tokens)", "minimum": 64, "maximum": 8000, "type": "integer", "description": "Maximum size of each chunk in tokens (OpenAI cl100k_base tokenizer, a good proxy for most embedding models). Chunks follow headings, paragraphs, lists, tables and code blocks and never cut through them unless a single block is larger than this limit.", "default": 800 }, "chunkOverlap": { "title": "Chunk overlap (tokens)", "minimum": 0, "maximum": 4000, "type": "integer", "description": "Tokens of context repeated at the start of the next chunk, taken as whole sentences or lines from the end of the previous chunk. Code blocks and tables are never cut for overlap. Cannot exceed half of the chunk size.", "default": 100 }, "minWordCount": { "title": "Minimum words per page", "minimum": 0, "maximum": 100000, "type": "integer", "description": "Pages whose main content has fewer words than this are skipped (and not charged). Useful to drop empty redirect, login or listing pages.", "default": 10 }, "includeLinks": { "title": "Keep links in Markdown", "type": "boolean", "description": "Keep hyperlinks as [text](absolute-url). Turn off to get plain text without URLs, which uses fewer tokens.", "default": true }, "includeImages": { "title": "Keep images in Markdown", "type": "boolean", "description": "Keep images as ![alt](absolute-url). Off by default because image links add tokens but no text for most RAG use cases.", "default": false }, "maxConcurrency": { "title": "Max concurrency", "minimum": 1, "maximum": 20, "type": "integer", "description": "Maximum number of pages fetched in parallel. The crawler starts slowly and halves its speed when a site answers with HTTP 429 or 503.", "default": 5 }, "requestDelaySecs": { "title": "Delay between requests (seconds)", "minimum": 0, "maximum": 30, "type": "number", "description": "Minimum pause between two requests to the same site. If robots.txt asks for a longer Crawl-delay, that value is used instead.", "default": 0.5 }, "proxyConfiguration": { "title": "Proxy configuration (optional)", "type": "object", "description": "Direct public connections only. Proxy routes are rejected because their target DNS cannot be pinned by this Actor. Leave useApifyProxy false and proxyUrls empty." } } }, "runsResponseSchema": { "type": "object", "properties": { "data": { "type": "object", "properties": { "id": { "type": "string" }, "actId": { "type": "string" }, "userId": { "type": "string" }, "startedAt": { "type": "string", "format": "date-time", "example": "2025-01-08T00:00:00.000Z" }, "finishedAt": { "type": "string", "format": "date-time", "example": "2025-01-08T00:00:00.000Z" }, "status": { "type": "string", "example": "READY" }, "meta": { "type": "object", "properties": { "origin": { "type": "string", "example": "API" }, "userAgent": { "type": "string" } } }, "stats": { "type": "object", "properties": { "inputBodyLen": { "type": "integer", "example": 2000 }, "rebootCount": { "type": "integer", "example": 0 }, "restartCount": { "type": "integer", "example": 0 }, "resurrectCount": { "type": "integer", "example": 0 }, "computeUnits": { "type": "integer", "example": 0 } } }, "options": { "type": "object", "properties": { "build": { "type": "string", "example": "latest" }, "timeoutSecs": { "type": "integer", "example": 300 }, "memoryMbytes": { "type": "integer", "example": 1024 }, "diskMbytes": { "type": "integer", "example": 2048 } } }, "buildId": { "type": "string" }, "defaultKeyValueStoreId": { "type": "string" }, "defaultDatasetId": { "type": "string" }, "defaultRequestQueueId": { "type": "string" }, "buildNumber": { "type": "string", "example": "1.0.0" }, "containerUrl": { "type": "string" }, "usage": { "type": "object", "properties": { "ACTOR_COMPUTE_UNITS": { "type": "integer", "example": 0 }, "DATASET_READS": { "type": "integer", "example": 0 }, "DATASET_WRITES": { "type": "integer", "example": 0 }, "KEY_VALUE_STORE_READS": { "type": "integer", "example": 0 }, "KEY_VALUE_STORE_WRITES": { "type": "integer", "example": 1 }, "KEY_VALUE_STORE_LISTS": { "type": "integer", "example": 0 }, "REQUEST_QUEUE_READS": { "type": "integer", "example": 0 }, "REQUEST_QUEUE_WRITES": { "type": "integer", "example": 0 }, "DATA_TRANSFER_INTERNAL_GBYTES": { "type": "integer", "example": 0 }, "DATA_TRANSFER_EXTERNAL_GBYTES": { "type": "integer", "example": 0 }, "PROXY_RESIDENTIAL_TRANSFER_GBYTES": { "type": "integer", "example": 0 }, "PROXY_SERPS": { "type": "integer", "example": 0 }, "PROXY_UNBLOCKER_UNITS": { "type": "integer", "example": 0 } } }, "usageTotalUsd": { "type": "number", "example": 0.00005 }, "usageUsd": { "type": "object", "properties": { "ACTOR_COMPUTE_UNITS": { "type": "integer", "example": 0 }, "DATASET_READS": { "type": "integer", "example": 0 }, "DATASET_WRITES": { "type": "integer", "example": 0 }, "KEY_VALUE_STORE_READS": { "type": "integer", "example": 0 }, "KEY_VALUE_STORE_WRITES": { "type": "number", "example": 0.00005 }, "KEY_VALUE_STORE_LISTS": { "type": "integer", "example": 0 }, "REQUEST_QUEUE_READS": { "type": "integer", "example": 0 }, "REQUEST_QUEUE_WRITES": { "type": "integer", "example": 0 }, "DATA_TRANSFER_INTERNAL_GBYTES": { "type": "integer", "example": 0 }, "DATA_TRANSFER_EXTERNAL_GBYTES": { "type": "integer", "example": 0 }, "PROXY_RESIDENTIAL_TRANSFER_GBYTES": { "type": "integer", "example": 0 }, "PROXY_SERPS": { "type": "integer", "example": 0 }, "PROXY_UNBLOCKER_UNITS": { "type": "integer", "example": 0 } } } } } } } } } }