{ "openapi": "3.0.1", "info": { "title": "Website to Markdown Crawler for LLM & RAG (Sitemaps, JS)", "description": "Crawl a whole website into clean Markdown and text for LLMs, RAG and vector databases. Fast HTTP crawling with an automatic headless-browser fallback for JavaScript pages. Sitemaps, llms.txt, globs, depth limits. Respects robots.txt.", "version": "0.1", "x-build-id": "z9xouIp3hqKn5g4ax" }, "servers": [ { "url": "https://api.apify.com/v2" } ], "paths": { "/acts/datlab~site-to-markdown-crawler/run-sync-get-dataset-items": { "post": { "operationId": "run-sync-get-dataset-items-datlab-site-to-markdown-crawler", "x-openai-isConsequential": false, "summary": "Executes an Actor, waits for its completion, and returns Actor's dataset items in response.", "tags": [ "Run Actor" ], "requestBody": { "required": true, "content": { "application/json": { "schema": { "$ref": "#/components/schemas/inputSchema" } } } }, "parameters": [ { "name": "token", "in": "query", "required": true, "schema": { "type": "string" }, "description": "Enter your Apify token here" } ], "responses": { "200": { "description": "OK" } } } }, "/acts/datlab~site-to-markdown-crawler/runs": { "post": { "operationId": "runs-sync-datlab-site-to-markdown-crawler", "x-openai-isConsequential": false, "summary": "Executes an Actor and returns information about the initiated run in response.", "tags": [ "Run Actor" ], "requestBody": { "required": true, "content": { "application/json": { "schema": { "$ref": "#/components/schemas/inputSchema" } } } }, "parameters": [ { "name": "token", "in": "query", "required": true, "schema": { "type": "string" }, "description": "Enter your Apify token here" } ], "responses": { "200": { "description": "OK", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/runsResponseSchema" } } } } } } }, "/acts/datlab~site-to-markdown-crawler/run-sync": { "post": { "operationId": "run-sync-datlab-site-to-markdown-crawler", "x-openai-isConsequential": false, "summary": "Executes an Actor, waits for completion, and returns the OUTPUT from Key-value store in response.", "tags": [ "Run Actor" ], "requestBody": { "required": true, "content": { "application/json": { "schema": { "$ref": "#/components/schemas/inputSchema" } } } }, "parameters": [ { "name": "token", "in": "query", "required": true, "schema": { "type": "string" }, "description": "Enter your Apify token here" } ], "responses": { "200": { "description": "OK" } } } } }, "components": { "schemas": { "inputSchema": { "type": "object", "required": [ "startUrls" ], "properties": { "startUrls": { "title": "Start URLs", "type": "array", "description": "Where to start. The crawler follows links on the same host whose path starts with a start URL's path (e.g. https://example.com/docs crawls /docs/... only), unless you set include globs.", "items": { "type": "object", "required": [ "url" ], "properties": { "url": { "type": "string", "title": "URL of a web page", "format": "uri" } } } }, "crawlerType": { "title": "Crawler type", "enum": [ "auto", "cheerio", "playwright:chromium" ], "type": "string", "description": "Auto crawls with plain HTTP and renders only client-side (JavaScript) pages with Chromium. You pay the browser price only for those pages.", "default": "auto" }, "includeUrlGlobs": { "title": "Include URLs (globs)", "type": "array", "description": "Optional. Only follow links matching at least one glob, e.g. https://example.com/{docs,blog}/**. Overrides the default start-URL path scope.", "default": [], "items": { "type": "object", "required": [ "glob" ], "properties": { "glob": { "type": "string", "title": "Glob of a web page" } } } }, "excludeUrlGlobs": { "title": "Exclude URLs (globs)", "type": "array", "description": "Never follow links matching these globs, e.g. **/tag/** or **/*?page=*.", "default": [], "items": { "type": "object", "required": [ "glob" ], "properties": { "glob": { "type": "string", "title": "Glob of a web page" } } } }, "maxCrawlDepth": { "title": "Max crawl depth", "minimum": 0, "type": "integer", "description": "How many links away from the start URLs to go. 0 = start URLs only.", "default": 20 }, "maxCrawlPages": { "title": "Max pages", "minimum": 1, "type": "integer", "description": "Maximum pages to crawl (including duplicates and failed pages). Caps your cost.", "default": 1000 }, "maxResults": { "title": "Max results", "minimum": 1, "type": "integer", "description": "Stop after saving this many results.", "default": 10000000 }, "useSitemaps": { "title": "Load URLs from sitemaps", "type": "boolean", "description": "Also discover pages from the site's sitemaps (robots.txt Sitemap entries and /sitemap.xml). Only in-scope URLs are used.", "default": false }, "useLlmsTxt": { "title": "Crawl /llms.txt and Markdown files", "type": "boolean", "description": "Fetch /llms.txt if the site has one and follow the Markdown files it links. Markdown is returned as-is.", "default": false }, "keepUrlFragments": { "title": "URL #fragments identify unique pages", "type": "boolean", "description": "Treat example.com/#a and example.com/#b as different pages (for hash-routed sites).", "default": false }, "ignoreCanonicalUrl": { "title": "Ignore canonical URLs", "type": "boolean", "description": "By default, pages with the same rel=canonical URL are saved once. Turn on to save every URL.", "default": false }, "maxConcurrency": { "title": "Max concurrency", "minimum": 1, "maximum": 50, "type": "integer", "description": "Parallel HTTP requests (browser pages are capped at 3).", "default": 10 }, "minDelayPerHostMs": { "title": "Minimum delay per host (ms)", "minimum": 0, "maximum": 60000, "type": "integer", "description": "Politeness: minimum gap between requests to the same host. robots.txt Crawl-delay is honored when larger.", "default": 250 }, "requestTimeoutSecs": { "title": "Page request timeout", "minimum": 5, "maximum": 600, "type": "integer", "description": "Maximum time to load one page.", "default": 60 }, "maxRequestRetries": { "title": "Max retries", "minimum": 0, "maximum": 10, "type": "integer", "description": "Retries on network errors and 5xx responses.", "default": 3 }, "dynamicContentWaitSecs": { "title": "Wait for dynamic content (browser)", "minimum": 0, "maximum": 60, "type": "integer", "description": "For browser-rendered pages: maximum wait for the network to go idle.", "default": 10 }, "waitForSelector": { "title": "Wait for selector (browser)", "type": "string", "description": "Optional. The page fails if this element doesn't appear." }, "softWaitForSelector": { "title": "Soft wait for selector (browser)", "type": "string", "description": "Optional. Wait for this element, but continue if it doesn't appear." }, "maxScrollHeightPixels": { "title": "Maximum scroll height (browser)", "minimum": 0, "type": "integer", "description": "Scroll down this far to trigger lazy-loaded content. 0 disables scrolling.", "default": 5000 }, "clickElementsCssSelector": { "title": "Expand clickable elements (browser)", "type": "string", "description": "Click matching elements (up to 50 per page) to expand collapsed content, e.g. [aria-expanded=\"false\"]." }, "removeCookieWarnings": { "title": "Remove cookie banners", "type": "boolean", "description": "Strip common cookie-consent banners from the output. Nothing is clicked or accepted.", "default": true }, "blockMedia": { "title": "Block images, video and fonts (browser)", "type": "boolean", "description": "Faster, cheaper rendering.", "default": true }, "keepElementsCssSelector": { "title": "Keep only these elements (CSS selector)", "type": "string", "description": "Optional, e.g. article, main .content." }, "removeElementsCssSelector": { "title": "Remove HTML elements (CSS selector)", "type": "string", "description": "Removed before conversion. Empty = default (nav, footer, scripts, dialogs, buttons...)." }, "htmlTransformer": { "title": "Main-content extraction", "enum": [ "readableTextIfPossible", "readableText", "none" ], "type": "string", "description": "Readable text uses Mozilla Readability. 'If possible' falls back to the whole cleaned page when Readability finds too little.", "default": "readableTextIfPossible" }, "readableTextCharThreshold": { "title": "Readable text minimum characters", "minimum": 0, "type": "integer", "description": "Shorter Readability results count as failed.", "default": 100 }, "aggressivePrune": { "title": "Remove duplicate text lines", "type": "boolean", "description": "Drop lines already seen on earlier pages (repeated headers, sidebars, banners). Useful for RAG.", "default": false }, "saveMarkdown": { "title": "Save Markdown", "type": "boolean", "description": "Include a markdown field.", "default": true }, "saveText": { "title": "Save plain text", "type": "boolean", "description": "Include a text field.", "default": true }, "saveHtml": { "title": "Save cleaned HTML to dataset", "type": "boolean", "description": "Include an html field (can make the dataset large).", "default": false }, "saveHtmlAsFile": { "title": "Save cleaned HTML to key-value store", "type": "boolean", "description": "Store HTML as a file and add htmlUrl to each item.", "default": false }, "saveLinks": { "title": "Save links", "type": "boolean", "description": "Include all links found on each page.", "default": false }, "saveScreenshots": { "title": "Save screenshots (browser pages only)", "type": "boolean", "description": "Store a viewport JPEG for browser-rendered pages and add screenshotUrl.", "default": false }, "saveFiles": { "title": "Save linked documents", "type": "boolean", "description": "Download same-site PDF, DOC(X), XLS(X), PPT(X), CSV, TXT, MD and JSON files linked from crawled pages (max 50 MB each) to the key-value store. Each saved file is charged as a file event.", "default": false }, "storeSkippedUrls": { "title": "Store skipped URLs", "type": "boolean", "description": "Save URLs skipped because of robots.txt or the page limit to the SKIPPED_URLS record.", "default": false }, "proxyConfiguration": { "title": "Proxy", "type": "object", "description": "Optional. Datacenter proxies only; residential and SERP groups are not supported.", "default": { "useApifyProxy": false } }, "debugMode": { "title": "Debug mode", "type": "boolean", "description": "Add response headers and page signals to each item.", "default": false } } }, "runsResponseSchema": { "type": "object", "properties": { "data": { "type": "object", "properties": { "id": { "type": "string" }, "actId": { "type": "string" }, "userId": { "type": "string" }, "startedAt": { "type": "string", "format": "date-time", "example": "2025-01-08T00:00:00.000Z" }, "finishedAt": { "type": "string", "format": "date-time", "example": "2025-01-08T00:00:00.000Z" }, "status": { "type": "string", "example": "READY" }, "meta": { "type": "object", "properties": { "origin": { "type": "string", "example": "API" }, "userAgent": { "type": "string" } } }, "stats": { "type": "object", "properties": { "inputBodyLen": { "type": "integer", "example": 2000 }, "rebootCount": { "type": "integer", "example": 0 }, "restartCount": { "type": "integer", "example": 0 }, "resurrectCount": { "type": "integer", "example": 0 }, "computeUnits": { "type": "integer", "example": 0 } } }, "options": { "type": "object", "properties": { "build": { "type": "string", "example": "latest" }, "timeoutSecs": { "type": "integer", "example": 300 }, "memoryMbytes": { "type": "integer", "example": 1024 }, "diskMbytes": { "type": "integer", "example": 2048 } } }, "buildId": { "type": "string" }, "defaultKeyValueStoreId": { "type": "string" }, "defaultDatasetId": { "type": "string" }, "defaultRequestQueueId": { "type": "string" }, "buildNumber": { "type": "string", "example": "1.0.0" }, "containerUrl": { "type": "string" }, "usage": { "type": "object", "properties": { "ACTOR_COMPUTE_UNITS": { "type": "integer", "example": 0 }, "DATASET_READS": { "type": "integer", "example": 0 }, "DATASET_WRITES": { "type": "integer", "example": 0 }, "KEY_VALUE_STORE_READS": { "type": "integer", "example": 0 }, "KEY_VALUE_STORE_WRITES": { "type": "integer", "example": 1 }, "KEY_VALUE_STORE_LISTS": { "type": "integer", "example": 0 }, "REQUEST_QUEUE_READS": { "type": "integer", "example": 0 }, "REQUEST_QUEUE_WRITES": { "type": "integer", "example": 0 }, "DATA_TRANSFER_INTERNAL_GBYTES": { "type": "integer", "example": 0 }, "DATA_TRANSFER_EXTERNAL_GBYTES": { "type": "integer", "example": 0 }, "PROXY_RESIDENTIAL_TRANSFER_GBYTES": { "type": "integer", "example": 0 }, "PROXY_SERPS": { "type": "integer", "example": 0 }, "PROXY_UNBLOCKER_UNITS": { "type": "integer", "example": 0 } } }, "usageTotalUsd": { "type": "number", "example": 0.00005 }, "usageUsd": { "type": "object", "properties": { "ACTOR_COMPUTE_UNITS": { "type": "integer", "example": 0 }, "DATASET_READS": { "type": "integer", "example": 0 }, "DATASET_WRITES": { "type": "integer", "example": 0 }, "KEY_VALUE_STORE_READS": { "type": "integer", "example": 0 }, "KEY_VALUE_STORE_WRITES": { "type": "number", "example": 0.00005 }, "KEY_VALUE_STORE_LISTS": { "type": "integer", "example": 0 }, "REQUEST_QUEUE_READS": { "type": "integer", "example": 0 }, "REQUEST_QUEUE_WRITES": { "type": "integer", "example": 0 }, "DATA_TRANSFER_INTERNAL_GBYTES": { "type": "integer", "example": 0 }, "DATA_TRANSFER_EXTERNAL_GBYTES": { "type": "integer", "example": 0 }, "PROXY_RESIDENTIAL_TRANSFER_GBYTES": { "type": "integer", "example": 0 }, "PROXY_SERPS": { "type": "integer", "example": 0 }, "PROXY_UNBLOCKER_UNITS": { "type": "integer", "example": 0 } } } } } } } } } }