{ "openapi": "3.0.1", "info": { "title": "Website to Markdown Crawler for LLMs and RAG", "description": "Crawl a whole website, docs section or blog and save every page as clean Markdown or text for LLMs, RAG, vector databases and AI chatbots. Depth, page and path limits, include and exclude globs. Stays on the site, respects robots.txt, no browser. $0.80 per 1,000 pages.", "version": "1.0", "x-build-id": "EPjVqdFjK76y53HMw" }, "servers": [ { "url": "https://api.apify.com/v2" } ], "paths": { "/acts/fecklesspleb~website-to-markdown-crawler/run-sync-get-dataset-items": { "post": { "operationId": "run-sync-get-dataset-items-fecklesspleb-website-to-markdown-crawler", "x-openai-isConsequential": false, "summary": "Executes an Actor, waits for its completion, and returns Actor's dataset items in response.", "tags": [ "Run Actor" ], "requestBody": { "required": true, "content": { "application/json": { "schema": { "$ref": "#/components/schemas/inputSchema" } } } }, "parameters": [ { "name": "token", "in": "query", "required": true, "schema": { "type": "string" }, "description": "Enter your Apify token here" } ], "responses": { "200": { "description": "OK" } } } }, "/acts/fecklesspleb~website-to-markdown-crawler/runs": { "post": { "operationId": "runs-sync-fecklesspleb-website-to-markdown-crawler", "x-openai-isConsequential": false, "summary": "Executes an Actor and returns information about the initiated run in response.", "tags": [ "Run Actor" ], "requestBody": { "required": true, "content": { "application/json": { "schema": { "$ref": "#/components/schemas/inputSchema" } } } }, "parameters": [ { "name": "token", "in": "query", "required": true, "schema": { "type": "string" }, "description": "Enter your Apify token here" } ], "responses": { "200": { "description": "OK", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/runsResponseSchema" } } } } } } }, "/acts/fecklesspleb~website-to-markdown-crawler/run-sync": { "post": { "operationId": "run-sync-fecklesspleb-website-to-markdown-crawler", "x-openai-isConsequential": false, "summary": "Executes an Actor, waits for completion, and returns the OUTPUT from Key-value store in response.", "tags": [ "Run Actor" ], "requestBody": { "required": true, "content": { "application/json": { "schema": { "$ref": "#/components/schemas/inputSchema" } } } }, "parameters": [ { "name": "token", "in": "query", "required": true, "schema": { "type": "string" }, "description": "Enter your Apify token here" } ], "responses": { "200": { "description": "OK" } } } } }, "components": { "schemas": { "inputSchema": { "type": "object", "required": [ "startUrls" ], "properties": { "startUrls": { "title": "Start URLs", "type": "array", "description": "Where the crawl begins, as objects: [{\"url\": \"https://docs.python.org/3/tutorial/\"}]. Links are followed on the same host only (www. is ignored). A URL without http(s):// gets https:// added. Start URLs are always fetched; scope rules and globs apply to the links found on them.", "default": [ { "url": "https://docs.python.org/3/tutorial/" } ], "items": { "type": "object", "required": [ "url" ], "properties": { "url": { "type": "string", "title": "URL of a web page", "format": "uri" } } } }, "maxPages": { "title": "Max pages", "minimum": 1, "type": "integer", "description": "Stop after saving this many pages. Default 10. This is the cost cap: at most maxPages x $0.0008 in page events per run. Failed, skipped and robots-blocked pages are free and are not counted.", "default": 10 }, "maxDepth": { "title": "Max link depth", "minimum": 0, "type": "integer", "description": "How many clicks away from a start URL to follow. Default 5. 0 saves only the start URLs.", "default": 5 }, "stayWithinPath": { "title": "Stay within the start URL's folder", "type": "boolean", "description": "If true (default), only URLs under the start URL's folder are crawled: with https://example.com/docs/ only /docs/... pages. Set false to crawl the whole host.", "default": true }, "includeUrlGlobs": { "title": "Only include URLs matching", "type": "array", "description": "Optional glob patterns matched against the full URL (tracking parameters and #fragments removed), e.g. [\"https://example.com/blog/*\"]. When set, only matching links are queued, so pages reachable only through non-matching pages are not found. Leave empty to include everything in scope.", "default": [], "items": { "type": "string" } }, "excludeUrlGlobs": { "title": "Exclude URLs matching", "type": "array", "description": "Glob patterns matched against the full URL; matching links are not queued, e.g. [\"*/tag/*\", \"*?page=*\"].", "default": [], "items": { "type": "string" } }, "contentMode": { "title": "Content", "enum": [ "main", "page", "full" ], "type": "string", "description": "\"main\" (default): main content only, boilerplate removed with trafilatura, with a whole-page fallback. \"page\": whole page without nav, header and footer. \"full\": whole page including navigation.", "default": "main" }, "outputFormat": { "title": "Output format", "enum": [ "markdown", "text" ], "type": "string", "description": "\"markdown\" (default) puts the content in the markdown field, keeping headings, lists, tables and links. \"text\" puts plain text in the text field instead.", "default": "markdown" }, "maxCharsPerPage": { "title": "Max characters per page", "minimum": 0, "type": "integer", "description": "Cut the markdown or text of each page to this many characters, e.g. 4000; truncated is then true. Default 0 = no limit.", "default": 0 }, "includeFailedPages": { "title": "Include failed pages", "type": "boolean", "description": "If true, pages that returned HTTP 400 or higher, or failed to load, get a free row with error set, useful for spotting broken links. Pages blocked by robots.txt, and hosts that could not be reached at all, get no row. Default false.", "default": false }, "maxConcurrency": { "title": "Max concurrency", "minimum": 1, "maximum": 20, "type": "integer", "description": "Parallel workers, 1 to 20. Default 5. Requests to one host are spaced at least 0.5 s apart, or by the robots.txt crawl delay.", "default": 5 } } }, "runsResponseSchema": { "type": "object", "properties": { "data": { "type": "object", "properties": { "id": { "type": "string" }, "actId": { "type": "string" }, "userId": { "type": "string" }, "startedAt": { "type": "string", "format": "date-time", "example": "2025-01-08T00:00:00.000Z" }, "finishedAt": { "type": "string", "format": "date-time", "example": "2025-01-08T00:00:00.000Z" }, "status": { "type": "string", "example": "READY" }, "meta": { "type": "object", "properties": { "origin": { "type": "string", "example": "API" }, "userAgent": { "type": "string" } } }, "stats": { "type": "object", "properties": { "inputBodyLen": { "type": "integer", "example": 2000 }, "rebootCount": { "type": "integer", "example": 0 }, "restartCount": { "type": "integer", "example": 0 }, "resurrectCount": { "type": "integer", "example": 0 }, "computeUnits": { "type": "integer", "example": 0 } } }, "options": { "type": "object", "properties": { "build": { "type": "string", "example": "latest" }, "timeoutSecs": { "type": "integer", "example": 300 }, "memoryMbytes": { "type": "integer", "example": 1024 }, "diskMbytes": { "type": "integer", "example": 2048 } } }, "buildId": { "type": "string" }, "defaultKeyValueStoreId": { "type": "string" }, "defaultDatasetId": { "type": "string" }, "defaultRequestQueueId": { "type": "string" }, "buildNumber": { "type": "string", "example": "1.0.0" }, "containerUrl": { "type": "string" }, "usage": { "type": "object", "properties": { "ACTOR_COMPUTE_UNITS": { "type": "integer", "example": 0 }, "DATASET_READS": { "type": "integer", "example": 0 }, "DATASET_WRITES": { "type": "integer", "example": 0 }, "KEY_VALUE_STORE_READS": { "type": "integer", "example": 0 }, "KEY_VALUE_STORE_WRITES": { "type": "integer", "example": 1 }, "KEY_VALUE_STORE_LISTS": { "type": "integer", "example": 0 }, "REQUEST_QUEUE_READS": { "type": "integer", "example": 0 }, "REQUEST_QUEUE_WRITES": { "type": "integer", "example": 0 }, "DATA_TRANSFER_INTERNAL_GBYTES": { "type": "integer", "example": 0 }, "DATA_TRANSFER_EXTERNAL_GBYTES": { "type": "integer", "example": 0 }, "PROXY_RESIDENTIAL_TRANSFER_GBYTES": { "type": "integer", "example": 0 }, "PROXY_SERPS": { "type": "integer", "example": 0 }, "PROXY_UNBLOCKER_UNITS": { "type": "integer", "example": 0 } } }, "usageTotalUsd": { "type": "number", "example": 0.00005 }, "usageUsd": { "type": "object", "properties": { "ACTOR_COMPUTE_UNITS": { "type": "integer", "example": 0 }, "DATASET_READS": { "type": "integer", "example": 0 }, "DATASET_WRITES": { "type": "integer", "example": 0 }, "KEY_VALUE_STORE_READS": { "type": "integer", "example": 0 }, "KEY_VALUE_STORE_WRITES": { "type": "number", "example": 0.00005 }, "KEY_VALUE_STORE_LISTS": { "type": "integer", "example": 0 }, "REQUEST_QUEUE_READS": { "type": "integer", "example": 0 }, "REQUEST_QUEUE_WRITES": { "type": "integer", "example": 0 }, "DATA_TRANSFER_INTERNAL_GBYTES": { "type": "integer", "example": 0 }, "DATA_TRANSFER_EXTERNAL_GBYTES": { "type": "integer", "example": 0 }, "PROXY_RESIDENTIAL_TRANSFER_GBYTES": { "type": "integer", "example": 0 }, "PROXY_SERPS": { "type": "integer", "example": 0 }, "PROXY_UNBLOCKER_UNITS": { "type": "integer", "example": 0 } } } } } } } } } }