{ "openapi": "3.0.1", "info": { "title": "Website to Markdown Crawler for LLM & RAG (fast, no browser)", "description": "Website to Markdown Crawler turns any site into clean Markdown per page for LLMs, RAG pipelines and vector DBs — no headless browser, $1 per 1,000 pages.", "version": "0.1", "x-build-id": "fVlL44yNOLmLVV1rH" }, "servers": [ { "url": "https://api.apify.com/v2" } ], "paths": { "/acts/webdatatools~website-to-markdown/run-sync-get-dataset-items": { "post": { "operationId": "run-sync-get-dataset-items-webdatatools-website-to-markdown", "x-openai-isConsequential": false, "summary": "Executes an Actor, waits for its completion, and returns Actor's dataset items in response.", "tags": [ "Run Actor" ], "requestBody": { "required": true, "content": { "application/json": { "schema": { "$ref": "#/components/schemas/inputSchema" } } } }, "parameters": [ { "name": "token", "in": "query", "required": true, "schema": { "type": "string" }, "description": "Enter your Apify token here" } ], "responses": { "200": { "description": "OK" } } } }, "/acts/webdatatools~website-to-markdown/runs": { "post": { "operationId": "runs-sync-webdatatools-website-to-markdown", "x-openai-isConsequential": false, "summary": "Executes an Actor and returns information about the initiated run in response.", "tags": [ "Run Actor" ], "requestBody": { "required": true, "content": { "application/json": { "schema": { "$ref": "#/components/schemas/inputSchema" } } } }, "parameters": [ { "name": "token", "in": "query", "required": true, "schema": { "type": "string" }, "description": "Enter your Apify token here" } ], "responses": { "200": { "description": "OK", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/runsResponseSchema" } } } } } } }, "/acts/webdatatools~website-to-markdown/run-sync": { "post": { "operationId": "run-sync-webdatatools-website-to-markdown", "x-openai-isConsequential": false, "summary": "Executes an Actor, waits for completion, and returns the OUTPUT from Key-value store in response.", "tags": [ "Run Actor" ], "requestBody": { "required": true, "content": { "application/json": { "schema": { "$ref": "#/components/schemas/inputSchema" } } } }, "parameters": [ { "name": "token", "in": "query", "required": true, "schema": { "type": "string" }, "description": "Enter your Apify token here" } ], "responses": { "200": { "description": "OK" } } } } }, "components": { "schemas": { "inputSchema": { "type": "object", "required": [ "startUrls" ], "properties": { "startUrls": { "title": "Start URLs", "type": "array", "description": "Enter the page(s) to start crawling from, e.g. https://docs.apify.com/platform. The crawler follows links from here in breadth-first order.", "default": [ { "url": "https://docs.apify.com/platform" } ], "items": { "type": "object", "required": [ "url" ], "properties": { "url": { "type": "string", "title": "URL of a web page", "format": "uri" } } } }, "maxPages": { "title": "Max pages", "minimum": 1, "maximum": 5000, "type": "integer", "description": "Enter the maximum number of pages to crawl and convert, e.g. 50. This is the billed unit ($1 per 1,000 pages) — the crawl stops exactly at this count.", "default": 50 }, "maxDepth": { "title": "Max link depth", "minimum": 0, "maximum": 10, "type": "integer", "description": "Enter how many links deep to follow from a start URL, e.g. 3. Set 0 to crawl only the start URL(s).", "default": 3 }, "sameDomainOnly": { "title": "Same domain only", "type": "boolean", "description": "Turn this on to only follow links on the same domain as the start URL (www. is treated as the same domain), and off to also follow links to other domains.", "default": true }, "includePathPrefixes": { "title": "Include path prefixes", "type": "array", "description": "Optional. Only crawl URLs whose path starts with one of these prefixes, e.g. /docs. Leave empty to crawl every path allowed by the other settings.", "default": [], "items": { "type": "string" } }, "excludePathPatterns": { "title": "Exclude URL patterns", "type": "array", "description": "Optional. Regular expressions tested against the full URL; a match is skipped, e.g. \\.(png|jpe?g)$ to skip images. Defaults cover binary files and login/signup pages.", "default": [ "\\.(png|jpe?g|gif|svg|webp|pdf|zip|mp4|css|js)$", "/login", "/signup", "\\?replytocom=" ], "items": { "type": "string" } }, "useSitemap": { "title": "Seed from sitemap.xml", "type": "boolean", "description": "Turn this on to also read /sitemap.xml on each start URL's domain and add its URLs (filtered by the settings above) to the crawl queue, up to Max pages.", "default": true }, "outputFormat": { "title": "Output format", "enum": [ "markdown", "text", "both" ], "type": "string", "description": "Choose what content to put in each row: Markdown only, plain text only, or both. Markdown preserves headings, lists, code blocks and tables.", "default": "markdown" }, "removeSelectors": { "title": "Extra CSS selectors to remove", "type": "array", "description": "Optional. Extra CSS selectors to strip before extracting content, e.g. .cookie-banner or #newsletter-signup, on top of the built-in nav/header/footer/aside removal.", "default": [], "items": { "type": "string" } }, "maxConcurrency": { "title": "Max concurrency", "minimum": 1, "maximum": 50, "type": "integer", "description": "Enter the maximum number of pages fetched in parallel, e.g. 10. Lower it if the target site rate-limits you.", "default": 10 }, "respectRobots": { "title": "Respect robots.txt", "type": "boolean", "description": "Turn this on to skip URLs disallowed by the site's robots.txt file (recommended and on by default).", "default": true }, "proxyConfiguration": { "title": "Proxy", "type": "object", "description": "Optional. Select a proxy configuration, e.g. Apify Proxy with the datacenter group, if the target site blocks requests from shared IPs.", "default": { "useApifyProxy": false } } } }, "runsResponseSchema": { "type": "object", "properties": { "data": { "type": "object", "properties": { "id": { "type": "string" }, "actId": { "type": "string" }, "userId": { "type": "string" }, "startedAt": { "type": "string", "format": "date-time", "example": "2025-01-08T00:00:00.000Z" }, "finishedAt": { "type": "string", "format": "date-time", "example": "2025-01-08T00:00:00.000Z" }, "status": { "type": "string", "example": "READY" }, "meta": { "type": "object", "properties": { "origin": { "type": "string", "example": "API" }, "userAgent": { "type": "string" } } }, "stats": { "type": "object", "properties": { "inputBodyLen": { "type": "integer", "example": 2000 }, "rebootCount": { "type": "integer", "example": 0 }, "restartCount": { "type": "integer", "example": 0 }, "resurrectCount": { "type": "integer", "example": 0 }, "computeUnits": { "type": "integer", "example": 0 } } }, "options": { "type": "object", "properties": { "build": { "type": "string", "example": "latest" }, "timeoutSecs": { "type": "integer", "example": 300 }, "memoryMbytes": { "type": "integer", "example": 1024 }, "diskMbytes": { "type": "integer", "example": 2048 } } }, "buildId": { "type": "string" }, "defaultKeyValueStoreId": { "type": "string" }, "defaultDatasetId": { "type": "string" }, "defaultRequestQueueId": { "type": "string" }, "buildNumber": { "type": "string", "example": "1.0.0" }, "containerUrl": { "type": "string" }, "usage": { "type": "object", "properties": { "ACTOR_COMPUTE_UNITS": { "type": "integer", "example": 0 }, "DATASET_READS": { "type": "integer", "example": 0 }, "DATASET_WRITES": { "type": "integer", "example": 0 }, "KEY_VALUE_STORE_READS": { "type": "integer", "example": 0 }, "KEY_VALUE_STORE_WRITES": { "type": "integer", "example": 1 }, "KEY_VALUE_STORE_LISTS": { "type": "integer", "example": 0 }, "REQUEST_QUEUE_READS": { "type": "integer", "example": 0 }, "REQUEST_QUEUE_WRITES": { "type": "integer", "example": 0 }, "DATA_TRANSFER_INTERNAL_GBYTES": { "type": "integer", "example": 0 }, "DATA_TRANSFER_EXTERNAL_GBYTES": { "type": "integer", "example": 0 }, "PROXY_RESIDENTIAL_TRANSFER_GBYTES": { "type": "integer", "example": 0 }, "PROXY_SERPS": { "type": "integer", "example": 0 } } }, "usageTotalUsd": { "type": "number", "example": 0.00005 }, "usageUsd": { "type": "object", "properties": { "ACTOR_COMPUTE_UNITS": { "type": "integer", "example": 0 }, "DATASET_READS": { "type": "integer", "example": 0 }, "DATASET_WRITES": { "type": "integer", "example": 0 }, "KEY_VALUE_STORE_READS": { "type": "integer", "example": 0 }, "KEY_VALUE_STORE_WRITES": { "type": "number", "example": 0.00005 }, "KEY_VALUE_STORE_LISTS": { "type": "integer", "example": 0 }, "REQUEST_QUEUE_READS": { "type": "integer", "example": 0 }, "REQUEST_QUEUE_WRITES": { "type": "integer", "example": 0 }, "DATA_TRANSFER_INTERNAL_GBYTES": { "type": "integer", "example": 0 }, "DATA_TRANSFER_EXTERNAL_GBYTES": { "type": "integer", "example": 0 }, "PROXY_RESIDENTIAL_TRANSFER_GBYTES": { "type": "integer", "example": 0 }, "PROXY_SERPS": { "type": "integer", "example": 0 } } } } } } } } } }