{ "openapi": "3.0.1", "info": { "title": "Website to Markdown Crawler for AI, RAG & LLMs", "description": "Crawl any website and get clean main-content Markdown for every page: no menus or footers. RAG-ready chunks, llms.txt, and an 'only changed pages' mode for cheap scheduled refreshes. $1 per 1,000 pages, all included.", "version": "0.1", "x-build-id": "RcRvgWwv4tpDrpShe" }, "servers": [ { "url": "https://api.apify.com/v2" } ], "paths": { "/acts/dima_kadirovich~website-to-markdown/run-sync-get-dataset-items": { "post": { "operationId": "run-sync-get-dataset-items-dima_kadirovich-website-to-markdown", "x-openai-isConsequential": false, "summary": "Executes an Actor, waits for its completion, and returns Actor's dataset items in response.", "tags": [ "Run Actor" ], "requestBody": { "required": true, "content": { "application/json": { "schema": { "$ref": "#/components/schemas/inputSchema" } } } }, "parameters": [ { "name": "token", "in": "query", "required": true, "schema": { "type": "string" }, "description": "Enter your Apify token here" } ], "responses": { "200": { "description": "OK" } } } }, "/acts/dima_kadirovich~website-to-markdown/runs": { "post": { "operationId": "runs-sync-dima_kadirovich-website-to-markdown", "x-openai-isConsequential": false, "summary": "Executes an Actor and returns information about the initiated run in response.", "tags": [ "Run Actor" ], "requestBody": { "required": true, "content": { "application/json": { "schema": { "$ref": "#/components/schemas/inputSchema" } } } }, "parameters": [ { "name": "token", "in": "query", "required": true, "schema": { "type": "string" }, "description": "Enter your Apify token here" } ], "responses": { "200": { "description": "OK", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/runsResponseSchema" } } } } } } }, "/acts/dima_kadirovich~website-to-markdown/run-sync": { "post": { "operationId": "run-sync-dima_kadirovich-website-to-markdown", "x-openai-isConsequential": false, "summary": "Executes an Actor, waits for completion, and returns the OUTPUT from Key-value store in response.", "tags": [ "Run Actor" ], "requestBody": { "required": true, "content": { "application/json": { "schema": { "$ref": "#/components/schemas/inputSchema" } } } }, "parameters": [ { "name": "token", "in": "query", "required": true, "schema": { "type": "string" }, "description": "Enter your Apify token here" } ], "responses": { "200": { "description": "OK" } } } } }, "components": { "schemas": { "inputSchema": { "type": "object", "required": [ "startUrls" ], "properties": { "startUrls": { "title": "Websites or start pages", "type": "array", "description": "Where to start crawling, for example a documentation homepage. The crawler follows links and the sitemap within the same website.", "items": { "type": "object", "required": [ "url" ], "properties": { "url": { "type": "string", "title": "URL of a web page", "format": "uri" } } } }, "maxPages": { "title": "Max pages", "minimum": 1, "maximum": 100000, "type": "integer", "description": "Upper limit of pages crawled.", "default": 100 }, "includeUrlPatterns": { "title": "Only crawl URLs matching", "type": "array", "description": "Regular expressions. If set, only URLs matching at least one are crawled, for example `/docs/` to get only documentation pages.", "items": { "type": "string" } }, "excludeUrlPatterns": { "title": "Skip URLs matching", "type": "array", "description": "Regular expressions for URLs to skip, for example `/blog/tag/` or `\\?page=`.", "items": { "type": "string" } }, "onlyChangedPages": { "title": "Only new or changed pages (refresh mode)", "type": "boolean", "description": "Remember each page's content from earlier runs with the same start URLs, and save only pages that are new or changed. Unchanged pages are skipped and not charged, which makes scheduled refreshes of your RAG index very cheap.", "default": false }, "chunkSize": { "title": "Chunk size (tokens)", "minimum": 0, "maximum": 8000, "type": "integer", "description": "Also split each page into chunks of about this many tokens for vector databases, keeping headings and code blocks together. Leave empty or 0 for no chunks. Typical: 500–1000." }, "chunkOverlap": { "title": "Chunk overlap (tokens)", "minimum": 0, "type": "integer", "description": "How many tokens each chunk repeats from the previous one, to keep context.", "default": 100 }, "generateLlmsTxt": { "title": "Create llms.txt and llms-full.txt", "type": "boolean", "description": "Create an llms.txt index of the crawled pages and an llms-full.txt with all content, following the llmstxt.org format.", "default": true }, "includeLinks": { "title": "Keep links in Markdown", "type": "boolean", "description": "Keep hyperlinks as [text](url). Turn off for plain text-like Markdown.", "default": true }, "useSitemap": { "title": "Use sitemap.xml", "type": "boolean", "description": "Also find pages from the website's sitemap.", "default": true }, "respectRobotsTxt": { "title": "Respect robots.txt", "type": "boolean", "description": "Skip pages that robots.txt disallows.", "default": true }, "maxDepth": { "title": "Max link depth", "minimum": 0, "type": "integer", "description": "How many clicks away from the start page to crawl. 0 = only the start pages.", "default": 20 }, "includeSubdomains": { "title": "Include subdomains", "type": "boolean", "description": "Also crawl subdomains like docs.example.com when starting from example.com.", "default": false }, "saveHtml": { "title": "Also save raw HTML", "type": "boolean", "description": "Add the original HTML of each page to the output.", "default": false }, "proxyConfiguration": { "title": "Proxy", "type": "object", "description": "Only needed if a website blocks the crawler.", "default": { "useApifyProxy": false } } } }, "runsResponseSchema": { "type": "object", "properties": { "data": { "type": "object", "properties": { "id": { "type": "string" }, "actId": { "type": "string" }, "userId": { "type": "string" }, "startedAt": { "type": "string", "format": "date-time", "example": "2025-01-08T00:00:00.000Z" }, "finishedAt": { "type": "string", "format": "date-time", "example": "2025-01-08T00:00:00.000Z" }, "status": { "type": "string", "example": "READY" }, "meta": { "type": "object", "properties": { "origin": { "type": "string", "example": "API" }, "userAgent": { "type": "string" } } }, "stats": { "type": "object", "properties": { "inputBodyLen": { "type": "integer", "example": 2000 }, "rebootCount": { "type": "integer", "example": 0 }, "restartCount": { "type": "integer", "example": 0 }, "resurrectCount": { "type": "integer", "example": 0 }, "computeUnits": { "type": "integer", "example": 0 } } }, "options": { "type": "object", "properties": { "build": { "type": "string", "example": "latest" }, "timeoutSecs": { "type": "integer", "example": 300 }, "memoryMbytes": { "type": "integer", "example": 1024 }, "diskMbytes": { "type": "integer", "example": 2048 } } }, "buildId": { "type": "string" }, "defaultKeyValueStoreId": { "type": "string" }, "defaultDatasetId": { "type": "string" }, "defaultRequestQueueId": { "type": "string" }, "buildNumber": { "type": "string", "example": "1.0.0" }, "containerUrl": { "type": "string" }, "usage": { "type": "object", "properties": { "ACTOR_COMPUTE_UNITS": { "type": "integer", "example": 0 }, "DATASET_READS": { "type": "integer", "example": 0 }, "DATASET_WRITES": { "type": "integer", "example": 0 }, "KEY_VALUE_STORE_READS": { "type": "integer", "example": 0 }, "KEY_VALUE_STORE_WRITES": { "type": "integer", "example": 1 }, "KEY_VALUE_STORE_LISTS": { "type": "integer", "example": 0 }, "REQUEST_QUEUE_READS": { "type": "integer", "example": 0 }, "REQUEST_QUEUE_WRITES": { "type": "integer", "example": 0 }, "DATA_TRANSFER_INTERNAL_GBYTES": { "type": "integer", "example": 0 }, "DATA_TRANSFER_EXTERNAL_GBYTES": { "type": "integer", "example": 0 }, "PROXY_RESIDENTIAL_TRANSFER_GBYTES": { "type": "integer", "example": 0 }, "PROXY_SERPS": { "type": "integer", "example": 0 }, "PROXY_UNBLOCKER_UNITS": { "type": "integer", "example": 0 } } }, "usageTotalUsd": { "type": "number", "example": 0.00005 }, "usageUsd": { "type": "object", "properties": { "ACTOR_COMPUTE_UNITS": { "type": "integer", "example": 0 }, "DATASET_READS": { "type": "integer", "example": 0 }, "DATASET_WRITES": { "type": "integer", "example": 0 }, "KEY_VALUE_STORE_READS": { "type": "integer", "example": 0 }, "KEY_VALUE_STORE_WRITES": { "type": "number", "example": 0.00005 }, "KEY_VALUE_STORE_LISTS": { "type": "integer", "example": 0 }, "REQUEST_QUEUE_READS": { "type": "integer", "example": 0 }, "REQUEST_QUEUE_WRITES": { "type": "integer", "example": 0 }, "DATA_TRANSFER_INTERNAL_GBYTES": { "type": "integer", "example": 0 }, "DATA_TRANSFER_EXTERNAL_GBYTES": { "type": "integer", "example": 0 }, "PROXY_RESIDENTIAL_TRANSFER_GBYTES": { "type": "integer", "example": 0 }, "PROXY_SERPS": { "type": "integer", "example": 0 }, "PROXY_UNBLOCKER_UNITS": { "type": "integer", "example": 0 } } } } } } } } } }