{ "openapi": "3.0.1", "info": { "title": "Website to Markdown Crawler | LLM & RAG | $0.5 / 1K", "description": "Convert any website to clean Markdown per page for LLM, RAG and vector databases. Main-content extraction, sitemap + llms.txt, glob filters, no browser. $0.50 per 1,000 pages, JavaScript-only pages are never billed.", "version": "1.0", "x-build-id": "waPPMZjiWd9vU5sIX" }, "servers": [ { "url": "https://api.apify.com/v2" } ], "paths": { "/acts/hridayrungta~website-to-markdown-crawler/run-sync-get-dataset-items": { "post": { "operationId": "run-sync-get-dataset-items-hridayrungta-website-to-markdown-crawler", "x-openai-isConsequential": false, "summary": "Executes an Actor, waits for its completion, and returns Actor's dataset items in response.", "tags": [ "Run Actor" ], "requestBody": { "required": true, "content": { "application/json": { "schema": { "$ref": "#/components/schemas/inputSchema" } } } }, "parameters": [ { "name": "token", "in": "query", "required": true, "schema": { "type": "string" }, "description": "Enter your Apify token here" } ], "responses": { "200": { "description": "OK" } } } }, "/acts/hridayrungta~website-to-markdown-crawler/runs": { "post": { "operationId": "runs-sync-hridayrungta-website-to-markdown-crawler", "x-openai-isConsequential": false, "summary": "Executes an Actor and returns information about the initiated run in response.", "tags": [ "Run Actor" ], "requestBody": { "required": true, "content": { "application/json": { "schema": { "$ref": "#/components/schemas/inputSchema" } } } }, "parameters": [ { "name": "token", "in": "query", "required": true, "schema": { "type": "string" }, "description": "Enter your Apify token here" } ], "responses": { "200": { "description": "OK", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/runsResponseSchema" } } } } } } }, "/acts/hridayrungta~website-to-markdown-crawler/run-sync": { "post": { "operationId": "run-sync-hridayrungta-website-to-markdown-crawler", "x-openai-isConsequential": false, "summary": "Executes an Actor, waits for completion, and returns the OUTPUT from Key-value store in response.", "tags": [ "Run Actor" ], "requestBody": { "required": true, "content": { "application/json": { "schema": { "$ref": "#/components/schemas/inputSchema" } } } }, "parameters": [ { "name": "token", "in": "query", "required": true, "schema": { "type": "string" }, "description": "Enter your Apify token here" } ], "responses": { "200": { "description": "OK" } } } } }, "components": { "schemas": { "inputSchema": { "type": "object", "required": [ "startUrls" ], "properties": { "startUrls": { "title": "Start URLs", "type": "array", "description": "Pages to start crawling from. The crawler follows links that stay inside the crawl scope below.", "items": { "type": "object", "required": [ "url" ], "properties": { "url": { "type": "string", "title": "URL of a web page", "format": "uri" } } } }, "maxPages": { "title": "Max pages (billed)", "minimum": 1, "maximum": 100000, "type": "integer", "description": "Cost cap: the crawl stops after this many pages were returned with real content. You are billed once per such page. Pages flagged as JavaScript-only, blocked or empty are never billed and do not count.", "default": 100 }, "maxDepth": { "title": "Max crawl depth", "minimum": 0, "maximum": 20, "type": "integer", "description": "How many link hops away from the start URLs to follow. 0 = only the start URLs. Pages found through sitemap.xml or llms.txt are not limited by depth.", "default": 3 }, "crawlScope": { "title": "Crawl scope", "enum": [ "same-hostname", "same-domain", "start-url-path" ], "type": "string", "description": "Which discovered links to follow. 'Same hostname' stays on the exact host of each start URL. 'Same domain' also allows subdomains. 'Under start URL path' only follows links below the folder of the start URL (good for one docs section).", "default": "same-hostname" }, "includeUrlGlobs": { "title": "Include URL patterns (glob)", "type": "array", "description": "Only follow URLs matching at least one pattern, e.g. https://example.com/docs/**. Leave empty to allow everything in scope. Start URLs are always crawled.", "items": { "type": "string" } }, "excludeUrlGlobs": { "title": "Exclude URL patterns (glob)", "type": "array", "description": "Never follow URLs matching any of these patterns, e.g. https://example.com/blog/tag/**.", "items": { "type": "string" } }, "useSitemaps": { "title": "Discover pages from sitemap.xml", "type": "boolean", "description": "Also read robots.txt Sitemap entries and /sitemap.xml (including sitemap indexes) to find pages that are not linked from the start page.", "default": false }, "useLlmsTxt": { "title": "Discover pages from llms.txt", "type": "boolean", "description": "Also read /llms.txt and queue the pages it links to.", "default": false }, "mainContentOnly": { "title": "Main content only", "type": "boolean", "description": "Strip navigation, headers, footers, cookie banners, ads and sidebars and return only the main content. Turn off to get the whole page body.", "default": true }, "includeImages": { "title": "Keep images in Markdown", "type": "boolean", "description": "Keep images as ![alt](absolute-url) in the Markdown. Turn off for text-only output (fewer tokens).", "default": true }, "includeLinks": { "title": "Output list of links", "type": "boolean", "description": "Add a 'links' array (url + anchor text) with every link found on each page.", "default": false }, "includeContentHash": { "title": "Add content hash", "type": "boolean", "description": "Add a 'contentHash' (SHA-256 prefix of the page text) so you can detect changed pages between crawls.", "default": false }, "includeSkippedPages": { "title": "Output unbilled pages too", "type": "boolean", "description": "Also save rows for pages that were skipped (JavaScript-only, blocked, thin, HTTP errors) with their contentStatus. These rows are free. Start URLs are always saved so you can see why they failed.", "default": false }, "respectRobotsTxt": { "title": "Respect robots.txt", "type": "boolean", "description": "Skip URLs that the site's robots.txt disallows for all crawlers.", "default": true }, "maxConcurrency": { "title": "Max concurrency", "minimum": 1, "maximum": 20, "type": "integer", "description": "Parallel requests (1-20). Lower it for small sites. The crawler backs off automatically on HTTP 429.", "default": 5 }, "minWords": { "title": "Minimum words per page", "minimum": 1, "maximum": 5000, "type": "integer", "description": "Pages with fewer extracted words are treated as empty (flagged, not billed).", "default": 20 }, "removeSelectors": { "title": "Extra CSS selectors to remove", "type": "array", "description": "Optional CSS selectors removed from every page before extraction, e.g. .promo-banner or #comments.", "items": { "type": "string" } }, "proxyConfiguration": { "title": "Proxy (optional)", "type": "object", "description": "Optional. By default no proxy is used; if a site answers 403/429, one retry goes through a free Apify datacenter proxy session. Set this to force a specific proxy group for all requests." } } }, "runsResponseSchema": { "type": "object", "properties": { "data": { "type": "object", "properties": { "id": { "type": "string" }, "actId": { "type": "string" }, "userId": { "type": "string" }, "startedAt": { "type": "string", "format": "date-time", "example": "2025-01-08T00:00:00.000Z" }, "finishedAt": { "type": "string", "format": "date-time", "example": "2025-01-08T00:00:00.000Z" }, "status": { "type": "string", "example": "READY" }, "meta": { "type": "object", "properties": { "origin": { "type": "string", "example": "API" }, "userAgent": { "type": "string" } } }, "stats": { "type": "object", "properties": { "inputBodyLen": { "type": "integer", "example": 2000 }, "rebootCount": { "type": "integer", "example": 0 }, "restartCount": { "type": "integer", "example": 0 }, "resurrectCount": { "type": "integer", "example": 0 }, "computeUnits": { "type": "integer", "example": 0 } } }, "options": { "type": "object", "properties": { "build": { "type": "string", "example": "latest" }, "timeoutSecs": { "type": "integer", "example": 300 }, "memoryMbytes": { "type": "integer", "example": 1024 }, "diskMbytes": { "type": "integer", "example": 2048 } } }, "buildId": { "type": "string" }, "defaultKeyValueStoreId": { "type": "string" }, "defaultDatasetId": { "type": "string" }, "defaultRequestQueueId": { "type": "string" }, "buildNumber": { "type": "string", "example": "1.0.0" }, "containerUrl": { "type": "string" }, "usage": { "type": "object", "properties": { "ACTOR_COMPUTE_UNITS": { "type": "integer", "example": 0 }, "DATASET_READS": { "type": "integer", "example": 0 }, "DATASET_WRITES": { "type": "integer", "example": 0 }, "KEY_VALUE_STORE_READS": { "type": "integer", "example": 0 }, "KEY_VALUE_STORE_WRITES": { "type": "integer", "example": 1 }, "KEY_VALUE_STORE_LISTS": { "type": "integer", "example": 0 }, "REQUEST_QUEUE_READS": { "type": "integer", "example": 0 }, "REQUEST_QUEUE_WRITES": { "type": "integer", "example": 0 }, "DATA_TRANSFER_INTERNAL_GBYTES": { "type": "integer", "example": 0 }, "DATA_TRANSFER_EXTERNAL_GBYTES": { "type": "integer", "example": 0 }, "PROXY_RESIDENTIAL_TRANSFER_GBYTES": { "type": "integer", "example": 0 }, "PROXY_SERPS": { "type": "integer", "example": 0 }, "PROXY_UNBLOCKER_UNITS": { "type": "integer", "example": 0 } } }, "usageTotalUsd": { "type": "number", "example": 0.00005 }, "usageUsd": { "type": "object", "properties": { "ACTOR_COMPUTE_UNITS": { "type": "integer", "example": 0 }, "DATASET_READS": { "type": "integer", "example": 0 }, "DATASET_WRITES": { "type": "integer", "example": 0 }, "KEY_VALUE_STORE_READS": { "type": "integer", "example": 0 }, "KEY_VALUE_STORE_WRITES": { "type": "number", "example": 0.00005 }, "KEY_VALUE_STORE_LISTS": { "type": "integer", "example": 0 }, "REQUEST_QUEUE_READS": { "type": "integer", "example": 0 }, "REQUEST_QUEUE_WRITES": { "type": "integer", "example": 0 }, "DATA_TRANSFER_INTERNAL_GBYTES": { "type": "integer", "example": 0 }, "DATA_TRANSFER_EXTERNAL_GBYTES": { "type": "integer", "example": 0 }, "PROXY_RESIDENTIAL_TRANSFER_GBYTES": { "type": "integer", "example": 0 }, "PROXY_SERPS": { "type": "integer", "example": 0 }, "PROXY_UNBLOCKER_UNITS": { "type": "integer", "example": 0 } } } } } } } } } }