{ "openapi": "3.0.1", "info": { "title": "Website to Markdown Crawler", "description": "Turn a whole website, docs site or blog into clean, LLM-ready Markdown and plain text, one record per page. Finds pages through links and sitemaps, filters URLs with include/exclude patterns and respects robots.txt. $0.001 per page.", "version": "0.0", "x-build-id": "bndVWtHD5HfVRgrEI" }, "servers": [ { "url": "https://api.apify.com/v2" } ], "paths": { "/acts/entranced_gelato~website-to-markdown-crawler/run-sync-get-dataset-items": { "post": { "operationId": "run-sync-get-dataset-items-entranced_gelato-website-to-markdown-crawler", "x-openai-isConsequential": false, "summary": "Executes an Actor, waits for its completion, and returns Actor's dataset items in response.", "tags": [ "Run Actor" ], "requestBody": { "required": true, "content": { "application/json": { "schema": { "$ref": "#/components/schemas/inputSchema" } } } }, "parameters": [ { "name": "token", "in": "query", "required": true, "schema": { "type": "string" }, "description": "Enter your Apify token here" } ], "responses": { "200": { "description": "OK" } } } }, "/acts/entranced_gelato~website-to-markdown-crawler/runs": { "post": { "operationId": "runs-sync-entranced_gelato-website-to-markdown-crawler", "x-openai-isConsequential": false, "summary": "Executes an Actor and returns information about the initiated run in response.", "tags": [ "Run Actor" ], "requestBody": { "required": true, "content": { "application/json": { "schema": { "$ref": "#/components/schemas/inputSchema" } } } }, "parameters": [ { "name": "token", "in": "query", "required": true, "schema": { "type": "string" }, "description": "Enter your Apify token here" } ], "responses": { "200": { "description": "OK", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/runsResponseSchema" } } } } } } }, "/acts/entranced_gelato~website-to-markdown-crawler/run-sync": { "post": { "operationId": "run-sync-entranced_gelato-website-to-markdown-crawler", "x-openai-isConsequential": false, "summary": "Executes an Actor, waits for completion, and returns the OUTPUT from Key-value store in response.", "tags": [ "Run Actor" ], "requestBody": { "required": true, "content": { "application/json": { "schema": { "$ref": "#/components/schemas/inputSchema" } } } }, "parameters": [ { "name": "token", "in": "query", "required": true, "schema": { "type": "string" }, "description": "Enter your Apify token here" } ], "responses": { "200": { "description": "OK" } } } } }, "components": { "schemas": { "inputSchema": { "type": "object", "required": [ "startUrl" ], "properties": { "startUrl": { "title": "Start URL", "type": "string", "description": "The website/section to crawl. The crawler follows internal links from here." }, "startUrls": { "title": "More start URLs", "type": "array", "description": "Optional. Crawl more sites or docs sections in the same run, one URL per line (up to 20 in total). Each is crawled in turn with the same settings; \"Max pages\" is a limit for the whole run, and a page is never returned or charged twice.", "items": { "type": "string" } }, "maxPages": { "title": "Max pages", "minimum": 1, "maximum": 2000, "type": "integer", "description": "Maximum number of pages to crawl (and pay for: $0.001 per page).", "default": 25 }, "maxDepth": { "title": "Max depth", "minimum": 0, "maximum": 20, "type": "integer", "description": "How many link-hops from the start URL to follow (0 = only the start page). Pages found in sitemaps count as one hop.", "default": 3 }, "sameDomainOnly": { "title": "Same domain only", "type": "boolean", "description": "Only follow links on the same domain as the start URL (or the domain it redirects to).", "default": true }, "useSitemap": { "title": "Use sitemaps", "type": "boolean", "description": "Also queue pages listed in the site's sitemaps (robots.txt 'Sitemap:' lines and /sitemap.xml, including sitemap indexes). Only URLs inside the start URL's section are used, e.g. a start URL of https://example.com/docs/ only takes sitemap pages under /docs/.", "default": true }, "includeUrlPatterns": { "title": "Include URL patterns", "type": "array", "description": "Optional. Only follow links and sitemap URLs that match at least one pattern (the start URL is always crawled). '*' matches within one path segment, '**' matches anything. Examples: https://example.com/docs/** , /blog/* , https://{docs,help}.example.com/**", "default": [], "items": { "type": "string" } }, "excludeUrlPatterns": { "title": "Exclude URL patterns", "type": "array", "description": "Optional. Never crawl URLs that match any of these patterns; exclusions win over inclusions. Examples: **/tag/** , **?page=* , /login** , **/*.pdf", "default": [], "items": { "type": "string" } }, "respectRobotsTxt": { "title": "Respect robots.txt", "type": "boolean", "description": "Skip pages that the site's robots.txt disallows, using its rules for AIDevsBot (or for all crawlers when it has none). On by default. Turn off only if you own the site or have the owner's permission.", "default": true }, "includeMarkdown": { "title": "Return Markdown", "type": "boolean", "description": "Include a clean Markdown version of each page (ideal for LLMs).", "default": true }, "maxCharsPerPage": { "title": "Max characters per page (0 = no limit)", "minimum": 0, "type": "integer", "description": "Optionally cap the length of content/markdown per page.", "default": 0 }, "proxyConfiguration": { "title": "Proxy configuration", "type": "object", "description": "Route requests through Apify Proxy so sites that throttle data-center traffic still load. Sites that block access are skipped, never bypassed. Uses Apify's standard datacenter proxies. Residential and custom proxies aren't supported.", "default": { "useApifyProxy": true } } } }, "runsResponseSchema": { "type": "object", "properties": { "data": { "type": "object", "properties": { "id": { "type": "string" }, "actId": { "type": "string" }, "userId": { "type": "string" }, "startedAt": { "type": "string", "format": "date-time", "example": "2025-01-08T00:00:00.000Z" }, "finishedAt": { "type": "string", "format": "date-time", "example": "2025-01-08T00:00:00.000Z" }, "status": { "type": "string", "example": "READY" }, "meta": { "type": "object", "properties": { "origin": { "type": "string", "example": "API" }, "userAgent": { "type": "string" } } }, "stats": { "type": "object", "properties": { "inputBodyLen": { "type": "integer", "example": 2000 }, "rebootCount": { "type": "integer", "example": 0 }, "restartCount": { "type": "integer", "example": 0 }, "resurrectCount": { "type": "integer", "example": 0 }, "computeUnits": { "type": "integer", "example": 0 } } }, "options": { "type": "object", "properties": { "build": { "type": "string", "example": "latest" }, "timeoutSecs": { "type": "integer", "example": 300 }, "memoryMbytes": { "type": "integer", "example": 1024 }, "diskMbytes": { "type": "integer", "example": 2048 } } }, "buildId": { "type": "string" }, "defaultKeyValueStoreId": { "type": "string" }, "defaultDatasetId": { "type": "string" }, "defaultRequestQueueId": { "type": "string" }, "buildNumber": { "type": "string", "example": "1.0.0" }, "containerUrl": { "type": "string" }, "usage": { "type": "object", "properties": { "ACTOR_COMPUTE_UNITS": { "type": "integer", "example": 0 }, "DATASET_READS": { "type": "integer", "example": 0 }, "DATASET_WRITES": { "type": "integer", "example": 0 }, "KEY_VALUE_STORE_READS": { "type": "integer", "example": 0 }, "KEY_VALUE_STORE_WRITES": { "type": "integer", "example": 1 }, "KEY_VALUE_STORE_LISTS": { "type": "integer", "example": 0 }, "REQUEST_QUEUE_READS": { "type": "integer", "example": 0 }, "REQUEST_QUEUE_WRITES": { "type": "integer", "example": 0 }, "DATA_TRANSFER_INTERNAL_GBYTES": { "type": "integer", "example": 0 }, "DATA_TRANSFER_EXTERNAL_GBYTES": { "type": "integer", "example": 0 }, "PROXY_RESIDENTIAL_TRANSFER_GBYTES": { "type": "integer", "example": 0 }, "PROXY_SERPS": { "type": "integer", "example": 0 }, "PROXY_UNBLOCKER_UNITS": { "type": "integer", "example": 0 } } }, "usageTotalUsd": { "type": "number", "example": 0.00005 }, "usageUsd": { "type": "object", "properties": { "ACTOR_COMPUTE_UNITS": { "type": "integer", "example": 0 }, "DATASET_READS": { "type": "integer", "example": 0 }, "DATASET_WRITES": { "type": "integer", "example": 0 }, "KEY_VALUE_STORE_READS": { "type": "integer", "example": 0 }, "KEY_VALUE_STORE_WRITES": { "type": "number", "example": 0.00005 }, "KEY_VALUE_STORE_LISTS": { "type": "integer", "example": 0 }, "REQUEST_QUEUE_READS": { "type": "integer", "example": 0 }, "REQUEST_QUEUE_WRITES": { "type": "integer", "example": 0 }, "DATA_TRANSFER_INTERNAL_GBYTES": { "type": "integer", "example": 0 }, "DATA_TRANSFER_EXTERNAL_GBYTES": { "type": "integer", "example": 0 }, "PROXY_RESIDENTIAL_TRANSFER_GBYTES": { "type": "integer", "example": 0 }, "PROXY_SERPS": { "type": "integer", "example": 0 }, "PROXY_UNBLOCKER_UNITS": { "type": "integer", "example": 0 } } } } } } } } } }