{ "openapi": "3.0.1", "info": { "title": "Website to Markdown Crawler for RAG: Fast, No Browser", "description": "Crawl a site into clean Markdown for RAG and LLMs without a browser. Sitemap first, robots.txt honored, main content only, heading chunks with token counts, and changed-pages-only re-crawls. JS-only pages are flagged, not charged. USD 1.20 per 1,000 pages.", "version": "1.0", "x-build-id": "XscRXYqAduRh6e7eW" }, "servers": [ { "url": "https://api.apify.com/v2" } ], "paths": { "/acts/jtpalms~website-to-markdown-crawler/run-sync-get-dataset-items": { "post": { "operationId": "run-sync-get-dataset-items-jtpalms-website-to-markdown-crawler", "x-openai-isConsequential": false, "summary": "Executes an Actor, waits for its completion, and returns Actor's dataset items in response.", "tags": [ "Run Actor" ], "requestBody": { "required": true, "content": { "application/json": { "schema": { "$ref": "#/components/schemas/inputSchema" } } } }, "parameters": [ { "name": "token", "in": "query", "required": true, "schema": { "type": "string" }, "description": "Enter your Apify token here" } ], "responses": { "200": { "description": "OK" } } } }, "/acts/jtpalms~website-to-markdown-crawler/runs": { "post": { "operationId": "runs-sync-jtpalms-website-to-markdown-crawler", "x-openai-isConsequential": false, "summary": "Executes an Actor and returns information about the initiated run in response.", "tags": [ "Run Actor" ], "requestBody": { "required": true, "content": { "application/json": { "schema": { "$ref": "#/components/schemas/inputSchema" } } } }, "parameters": [ { "name": "token", "in": "query", "required": true, "schema": { "type": "string" }, "description": "Enter your Apify token here" } ], "responses": { "200": { "description": "OK", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/runsResponseSchema" } } } } } } }, "/acts/jtpalms~website-to-markdown-crawler/run-sync": { "post": { "operationId": "run-sync-jtpalms-website-to-markdown-crawler", "x-openai-isConsequential": false, "summary": "Executes an Actor, waits for completion, and returns the OUTPUT from Key-value store in response.", "tags": [ "Run Actor" ], "requestBody": { "required": true, "content": { "application/json": { "schema": { "$ref": "#/components/schemas/inputSchema" } } } }, "parameters": [ { "name": "token", "in": "query", "required": true, "schema": { "type": "string" }, "description": "Enter your Apify token here" } ], "responses": { "200": { "description": "OK" } } } } }, "components": { "schemas": { "inputSchema": { "type": "object", "required": [ "startUrls" ], "properties": { "startUrls": { "title": "Start URLs", "type": "array", "description": "Pages or site roots to crawl, one per line. A URL with a path (https://example.com/docs/intro) keeps the crawl inside that folder (/docs/) unless you turn off \"Stay in the start folder\". The default is one page of the public Crawlee documentation.", "items": { "type": "object", "required": [ "url" ], "properties": { "url": { "type": "string", "title": "URL of a web page", "format": "uri" } } } }, "crawlScope": { "title": "How to find pages", "enum": [ "sitemap_then_links", "sitemap", "links_only", "single_pages" ], "type": "string", "description": "sitemap_then_links: read the sitemap, and follow links only when the sitemap lists fewer pages than your limit. sitemap: sitemap pages only. links_only: follow links from the start URLs. single_pages: only the start URLs.", "default": "sitemap_then_links" }, "maxPages": { "title": "Max pages", "minimum": 1, "maximum": 50000, "type": "integer", "description": "Hard cap on pages requested in this run, across all sites (1 to 50,000). Pages that fail still count, because they were requested.", "default": 100 }, "maxDepth": { "title": "Max link depth", "minimum": 0, "maximum": 20, "type": "integer", "description": "How many links deep to follow from a start URL when following links (0 to 20). Sitemap pages and start URLs are depth 0.", "default": 3 }, "stayInStartFolder": { "title": "Stay in the start folder", "type": "boolean", "description": "When a start URL has a path, only crawl pages under its folder (the start URL https://example.com/docs/intro covers /docs/). Turn off to crawl the whole site. Pages are always on the same host as a start URL.", "default": true }, "includeUrlPatterns": { "title": "Only these URLs", "type": "array", "description": "Optional. Crawl only URLs that match at least one pattern. A pattern is a glob (* matches inside one path segment, ** across segments, for example **/docs/**), a plain text that the URL must contain (for example /blog/), or a regular expression written as re:^https://example\\.com/(en|de)/.", "items": { "type": "string" } }, "excludeUrlPatterns": { "title": "Skip these URLs", "type": "array", "description": "URLs that match any pattern are skipped (same pattern types as above). The defaults skip author, team, people, staff and contact pages, which are mostly personal details. The prefill also skips the old versions of the Crawlee docs.", "default": [ "**/author/**", "**/author", "**/authors/**", "**/authors", "**/team/**", "**/team", "**/people/**", "**/people", "**/staff/**", "**/staff", "**/contact", "**/contact/**", "**/contact-us", "**/contact-us/**" ], "items": { "type": "string" } }, "contentMode": { "title": "Content mode", "enum": [ "page", "article" ], "type": "string", "description": "page: the cleaned page body without navigation, footer, sidebar and cookie banners (best for documentation). article: Readability's main article (best for blogs and news); falls back to page mode when no article is found.", "default": "page" }, "outputFormat": { "title": "Output format", "enum": [ "markdown", "markdown_and_text" ], "type": "string", "description": "markdown: the markdown field. markdown_and_text: also a plain text field with the same reading order and no Markdown syntax.", "default": "markdown" }, "chunkMarkdown": { "title": "Chunk for RAG", "type": "boolean", "description": "Add chunks: the Markdown split at headings into pieces of at most the token limit below, each with its heading path and a token estimate.", "default": false }, "maxChunkTokens": { "title": "Max tokens per chunk", "minimum": 50, "maximum": 4000, "type": "integer", "description": "Estimated tokens per chunk (characters divided by 4, an estimate, not a tokenizer count). 50 to 4,000.", "default": 500 }, "includeImages": { "title": "Keep images", "type": "boolean", "description": "Keep images as Markdown image links with their alt text. Off by default: images are dropped.", "default": false }, "includeLinks": { "title": "List links", "type": "boolean", "description": "Add links: the internal and external links found in the page content (deduplicated, with link text). The counts are always included.", "default": false }, "onlyChanged": { "title": "Only changed pages", "type": "boolean", "description": "Skip pages whose Markdown is unchanged since your last only-changed run on the same site. Unchanged pages are not output and not charged. The first run outputs everything as new. State is kept in the key-value store website-to-markdown-crawler-state.", "default": false }, "useLlmsTxt": { "title": "Also use llms.txt", "type": "boolean", "description": "Also seed the crawl from the site's /llms.txt (the curated list of pages some sites publish for AI tools).", "default": false }, "minWords": { "title": "Minimum words", "minimum": 0, "maximum": 10000, "type": "integer", "description": "A page with fewer words than this is not charged (and with a JavaScript app shell it is flagged needsJavaScript). 0 to 10,000.", "default": 30 }, "requestsPerSecondPerHost": { "title": "Requests per second per site", "minimum": 0.1, "maximum": 2, "type": "number", "description": "Politeness limit for each site (0.1 to 2). Never more than 2 requests at once and never closer than 500 ms apart. A Crawl-delay in robots.txt slows this further.", "default": 1 }, "ignoreNoindex": { "title": "Include noindex pages", "type": "boolean", "description": "Pages marked noindex (meta robots or X-Robots-Tag) are skipped by default. Turn on to include them. robots.txt is always respected.", "default": false }, "maxSitemapUrls": { "title": "Max sitemap URLs", "minimum": 1, "maximum": 100000, "type": "integer", "description": "Most URLs read from a site's sitemaps (1 to 100,000). Only URLs in scope are kept.", "default": 5000 } } }, "runsResponseSchema": { "type": "object", "properties": { "data": { "type": "object", "properties": { "id": { "type": "string" }, "actId": { "type": "string" }, "userId": { "type": "string" }, "startedAt": { "type": "string", "format": "date-time", "example": "2025-01-08T00:00:00.000Z" }, "finishedAt": { "type": "string", "format": "date-time", "example": "2025-01-08T00:00:00.000Z" }, "status": { "type": "string", "example": "READY" }, "meta": { "type": "object", "properties": { "origin": { "type": "string", "example": "API" }, "userAgent": { "type": "string" } } }, "stats": { "type": "object", "properties": { "inputBodyLen": { "type": "integer", "example": 2000 }, "rebootCount": { "type": "integer", "example": 0 }, "restartCount": { "type": "integer", "example": 0 }, "resurrectCount": { "type": "integer", "example": 0 }, "computeUnits": { "type": "integer", "example": 0 } } }, "options": { "type": "object", "properties": { "build": { "type": "string", "example": "latest" }, "timeoutSecs": { "type": "integer", "example": 300 }, "memoryMbytes": { "type": "integer", "example": 1024 }, "diskMbytes": { "type": "integer", "example": 2048 } } }, "buildId": { "type": "string" }, "defaultKeyValueStoreId": { "type": "string" }, "defaultDatasetId": { "type": "string" }, "defaultRequestQueueId": { "type": "string" }, "buildNumber": { "type": "string", "example": "1.0.0" }, "containerUrl": { "type": "string" }, "usage": { "type": "object", "properties": { "ACTOR_COMPUTE_UNITS": { "type": "integer", "example": 0 }, "DATASET_READS": { "type": "integer", "example": 0 }, "DATASET_WRITES": { "type": "integer", "example": 0 }, "KEY_VALUE_STORE_READS": { "type": "integer", "example": 0 }, "KEY_VALUE_STORE_WRITES": { "type": "integer", "example": 1 }, "KEY_VALUE_STORE_LISTS": { "type": "integer", "example": 0 }, "REQUEST_QUEUE_READS": { "type": "integer", "example": 0 }, "REQUEST_QUEUE_WRITES": { "type": "integer", "example": 0 }, "DATA_TRANSFER_INTERNAL_GBYTES": { "type": "integer", "example": 0 }, "DATA_TRANSFER_EXTERNAL_GBYTES": { "type": "integer", "example": 0 }, "PROXY_RESIDENTIAL_TRANSFER_GBYTES": { "type": "integer", "example": 0 }, "PROXY_SERPS": { "type": "integer", "example": 0 }, "PROXY_UNBLOCKER_UNITS": { "type": "integer", "example": 0 } } }, "usageTotalUsd": { "type": "number", "example": 0.00005 }, "usageUsd": { "type": "object", "properties": { "ACTOR_COMPUTE_UNITS": { "type": "integer", "example": 0 }, "DATASET_READS": { "type": "integer", "example": 0 }, "DATASET_WRITES": { "type": "integer", "example": 0 }, "KEY_VALUE_STORE_READS": { "type": "integer", "example": 0 }, "KEY_VALUE_STORE_WRITES": { "type": "number", "example": 0.00005 }, "KEY_VALUE_STORE_LISTS": { "type": "integer", "example": 0 }, "REQUEST_QUEUE_READS": { "type": "integer", "example": 0 }, "REQUEST_QUEUE_WRITES": { "type": "integer", "example": 0 }, "DATA_TRANSFER_INTERNAL_GBYTES": { "type": "integer", "example": 0 }, "DATA_TRANSFER_EXTERNAL_GBYTES": { "type": "integer", "example": 0 }, "PROXY_RESIDENTIAL_TRANSFER_GBYTES": { "type": "integer", "example": 0 }, "PROXY_SERPS": { "type": "integer", "example": 0 }, "PROXY_UNBLOCKER_UNITS": { "type": "integer", "example": 0 } } } } } } } } } }