{ "openapi": "3.0.1", "info": { "title": "Website Content Extractor - Markdown for LLM, RAG, Vector DB", "description": "Readable text from 73.1% of 52 live pages over plain HTTP, including 10 of 10 documentation sites. Website to LLM text and a website to markdown converter in one run: clean Markdown, chunks with overlap for a vector database, 46 flat fields, and what changed since the last run.", "version": "0.1", "x-build-id": "2Jjc5Wbb2sovHxdgL" }, "servers": [ { "url": "https://api.apify.com/v2" } ], "paths": { "/acts/snow_leo_data~website-to-markdown/run-sync-get-dataset-items": { "post": { "operationId": "run-sync-get-dataset-items-snow_leo_data-website-to-markdown", "x-openai-isConsequential": false, "summary": "Executes an Actor, waits for its completion, and returns Actor's dataset items in response.", "tags": [ "Run Actor" ], "requestBody": { "required": true, "content": { "application/json": { "schema": { "$ref": "#/components/schemas/inputSchema" } } } }, "parameters": [ { "name": "token", "in": "query", "required": true, "schema": { "type": "string" }, "description": "Enter your Apify token here" } ], "responses": { "200": { "description": "OK" } } } }, "/acts/snow_leo_data~website-to-markdown/runs": { "post": { "operationId": "runs-sync-snow_leo_data-website-to-markdown", "x-openai-isConsequential": false, "summary": "Executes an Actor and returns information about the initiated run in response.", "tags": [ "Run Actor" ], "requestBody": { "required": true, "content": { "application/json": { "schema": { "$ref": "#/components/schemas/inputSchema" } } } }, "parameters": [ { "name": "token", "in": "query", "required": true, "schema": { "type": "string" }, "description": "Enter your Apify token here" } ], "responses": { "200": { "description": "OK", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/runsResponseSchema" } } } } } } }, "/acts/snow_leo_data~website-to-markdown/run-sync": { "post": { "operationId": "run-sync-snow_leo_data-website-to-markdown", "x-openai-isConsequential": false, "summary": "Executes an Actor, waits for completion, and returns the OUTPUT from Key-value store in response.", "tags": [ "Run Actor" ], "requestBody": { "required": true, "content": { "application/json": { "schema": { "$ref": "#/components/schemas/inputSchema" } } } }, "parameters": [ { "name": "token", "in": "query", "required": true, "schema": { "type": "string" }, "description": "Enter your Apify token here" } ], "responses": { "200": { "description": "OK" } } } } }, "components": { "schemas": { "inputSchema": { "type": "object", "properties": { "startUrls": { "title": "🔗 URLs to extract", "type": "array", "description": "Pages to turn into clean text and Markdown. Paste a list of links, or one address plus a crawl depth below to walk a whole site. A bare domain such as example.com works too.", "default": [ { "url": "https://docs.apify.com/platform/actors" }, { "url": "https://en.wikipedia.org/wiki/Retrieval-augmented_generation" } ], "items": { "type": "object", "required": [ "url" ], "properties": { "url": { "type": "string", "title": "URL of a web page", "format": "uri" } } } }, "maxItems": { "title": "Maximum rows", "minimum": 0, "maximum": 100000, "type": "integer", "description": "Hard cap on delivered rows, and on what you are charged. It limits the OUTPUT, not the crawl: filtered rows do not eat the budget. 0 means no cap.", "default": 100 }, "outputFormat": { "title": "Output format", "enum": [ "both", "markdown", "text" ], "type": "string", "description": "Markdown keeps headings, lists, tables and code blocks - the structure a RAG pipeline chunks on. Plain text is smaller. Both is the default.", "default": "both" }, "crawlDepth": { "title": "🕸 Crawl depth", "minimum": 0, "maximum": 10, "type": "integer", "description": "0 opens only the addresses you gave. 1 also opens the links found on them, 2 goes one level further, and so on. Every crawl obeys the site's robots.txt.", "default": 0 }, "maxPages": { "title": "Maximum pages to open", "minimum": 0, "maximum": 50000, "type": "integer", "description": "Budget for the whole run, counted in requests actually sent. Protects you from a site with an infinite calendar. 0 means no budget.", "default": 50 }, "maxPagesPerDomain": { "title": "Maximum pages per domain", "minimum": 0, "maximum": 50000, "type": "integer", "description": "Keeps one large site from eating the whole budget when you crawl several. 0 means no per-domain limit.", "default": 0 }, "stayOnDomain": { "title": "Stay on the starting domain", "type": "boolean", "description": "Off lets the crawl follow links anywhere, which on a news site means the whole internet. Keep it on unless you know why not.", "default": true }, "includeSubdomains": { "title": "Allow subdomains", "type": "boolean", "description": "Treats docs.example.com and example.com as the same site. Most documentation lives on a subdomain.", "default": true }, "includeUrlPatterns": { "title": "Only crawl URLs matching", "type": "array", "description": "Wildcard patterns such as https://example.com/docs/* . Applied to links found while crawling, never to the URLs you gave yourself.", "default": [], "items": { "type": "string" } }, "excludeUrlPatterns": { "title": "Never crawl URLs matching", "type": "array", "description": "Wildcard patterns such as */tag/* or */page/* . Applied to links found while crawling.", "default": [], "items": { "type": "string" } }, "useSitemap": { "title": "Seed the crawl from sitemap.xml", "type": "boolean", "description": "Reads the site's own sitemap and queues the addresses in it. This is how you reach a site whose menu is drawn by JavaScript: the links are not in the HTML, but the sitemap lists every page. Only used when crawl depth is above 0.", "default": false }, "chunkText": { "title": "✂️ Split into chunks for a vector database", "type": "boolean", "description": "Adds a `chunks` field: the page cut on heading, paragraph and sentence boundaries, never mid-word and never inside a code block. Each chunk carries its heading path, so a match can be cited.", "default": false }, "chunkSize": { "title": "Chunk size", "minimum": 200, "maximum": 20000, "type": "integer", "description": "Target size of one chunk in characters. Roughly four characters per token, so 1200 is about 300 tokens.", "default": 1200 }, "chunkOverlap": { "title": "Chunk overlap", "minimum": 0, "maximum": 10000, "type": "integer", "description": "How much of the previous chunk is repeated at the start of the next one, so a thought split across the boundary is still findable whole. Capped at half the chunk size.", "default": 150 }, "keepNavigation": { "title": "Keep menus, footers and banners", "type": "boolean", "description": "Off (the default) removes navigation, footers, cookie banners and 'related articles' - they repeat on every page of a site and pollute a vector search. On returns the whole page body.", "default": false }, "keepLinksInMarkdown": { "title": "Keep links in Markdown", "type": "boolean", "description": "Off strips [text](url) down to the text. The `links` field still lists every link either way.", "default": true }, "keepImagesInMarkdown": { "title": "Keep images in Markdown", "type": "boolean", "description": "Off strips ![alt](url) out of the Markdown. The `images` field still lists them.", "default": true }, "compactOutput": { "title": "Compact rows (for AI agents)", "type": "boolean", "description": "Drops link lists, image lists, og tags and heading outlines, and removes empty fields. Keeps the row small enough for a model context window.", "default": false }, "includeHtml": { "title": "Also include the raw HTML", "type": "boolean", "description": "Adds an `html` field with the page source as downloaded. Rows get much larger, so leave it off unless you post-process the HTML yourself.", "default": false }, "removeElementsSelector": { "title": "Remove these elements", "type": "array", "description": "Simple selectors only: a tag name (aside), a class (.cookie-bar) or an id (#sidebar). Combinators, attributes and pseudo-classes are not supported - they would fail silently, so they are refused instead.", "default": [], "items": { "type": "string" } }, "keepElementsSelector": { "title": "Keep only these elements", "type": "array", "description": "Same simple selectors. When set, the text is taken from these elements only and the automatic article detection is switched off - your choice wins over ours.", "default": [], "items": { "type": "string" } }, "customHttpHeaders": { "title": "Extra HTTP headers", "type": "object", "description": "Sent with every request, for example an Accept-Language or an API key your own site expects. Measured on bbb.org: an extra Accept-Language turns a working request into a 403, so add headers only when you know the site wants them.", "default": {} }, "minWordsPerPage": { "title": "Minimum words per page", "minimum": 0, "maximum": 100000, "type": "integer", "description": "Drops pages with less text than this. Use it to throw away the JavaScript-only pages this Actor cannot read: measured from Apify servers, 3 of 52 live pages came back as an empty shell. 0 keeps everything.", "default": 0 }, "skipFailedPages": { "title": "Drop pages that did not open", "type": "boolean", "description": "By default a page that answered 403 or timed out still gets a row, so you can see which address failed and why. Turn this on to leave them out entirely.", "default": false }, "onlyLanguages": { "title": "Only these languages", "type": "array", "description": "Two-letter codes such as en, de, fr, taken from the page's own language tag. Pages that declare no language are kept: missing data is not a mismatch.", "default": [], "items": { "type": "string" } }, "urlMustContain": { "title": "URL must contain", "type": "array", "description": "Keep only rows whose final URL contains one of these strings.", "default": [], "items": { "type": "string" } }, "urlMustNotContain": { "title": "URL must not contain", "type": "array", "description": "Drop rows whose final URL contains any of these strings.", "default": [], "items": { "type": "string" } }, "onlyChanged": { "title": "🔁 Monitor mode: only new and changed pages", "type": "boolean", "description": "Remembers a fingerprint of each page's text between runs and marks every row NEW, UPDATED or UNCHANGED. Unchanged pages are not delivered and not charged, so a scheduled run re-indexes only the delta in your vector database.", "default": false }, "emitUnchanged": { "title": "Deliver unchanged pages too", "type": "boolean", "description": "Keeps unchanged rows in the output, still labelled UNCHANGED. Useful for a full re-export while keeping the labels.", "default": false }, "monitorStoreName": { "title": "Memory name", "type": "string", "description": "Name of the storage holding the fingerprints. Give separate pipelines separate names so they do not share memory.", "default": "" }, "concurrency": { "title": "Parallel requests", "minimum": 1, "maximum": 10, "type": "integer", "description": "How many pages are opened at once. Requests to the same host are spaced out regardless, and a Crawl-delay in robots.txt is obeyed.", "default": 5 }, "requestTimeoutSecs": { "title": "Request timeout", "minimum": 5, "maximum": 120, "type": "integer", "description": "How long to wait for one page before giving up on it.", "default": 25 } } }, "runsResponseSchema": { "type": "object", "properties": { "data": { "type": "object", "properties": { "id": { "type": "string" }, "actId": { "type": "string" }, "userId": { "type": "string" }, "startedAt": { "type": "string", "format": "date-time", "example": "2025-01-08T00:00:00.000Z" }, "finishedAt": { "type": "string", "format": "date-time", "example": "2025-01-08T00:00:00.000Z" }, "status": { "type": "string", "example": "READY" }, "meta": { "type": "object", "properties": { "origin": { "type": "string", "example": "API" }, "userAgent": { "type": "string" } } }, "stats": { "type": "object", "properties": { "inputBodyLen": { "type": "integer", "example": 2000 }, "rebootCount": { "type": "integer", "example": 0 }, "restartCount": { "type": "integer", "example": 0 }, "resurrectCount": { "type": "integer", "example": 0 }, "computeUnits": { "type": "integer", "example": 0 } } }, "options": { "type": "object", "properties": { "build": { "type": "string", "example": "latest" }, "timeoutSecs": { "type": "integer", "example": 300 }, "memoryMbytes": { "type": "integer", "example": 1024 }, "diskMbytes": { "type": "integer", "example": 2048 } } }, "buildId": { "type": "string" }, "defaultKeyValueStoreId": { "type": "string" }, "defaultDatasetId": { "type": "string" }, "defaultRequestQueueId": { "type": "string" }, "buildNumber": { "type": "string", "example": "1.0.0" }, "containerUrl": { "type": "string" }, "usage": { "type": "object", "properties": { "ACTOR_COMPUTE_UNITS": { "type": "integer", "example": 0 }, "DATASET_READS": { "type": "integer", "example": 0 }, "DATASET_WRITES": { "type": "integer", "example": 0 }, "KEY_VALUE_STORE_READS": { "type": "integer", "example": 0 }, "KEY_VALUE_STORE_WRITES": { "type": "integer", "example": 1 }, "KEY_VALUE_STORE_LISTS": { "type": "integer", "example": 0 }, "REQUEST_QUEUE_READS": { "type": "integer", "example": 0 }, "REQUEST_QUEUE_WRITES": { "type": "integer", "example": 0 }, "DATA_TRANSFER_INTERNAL_GBYTES": { "type": "integer", "example": 0 }, "DATA_TRANSFER_EXTERNAL_GBYTES": { "type": "integer", "example": 0 }, "PROXY_RESIDENTIAL_TRANSFER_GBYTES": { "type": "integer", "example": 0 }, "PROXY_SERPS": { "type": "integer", "example": 0 }, "PROXY_UNBLOCKER_UNITS": { "type": "integer", "example": 0 } } }, "usageTotalUsd": { "type": "number", "example": 0.00005 }, "usageUsd": { "type": "object", "properties": { "ACTOR_COMPUTE_UNITS": { "type": "integer", "example": 0 }, "DATASET_READS": { "type": "integer", "example": 0 }, "DATASET_WRITES": { "type": "integer", "example": 0 }, "KEY_VALUE_STORE_READS": { "type": "integer", "example": 0 }, "KEY_VALUE_STORE_WRITES": { "type": "number", "example": 0.00005 }, "KEY_VALUE_STORE_LISTS": { "type": "integer", "example": 0 }, "REQUEST_QUEUE_READS": { "type": "integer", "example": 0 }, "REQUEST_QUEUE_WRITES": { "type": "integer", "example": 0 }, "DATA_TRANSFER_INTERNAL_GBYTES": { "type": "integer", "example": 0 }, "DATA_TRANSFER_EXTERNAL_GBYTES": { "type": "integer", "example": 0 }, "PROXY_RESIDENTIAL_TRANSFER_GBYTES": { "type": "integer", "example": 0 }, "PROXY_SERPS": { "type": "integer", "example": 0 }, "PROXY_UNBLOCKER_UNITS": { "type": "integer", "example": 0 } } } } } } } } } }