{ "openapi": "3.0.1", "info": { "title": "Website to Markdown — Content Extractor for LLMs & RAG", "description": "Turn any website into clean Markdown and plain text for RAG pipelines, vector stores and LLM agents. Reads the sitemap or crawls links, strips navigation, cookie bars and footers, and returns one row per page with metadata, headings, links and optional chunks. Static HTML, no browser, no API key.", "version": "0.1", "x-build-id": "YzTQ1EJcvIImlsqgL" }, "servers": [ { "url": "https://api.apify.com/v2" } ], "paths": { "/acts/insight.solutions~website-content-extractor/run-sync-get-dataset-items": { "post": { "operationId": "run-sync-get-dataset-items-insight.solutions-website-content-extractor", "x-openai-isConsequential": false, "summary": "Executes an Actor, waits for its completion, and returns Actor's dataset items in response.", "tags": [ "Run Actor" ], "requestBody": { "required": true, "content": { "application/json": { "schema": { "$ref": "#/components/schemas/inputSchema" } } } }, "parameters": [ { "name": "token", "in": "query", "required": true, "schema": { "type": "string" }, "description": "Enter your Apify token here" } ], "responses": { "200": { "description": "OK" } } } }, "/acts/insight.solutions~website-content-extractor/runs": { "post": { "operationId": "runs-sync-insight.solutions-website-content-extractor", "x-openai-isConsequential": false, "summary": "Executes an Actor and returns information about the initiated run in response.", "tags": [ "Run Actor" ], "requestBody": { "required": true, "content": { "application/json": { "schema": { "$ref": "#/components/schemas/inputSchema" } } } }, "parameters": [ { "name": "token", "in": "query", "required": true, "schema": { "type": "string" }, "description": "Enter your Apify token here" } ], "responses": { "200": { "description": "OK", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/runsResponseSchema" } } } } } } }, "/acts/insight.solutions~website-content-extractor/run-sync": { "post": { "operationId": "run-sync-insight.solutions-website-content-extractor", "x-openai-isConsequential": false, "summary": "Executes an Actor, waits for completion, and returns the OUTPUT from Key-value store in response.", "tags": [ "Run Actor" ], "requestBody": { "required": true, "content": { "application/json": { "schema": { "$ref": "#/components/schemas/inputSchema" } } } }, "parameters": [ { "name": "token", "in": "query", "required": true, "schema": { "type": "string" }, "description": "Enter your Apify token here" } ], "responses": { "200": { "description": "OK" } } } } }, "components": { "schemas": { "inputSchema": { "type": "object", "properties": { "startUrls": { "title": "Start URLs or domains", "type": "array", "description": "What to read. An entry with **no path** — `apify.com` — means *the site*: the Actor reads `robots.txt`, follows the `Sitemap:` lines it finds, and extracts up to **Maximum pages** of it; a site that publishes no sitemap is crawled breadth-first from its home page. An entry **with a path** — `https://docs.apify.com/platform/actors` — means *that page*, plus the pages it links to when **Follow links** is on. Entries on one host are one website, one crawl, one summary row.", "items": { "type": "string" } }, "crawlLinks": { "title": "Follow links", "type": "boolean", "description": "On by default. Every page that is extracted has its same-host links read and queued, down to **Maximum depth**. Turn it off to extract exactly the URLs you listed and nothing else — useful when you already have the list and want the cost to equal the number of URLs in it. Links are read from the whole page, including the navigation this Actor strips out of the text: that is where a site keeps the links to the rest of itself.", "default": true }, "maxPages": { "title": "Maximum pages per website", "minimum": 1, "maximum": 5000, "type": "integer", "description": "How many pages to extract per website. This is the number that sets the price of a run: every extracted page is one `page` event. Sitemap URLs are taken in the order the sitemap lists them; a link crawl takes them breadth-first, shallowest links first. Pages that turn out to be PDFs, error pages, bot walls or empty shells are not counted against this and are not charged.", "default": 50 }, "maxDepth": { "title": "Maximum depth", "minimum": 0, "maximum": 10, "type": "integer", "description": "How far the link crawl may travel from where it started. A URL you listed, and a URL that came out of a sitemap, is depth 0; a link found on it is depth 1. Set it to 0 to follow nothing, 1 for a page and its immediate links. Only used when **Follow links** is on.", "default": 3 }, "output": { "title": "What to return", "enum": [ "both", "markdown", "text" ], "type": "string", "description": "`both` gives each row a `markdown` field and a `text` field. `markdown` fills only `markdown` — headings, lists, tables, fenced code, links. `text` fills only `text` — the same content with the syntax stripped, paragraphs separated by blank lines. Both are produced from one walk of the page, so they can never disagree about what it said; choosing one only leaves the other column null and the dataset smaller.", "default": "both" }, "includeUrlPatterns": { "title": "Only extract URLs matching", "type": "array", "description": "Optional. When any pattern is given, a discovered URL must match at least one of them to be extracted. Two spellings are accepted: a **glob** matched against the whole URL (`https://example.com/blog/*`, `*/docs/*`), where `*` is any run of characters and `?` is one; or a **regular expression** wrapped in slashes (`/\\/blog\\/\\d{4}\\//i`). Patterns apply to URLs the Actor *discovers* — never to a URL you typed into **Start URLs**, which is always read.", "items": { "type": "string" } }, "excludeUrlPatterns": { "title": "Never extract URLs matching", "type": "array", "description": "Optional, and it wins over the include list. Same glob-or-regex spelling. Use it to keep tag archives, paginated listings, author pages and print views out of a crawl: `*/tag/*`, `*?page=*`, `/\\/(author|print)\\//`. A pattern that will not compile as a regular expression gets its own free diagnostic row rather than being silently ignored.", "items": { "type": "string" } }, "includeHtml": { "title": "Also return the extracted HTML", "type": "boolean", "description": "Off by default. When on, each row carries an `html` field holding the markup of the content region the Actor chose — not the whole document, and with scripts, styles, navigation and the rest of the boilerplate already removed. Useful when you want to run your own parser over the same region this Actor picked. It makes the dataset several times larger.", "default": false }, "keepLinks": { "title": "Keep links", "type": "boolean", "description": "On by default: an anchor becomes `[text](https://absolute/url)` in the Markdown, with relative URLs resolved against the page they were found on. Turn it off and the anchor text stays while the URL goes — often what you want before embedding, because a URL costs tokens and carries little meaning. The `links` column lists every link either way, so nothing is lost.", "default": true }, "keepImages": { "title": "Keep images", "type": "boolean", "description": "On by default: an image becomes `![alt](https://absolute/src)`. Turn it off and only the alt text survives in the body. Spacers, 1×1 tracking beacons and `data:` URIs are dropped either way. The `images` column lists up to 100 images with their alt text regardless of this setting.", "default": true }, "chunkSize": { "title": "Chunk size (characters)", "minimum": 0, "maximum": 10000, "type": "integer", "description": "Leave at 0 for no chunking. Set it between 500 and 10,000 and each row gains a `chunks` array — `{index, text, headingPath, charCount, tokenEstimate}` — ready to embed. Chunks break at block boundaries, so a chunk never starts mid-sentence; a fenced code block or a table is never split; and `headingPath` carries the real `H1 > H2 > H3` trail. Chunking is included in the per-page price. See *Chunking for RAG* in the README for the one documented size exception.", "default": 0 }, "chunkOverlap": { "title": "Chunk overlap (characters)", "minimum": 0, "maximum": 200, "type": "integer", "description": "How much of the end of one chunk to repeat at the start of the next, so a fact that straddles a boundary is retrievable from either side. Only used when **Chunk size** is set, and capped at half the chunk size. The repeated text is cut back to a word boundary, and never opens a code fence it does not close.", "default": 0 }, "respectRobots": { "title": "Respect robots.txt", "type": "boolean", "description": "On by default, and worth leaving on. The Actor reads the `User-agent: *` group of each site's `robots.txt` and skips any URL it disallows, using the same longest-match-wins rule search engines use, with `*` and `$` patterns supported. Skipped URLs are counted on the `site` row as `robotsBlocked` and are never fetched or billed. Turn it off only for a site you own or have permission to crawl.", "default": true }, "maxSitemapUrls": { "title": "Maximum sitemap URLs to collect", "minimum": 1, "maximum": 5000, "type": "integer", "description": "How many URLs to read out of a site's sitemaps before stopping. Collection also stops as soon as there are enough URLs to fill **Maximum pages**, because reading a news site's whole sitemap tree to extract fifty pages would be slow as well as rude. Sitemap indexes are followed three levels deep, gzip children included, and at most 50 sitemap files are fetched per website.", "default": 5000 }, "maxConcurrency": { "title": "Pages in parallel", "minimum": 1, "maximum": 16, "type": "integer", "description": "How many pages of one website are fetched at once. Each parallel worker keeps its own proxy session, so a page that refuses one exit IP does not burn the address the others are using. Requests to a single host are spaced 250–600 ms apart whatever this is set to, so raising it buys parallel parsing rather than a burst of simultaneous requests. Websites themselves are always read one at a time.", "default": 8 }, "maxRunSecs": { "title": "Maximum run time (seconds)", "minimum": 30, "maximum": 3600, "type": "integer", "description": "Wall-clock budget for the whole run. When it is reached the Actor stops fetching, keeps and bills for every row it already wrote, and files a free diagnostic row saying how many URLs it never reached. Nothing is charged for work that did not happen.", "default": 240 }, "proxyConfiguration": { "title": "Proxy configuration", "type": "object", "description": "Ordinary websites do not refuse datacenter address ranges the way the big platforms do, so the default is Apify's datacenter proxy and its cost is already inside the per-page price. If you are reading a site behind an aggressive bot wall, switch the group to RESIDENTIAL here and nothing else changes — a page that comes back as an anti-bot interstitial is already retried once from a fresh exit IP. Clearing the field sends requests from the run's own address.", "default": { "useApifyProxy": true } } } }, "runsResponseSchema": { "type": "object", "properties": { "data": { "type": "object", "properties": { "id": { "type": "string" }, "actId": { "type": "string" }, "userId": { "type": "string" }, "startedAt": { "type": "string", "format": "date-time", "example": "2025-01-08T00:00:00.000Z" }, "finishedAt": { "type": "string", "format": "date-time", "example": "2025-01-08T00:00:00.000Z" }, "status": { "type": "string", "example": "READY" }, "meta": { "type": "object", "properties": { "origin": { "type": "string", "example": "API" }, "userAgent": { "type": "string" } } }, "stats": { "type": "object", "properties": { "inputBodyLen": { "type": "integer", "example": 2000 }, "rebootCount": { "type": "integer", "example": 0 }, "restartCount": { "type": "integer", "example": 0 }, "resurrectCount": { "type": "integer", "example": 0 }, "computeUnits": { "type": "integer", "example": 0 } } }, "options": { "type": "object", "properties": { "build": { "type": "string", "example": "latest" }, "timeoutSecs": { "type": "integer", "example": 300 }, "memoryMbytes": { "type": "integer", "example": 1024 }, "diskMbytes": { "type": "integer", "example": 2048 } } }, "buildId": { "type": "string" }, "defaultKeyValueStoreId": { "type": "string" }, "defaultDatasetId": { "type": "string" }, "defaultRequestQueueId": { "type": "string" }, "buildNumber": { "type": "string", "example": "1.0.0" }, "containerUrl": { "type": "string" }, "usage": { "type": "object", "properties": { "ACTOR_COMPUTE_UNITS": { "type": "integer", "example": 0 }, "DATASET_READS": { "type": "integer", "example": 0 }, "DATASET_WRITES": { "type": "integer", "example": 0 }, "KEY_VALUE_STORE_READS": { "type": "integer", "example": 0 }, "KEY_VALUE_STORE_WRITES": { "type": "integer", "example": 1 }, "KEY_VALUE_STORE_LISTS": { "type": "integer", "example": 0 }, "REQUEST_QUEUE_READS": { "type": "integer", "example": 0 }, "REQUEST_QUEUE_WRITES": { "type": "integer", "example": 0 }, "DATA_TRANSFER_INTERNAL_GBYTES": { "type": "integer", "example": 0 }, "DATA_TRANSFER_EXTERNAL_GBYTES": { "type": "integer", "example": 0 }, "PROXY_RESIDENTIAL_TRANSFER_GBYTES": { "type": "integer", "example": 0 }, "PROXY_SERPS": { "type": "integer", "example": 0 }, "PROXY_UNBLOCKER_UNITS": { "type": "integer", "example": 0 } } }, "usageTotalUsd": { "type": "number", "example": 0.00005 }, "usageUsd": { "type": "object", "properties": { "ACTOR_COMPUTE_UNITS": { "type": "integer", "example": 0 }, "DATASET_READS": { "type": "integer", "example": 0 }, "DATASET_WRITES": { "type": "integer", "example": 0 }, "KEY_VALUE_STORE_READS": { "type": "integer", "example": 0 }, "KEY_VALUE_STORE_WRITES": { "type": "number", "example": 0.00005 }, "KEY_VALUE_STORE_LISTS": { "type": "integer", "example": 0 }, "REQUEST_QUEUE_READS": { "type": "integer", "example": 0 }, "REQUEST_QUEUE_WRITES": { "type": "integer", "example": 0 }, "DATA_TRANSFER_INTERNAL_GBYTES": { "type": "integer", "example": 0 }, "DATA_TRANSFER_EXTERNAL_GBYTES": { "type": "integer", "example": 0 }, "PROXY_RESIDENTIAL_TRANSFER_GBYTES": { "type": "integer", "example": 0 }, "PROXY_SERPS": { "type": "integer", "example": 0 }, "PROXY_UNBLOCKER_UNITS": { "type": "integer", "example": 0 } } } } } } } } } }