{
"openapi": "3.0.1",
"info": {
"title": "Website to Markdown Crawler for LLM & RAG",
"description": "Crawl a website, sitemap or list of URLs and convert pages to clean Markdown for LLMs, RAG and AI agents. Readability main-content extraction, metadata, optional chunking. Fast HTTP crawler, $1 per 1,000 pages.",
"version": "1.0",
"x-build-id": "NbXX6xEWNEE1hI0nS"
},
"servers": [
{
"url": "https://api.apify.com/v2"
}
],
"paths": {
"/acts/parth-tools~website-to-markdown-crawler/run-sync-get-dataset-items": {
"post": {
"operationId": "run-sync-get-dataset-items-parth-tools-website-to-markdown-crawler",
"x-openai-isConsequential": false,
"summary": "Executes an Actor, waits for its completion, and returns Actor's dataset items in response.",
"tags": [
"Run Actor"
],
"requestBody": {
"required": true,
"content": {
"application/json": {
"schema": {
"$ref": "#/components/schemas/inputSchema"
}
}
}
},
"parameters": [
{
"name": "token",
"in": "query",
"required": true,
"schema": {
"type": "string"
},
"description": "Enter your Apify token here"
}
],
"responses": {
"200": {
"description": "OK"
}
}
}
},
"/acts/parth-tools~website-to-markdown-crawler/runs": {
"post": {
"operationId": "runs-sync-parth-tools-website-to-markdown-crawler",
"x-openai-isConsequential": false,
"summary": "Executes an Actor and returns information about the initiated run in response.",
"tags": [
"Run Actor"
],
"requestBody": {
"required": true,
"content": {
"application/json": {
"schema": {
"$ref": "#/components/schemas/inputSchema"
}
}
}
},
"parameters": [
{
"name": "token",
"in": "query",
"required": true,
"schema": {
"type": "string"
},
"description": "Enter your Apify token here"
}
],
"responses": {
"200": {
"description": "OK",
"content": {
"application/json": {
"schema": {
"$ref": "#/components/schemas/runsResponseSchema"
}
}
}
}
}
}
},
"/acts/parth-tools~website-to-markdown-crawler/run-sync": {
"post": {
"operationId": "run-sync-parth-tools-website-to-markdown-crawler",
"x-openai-isConsequential": false,
"summary": "Executes an Actor, waits for completion, and returns the OUTPUT from Key-value store in response.",
"tags": [
"Run Actor"
],
"requestBody": {
"required": true,
"content": {
"application/json": {
"schema": {
"$ref": "#/components/schemas/inputSchema"
}
}
}
},
"parameters": [
{
"name": "token",
"in": "query",
"required": true,
"schema": {
"type": "string"
},
"description": "Enter your Apify token here"
}
],
"responses": {
"200": {
"description": "OK"
}
}
}
}
},
"components": {
"schemas": {
"inputSchema": {
"type": "object",
"required": [
"startUrls"
],
"properties": {
"startUrls": {
"title": "Start URLs",
"type": "array",
"description": "Pages or websites to convert. In crawl mode, links are followed from these pages.",
"items": {
"type": "object",
"required": [
"url"
],
"properties": {
"url": {
"type": "string",
"title": "URL of a web page",
"format": "uri"
}
}
}
},
"crawlMode": {
"title": "Mode",
"enum": [
"single",
"crawl",
"sitemap"
],
"type": "string",
"description": "Single pages: convert only the URLs you list. Crawl: follow links from the start URLs. Sitemap: take every URL from the site's sitemap.xml.",
"default": "crawl"
},
"maxPages": {
"title": "Max pages",
"minimum": 1,
"maximum": 100000,
"type": "integer",
"description": "Maximum number of pages to convert in total. Each converted page is billed as one Page converted event.",
"default": 50
},
"maxDepth": {
"title": "Max crawl depth",
"minimum": 0,
"maximum": 50,
"type": "integer",
"description": "How many link hops from the start URLs to follow (crawl mode).",
"default": 3
},
"crawlScope": {
"title": "Crawl scope",
"enum": [
"host",
"domain",
"path"
],
"type": "string",
"description": "Which links to follow: same hostname, same registered domain (incl. subdomains), or only URLs under the start URL's path (e.g. /docs/).",
"default": "host"
},
"includeUrlPatterns": {
"title": "Only URLs matching",
"type": "array",
"description": "Glob patterns like https://example.com/docs/** or /regex/.",
"items": {
"type": "string"
}
},
"excludeUrlPatterns": {
"title": "Skip URLs matching",
"type": "array",
"description": "Glob patterns or /regex/ for URLs to skip.",
"items": {
"type": "string"
}
},
"contentMode": {
"title": "Content extraction",
"enum": [
"main",
"full"
],
"type": "string",
"description": "Main content uses Mozilla Readability (the Firefox Reader View engine) to keep the article/body and drop menus, footers, cookie bars and ads. Full page keeps everything except navigation, header, footer and sidebars.",
"default": "main"
},
"removeSelectors": {
"title": "Remove CSS selectors",
"type": "string",
"description": "Extra elements to remove before conversion, e.g. .promo, #related-posts."
},
"keepLinks": {
"title": "Keep links",
"type": "boolean",
"description": "Keep hyperlinks as Markdown links (absolute URLs). Off = plain text.",
"default": true
},
"keepImages": {
"title": "Keep images",
"type": "boolean",
"description": "Keep images as Markdown image tags.",
"default": false
},
"chunkSize": {
"title": "Chunk size (characters)",
"minimum": 0,
"maximum": 100000,
"type": "integer",
"description": "Split each page's Markdown into chunks of about this many characters for RAG / vector databases (heading- and paragraph-aware). 0 = no chunking. ~4 characters ≈ 1 token.",
"default": 0
},
"chunkOverlap": {
"title": "Chunk overlap (characters)",
"minimum": 0,
"maximum": 5000,
"type": "integer",
"description": "Characters repeated between consecutive chunks.",
"default": 200
},
"minWordCount": {
"title": "Skip pages under N words",
"minimum": 0,
"type": "integer",
"description": "Pages with fewer words are skipped (not saved, not billed).",
"default": 0
},
"includeText": {
"title": "Also output plain text",
"type": "boolean",
"description": "Add a text field without Markdown syntax.",
"default": false
},
"includeHtml": {
"title": "Also output cleaned HTML",
"type": "boolean",
"description": "Add the cleaned content HTML.",
"default": false
},
"ignoreQueryStrings": {
"title": "Ignore query strings",
"type": "boolean",
"description": "Treat URLs that differ only in ?query as the same page.",
"default": true
},
"respectRobotsTxt": {
"title": "Respect robots.txt",
"type": "boolean",
"description": "Skip URLs disallowed by robots.txt for followed links.",
"default": true
},
"userAgent": {
"title": "User agent",
"enum": [
"browser",
"bot"
],
"type": "string",
"description": "Request identity.",
"default": "browser"
},
"maxConcurrency": {
"title": "Max concurrency",
"minimum": 1,
"maximum": 30,
"type": "integer",
"description": "Parallel requests.",
"default": 5
}
}
},
"runsResponseSchema": {
"type": "object",
"properties": {
"data": {
"type": "object",
"properties": {
"id": {
"type": "string"
},
"actId": {
"type": "string"
},
"userId": {
"type": "string"
},
"startedAt": {
"type": "string",
"format": "date-time",
"example": "2025-01-08T00:00:00.000Z"
},
"finishedAt": {
"type": "string",
"format": "date-time",
"example": "2025-01-08T00:00:00.000Z"
},
"status": {
"type": "string",
"example": "READY"
},
"meta": {
"type": "object",
"properties": {
"origin": {
"type": "string",
"example": "API"
},
"userAgent": {
"type": "string"
}
}
},
"stats": {
"type": "object",
"properties": {
"inputBodyLen": {
"type": "integer",
"example": 2000
},
"rebootCount": {
"type": "integer",
"example": 0
},
"restartCount": {
"type": "integer",
"example": 0
},
"resurrectCount": {
"type": "integer",
"example": 0
},
"computeUnits": {
"type": "integer",
"example": 0
}
}
},
"options": {
"type": "object",
"properties": {
"build": {
"type": "string",
"example": "latest"
},
"timeoutSecs": {
"type": "integer",
"example": 300
},
"memoryMbytes": {
"type": "integer",
"example": 1024
},
"diskMbytes": {
"type": "integer",
"example": 2048
}
}
},
"buildId": {
"type": "string"
},
"defaultKeyValueStoreId": {
"type": "string"
},
"defaultDatasetId": {
"type": "string"
},
"defaultRequestQueueId": {
"type": "string"
},
"buildNumber": {
"type": "string",
"example": "1.0.0"
},
"containerUrl": {
"type": "string"
},
"usage": {
"type": "object",
"properties": {
"ACTOR_COMPUTE_UNITS": {
"type": "integer",
"example": 0
},
"DATASET_READS": {
"type": "integer",
"example": 0
},
"DATASET_WRITES": {
"type": "integer",
"example": 0
},
"KEY_VALUE_STORE_READS": {
"type": "integer",
"example": 0
},
"KEY_VALUE_STORE_WRITES": {
"type": "integer",
"example": 1
},
"KEY_VALUE_STORE_LISTS": {
"type": "integer",
"example": 0
},
"REQUEST_QUEUE_READS": {
"type": "integer",
"example": 0
},
"REQUEST_QUEUE_WRITES": {
"type": "integer",
"example": 0
},
"DATA_TRANSFER_INTERNAL_GBYTES": {
"type": "integer",
"example": 0
},
"DATA_TRANSFER_EXTERNAL_GBYTES": {
"type": "integer",
"example": 0
},
"PROXY_RESIDENTIAL_TRANSFER_GBYTES": {
"type": "integer",
"example": 0
},
"PROXY_SERPS": {
"type": "integer",
"example": 0
},
"PROXY_UNBLOCKER_UNITS": {
"type": "integer",
"example": 0
}
}
},
"usageTotalUsd": {
"type": "number",
"example": 0.00005
},
"usageUsd": {
"type": "object",
"properties": {
"ACTOR_COMPUTE_UNITS": {
"type": "integer",
"example": 0
},
"DATASET_READS": {
"type": "integer",
"example": 0
},
"DATASET_WRITES": {
"type": "integer",
"example": 0
},
"KEY_VALUE_STORE_READS": {
"type": "integer",
"example": 0
},
"KEY_VALUE_STORE_WRITES": {
"type": "number",
"example": 0.00005
},
"KEY_VALUE_STORE_LISTS": {
"type": "integer",
"example": 0
},
"REQUEST_QUEUE_READS": {
"type": "integer",
"example": 0
},
"REQUEST_QUEUE_WRITES": {
"type": "integer",
"example": 0
},
"DATA_TRANSFER_INTERNAL_GBYTES": {
"type": "integer",
"example": 0
},
"DATA_TRANSFER_EXTERNAL_GBYTES": {
"type": "integer",
"example": 0
},
"PROXY_RESIDENTIAL_TRANSFER_GBYTES": {
"type": "integer",
"example": 0
},
"PROXY_SERPS": {
"type": "integer",
"example": 0
},
"PROXY_UNBLOCKER_UNITS": {
"type": "integer",
"example": 0
}
}
}
}
}
}
}
}
}
}