{ "openapi": "3.0.1", "info": { "title": "Website to Markdown Crawler - LLM & RAG Ready (moved)", "description": "Moved: this actor is no longer actively maintained. Use the maintained version at https://apify.com/digivibetech/website-to-markdown-crawler. Crawl documentation sites, blogs and knowledge bases into clean Markdown for LLMs and RAG.", "version": "1.0", "x-build-id": "G9r6iOSpE2vMeBFJd" }, "servers": [ { "url": "https://api.apify.com/v2" } ], "paths": { "/acts/sophisticated_occasion~website-to-markdown-crawler/run-sync-get-dataset-items": { "post": { "operationId": "run-sync-get-dataset-items-sophisticated_occasion-website-to-markdown-crawler", "x-openai-isConsequential": false, "summary": "Executes an Actor, waits for its completion, and returns Actor's dataset items in response.", "tags": [ "Run Actor" ], "requestBody": { "required": true, "content": { "application/json": { "schema": { "$ref": "#/components/schemas/inputSchema" } } } }, "parameters": [ { "name": "token", "in": "query", "required": true, "schema": { "type": "string" }, "description": "Enter your Apify token here" } ], "responses": { "200": { "description": "OK" } } } }, "/acts/sophisticated_occasion~website-to-markdown-crawler/runs": { "post": { "operationId": "runs-sync-sophisticated_occasion-website-to-markdown-crawler", "x-openai-isConsequential": false, "summary": "Executes an Actor and returns information about the initiated run in response.", "tags": [ "Run Actor" ], "requestBody": { "required": true, "content": { "application/json": { "schema": { "$ref": "#/components/schemas/inputSchema" } } } }, "parameters": [ { "name": "token", "in": "query", "required": true, "schema": { "type": "string" }, "description": "Enter your Apify token here" } ], "responses": { "200": { "description": "OK", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/runsResponseSchema" } } } } } } }, "/acts/sophisticated_occasion~website-to-markdown-crawler/run-sync": { "post": { "operationId": "run-sync-sophisticated_occasion-website-to-markdown-crawler", "x-openai-isConsequential": false, "summary": "Executes an Actor, waits for completion, and returns the OUTPUT from Key-value store in response.", "tags": [ "Run Actor" ], "requestBody": { "required": true, "content": { "application/json": { "schema": { "$ref": "#/components/schemas/inputSchema" } } } }, "parameters": [ { "name": "token", "in": "query", "required": true, "schema": { "type": "string" }, "description": "Enter your Apify token here" } ], "responses": { "200": { "description": "OK" } } } } }, "components": { "schemas": { "inputSchema": { "type": "object", "required": [ "startUrls" ], "properties": { "startUrls": { "title": "Start URLs", "type": "array", "description": "Docs sites, blogs or knowledge bases (up to 200). Each start URL is crawled within its own host (and path, see below).", "items": { "type": "string" } }, "maxPagesPerSite": { "title": "Max pages per site", "minimum": 1, "maximum": 5000, "type": "integer", "description": "Hard limit per start URL, so the cost is predictable: at most this many pages x $0.001.", "default": 50 }, "maxDepth": { "title": "Max link depth", "minimum": 0, "maximum": 20, "type": "integer", "description": "0 = only the start URL.", "default": 5 }, "stayWithinPath": { "title": "Stay within the start URL's folder", "type": "boolean", "description": "For https://site.com/docs/intro only /docs/... pages are crawled.", "default": true }, "useSitemap": { "title": "Seed from sitemap.xml", "type": "boolean", "description": "Also queue in-scope URLs from the site's sitemaps (listed in robots.txt or /sitemap.xml).", "default": false }, "includeUrlRegex": { "title": "Only URLs matching (regex)", "type": "string", "description": "Optional regular expression; other URLs are not crawled." }, "excludeUrlRegex": { "title": "Skip URLs matching (regex)", "type": "string", "description": "Optional regular expression, e.g. /tag/|/page/\\d+" }, "chunkSize": { "title": "Chunk size for RAG (characters)", "minimum": 0, "maximum": 20000, "type": "integer", "description": "0 = off. Otherwise each item also has chunks[] split on paragraph boundaries. No extra cost.", "default": 0 }, "includeLinks": { "title": "Keep links in Markdown", "type": "boolean", "description": "Keep [text](url) links in the Markdown.", "default": true }, "redactContactDetails": { "title": "Redact email addresses and phone numbers", "type": "boolean", "description": "Replaces emails and phone numbers in the Markdown with [redacted] (privacy by default).", "default": true }, "syncName": { "title": "Sync name (only new and changed pages)", "type": "string", "description": "Optional. Give the crawl a name, for example docs-sync. The next run with the same name returns and charges only pages that are new or whose content changed; unchanged pages are skipped for free. SYNC_REPORT lists new, changed and removed URLs, so you can update or delete them in your vector database. Letters, digits and '-'." }, "removeRepeatedBlocks": { "title": "Remove text repeated across pages", "type": "boolean", "description": "Drops paragraphs that already appeared on an earlier page of the same site, such as newsletter boxes, 'related articles' lists and shared disclaimers. Your vector database then holds each text once instead of on every page. Headings and code blocks are always kept.", "default": false } } }, "runsResponseSchema": { "type": "object", "properties": { "data": { "type": "object", "properties": { "id": { "type": "string" }, "actId": { "type": "string" }, "userId": { "type": "string" }, "startedAt": { "type": "string", "format": "date-time", "example": "2025-01-08T00:00:00.000Z" }, "finishedAt": { "type": "string", "format": "date-time", "example": "2025-01-08T00:00:00.000Z" }, "status": { "type": "string", "example": "READY" }, "meta": { "type": "object", "properties": { "origin": { "type": "string", "example": "API" }, "userAgent": { "type": "string" } } }, "stats": { "type": "object", "properties": { "inputBodyLen": { "type": "integer", "example": 2000 }, "rebootCount": { "type": "integer", "example": 0 }, "restartCount": { "type": "integer", "example": 0 }, "resurrectCount": { "type": "integer", "example": 0 }, "computeUnits": { "type": "integer", "example": 0 } } }, "options": { "type": "object", "properties": { "build": { "type": "string", "example": "latest" }, "timeoutSecs": { "type": "integer", "example": 300 }, "memoryMbytes": { "type": "integer", "example": 1024 }, "diskMbytes": { "type": "integer", "example": 2048 } } }, "buildId": { "type": "string" }, "defaultKeyValueStoreId": { "type": "string" }, "defaultDatasetId": { "type": "string" }, "defaultRequestQueueId": { "type": "string" }, "buildNumber": { "type": "string", "example": "1.0.0" }, "containerUrl": { "type": "string" }, "usage": { "type": "object", "properties": { "ACTOR_COMPUTE_UNITS": { "type": "integer", "example": 0 }, "DATASET_READS": { "type": "integer", "example": 0 }, "DATASET_WRITES": { "type": "integer", "example": 0 }, "KEY_VALUE_STORE_READS": { "type": "integer", "example": 0 }, "KEY_VALUE_STORE_WRITES": { "type": "integer", "example": 1 }, "KEY_VALUE_STORE_LISTS": { "type": "integer", "example": 0 }, "REQUEST_QUEUE_READS": { "type": "integer", "example": 0 }, "REQUEST_QUEUE_WRITES": { "type": "integer", "example": 0 }, "DATA_TRANSFER_INTERNAL_GBYTES": { "type": "integer", "example": 0 }, "DATA_TRANSFER_EXTERNAL_GBYTES": { "type": "integer", "example": 0 }, "PROXY_RESIDENTIAL_TRANSFER_GBYTES": { "type": "integer", "example": 0 }, "PROXY_SERPS": { "type": "integer", "example": 0 }, "PROXY_UNBLOCKER_UNITS": { "type": "integer", "example": 0 } } }, "usageTotalUsd": { "type": "number", "example": 0.00005 }, "usageUsd": { "type": "object", "properties": { "ACTOR_COMPUTE_UNITS": { "type": "integer", "example": 0 }, "DATASET_READS": { "type": "integer", "example": 0 }, "DATASET_WRITES": { "type": "integer", "example": 0 }, "KEY_VALUE_STORE_READS": { "type": "integer", "example": 0 }, "KEY_VALUE_STORE_WRITES": { "type": "number", "example": 0.00005 }, "KEY_VALUE_STORE_LISTS": { "type": "integer", "example": 0 }, "REQUEST_QUEUE_READS": { "type": "integer", "example": 0 }, "REQUEST_QUEUE_WRITES": { "type": "integer", "example": 0 }, "DATA_TRANSFER_INTERNAL_GBYTES": { "type": "integer", "example": 0 }, "DATA_TRANSFER_EXTERNAL_GBYTES": { "type": "integer", "example": 0 }, "PROXY_RESIDENTIAL_TRANSFER_GBYTES": { "type": "integer", "example": 0 }, "PROXY_SERPS": { "type": "integer", "example": 0 }, "PROXY_UNBLOCKER_UNITS": { "type": "integer", "example": 0 } } } } } } } } } }