{ "openapi": "3.0.1", "info": { "title": "Website to Markdown & RAG Chunks for AI", "description": "Crawl public websites over HTTP and get clean Markdown plus ready-to-embed chunks with heading path, token estimate and hash. Sitemaps, robots.txt and a changed-pages mode. You only pay for pages that returned text.", "version": "0.3", "x-build-id": "5crhjPI7WKfkjzvaI" }, "servers": [ { "url": "https://api.apify.com/v2" } ], "paths": { "/acts/belgair~website-markdown-rag/run-sync-get-dataset-items": { "post": { "operationId": "run-sync-get-dataset-items-belgair-website-markdown-rag", "x-openai-isConsequential": false, "summary": "Executes an Actor, waits for its completion, and returns Actor's dataset items in response.", "tags": [ "Run Actor" ], "requestBody": { "required": true, "content": { "application/json": { "schema": { "$ref": "#/components/schemas/inputSchema" } } } }, "parameters": [ { "name": "token", "in": "query", "required": true, "schema": { "type": "string" }, "description": "Enter your Apify token here" } ], "responses": { "200": { "description": "OK" } } } }, "/acts/belgair~website-markdown-rag/runs": { "post": { "operationId": "runs-sync-belgair-website-markdown-rag", "x-openai-isConsequential": false, "summary": "Executes an Actor and returns information about the initiated run in response.", "tags": [ "Run Actor" ], "requestBody": { "required": true, "content": { "application/json": { "schema": { "$ref": "#/components/schemas/inputSchema" } } } }, "parameters": [ { "name": "token", "in": "query", "required": true, "schema": { "type": "string" }, "description": "Enter your Apify token here" } ], "responses": { "200": { "description": "OK", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/runsResponseSchema" } } } } } } }, "/acts/belgair~website-markdown-rag/run-sync": { "post": { "operationId": "run-sync-belgair-website-markdown-rag", "x-openai-isConsequential": false, "summary": "Executes an Actor, waits for completion, and returns the OUTPUT from Key-value store in response.", "tags": [ "Run Actor" ], "requestBody": { "required": true, "content": { "application/json": { "schema": { "$ref": "#/components/schemas/inputSchema" } } } }, "parameters": [ { "name": "token", "in": "query", "required": true, "schema": { "type": "string" }, "description": "Enter your Apify token here" } ], "responses": { "200": { "description": "OK" } } } } }, "components": { "schemas": { "inputSchema": { "type": "object", "required": [ "startUrls" ], "properties": { "startUrls": { "title": "Start URLs", "type": "array", "description": "Pages to start from, on public websites you are allowed to crawl. The crawler follows links and the sitemap from there, within the scope below. Up to 100 start URLs.", "items": { "type": "object", "required": [ "url" ], "properties": { "url": { "type": "string", "title": "URL of a web page", "format": "uri" } } } }, "scope": { "title": "Scope", "enum": [ "folder", "host", "domain" ], "type": "string", "description": "Which links to follow. Folder (default): the same host and the folder of the start URL, so https://example.com/docs/intro crawls everything under /docs/. Host: the whole host. Domain: the host and its subdomains.", "default": "folder" }, "pathPrefix": { "title": "Path prefix (optional)", "type": "string", "description": "Only crawl paths that start with this, for example /blog/ or /docs/v2/. Replaces the folder of the start URL. Leave empty to use the scope above." }, "excludePaths": { "title": "Skip paths (optional)", "type": "array", "description": "Skip URLs whose path starts with one of these, for example /blog/tag/ or /docs/v1/. Use * for any characters, for example /*/print/. Skipped URLs count as outside the scope in RUN_SUMMARY and cost nothing.", "items": { "type": "string" } }, "maxResults": { "title": "Max pages", "minimum": 1, "maximum": 10000, "type": "integer", "description": "Stop after this many pages with text. This is also the most you can be charged for in one run: empty, blocked, failed and duplicate pages are free.", "default": 100 }, "chunkSize": { "title": "Chunk size (tokens)", "minimum": 100, "maximum": 8000, "type": "integer", "description": "Target size of each chunk, in estimated tokens (about 4 characters per token). Chunks follow the headings of the page; a section that is too long is split by paragraph, list item, table row, code line or sentence.", "default": 500 }, "chunkOverlap": { "title": "Chunk overlap (tokens)", "minimum": 0, "maximum": 1000, "type": "integer", "description": "How much of the previous chunk is repeated at the start of the next one, within the same section. Whole blocks only, so code and tables stay intact. 0 = no overlap. At most half the chunk size.", "default": 50 }, "onlyChanged": { "title": "Only changed pages", "type": "boolean", "description": "Return only pages that are new or whose clean text changed since the last run with the same start URLs, scope, path prefix, skip paths and link depth (each combination keeps its own memory). The first run returns every page. Pages that did not change are checked but not delivered and not charged. Off = every run returns all pages again.", "default": false }, "useSitemaps": { "title": "Use sitemaps", "type": "boolean", "description": "Read the sitemaps listed in robots.txt (or /sitemap.xml), including sitemap indexes, and queue the pages that fall within the scope. Order: the start URLs first, then the links on those pages (links in the main content before menu links), then the sitemap, then deeper links.", "default": true }, "sitemapUrls": { "title": "Extra sitemap URLs (optional)", "type": "array", "description": "Sitemaps to read in addition to the ones found automatically, for example https://example.com/sitemap-docs.xml.", "items": { "type": "string" } }, "maxDepth": { "title": "Max link depth", "minimum": 0, "maximum": 50, "type": "integer", "description": "How many clicks away from a start URL the crawler goes. Start URLs are depth 0, pages from the sitemap depth 1.", "default": 10 } } }, "runsResponseSchema": { "type": "object", "properties": { "data": { "type": "object", "properties": { "id": { "type": "string" }, "actId": { "type": "string" }, "userId": { "type": "string" }, "startedAt": { "type": "string", "format": "date-time", "example": "2025-01-08T00:00:00.000Z" }, "finishedAt": { "type": "string", "format": "date-time", "example": "2025-01-08T00:00:00.000Z" }, "status": { "type": "string", "example": "READY" }, "meta": { "type": "object", "properties": { "origin": { "type": "string", "example": "API" }, "userAgent": { "type": "string" } } }, "stats": { "type": "object", "properties": { "inputBodyLen": { "type": "integer", "example": 2000 }, "rebootCount": { "type": "integer", "example": 0 }, "restartCount": { "type": "integer", "example": 0 }, "resurrectCount": { "type": "integer", "example": 0 }, "computeUnits": { "type": "integer", "example": 0 } } }, "options": { "type": "object", "properties": { "build": { "type": "string", "example": "latest" }, "timeoutSecs": { "type": "integer", "example": 300 }, "memoryMbytes": { "type": "integer", "example": 1024 }, "diskMbytes": { "type": "integer", "example": 2048 } } }, "buildId": { "type": "string" }, "defaultKeyValueStoreId": { "type": "string" }, "defaultDatasetId": { "type": "string" }, "defaultRequestQueueId": { "type": "string" }, "buildNumber": { "type": "string", "example": "1.0.0" }, "containerUrl": { "type": "string" }, "usage": { "type": "object", "properties": { "ACTOR_COMPUTE_UNITS": { "type": "integer", "example": 0 }, "DATASET_READS": { "type": "integer", "example": 0 }, "DATASET_WRITES": { "type": "integer", "example": 0 }, "KEY_VALUE_STORE_READS": { "type": "integer", "example": 0 }, "KEY_VALUE_STORE_WRITES": { "type": "integer", "example": 1 }, "KEY_VALUE_STORE_LISTS": { "type": "integer", "example": 0 }, "REQUEST_QUEUE_READS": { "type": "integer", "example": 0 }, "REQUEST_QUEUE_WRITES": { "type": "integer", "example": 0 }, "DATA_TRANSFER_INTERNAL_GBYTES": { "type": "integer", "example": 0 }, "DATA_TRANSFER_EXTERNAL_GBYTES": { "type": "integer", "example": 0 }, "PROXY_RESIDENTIAL_TRANSFER_GBYTES": { "type": "integer", "example": 0 }, "PROXY_SERPS": { "type": "integer", "example": 0 }, "PROXY_UNBLOCKER_UNITS": { "type": "integer", "example": 0 } } }, "usageTotalUsd": { "type": "number", "example": 0.00005 }, "usageUsd": { "type": "object", "properties": { "ACTOR_COMPUTE_UNITS": { "type": "integer", "example": 0 }, "DATASET_READS": { "type": "integer", "example": 0 }, "DATASET_WRITES": { "type": "integer", "example": 0 }, "KEY_VALUE_STORE_READS": { "type": "integer", "example": 0 }, "KEY_VALUE_STORE_WRITES": { "type": "number", "example": 0.00005 }, "KEY_VALUE_STORE_LISTS": { "type": "integer", "example": 0 }, "REQUEST_QUEUE_READS": { "type": "integer", "example": 0 }, "REQUEST_QUEUE_WRITES": { "type": "integer", "example": 0 }, "DATA_TRANSFER_INTERNAL_GBYTES": { "type": "integer", "example": 0 }, "DATA_TRANSFER_EXTERNAL_GBYTES": { "type": "integer", "example": 0 }, "PROXY_RESIDENTIAL_TRANSFER_GBYTES": { "type": "integer", "example": 0 }, "PROXY_SERPS": { "type": "integer", "example": 0 }, "PROXY_UNBLOCKER_UNITS": { "type": "integer", "example": 0 } } } } } } } } } }