{ "openapi": "3.0.1", "info": { "title": "Website Content Crawler API: Web Pages to Markdown for RAG", "description": "Website content crawler API: turn any web page or whole site into clean Markdown and text for RAG, LLMs and AI agents. Crawls links and sitemaps, renders JavaScript pages when needed. RAG Web Browser alternative at $1 per 1,000 pages.", "version": "0.1", "x-build-id": "aavMGdNTidrNaZr9C" }, "servers": [ { "url": "https://api.apify.com/v2" } ], "paths": { "/acts/sauliusautomatesit~website-content-crawler-api/run-sync-get-dataset-items": { "post": { "operationId": "run-sync-get-dataset-items-sauliusautomatesit-website-content-crawler-api", "x-openai-isConsequential": false, "summary": "Executes an Actor, waits for its completion, and returns Actor's dataset items in response.", "tags": [ "Run Actor" ], "requestBody": { "required": true, "content": { "application/json": { "schema": { "$ref": "#/components/schemas/inputSchema" } } } }, "parameters": [ { "name": "token", "in": "query", "required": true, "schema": { "type": "string" }, "description": "Enter your Apify token here" } ], "responses": { "200": { "description": "OK" } } } }, "/acts/sauliusautomatesit~website-content-crawler-api/runs": { "post": { "operationId": "runs-sync-sauliusautomatesit-website-content-crawler-api", "x-openai-isConsequential": false, "summary": "Executes an Actor and returns information about the initiated run in response.", "tags": [ "Run Actor" ], "requestBody": { "required": true, "content": { "application/json": { "schema": { "$ref": "#/components/schemas/inputSchema" } } } }, "parameters": [ { "name": "token", "in": "query", "required": true, "schema": { "type": "string" }, "description": "Enter your Apify token here" } ], "responses": { "200": { "description": "OK", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/runsResponseSchema" } } } } } } }, "/acts/sauliusautomatesit~website-content-crawler-api/run-sync": { "post": { "operationId": "run-sync-sauliusautomatesit-website-content-crawler-api", "x-openai-isConsequential": false, "summary": "Executes an Actor, waits for completion, and returns the OUTPUT from Key-value store in response.", "tags": [ "Run Actor" ], "requestBody": { "required": true, "content": { "application/json": { "schema": { "$ref": "#/components/schemas/inputSchema" } } } }, "parameters": [ { "name": "token", "in": "query", "required": true, "schema": { "type": "string" }, "description": "Enter your Apify token here" } ], "responses": { "200": { "description": "OK" } } } } }, "components": { "schemas": { "inputSchema": { "type": "object", "required": [ "startUrls" ], "properties": { "startUrls": { "title": "Start URLs", "type": "array", "description": "Web pages to read. With crawl depth 0 (default) you get exactly these pages; raise the depth to follow their links. A bare domain (apify.com) works too.", "items": { "type": "string" } }, "maxCrawlDepth": { "title": "Crawl depth", "minimum": 0, "maximum": 20, "type": "integer", "description": "0 reads only the start URLs. 1 also reads the pages they link to, 2 the pages those link to, and so on. Links are followed only on the same site and inside the start URL's folder (https://docs.apify.com/platform/actors crawls https://docs.apify.com/platform/...).", "default": 0 }, "maxPagesPerStartUrl": { "title": "Max pages per start URL", "minimum": 1, "type": "integer", "description": "Stop crawling a start URL's site after this many pages.", "default": 100 }, "maxPages": { "title": "Max pages in total", "minimum": 0, "type": "integer", "description": "Stop the run after this many charged pages. 0 means no limit beyond the per start URL limit.", "default": 0 }, "useSitemaps": { "title": "Use sitemaps", "type": "boolean", "description": "Also read the pages listed in the site's sitemap.xml (and sitemaps named in robots.txt), within the start URL's folder. The fastest way to get a whole site or docs section. Used when Max pages per start URL is 10 or more.", "default": false }, "urlMustContain": { "title": "Only URLs containing", "type": "array", "description": "Crawl only links whose URL contains at least one of these texts, for example /blog/ or /docs/. Start URLs are always read.", "items": { "type": "string" } }, "urlMustNotContain": { "title": "Skip URLs containing", "type": "array", "description": "Do not crawl links whose URL contains any of these texts, for example /tag/ or ?page=.", "items": { "type": "string" } }, "mainContentOnly": { "title": "Main content only", "type": "boolean", "description": "On articles and docs, keep the main text and drop sidebars and related links (Mozilla Readability). Pages where the main block is only a small part of the text, like home pages, keep everything except navigation, footers, scripts and cookie banners.", "default": true }, "removeSelectors": { "title": "Remove elements (CSS selector)", "type": "string", "description": "Extra elements to drop before conversion, for example .sidebar, .ads, #comments" }, "includeText": { "title": "Include plain text", "type": "boolean", "description": "Add a text field with the page's text without Markdown.", "default": false }, "includeHtml": { "title": "Include cleaned HTML", "type": "boolean", "description": "Add the cleaned HTML that the Markdown was made from.", "default": false }, "includeLinks": { "title": "Include links", "type": "boolean", "description": "Add every link found on the page (absolute URLs).", "default": false }, "maxConcurrency": { "title": "Pages in parallel", "minimum": 1, "maximum": 100, "type": "integer", "description": "How many pages to load at once. Capped at one per 128 MB of run memory (8 at the default 1 GB). A run gets one CPU core per 4 GB of memory, so give big crawls 2 to 4 GB: same price, faster.", "default": 20 } } }, "runsResponseSchema": { "type": "object", "properties": { "data": { "type": "object", "properties": { "id": { "type": "string" }, "actId": { "type": "string" }, "userId": { "type": "string" }, "startedAt": { "type": "string", "format": "date-time", "example": "2025-01-08T00:00:00.000Z" }, "finishedAt": { "type": "string", "format": "date-time", "example": "2025-01-08T00:00:00.000Z" }, "status": { "type": "string", "example": "READY" }, "meta": { "type": "object", "properties": { "origin": { "type": "string", "example": "API" }, "userAgent": { "type": "string" } } }, "stats": { "type": "object", "properties": { "inputBodyLen": { "type": "integer", "example": 2000 }, "rebootCount": { "type": "integer", "example": 0 }, "restartCount": { "type": "integer", "example": 0 }, "resurrectCount": { "type": "integer", "example": 0 }, "computeUnits": { "type": "integer", "example": 0 } } }, "options": { "type": "object", "properties": { "build": { "type": "string", "example": "latest" }, "timeoutSecs": { "type": "integer", "example": 300 }, "memoryMbytes": { "type": "integer", "example": 1024 }, "diskMbytes": { "type": "integer", "example": 2048 } } }, "buildId": { "type": "string" }, "defaultKeyValueStoreId": { "type": "string" }, "defaultDatasetId": { "type": "string" }, "defaultRequestQueueId": { "type": "string" }, "buildNumber": { "type": "string", "example": "1.0.0" }, "containerUrl": { "type": "string" }, "usage": { "type": "object", "properties": { "ACTOR_COMPUTE_UNITS": { "type": "integer", "example": 0 }, "DATASET_READS": { "type": "integer", "example": 0 }, "DATASET_WRITES": { "type": "integer", "example": 0 }, "KEY_VALUE_STORE_READS": { "type": "integer", "example": 0 }, "KEY_VALUE_STORE_WRITES": { "type": "integer", "example": 1 }, "KEY_VALUE_STORE_LISTS": { "type": "integer", "example": 0 }, "REQUEST_QUEUE_READS": { "type": "integer", "example": 0 }, "REQUEST_QUEUE_WRITES": { "type": "integer", "example": 0 }, "DATA_TRANSFER_INTERNAL_GBYTES": { "type": "integer", "example": 0 }, "DATA_TRANSFER_EXTERNAL_GBYTES": { "type": "integer", "example": 0 }, "PROXY_RESIDENTIAL_TRANSFER_GBYTES": { "type": "integer", "example": 0 }, "PROXY_SERPS": { "type": "integer", "example": 0 }, "PROXY_UNBLOCKER_UNITS": { "type": "integer", "example": 0 } } }, "usageTotalUsd": { "type": "number", "example": 0.00005 }, "usageUsd": { "type": "object", "properties": { "ACTOR_COMPUTE_UNITS": { "type": "integer", "example": 0 }, "DATASET_READS": { "type": "integer", "example": 0 }, "DATASET_WRITES": { "type": "integer", "example": 0 }, "KEY_VALUE_STORE_READS": { "type": "integer", "example": 0 }, "KEY_VALUE_STORE_WRITES": { "type": "number", "example": 0.00005 }, "KEY_VALUE_STORE_LISTS": { "type": "integer", "example": 0 }, "REQUEST_QUEUE_READS": { "type": "integer", "example": 0 }, "REQUEST_QUEUE_WRITES": { "type": "integer", "example": 0 }, "DATA_TRANSFER_INTERNAL_GBYTES": { "type": "integer", "example": 0 }, "DATA_TRANSFER_EXTERNAL_GBYTES": { "type": "integer", "example": 0 }, "PROXY_RESIDENTIAL_TRANSFER_GBYTES": { "type": "integer", "example": 0 }, "PROXY_SERPS": { "type": "integer", "example": 0 }, "PROXY_UNBLOCKER_UNITS": { "type": "integer", "example": 0 } } } } } } } } } }