{ "openapi": "3.0.1", "info": { "title": "Web Crawler: Website to Markdown for LLM & RAG", "description": "Website content crawler for AI. Crawl any site into clean Markdown, text and ready-to-embed RAG chunks for LLMs and vector databases. Removes navigation, footers and cookie banners, renders JavaScript only when needed. $2 per 1,000 pages.", "version": "1.0", "x-build-id": "JgjJqsm1Q14lxlZSD" }, "servers": [ { "url": "https://api.apify.com/v2" } ], "paths": { "/acts/scrape-lads~website-content-crawler-rag/run-sync-get-dataset-items": { "post": { "operationId": "run-sync-get-dataset-items-scrape-lads-website-content-crawler-rag", "x-openai-isConsequential": false, "summary": "Executes an Actor, waits for its completion, and returns Actor's dataset items in response.", "tags": [ "Run Actor" ], "requestBody": { "required": true, "content": { "application/json": { "schema": { "$ref": "#/components/schemas/inputSchema" } } } }, "parameters": [ { "name": "token", "in": "query", "required": true, "schema": { "type": "string" }, "description": "Enter your Apify token here" } ], "responses": { "200": { "description": "OK" } } } }, "/acts/scrape-lads~website-content-crawler-rag/runs": { "post": { "operationId": "runs-sync-scrape-lads-website-content-crawler-rag", "x-openai-isConsequential": false, "summary": "Executes an Actor and returns information about the initiated run in response.", "tags": [ "Run Actor" ], "requestBody": { "required": true, "content": { "application/json": { "schema": { "$ref": "#/components/schemas/inputSchema" } } } }, "parameters": [ { "name": "token", "in": "query", "required": true, "schema": { "type": "string" }, "description": "Enter your Apify token here" } ], "responses": { "200": { "description": "OK", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/runsResponseSchema" } } } } } } }, "/acts/scrape-lads~website-content-crawler-rag/run-sync": { "post": { "operationId": "run-sync-scrape-lads-website-content-crawler-rag", "x-openai-isConsequential": false, "summary": "Executes an Actor, waits for completion, and returns the OUTPUT from Key-value store in response.", "tags": [ "Run Actor" ], "requestBody": { "required": true, "content": { "application/json": { "schema": { "$ref": "#/components/schemas/inputSchema" } } } }, "parameters": [ { "name": "token", "in": "query", "required": true, "schema": { "type": "string" }, "description": "Enter your Apify token here" } ], "responses": { "200": { "description": "OK" } } } } }, "components": { "schemas": { "inputSchema": { "type": "object", "required": [ "startUrls" ], "properties": { "startUrls": { "title": "Start URLs", "type": "array", "description": "Websites or sections to crawl. With the default scope the crawler stays under each URL's path, so https://docs.apify.com/academy only crawls pages under /academy.", "items": { "type": "object", "required": [ "url" ], "properties": { "url": { "type": "string", "title": "URL of a web page", "format": "uri" } } } }, "maxPages": { "title": "Max pages", "minimum": 1, "maximum": 100000, "type": "integer", "description": "Stop after this many pages are saved. Each saved page is one billable event, so this is also your price cap.", "default": 50 }, "maxCrawlDepth": { "title": "Max crawl depth", "minimum": 0, "maximum": 100, "type": "integer", "description": "How many links away from a start URL to follow. 0 crawls only the start URLs.", "default": 5 }, "crawlScope": { "title": "Crawl scope", "enum": [ "path", "hostname", "domain" ], "type": "string", "description": "Which discovered links to follow. \"Same path\" stays under the start URL's folder, \"Same hostname\" allows the whole host, \"Same domain\" also allows subdomains.", "default": "path" }, "includeUrlGlobs": { "title": "Include URLs (globs)", "type": "array", "description": "Optional. Only follow links matching one of these glob patterns, e.g. https://example.com/docs/**. Applied on top of the crawl scope.", "default": [], "items": { "type": "object", "required": [ "glob" ], "properties": { "glob": { "type": "string", "title": "Glob of a web page" } } } }, "excludeUrlGlobs": { "title": "Exclude URLs (globs)", "type": "array", "description": "Optional. Never follow links matching these glob patterns, e.g. **/changelog/** or **/*?page=*.", "default": [], "items": { "type": "object", "required": [ "glob" ], "properties": { "glob": { "type": "string", "title": "Glob of a web page" } } } }, "crawlerType": { "title": "Crawler type", "enum": [ "auto", "http", "browser" ], "type": "string", "description": "\"Auto\" fetches pages over fast HTTP and only renders a page in a headless browser when it looks JavaScript-rendered (empty app shell). \"HTTP only\" never uses a browser. \"Browser\" renders every page in headless Chrome (slower, $0.004 per page).", "default": "auto" }, "outputFormats": { "title": "Output formats", "type": "array", "description": "Which content fields to save for each page.", "items": { "type": "string", "enum": [ "markdown", "text", "html" ], "enumTitles": [ "Markdown", "Plain text", "Cleaned HTML" ] }, "default": [ "markdown", "text" ] }, "removeBoilerplate": { "title": "Remove boilerplate", "type": "boolean", "description": "Strip navigation, headers, footers, sidebars, cookie banners and share widgets, and keep the main content only.", "default": true }, "removeElementsCssSelector": { "title": "Extra elements to remove (CSS selector)", "type": "string", "description": "Optional CSS selector of additional elements to delete before conversion, e.g. .ad-slot, #comments." }, "chunkSize": { "title": "Chunk size (characters)", "minimum": 0, "maximum": 100000, "type": "integer", "description": "Set above 0 to split each page into chunks of at most this many characters, along headings and paragraphs, ready for embeddings. 1000 to 2000 works well for most vector databases. 0 turns chunking off.", "default": 0 }, "chunkOverlap": { "title": "Chunk overlap (characters)", "minimum": 0, "maximum": 50000, "type": "integer", "description": "Characters from the end of the previous chunk repeated at the start of the next one, so context is not lost at the boundary. Capped at half the chunk size.", "default": 200 }, "useSitemaps": { "title": "Discover pages from sitemaps", "type": "boolean", "description": "Also read robots.txt Sitemap entries and /sitemap.xml to find pages that are not linked. In-scope sitemap URLs are crawled alongside discovered links.", "default": false }, "respectRobotsTxt": { "title": "Respect robots.txt", "type": "boolean", "description": "Skip pages that the site's robots.txt disallows.", "default": true }, "maxConcurrency": { "title": "Max concurrency", "minimum": 1, "maximum": 50, "type": "integer", "description": "Maximum parallel HTTP requests. Browser pages are always limited to a few at a time.", "default": 10 } } }, "runsResponseSchema": { "type": "object", "properties": { "data": { "type": "object", "properties": { "id": { "type": "string" }, "actId": { "type": "string" }, "userId": { "type": "string" }, "startedAt": { "type": "string", "format": "date-time", "example": "2025-01-08T00:00:00.000Z" }, "finishedAt": { "type": "string", "format": "date-time", "example": "2025-01-08T00:00:00.000Z" }, "status": { "type": "string", "example": "READY" }, "meta": { "type": "object", "properties": { "origin": { "type": "string", "example": "API" }, "userAgent": { "type": "string" } } }, "stats": { "type": "object", "properties": { "inputBodyLen": { "type": "integer", "example": 2000 }, "rebootCount": { "type": "integer", "example": 0 }, "restartCount": { "type": "integer", "example": 0 }, "resurrectCount": { "type": "integer", "example": 0 }, "computeUnits": { "type": "integer", "example": 0 } } }, "options": { "type": "object", "properties": { "build": { "type": "string", "example": "latest" }, "timeoutSecs": { "type": "integer", "example": 300 }, "memoryMbytes": { "type": "integer", "example": 1024 }, "diskMbytes": { "type": "integer", "example": 2048 } } }, "buildId": { "type": "string" }, "defaultKeyValueStoreId": { "type": "string" }, "defaultDatasetId": { "type": "string" }, "defaultRequestQueueId": { "type": "string" }, "buildNumber": { "type": "string", "example": "1.0.0" }, "containerUrl": { "type": "string" }, "usage": { "type": "object", "properties": { "ACTOR_COMPUTE_UNITS": { "type": "integer", "example": 0 }, "DATASET_READS": { "type": "integer", "example": 0 }, "DATASET_WRITES": { "type": "integer", "example": 0 }, "KEY_VALUE_STORE_READS": { "type": "integer", "example": 0 }, "KEY_VALUE_STORE_WRITES": { "type": "integer", "example": 1 }, "KEY_VALUE_STORE_LISTS": { "type": "integer", "example": 0 }, "REQUEST_QUEUE_READS": { "type": "integer", "example": 0 }, "REQUEST_QUEUE_WRITES": { "type": "integer", "example": 0 }, "DATA_TRANSFER_INTERNAL_GBYTES": { "type": "integer", "example": 0 }, "DATA_TRANSFER_EXTERNAL_GBYTES": { "type": "integer", "example": 0 }, "PROXY_RESIDENTIAL_TRANSFER_GBYTES": { "type": "integer", "example": 0 }, "PROXY_SERPS": { "type": "integer", "example": 0 }, "PROXY_UNBLOCKER_UNITS": { "type": "integer", "example": 0 } } }, "usageTotalUsd": { "type": "number", "example": 0.00005 }, "usageUsd": { "type": "object", "properties": { "ACTOR_COMPUTE_UNITS": { "type": "integer", "example": 0 }, "DATASET_READS": { "type": "integer", "example": 0 }, "DATASET_WRITES": { "type": "integer", "example": 0 }, "KEY_VALUE_STORE_READS": { "type": "integer", "example": 0 }, "KEY_VALUE_STORE_WRITES": { "type": "number", "example": 0.00005 }, "KEY_VALUE_STORE_LISTS": { "type": "integer", "example": 0 }, "REQUEST_QUEUE_READS": { "type": "integer", "example": 0 }, "REQUEST_QUEUE_WRITES": { "type": "integer", "example": 0 }, "DATA_TRANSFER_INTERNAL_GBYTES": { "type": "integer", "example": 0 }, "DATA_TRANSFER_EXTERNAL_GBYTES": { "type": "integer", "example": 0 }, "PROXY_RESIDENTIAL_TRANSFER_GBYTES": { "type": "integer", "example": 0 }, "PROXY_SERPS": { "type": "integer", "example": 0 }, "PROXY_UNBLOCKER_UNITS": { "type": "integer", "example": 0 } } } } } } } } } }