{ "openapi": "3.0.1", "info": { "title": "Domain-Scoped Site Crawler for Markdown Datasets", "description": "Crawl a website from one or more start URLs, staying on the same domain, and get back clean Markdown, text, or HTML per page with title, description, and crawl depth. Built for feeding LLM, RAG, or knowledge-base pipelines.", "version": "1.1", "x-build-id": "mlbNAEkzbrejmXih5" }, "servers": [ { "url": "https://api.apify.com/v2" } ], "paths": { "/acts/vexaloom-labs~domain-scoped-site-crawler-for-markdown-datasets/run-sync-get-dataset-items": { "post": { "operationId": "run-sync-get-dataset-items-vexaloom-labs-domain-scoped-site-crawler-for-markdown-datasets", "x-openai-isConsequential": false, "summary": "Executes an Actor, waits for its completion, and returns Actor's dataset items in response.", "tags": [ "Run Actor" ], "requestBody": { "required": true, "content": { "application/json": { "schema": { "$ref": "#/components/schemas/inputSchema" } } } }, "parameters": [ { "name": "token", "in": "query", "required": true, "schema": { "type": "string" }, "description": "Enter your Apify token here" } ], "responses": { "200": { "description": "OK" } } } }, "/acts/vexaloom-labs~domain-scoped-site-crawler-for-markdown-datasets/runs": { "post": { "operationId": "runs-sync-vexaloom-labs-domain-scoped-site-crawler-for-markdown-datasets", "x-openai-isConsequential": false, "summary": "Executes an Actor and returns information about the initiated run in response.", "tags": [ "Run Actor" ], "requestBody": { "required": true, "content": { "application/json": { "schema": { "$ref": "#/components/schemas/inputSchema" } } } }, "parameters": [ { "name": "token", "in": "query", "required": true, "schema": { "type": "string" }, "description": "Enter your Apify token here" } ], "responses": { "200": { "description": "OK", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/runsResponseSchema" } } } } } } }, "/acts/vexaloom-labs~domain-scoped-site-crawler-for-markdown-datasets/run-sync": { "post": { "operationId": "run-sync-vexaloom-labs-domain-scoped-site-crawler-for-markdown-datasets", "x-openai-isConsequential": false, "summary": "Executes an Actor, waits for completion, and returns the OUTPUT from Key-value store in response.", "tags": [ "Run Actor" ], "requestBody": { "required": true, "content": { "application/json": { "schema": { "$ref": "#/components/schemas/inputSchema" } } } }, "parameters": [ { "name": "token", "in": "query", "required": true, "schema": { "type": "string" }, "description": "Enter your Apify token here" } ], "responses": { "200": { "description": "OK" } } } } }, "components": { "schemas": { "inputSchema": { "type": "object", "required": [ "startUrls" ], "properties": { "startUrls": { "title": "Start URLs", "type": "array", "description": "One or more URLs to start crawling from. The crawler follows links found on these pages and stays on the domain of the first start URL.", "items": { "type": "object", "required": [ "url" ], "properties": { "url": { "type": "string", "title": "URL of a web page", "format": "uri" } } } }, "maxCrawlDepth": { "title": "Max Crawl Depth", "minimum": 0, "maximum": 100, "type": "integer", "description": "How many links deep to follow from the start URLs. 0 means only the start URLs are crawled.", "default": 10 }, "maxCrawlPages": { "title": "Max Pages", "minimum": 1, "maximum": 100000, "type": "integer", "description": "Maximum number of pages (and files) to store. The crawl stops when this limit is reached.", "default": 20 }, "useSitemaps": { "title": "Use Sitemaps", "type": "boolean", "description": "Look for sitemaps (from robots.txt and /sitemap.xml) and enqueue the URLs they list. Finds pages that are not linked from the start URLs.", "default": false }, "useLlmsTxt": { "title": "Use llms.txt", "type": "boolean", "description": "Look for /llms.txt at the start URL domains and enqueue the files listed there (common on documentation sites).", "default": false }, "respectRobotsTxtFile": { "title": "Respect robots.txt", "type": "boolean", "description": "Skip discovered pages that the site's robots.txt disallows. Start URLs are always crawled.", "default": true }, "crawlOnlyPagesChangedSince": { "title": "Crawl Only Pages Changed Since", "type": "string", "description": "Optional date (YYYY-MM-DD or full ISO date-time). Crawls only pages that may have changed since then and outputs a delta. Leave empty to crawl everything.", "default": "" }, "includeUrlGlobs": { "title": "Include URL Patterns (globs)", "type": "array", "description": "Only crawl discovered URLs matching these glob patterns (e.g. https://example.com/docs/*). Leave empty to allow everything on the domain.", "default": [], "items": { "type": "object", "required": [ "glob" ], "properties": { "glob": { "type": "string", "title": "Glob of a web page" } } } }, "excludeUrlGlobs": { "title": "Exclude URL Patterns (globs)", "type": "array", "description": "Skip discovered URLs matching any of these glob patterns (e.g. *login*, *.pdf).", "default": [], "items": { "type": "object", "required": [ "glob" ], "properties": { "glob": { "type": "string", "title": "Glob of a web page" } } } }, "includeUrlRegexps": { "title": "Include URL Patterns (regex)", "type": "array", "description": "Only crawl discovered URLs matching at least one of these regular expressions. Combined with glob patterns (both must pass when both are set).", "default": [], "items": { "type": "string" } }, "excludeUrlRegexps": { "title": "Exclude URL Patterns (regex)", "type": "array", "description": "Skip discovered URLs matching any of these regular expressions.", "default": [], "items": { "type": "string" } }, "keepUrlFragments": { "title": "Keep URL Fragments", "type": "boolean", "description": "Treat URLs that differ only in #fragment as different pages. Enable for single-page sites that use fragments as routes.", "default": false }, "ignoreCanonicalUrl": { "title": "Ignore canonical URL", "type": "boolean", "description": "When disabled, pages whose canonical URL points to a page that was already crawled are skipped as duplicates.", "default": false }, "outputFormat": { "title": "Output Format", "enum": [ "markdown", "text", "html" ], "type": "string", "description": "How page content is returned. Markdown keeps headings, lists, tables, and links — best for LLMs and RAG.", "default": "markdown" }, "htmlTransformer": { "title": "HTML Transformer", "enum": [ "auto", "trafilatura", "readability", "basic", "none" ], "type": "string", "description": "How the main content is isolated from navigation, footers, ads, and cookie banners.", "default": "auto" }, "includeImages": { "title": "Include Images in Markdown", "type": "boolean", "description": "Add image references to the markdown field. Image URLs are always collected in the imageUrls field.", "default": false }, "chunkMarkdown": { "title": "Split Content into Chunks", "type": "boolean", "description": "Emit a chunks array per page — heading-aware Markdown pieces ready for embeddings. Code blocks and tables are never split, each piece carries its heading path and character count, and consecutive pieces overlap so context is kept at the boundaries.", "default": false }, "chunkMaxChars": { "title": "Chunk Size (characters)", "minimum": 200, "maximum": 10000, "type": "integer", "description": "Target size of each chunk when chunkMarkdown is on. Code blocks and tables stay whole even when larger than this.", "default": 2000 }, "chunkOverlapChars": { "title": "Chunk Overlap (characters)", "minimum": 0, "maximum": 1000, "type": "integer", "description": "Approximate overlap between consecutive chunks when chunkMarkdown is on, so context is not lost at chunk boundaries.", "default": 200 }, "keepElementsCssSelector": { "title": "Keep HTML Elements (CSS selector)", "type": "string", "description": "Keep only the elements matching this CSS selector (e.g. article, main .content) before extraction. Leave empty to use the whole page.", "default": "" }, "removeElementsCssSelector": { "title": "Remove HTML Elements (CSS selector)", "type": "string", "description": "Remove elements matching this CSS selector (e.g. .sidebar, .ad, nav) before extraction.", "default": "" }, "parsePdfText": { "title": "Extract PDF Text", "type": "boolean", "description": "When a crawled URL serves a PDF, extract its text into the text field as a file record instead of skipping it.", "default": true }, "clickElementsCssSelector": { "title": "Click Elements (CSS selector)", "type": "string", "description": "Click elements matching this CSS selector (e.g. [aria-expanded=\"false\"]) before extraction to expand collapsed content.", "default": "" }, "takeScreenshots": { "title": "Take Page Screenshots", "type": "boolean", "description": "Render every stored page in the browser and capture a full-page JPEG (quality 80) of it, putting its public URL in the screenshot field. Screenshots are held in memory until pushed and add roughly 0.3-1 MB and a fraction of a second per page; screenshot capture always runs after the page is rendered.", "default": false }, "customHttpHeaders": { "title": "Custom HTTP Headers", "type": "array", "description": "Extra request headers for all engines and discovery requests, as a list of {\"key\", \"value\"} pairs (e.g. an Authorization token or a specific User-Agent for sites that ask for one). Transport/framing and cookie headers (Host, Cookie, Content-Length, Connection, ...) are rejected so the actor stays cookieless.", "items": { "type": "object", "properties": { "key": { "type": "string", "title": "Header name", "description": "HTTP header name, e.g. Authorization or Accept-Language." }, "value": { "type": "string", "title": "Header value", "description": "Header value sent with every request." } }, "required": [ "key", "value" ] }, "default": [] }, "storeSkippedUrls": { "title": "Store Skipped URLs", "type": "boolean", "description": "Save a SKIPPED_URLS record to the key-value store listing skipped URLs grouped by reason (robots, blocked, non-HTML, errors, duplicates).", "default": false } } }, "runsResponseSchema": { "type": "object", "properties": { "data": { "type": "object", "properties": { "id": { "type": "string" }, "actId": { "type": "string" }, "userId": { "type": "string" }, "startedAt": { "type": "string", "format": "date-time", "example": "2025-01-08T00:00:00.000Z" }, "finishedAt": { "type": "string", "format": "date-time", "example": "2025-01-08T00:00:00.000Z" }, "status": { "type": "string", "example": "READY" }, "meta": { "type": "object", "properties": { "origin": { "type": "string", "example": "API" }, "userAgent": { "type": "string" } } }, "stats": { "type": "object", "properties": { "inputBodyLen": { "type": "integer", "example": 2000 }, "rebootCount": { "type": "integer", "example": 0 }, "restartCount": { "type": "integer", "example": 0 }, "resurrectCount": { "type": "integer", "example": 0 }, "computeUnits": { "type": "integer", "example": 0 } } }, "options": { "type": "object", "properties": { "build": { "type": "string", "example": "latest" }, "timeoutSecs": { "type": "integer", "example": 300 }, "memoryMbytes": { "type": "integer", "example": 1024 }, "diskMbytes": { "type": "integer", "example": 2048 } } }, "buildId": { "type": "string" }, "defaultKeyValueStoreId": { "type": "string" }, "defaultDatasetId": { "type": "string" }, "defaultRequestQueueId": { "type": "string" }, "buildNumber": { "type": "string", "example": "1.0.0" }, "containerUrl": { "type": "string" }, "usage": { "type": "object", "properties": { "ACTOR_COMPUTE_UNITS": { "type": "integer", "example": 0 }, "DATASET_READS": { "type": "integer", "example": 0 }, "DATASET_WRITES": { "type": "integer", "example": 0 }, "KEY_VALUE_STORE_READS": { "type": "integer", "example": 0 }, "KEY_VALUE_STORE_WRITES": { "type": "integer", "example": 1 }, "KEY_VALUE_STORE_LISTS": { "type": "integer", "example": 0 }, "REQUEST_QUEUE_READS": { "type": "integer", "example": 0 }, "REQUEST_QUEUE_WRITES": { "type": "integer", "example": 0 }, "DATA_TRANSFER_INTERNAL_GBYTES": { "type": "integer", "example": 0 }, "DATA_TRANSFER_EXTERNAL_GBYTES": { "type": "integer", "example": 0 }, "PROXY_RESIDENTIAL_TRANSFER_GBYTES": { "type": "integer", "example": 0 }, "PROXY_SERPS": { "type": "integer", "example": 0 }, "PROXY_UNBLOCKER_UNITS": { "type": "integer", "example": 0 } } }, "usageTotalUsd": { "type": "number", "example": 0.00005 }, "usageUsd": { "type": "object", "properties": { "ACTOR_COMPUTE_UNITS": { "type": "integer", "example": 0 }, "DATASET_READS": { "type": "integer", "example": 0 }, "DATASET_WRITES": { "type": "integer", "example": 0 }, "KEY_VALUE_STORE_READS": { "type": "integer", "example": 0 }, "KEY_VALUE_STORE_WRITES": { "type": "number", "example": 0.00005 }, "KEY_VALUE_STORE_LISTS": { "type": "integer", "example": 0 }, "REQUEST_QUEUE_READS": { "type": "integer", "example": 0 }, "REQUEST_QUEUE_WRITES": { "type": "integer", "example": 0 }, "DATA_TRANSFER_INTERNAL_GBYTES": { "type": "integer", "example": 0 }, "DATA_TRANSFER_EXTERNAL_GBYTES": { "type": "integer", "example": 0 }, "PROXY_RESIDENTIAL_TRANSFER_GBYTES": { "type": "integer", "example": 0 }, "PROXY_SERPS": { "type": "integer", "example": 0 }, "PROXY_UNBLOCKER_UNITS": { "type": "integer", "example": 0 } } } } } } } } } }