{ "openapi": "3.0.1", "info": { "title": "Website Content Scraper", "description": "Extract clean Markdown, plain text, linked files, and RAG-ready chunks from websites, documentation, help centers, knowledge bases, and authenticated portals. Preserve structure, metadata, URLs, and crawl context for AI search, training, and retrieval workflows.", "version": "0.1", "x-build-id": "EUKMqJlsWpqLZFmG5" }, "servers": [ { "url": "https://api.apify.com/v2" } ], "paths": { "/acts/qaseemiqbal~website-content-scraper/run-sync-get-dataset-items": { "post": { "operationId": "run-sync-get-dataset-items-qaseemiqbal-website-content-scraper", "x-openai-isConsequential": false, "summary": "Executes an Actor, waits for its completion, and returns Actor's dataset items in response.", "tags": [ "Run Actor" ], "requestBody": { "required": true, "content": { "application/json": { "schema": { "$ref": "#/components/schemas/inputSchema" } } } }, "parameters": [ { "name": "token", "in": "query", "required": true, "schema": { "type": "string" }, "description": "Enter your Apify token here" } ], "responses": { "200": { "description": "OK" } } } }, "/acts/qaseemiqbal~website-content-scraper/runs": { "post": { "operationId": "runs-sync-qaseemiqbal-website-content-scraper", "x-openai-isConsequential": false, "summary": "Executes an Actor and returns information about the initiated run in response.", "tags": [ "Run Actor" ], "requestBody": { "required": true, "content": { "application/json": { "schema": { "$ref": "#/components/schemas/inputSchema" } } } }, "parameters": [ { "name": "token", "in": "query", "required": true, "schema": { "type": "string" }, "description": "Enter your Apify token here" } ], "responses": { "200": { "description": "OK", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/runsResponseSchema" } } } } } } }, "/acts/qaseemiqbal~website-content-scraper/run-sync": { "post": { "operationId": "run-sync-qaseemiqbal-website-content-scraper", "x-openai-isConsequential": false, "summary": "Executes an Actor, waits for completion, and returns the OUTPUT from Key-value store in response.", "tags": [ "Run Actor" ], "requestBody": { "required": true, "content": { "application/json": { "schema": { "$ref": "#/components/schemas/inputSchema" } } } }, "parameters": [ { "name": "token", "in": "query", "required": true, "schema": { "type": "string" }, "description": "Enter your Apify token here" } ], "responses": { "200": { "description": "OK" } } } } }, "components": { "schemas": { "inputSchema": { "type": "object", "required": [ "startUrls" ], "properties": { "startUrls": { "title": "Start URLs", "type": "array", "description": "One or more pages, website sections, or documents to crawl.", "items": { "type": "object", "required": [ "url" ], "properties": { "url": { "type": "string", "title": "URL of a web page", "format": "uri" } } } }, "preset": { "title": "Workflow preset", "enum": [ "simple", "rag", "docsMigration", "debug", "incremental", "authenticatedPortal" ], "type": "string", "description": "Applies sensible defaults for common workflows. Explicit fields still override the preset.", "default": "simple" }, "crawlerType": { "title": "Crawler type", "enum": [ "adaptive", "cheerio", "playwright-firefox", "playwright-chromium" ], "type": "string", "description": "Choose whether to use fast HTTP crawling, browser rendering, or adaptive HTTP with browser fallback. Fast HTTP parser is the cheapest option.", "default": "cheerio" }, "maxBrowserFallbacks": { "title": "Maximum adaptive browser fallbacks", "minimum": 0, "type": "integer", "description": "Maximum number of pages adaptive mode may retry in a browser. Lower values keep runs cheaper.", "default": 2 }, "crawlScope": { "title": "Crawl scope", "enum": [ "startUrlPath", "sameHostname", "sameDomain", "customGlobs", "exactUrlsOnly" ], "type": "string", "description": "Controls which discovered URLs are considered in scope for crawling.", "default": "startUrlPath" }, "includeUrlGlobs": { "title": "Include URL globs", "type": "array", "description": "Only URLs matching at least one glob are crawled when provided, or when crawl scope is customGlobs.", "default": [], "items": { "type": "string" } }, "excludeUrlGlobs": { "title": "Exclude URL globs", "type": "array", "description": "URLs matching any glob are skipped.", "default": [], "items": { "type": "string" } }, "maxCrawlPages": { "title": "Maximum pages to crawl", "minimum": 1, "type": "integer", "description": "Maximum number of page requests the crawler will process before stopping.", "default": 25 }, "maxCrawlDepth": { "title": "Maximum crawl depth", "minimum": 0, "type": "integer", "description": "Maximum link depth from the start URLs. Use 0 to crawl only the start URLs.", "default": 3 }, "maxResults": { "title": "Maximum saved page records", "minimum": 1, "type": "integer", "description": "Maximum number of page records to save to the default dataset.", "default": 25 }, "maxConcurrency": { "title": "Maximum concurrency", "minimum": 1, "maximum": 100, "type": "integer", "description": "Maximum number of requests processed in parallel. Lower concurrency is gentler and can reduce memory usage.", "default": 4 }, "requestHandlerTimeoutSecs": { "title": "Request timeout", "minimum": 10, "type": "integer", "description": "Maximum time allowed for each page or file request handler.", "default": 45 }, "respectRobotsTxtFile": { "title": "Respect robots.txt", "type": "boolean", "description": "Recommended for broad public crawls.", "default": true }, "discoverSitemaps": { "title": "Discover sitemaps", "type": "boolean", "description": "Discover URLs from sitemap.xml files and sitemap declarations in robots.txt. Turn on for broader crawls.", "default": false }, "discoverLlmsTxt": { "title": "Discover llms.txt", "type": "boolean", "description": "Discover URLs from llms.txt and llms-full.txt files when they are available. Turn on when you want extra discovery.", "default": false }, "discoverLlmsFullTxt": { "title": "Discover llms-full.txt", "type": "boolean", "description": "Also fetch llms-full.txt when discovering LLM-oriented links. This can be large, so it is disabled by default.", "default": false }, "discoveryOnly": { "title": "Discovery-only dry run", "type": "boolean", "description": "Emit discovered URL decisions without saving page content.", "default": false }, "saveMarkdown": { "title": "Save Markdown", "type": "boolean", "description": "Save cleaned Markdown for each page record.", "default": true }, "saveText": { "title": "Save plain text", "type": "boolean", "description": "Save cleaned plain text for each page record.", "default": false }, "saveHtml": { "title": "Save cleaned HTML", "type": "boolean", "description": "Save cleaned HTML for each page record. Useful for debugging and migration workflows.", "default": false }, "saveScreenshots": { "title": "Save screenshots", "type": "boolean", "description": "Save screenshots for browser-rendered pages to the default key-value store.", "default": false }, "createChunks": { "title": "Create RAG chunks", "type": "boolean", "description": "Create RAG-ready chunk records with stable IDs, hashes, and token estimates.", "default": false }, "outputMode": { "title": "Output mode", "enum": [ "fullContentAndChunks", "fullContent", "chunksOnly" ], "type": "string", "description": "Choose whether to emit full page content, chunk records, or both.", "default": "fullContent" }, "chunkTargetTokens": { "title": "Chunk target tokens", "minimum": 100, "type": "integer", "description": "Target token count for each generated RAG chunk.", "default": 700 }, "chunkOverlapTokens": { "title": "Chunk overlap tokens", "minimum": 0, "type": "integer", "description": "Approximate token overlap between adjacent chunks when splitting long sections.", "default": 40 }, "chunkMaxChars": { "title": "Chunk maximum characters", "minimum": 500, "type": "integer", "description": "Hard character limit for a generated chunk.", "default": 5000 }, "incrementalMode": { "title": "Incremental mode", "enum": [ "disabled", "readState", "readWriteState" ], "type": "string", "description": "Controls whether previous crawl state is read and updated in the key-value store.", "default": "disabled" }, "stateKey": { "title": "State key", "type": "string", "description": "Stable key used to store URL and content hashes between scheduled runs.", "default": "default" }, "skipUnchanged": { "title": "Skip unchanged content", "type": "boolean", "description": "Skip writing unchanged page and file records when incremental state shows the content did not change.", "default": true }, "emitUnchangedRecords": { "title": "Emit unchanged records", "type": "boolean", "description": "Write unchanged records to the dataset instead of only counting them in the summary.", "default": false }, "emitDeletedRecords": { "title": "Emit deletion tombstones", "type": "boolean", "description": "Emit tombstone records for documents that existed in prior state but were not seen in this run.", "default": false }, "saveFiles": { "title": "Download linked files", "type": "boolean", "description": "Download supported linked files to the default key-value store. Leave off for the cheapest page-only crawls.", "default": false }, "parseFiles": { "title": "Parse linked files", "type": "boolean", "description": "Extract text from supported linked files and emit file/chunk records. Leave off unless you need linked documents.", "default": false }, "fileMaxSizeMb": { "title": "Maximum file size in MB", "minimum": 1, "type": "integer", "description": "Maximum file size that can be downloaded or parsed.", "default": 10 }, "maxFiles": { "title": "Maximum files", "minimum": 0, "type": "integer", "description": "Maximum number of linked files to process during the run.", "default": 0 }, "removeElementsCssSelector": { "title": "Remove elements CSS selector", "type": "string", "description": "Additional CSS selectors to remove before extraction.", "default": "" }, "keepElementsCssSelector": { "title": "Keep elements CSS selector", "type": "string", "description": "When set, extraction keeps only matching elements.", "default": "" }, "waitForSelector": { "title": "Wait for selector", "type": "string", "description": "Browser modes wait for this selector before extracting content.", "default": "" }, "dynamicContentWaitSecs": { "title": "Dynamic content wait", "minimum": 0, "type": "integer", "description": "Extra wait time in browser mode before extracting content.", "default": 0 }, "scrollToBottom": { "title": "Scroll to bottom", "type": "boolean", "description": "Scroll pages in browser mode to load infinite or lazy-loaded content.", "default": false }, "clickElementsCssSelector": { "title": "Click expandable elements selector", "type": "string", "description": "CSS selector for expandable elements to click before extraction in browser mode.", "default": "" }, "closeCookieModals": { "title": "Close cookie modals", "type": "boolean", "description": "Attempt to close common cookie consent modals in browser mode.", "default": false }, "blockMedia": { "title": "Block media requests", "type": "boolean", "description": "Block image, font, and media requests in browser mode to reduce cost and speed up crawls.", "default": true }, "requestHeaders": { "title": "Request headers", "type": "object", "description": "Optional headers for authenticated or signed requests. Values are treated as secrets." }, "cookies": { "title": "Cookies", "type": "string", "description": "Cookie header string for authenticated crawls. Stored as a secret input and never emitted." }, "loginValidationUrl": { "title": "Login validation URL", "type": "string", "description": "Optional URL to validate authenticated access before the crawl.", "default": "" }, "debugMode": { "title": "Debug mode", "type": "boolean", "description": "Enable extra diagnostics such as skipped URL records and debug artifacts.", "default": false }, "saveSkippedUrls": { "title": "Save skipped URL diagnostics", "type": "boolean", "description": "Write skipped URL diagnostics to the default dataset.", "default": false }, "integrationTarget": { "title": "Integration handoff", "enum": [ "none", "customWebhook" ], "type": "string", "description": "Optional post-run integration handoff target.", "default": "none" }, "customWebhookUrl": { "title": "Custom webhook URL", "type": "string", "description": "Webhook URL used when the custom webhook integration target is selected.", "default": "" }, "proxyConfiguration": { "title": "Proxy configuration", "type": "object", "description": "Proxy settings used by Crawlee and Apify when fetching pages.", "default": { "useApifyProxy": false } } } }, "runsResponseSchema": { "type": "object", "properties": { "data": { "type": "object", "properties": { "id": { "type": "string" }, "actId": { "type": "string" }, "userId": { "type": "string" }, "startedAt": { "type": "string", "format": "date-time", "example": "2025-01-08T00:00:00.000Z" }, "finishedAt": { "type": "string", "format": "date-time", "example": "2025-01-08T00:00:00.000Z" }, "status": { "type": "string", "example": "READY" }, "meta": { "type": "object", "properties": { "origin": { "type": "string", "example": "API" }, "userAgent": { "type": "string" } } }, "stats": { "type": "object", "properties": { "inputBodyLen": { "type": "integer", "example": 2000 }, "rebootCount": { "type": "integer", "example": 0 }, "restartCount": { "type": "integer", "example": 0 }, "resurrectCount": { "type": "integer", "example": 0 }, "computeUnits": { "type": "integer", "example": 0 } } }, "options": { "type": "object", "properties": { "build": { "type": "string", "example": "latest" }, "timeoutSecs": { "type": "integer", "example": 300 }, "memoryMbytes": { "type": "integer", "example": 1024 }, "diskMbytes": { "type": "integer", "example": 2048 } } }, "buildId": { "type": "string" }, "defaultKeyValueStoreId": { "type": "string" }, "defaultDatasetId": { "type": "string" }, "defaultRequestQueueId": { "type": "string" }, "buildNumber": { "type": "string", "example": "1.0.0" }, "containerUrl": { "type": "string" }, "usage": { "type": "object", "properties": { "ACTOR_COMPUTE_UNITS": { "type": "integer", "example": 0 }, "DATASET_READS": { "type": "integer", "example": 0 }, "DATASET_WRITES": { "type": "integer", "example": 0 }, "KEY_VALUE_STORE_READS": { "type": "integer", "example": 0 }, "KEY_VALUE_STORE_WRITES": { "type": "integer", "example": 1 }, "KEY_VALUE_STORE_LISTS": { "type": "integer", "example": 0 }, "REQUEST_QUEUE_READS": { "type": "integer", "example": 0 }, "REQUEST_QUEUE_WRITES": { "type": "integer", "example": 0 }, "DATA_TRANSFER_INTERNAL_GBYTES": { "type": "integer", "example": 0 }, "DATA_TRANSFER_EXTERNAL_GBYTES": { "type": "integer", "example": 0 }, "PROXY_RESIDENTIAL_TRANSFER_GBYTES": { "type": "integer", "example": 0 }, "PROXY_SERPS": { "type": "integer", "example": 0 } } }, "usageTotalUsd": { "type": "number", "example": 0.00005 }, "usageUsd": { "type": "object", "properties": { "ACTOR_COMPUTE_UNITS": { "type": "integer", "example": 0 }, "DATASET_READS": { "type": "integer", "example": 0 }, "DATASET_WRITES": { "type": "integer", "example": 0 }, "KEY_VALUE_STORE_READS": { "type": "integer", "example": 0 }, "KEY_VALUE_STORE_WRITES": { "type": "number", "example": 0.00005 }, "KEY_VALUE_STORE_LISTS": { "type": "integer", "example": 0 }, "REQUEST_QUEUE_READS": { "type": "integer", "example": 0 }, "REQUEST_QUEUE_WRITES": { "type": "integer", "example": 0 }, "DATA_TRANSFER_INTERNAL_GBYTES": { "type": "integer", "example": 0 }, "DATA_TRANSFER_EXTERNAL_GBYTES": { "type": "integer", "example": 0 }, "PROXY_RESIDENTIAL_TRANSFER_GBYTES": { "type": "integer", "example": 0 }, "PROXY_SERPS": { "type": "integer", "example": 0 } } } } } } } } } }