{ "openapi": "3.0.1", "info": { "title": "Website Content Crawler: Markdown for AI and RAG", "description": "Turn websites you own or have permission to process into Markdown for AI and RAG. Keep headings, tables and code, with optional chunks linked to each source page. Set a page limit; JavaScript rendering is not supported.", "version": "0.1", "x-build-id": "Zv11DEnifWQdvekhl" }, "servers": [ { "url": "https://api.apify.com/v2" } ], "paths": { "/acts/automa-flow~website-content-crawler/run-sync-get-dataset-items": { "post": { "operationId": "run-sync-get-dataset-items-automa-flow-website-content-crawler", "x-openai-isConsequential": false, "summary": "Executes an Actor, waits for its completion, and returns Actor's dataset items in response.", "tags": [ "Run Actor" ], "requestBody": { "required": true, "content": { "application/json": { "schema": { "$ref": "#/components/schemas/inputSchema" } } } }, "parameters": [ { "name": "token", "in": "query", "required": true, "schema": { "type": "string" }, "description": "Enter your Apify token here" } ], "responses": { "200": { "description": "OK" } } } }, "/acts/automa-flow~website-content-crawler/runs": { "post": { "operationId": "runs-sync-automa-flow-website-content-crawler", "x-openai-isConsequential": false, "summary": "Executes an Actor and returns information about the initiated run in response.", "tags": [ "Run Actor" ], "requestBody": { "required": true, "content": { "application/json": { "schema": { "$ref": "#/components/schemas/inputSchema" } } } }, "parameters": [ { "name": "token", "in": "query", "required": true, "schema": { "type": "string" }, "description": "Enter your Apify token here" } ], "responses": { "200": { "description": "OK", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/runsResponseSchema" } } } } } } }, "/acts/automa-flow~website-content-crawler/run-sync": { "post": { "operationId": "run-sync-automa-flow-website-content-crawler", "x-openai-isConsequential": false, "summary": "Executes an Actor, waits for completion, and returns the OUTPUT from Key-value store in response.", "tags": [ "Run Actor" ], "requestBody": { "required": true, "content": { "application/json": { "schema": { "$ref": "#/components/schemas/inputSchema" } } } }, "parameters": [ { "name": "token", "in": "query", "required": true, "schema": { "type": "string" }, "description": "Enter your Apify token here" } ], "responses": { "200": { "description": "OK" } } } } }, "components": { "schemas": { "inputSchema": { "type": "object", "required": [ "sources" ], "properties": { "sources": { "title": "Authorized sources", "minItems": 1, "maxItems": 10, "type": "array", "description": "One to 10 public HTTP(S) start URLs you own or are authorized to process. Each source needs an explicit rightsBasis. Exact duplicate seeds are dropped; conflicting scope or rights on the same URL are rejected.", "items": { "type": "object", "required": [ "url", "rightsBasis" ], "additionalProperties": false, "properties": { "url": { "title": "Start URL", "description": "Public http:// or https:// HTML page on port 80 or 443. Credentials, fragments-only, file downloads, localhost, private networks and secret query parameters are rejected.", "type": "string", "editor": "textfield", "minLength": 1, "maxLength": 2048, "pattern": "^[Hh][Tt][Tt][Pp][Ss]?://" }, "scope": { "title": "Crawl scope", "description": "page crawls only this URL. subtree crawls the seed path and nested paths ( /docs allows /docs/... but not /docs-old ). origin crawls the exact origin, not sibling subdomains.", "type": "string", "editor": "select", "enum": [ "page", "subtree", "origin" ], "enumTitles": [ "This page only", "This path and below", "Entire origin" ], "default": "subtree" }, "rightsBasis": { "title": "Rights basis", "description": "Your declaration of why this source may be processed. owned = the content belongs to the customer. permission and licensed require rightsReference at runtime (Apify Input Schema cannot express that cross-field rule, so Cloud may start the run and the Actor then fails validation). This is provenance, not a legal check.", "type": "string", "editor": "select", "enum": [ "owned", "permission", "licensed" ], "enumTitles": [ "Owned by the customer", "Permission granted", "Licensed use" ] }, "rightsReference": { "title": "Rights reference", "description": "Required at runtime when rightsBasis is permission or licensed. Non-secret URL or identifier of the terms or permission. The Actor does not fetch this value. Cross-field requirement is enforced in the Actor, not by the Input Schema dialect.", "type": "string", "editor": "textfield", "maxLength": 2000 }, "attribution": { "title": "Attribution", "description": "Optional non-secret attribution copied into the rights manifest and each PAGE.", "type": "string", "editor": "textarea", "maxLength": 2000 }, "licenseNotice": { "title": "License notice", "description": "Optional accompanying notice required by a license. Stored once in RIGHTS_MANIFEST.", "type": "string", "editor": "textarea", "maxLength": 20000 } } } }, "sitemapUrls": { "title": "Sitemap URLs", "maxItems": 10, "type": "array", "description": "Optional XML sitemaps on an authorized origin. Other origins never expand permissions. Ignored when maxDepth is 0.", "items": { "type": "string", "minLength": 1, "maxLength": 2048, "pattern": "^[Hh][Tt][Tt][Pp][Ss]?://" }, "default": [] }, "discoverSitemaps": { "title": "Discover sitemaps", "type": "boolean", "description": "Read Sitemap from robots.txt and /sitemap.xml on authorized origins. Does not try dozens of conventional paths. Disabled when maxDepth is 0.", "default": true }, "includeUrlGlobs": { "title": "Include URL globs", "maxItems": 20, "type": "array", "description": "Optional glob patterns (* and ?) matched against the full URL and path. Empty means every in-scope URL is eligible. Regular expressions are rejected.", "items": { "type": "string", "minLength": 1, "maxLength": 2048 }, "default": [] }, "excludeUrlGlobs": { "title": "Exclude URL globs", "maxItems": 20, "type": "array", "description": "Glob exclusions. These take priority over includeUrlGlobs.", "items": { "type": "string", "minLength": 1, "maxLength": 2048 }, "default": [] }, "maxPages": { "title": "Maximum pages", "minimum": 1, "maximum": 1500, "type": "integer", "description": "Limit on unique document URLs taken into processing, including failed attempts. Retries of the same URL do not consume another slot. This is not a promise of that many successful PAGE rows.", "default": 25 }, "maxDepth": { "title": "Maximum link depth", "minimum": 0, "maximum": 10, "type": "integer", "description": "Link-follow depth. Explicit seeds are depth 0. Sitemap-discovered URLs are depth 1. maxDepth 0 processes only seeds and disables sitemap discovery.", "default": 5 }, "renderingMode": { "title": "Rendering mode", "enum": [ "http" ], "type": "string", "description": "HTTP-only extraction. JavaScript rendering is not available in this build; it will be added only after browser egress is independently verified.", "default": "http" }, "contentSelector": { "title": "Content CSS selector", "maxLength": 300, "type": "string", "description": "Optional CSS selector for the main text. If it matches nothing, extraction fails with EXTRACTION_FAILED and the run fails when no pages succeed (it is not treated as a verified empty site). JavaScript expressions are not accepted." }, "removeSelectors": { "title": "Remove CSS selectors", "maxItems": 20, "type": "array", "description": "Optional CSS selectors removed from the chosen content root before Markdown conversion.", "items": { "type": "string", "minLength": 1, "maxLength": 300 }, "default": [] }, "includeChunks": { "title": "Include chunks", "type": "boolean", "description": "Attach structural heading-based chunks on each PAGE. Chunks are not billed separately and do not use an LLM.", "default": true }, "chunkMaxChars": { "title": "Chunk size (characters)", "minimum": 1000, "maximum": 12000, "type": "integer", "description": "Soft Unicode-character limit for a chunk body. This is not a tokenizer or model context-window guarantee. Oversized code or table lines are kept and marked oversized.", "default": 6000 }, "chunkOverlapChars": { "title": "Chunk overlap (characters)", "minimum": 0, "maximum": 1000, "type": "integer", "description": "Prose overlap only. Must be less than one quarter of chunkMaxChars. Tables and code are not arbitrarily duplicated.", "default": 300 }, "maxDownloadMiB": { "title": "Download budget (MiB)", "minimum": 10, "maximum": 2048, "type": "integer", "description": "Decoded-byte ceiling for the run, including robots and sitemaps.", "default": 100 }, "securityMode": { "title": "Secure-for-AI mode", "enum": [ "off", "detect", "sanitize" ], "type": "string", "description": "Optional pattern check before RAG ingestion. off keeps the crawl unchanged. detect adds findings and keeps the original Markdown. sanitize replaces known secrets, emails, phones and card numbers, and quarantines pages that speak to an AI model or tell it to ignore its instructions (not billed). Catches common attacks, mainly in English; a reworded attack can pass. Not DLP or a safety guarantee.", "default": "off" } } }, "runsResponseSchema": { "type": "object", "properties": { "data": { "type": "object", "properties": { "id": { "type": "string" }, "actId": { "type": "string" }, "userId": { "type": "string" }, "startedAt": { "type": "string", "format": "date-time", "example": "2025-01-08T00:00:00.000Z" }, "finishedAt": { "type": "string", "format": "date-time", "example": "2025-01-08T00:00:00.000Z" }, "status": { "type": "string", "example": "READY" }, "meta": { "type": "object", "properties": { "origin": { "type": "string", "example": "API" }, "userAgent": { "type": "string" } } }, "stats": { "type": "object", "properties": { "inputBodyLen": { "type": "integer", "example": 2000 }, "rebootCount": { "type": "integer", "example": 0 }, "restartCount": { "type": "integer", "example": 0 }, "resurrectCount": { "type": "integer", "example": 0 }, "computeUnits": { "type": "integer", "example": 0 } } }, "options": { "type": "object", "properties": { "build": { "type": "string", "example": "latest" }, "timeoutSecs": { "type": "integer", "example": 300 }, "memoryMbytes": { "type": "integer", "example": 1024 }, "diskMbytes": { "type": "integer", "example": 2048 } } }, "buildId": { "type": "string" }, "defaultKeyValueStoreId": { "type": "string" }, "defaultDatasetId": { "type": "string" }, "defaultRequestQueueId": { "type": "string" }, "buildNumber": { "type": "string", "example": "1.0.0" }, "containerUrl": { "type": "string" }, "usage": { "type": "object", "properties": { "ACTOR_COMPUTE_UNITS": { "type": "integer", "example": 0 }, "DATASET_READS": { "type": "integer", "example": 0 }, "DATASET_WRITES": { "type": "integer", "example": 0 }, "KEY_VALUE_STORE_READS": { "type": "integer", "example": 0 }, "KEY_VALUE_STORE_WRITES": { "type": "integer", "example": 1 }, "KEY_VALUE_STORE_LISTS": { "type": "integer", "example": 0 }, "REQUEST_QUEUE_READS": { "type": "integer", "example": 0 }, "REQUEST_QUEUE_WRITES": { "type": "integer", "example": 0 }, "DATA_TRANSFER_INTERNAL_GBYTES": { "type": "integer", "example": 0 }, "DATA_TRANSFER_EXTERNAL_GBYTES": { "type": "integer", "example": 0 }, "PROXY_RESIDENTIAL_TRANSFER_GBYTES": { "type": "integer", "example": 0 }, "PROXY_SERPS": { "type": "integer", "example": 0 }, "PROXY_UNBLOCKER_UNITS": { "type": "integer", "example": 0 } } }, "usageTotalUsd": { "type": "number", "example": 0.00005 }, "usageUsd": { "type": "object", "properties": { "ACTOR_COMPUTE_UNITS": { "type": "integer", "example": 0 }, "DATASET_READS": { "type": "integer", "example": 0 }, "DATASET_WRITES": { "type": "integer", "example": 0 }, "KEY_VALUE_STORE_READS": { "type": "integer", "example": 0 }, "KEY_VALUE_STORE_WRITES": { "type": "number", "example": 0.00005 }, "KEY_VALUE_STORE_LISTS": { "type": "integer", "example": 0 }, "REQUEST_QUEUE_READS": { "type": "integer", "example": 0 }, "REQUEST_QUEUE_WRITES": { "type": "integer", "example": 0 }, "DATA_TRANSFER_INTERNAL_GBYTES": { "type": "integer", "example": 0 }, "DATA_TRANSFER_EXTERNAL_GBYTES": { "type": "integer", "example": 0 }, "PROXY_RESIDENTIAL_TRANSFER_GBYTES": { "type": "integer", "example": 0 }, "PROXY_SERPS": { "type": "integer", "example": 0 }, "PROXY_UNBLOCKER_UNITS": { "type": "integer", "example": 0 } } } } } } } } } }