{ "openapi": "3.0.1", "info": { "title": "Website Content Crawler: Markdown for AI and RAG", "description": "Crawl sites you own or are authorized to process into clean Markdown for AI and RAG. Keep headings, tables and code, emit source-linked chunks, and cap pages with HTTP-first limits.", "version": "0.1", "x-build-id": "ERbzEPjB66G37AP9R" }, "servers": [ { "url": "https://api.apify.com/v2" } ], "paths": { "/acts/automa-flow~website-content-crawler/run-sync-get-dataset-items": { "post": { "operationId": "run-sync-get-dataset-items-automa-flow-website-content-crawler", "x-openai-isConsequential": false, "summary": "Executes an Actor, waits for its completion, and returns Actor's dataset items in response.", "tags": [ "Run Actor" ], "requestBody": { "required": true, "content": { "application/json": { "schema": { "$ref": "#/components/schemas/inputSchema" } } } }, "parameters": [ { "name": "token", "in": "query", "required": true, "schema": { "type": "string" }, "description": "Enter your Apify token here" } ], "responses": { "200": { "description": "OK" } } } }, "/acts/automa-flow~website-content-crawler/runs": { "post": { "operationId": "runs-sync-automa-flow-website-content-crawler", "x-openai-isConsequential": false, "summary": "Executes an Actor and returns information about the initiated run in response.", "tags": [ "Run Actor" ], "requestBody": { "required": true, "content": { "application/json": { "schema": { "$ref": "#/components/schemas/inputSchema" } } } }, "parameters": [ { "name": "token", "in": "query", "required": true, "schema": { "type": "string" }, "description": "Enter your Apify token here" } ], "responses": { "200": { "description": "OK", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/runsResponseSchema" } } } } } } }, "/acts/automa-flow~website-content-crawler/run-sync": { "post": { "operationId": "run-sync-automa-flow-website-content-crawler", "x-openai-isConsequential": false, "summary": "Executes an Actor, waits for completion, and returns the OUTPUT from Key-value store in response.", "tags": [ "Run Actor" ], "requestBody": { "required": true, "content": { "application/json": { "schema": { "$ref": "#/components/schemas/inputSchema" } } } }, "parameters": [ { "name": "token", "in": "query", "required": true, "schema": { "type": "string" }, "description": "Enter your Apify token here" } ], "responses": { "200": { "description": "OK" } } } } }, "components": { "schemas": { "inputSchema": { "type": "object", "required": [ "sources" ], "properties": { "sources": { "title": "Authorized sources", "minItems": 1, "maxItems": 10, "type": "array", "description": "One to 10 public HTTP(S) start URLs you own or are authorized to process. Each source needs an explicit rightsBasis. Exact duplicate seeds are dropped; conflicting scope or rights on the same URL are rejected.", "items": { "type": "object", "required": [ "url", "rightsBasis" ], "additionalProperties": false, "properties": { "url": { "title": "Start URL", "description": "Public http:// or https:// HTML page on port 80 or 443. Credentials, fragments-only, file downloads, localhost, private networks and secret query parameters are rejected.", "type": "string", "editor": "textfield", "minLength": 1, "maxLength": 2048, "pattern": "^[Hh][Tt][Tt][Pp][Ss]?://" }, "scope": { "title": "Crawl scope", "description": "page crawls only this URL. subtree crawls the seed path and nested paths ( /docs allows /docs/... but not /docs-old ). origin crawls the exact origin, not sibling subdomains.", "type": "string", "editor": "select", "enum": [ "page", "subtree", "origin" ], "enumTitles": [ "This page only", "This path and below", "Entire origin" ], "default": "subtree" }, "rightsBasis": { "title": "Rights basis", "description": "Your declaration of why this source may be processed. owned = the content belongs to the customer. permission and licensed require rightsReference at runtime (Apify Input Schema cannot express that cross-field rule, so Cloud may start the run and the Actor then fails validation). This is provenance, not a legal check.", "type": "string", "editor": "select", "enum": [ "owned", "permission", "licensed" ], "enumTitles": [ "Owned by the customer", "Permission granted", "Licensed use" ] }, "rightsReference": { "title": "Rights reference", "description": "Required at runtime when rightsBasis is permission or licensed. Non-secret URL or identifier of the terms or permission. The Actor does not fetch this value. Cross-field requirement is enforced in the Actor, not by the Input Schema dialect.", "type": "string", "editor": "textfield", "maxLength": 2000 }, "attribution": { "title": "Attribution", "description": "Optional non-secret attribution copied into the rights manifest and each PAGE.", "type": "string", "editor": "textarea", "maxLength": 2000 }, "licenseNotice": { "title": "License notice", "description": "Optional accompanying notice required by a license. Stored once in RIGHTS_MANIFEST.", "type": "string", "editor": "textarea", "maxLength": 20000 } } } }, "sitemapUrls": { "title": "Sitemap URLs", "maxItems": 10, "type": "array", "description": "Optional XML sitemaps on an authorized origin. Other origins never expand permissions. Ignored when maxDepth is 0.", "items": { "type": "string", "minLength": 1, "maxLength": 2048, "pattern": "^[Hh][Tt][Tt][Pp][Ss]?://" }, "default": [] }, "discoverSitemaps": { "title": "Discover sitemaps", "type": "boolean", "description": "Read Sitemap from robots.txt and /sitemap.xml on authorized origins. Does not try dozens of conventional paths. Disabled when maxDepth is 0.", "default": true }, "includeUrlGlobs": { "title": "Include URL globs", "maxItems": 20, "type": "array", "description": "Optional glob patterns (* and ?) matched against the full URL and path. Empty means every in-scope URL is eligible. Regular expressions are rejected.", "items": { "type": "string", "minLength": 1, "maxLength": 2048 }, "default": [] }, "excludeUrlGlobs": { "title": "Exclude URL globs", "maxItems": 20, "type": "array", "description": "Glob exclusions. These take priority over includeUrlGlobs.", "items": { "type": "string", "minLength": 1, "maxLength": 2048 }, "default": [] }, "maxPages": { "title": "Maximum pages", "minimum": 1, "maximum": 1500, "type": "integer", "description": "Limit on unique document URLs taken into processing, including failed attempts. Retries of the same URL do not consume another slot. This is not a promise of that many successful PAGE rows.", "default": 25 }, "maxDepth": { "title": "Maximum link depth", "minimum": 0, "maximum": 10, "type": "integer", "description": "Link-follow depth. Explicit seeds are depth 0. Sitemap-discovered URLs are depth 1. maxDepth 0 processes only seeds and disables sitemap discovery.", "default": 5 }, "renderingMode": { "title": "Rendering mode", "enum": [ "http" ], "type": "string", "description": "HTTP-only extraction. JavaScript rendering is not available in this build; it will be added only after browser egress is independently verified.", "default": "http" }, "contentSelector": { "title": "Content CSS selector", "maxLength": 300, "type": "string", "description": "Optional CSS selector for the main text. If it matches nothing, extraction fails with EXTRACTION_FAILED and the run fails when no pages succeed (it is not treated as a verified empty site). JavaScript expressions are not accepted." }, "removeSelectors": { "title": "Remove CSS selectors", "maxItems": 20, "type": "array", "description": "Optional CSS selectors removed from the chosen content root before Markdown conversion.", "items": { "type": "string", "minLength": 1, "maxLength": 300 }, "default": [] }, "includeChunks": { "title": "Include chunks", "type": "boolean", "description": "Attach structural heading-based chunks on each PAGE. Chunks are not billed separately and do not use an LLM.", "default": true }, "chunkMaxChars": { "title": "Chunk size (characters)", "minimum": 1000, "maximum": 12000, "type": "integer", "description": "Soft Unicode-character limit for a chunk body. This is not a tokenizer or model context-window guarantee. Oversized code or table lines are kept and marked oversized.", "default": 6000 }, "chunkOverlapChars": { "title": "Chunk overlap (characters)", "minimum": 0, "maximum": 1000, "type": "integer", "description": "Prose overlap only. Must be less than one quarter of chunkMaxChars. Tables and code are not arbitrarily duplicated.", "default": 300 }, "maxDownloadMiB": { "title": "Download budget (MiB)", "minimum": 10, "maximum": 2048, "type": "integer", "description": "Decoded-byte ceiling for the run, including robots and sitemaps.", "default": 100 } } }, "runsResponseSchema": { "type": "object", "properties": { "data": { "type": "object", "properties": { "id": { "type": "string" }, "actId": { "type": "string" }, "userId": { "type": "string" }, "startedAt": { "type": "string", "format": "date-time", "example": "2025-01-08T00:00:00.000Z" }, "finishedAt": { "type": "string", "format": "date-time", "example": "2025-01-08T00:00:00.000Z" }, "status": { "type": "string", "example": "READY" }, "meta": { "type": "object", "properties": { "origin": { "type": "string", "example": "API" }, "userAgent": { "type": "string" } } }, "stats": { "type": "object", "properties": { "inputBodyLen": { "type": "integer", "example": 2000 }, "rebootCount": { "type": "integer", "example": 0 }, "restartCount": { "type": "integer", "example": 0 }, "resurrectCount": { "type": "integer", "example": 0 }, "computeUnits": { "type": "integer", "example": 0 } } }, "options": { "type": "object", "properties": { "build": { "type": "string", "example": "latest" }, "timeoutSecs": { "type": "integer", "example": 300 }, "memoryMbytes": { "type": "integer", "example": 1024 }, "diskMbytes": { "type": "integer", "example": 2048 } } }, "buildId": { "type": "string" }, "defaultKeyValueStoreId": { "type": "string" }, "defaultDatasetId": { "type": "string" }, "defaultRequestQueueId": { "type": "string" }, "buildNumber": { "type": "string", "example": "1.0.0" }, "containerUrl": { "type": "string" }, "usage": { "type": "object", "properties": { "ACTOR_COMPUTE_UNITS": { "type": "integer", "example": 0 }, "DATASET_READS": { "type": "integer", "example": 0 }, "DATASET_WRITES": { "type": "integer", "example": 0 }, "KEY_VALUE_STORE_READS": { "type": "integer", "example": 0 }, "KEY_VALUE_STORE_WRITES": { "type": "integer", "example": 1 }, "KEY_VALUE_STORE_LISTS": { "type": "integer", "example": 0 }, "REQUEST_QUEUE_READS": { "type": "integer", "example": 0 }, "REQUEST_QUEUE_WRITES": { "type": "integer", "example": 0 }, "DATA_TRANSFER_INTERNAL_GBYTES": { "type": "integer", "example": 0 }, "DATA_TRANSFER_EXTERNAL_GBYTES": { "type": "integer", "example": 0 }, "PROXY_RESIDENTIAL_TRANSFER_GBYTES": { "type": "integer", "example": 0 }, "PROXY_SERPS": { "type": "integer", "example": 0 } } }, "usageTotalUsd": { "type": "number", "example": 0.00005 }, "usageUsd": { "type": "object", "properties": { "ACTOR_COMPUTE_UNITS": { "type": "integer", "example": 0 }, "DATASET_READS": { "type": "integer", "example": 0 }, "DATASET_WRITES": { "type": "integer", "example": 0 }, "KEY_VALUE_STORE_READS": { "type": "integer", "example": 0 }, "KEY_VALUE_STORE_WRITES": { "type": "number", "example": 0.00005 }, "KEY_VALUE_STORE_LISTS": { "type": "integer", "example": 0 }, "REQUEST_QUEUE_READS": { "type": "integer", "example": 0 }, "REQUEST_QUEUE_WRITES": { "type": "integer", "example": 0 }, "DATA_TRANSFER_INTERNAL_GBYTES": { "type": "integer", "example": 0 }, "DATA_TRANSFER_EXTERNAL_GBYTES": { "type": "integer", "example": 0 }, "PROXY_RESIDENTIAL_TRANSFER_GBYTES": { "type": "integer", "example": 0 }, "PROXY_SERPS": { "type": "integer", "example": 0 } } } } } } } } } }