{ "openapi": "3.0.1", "info": { "title": "Website to RAG Chunks - Re-crawl and Update Only What Changed", "description": "Stop re-embedding your whole site. Crawl docs or any site into heading-aware, token-counted RAG chunks; on re-runs get only new and changed chunks plus the ids to delete from your vector DB. Sitemap and robots.txt aware. JSON, JSONL, Markdown. No embedding or API key.", "version": "2.0", "x-build-id": "LcF4PYh5Pmfs4wfPF" }, "servers": [ { "url": "https://api.apify.com/v2" } ], "paths": { "/acts/that_red_bird~website-to-rag-incremental/run-sync-get-dataset-items": { "post": { "operationId": "run-sync-get-dataset-items-that_red_bird-website-to-rag-incremental", "x-openai-isConsequential": false, "summary": "Executes an Actor, waits for its completion, and returns Actor's dataset items in response.", "tags": [ "Run Actor" ], "requestBody": { "required": true, "content": { "application/json": { "schema": { "$ref": "#/components/schemas/inputSchema" } } } }, "parameters": [ { "name": "token", "in": "query", "required": true, "schema": { "type": "string" }, "description": "Enter your Apify token here" } ], "responses": { "200": { "description": "OK" } } } }, "/acts/that_red_bird~website-to-rag-incremental/runs": { "post": { "operationId": "runs-sync-that_red_bird-website-to-rag-incremental", "x-openai-isConsequential": false, "summary": "Executes an Actor and returns information about the initiated run in response.", "tags": [ "Run Actor" ], "requestBody": { "required": true, "content": { "application/json": { "schema": { "$ref": "#/components/schemas/inputSchema" } } } }, "parameters": [ { "name": "token", "in": "query", "required": true, "schema": { "type": "string" }, "description": "Enter your Apify token here" } ], "responses": { "200": { "description": "OK", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/runsResponseSchema" } } } } } } }, "/acts/that_red_bird~website-to-rag-incremental/run-sync": { "post": { "operationId": "run-sync-that_red_bird-website-to-rag-incremental", "x-openai-isConsequential": false, "summary": "Executes an Actor, waits for completion, and returns the OUTPUT from Key-value store in response.", "tags": [ "Run Actor" ], "requestBody": { "required": true, "content": { "application/json": { "schema": { "$ref": "#/components/schemas/inputSchema" } } } }, "parameters": [ { "name": "token", "in": "query", "required": true, "schema": { "type": "string" }, "description": "Enter your Apify token here" } ], "responses": { "200": { "description": "OK" } } } } }, "components": { "schemas": { "inputSchema": { "type": "object", "properties": { "siteUrl": { "title": "Site URL", "type": "string", "description": "The site or docs section to turn into RAG chunks, e.g. https://docs.example.com. End it with a slash and a folder (https://example.com/docs/) to crawl only that folder. Pages come from sitemap.xml (also those listed in robots.txt), or from following same-origin links if there is no sitemap.", "default": "https://docs.apify.com" }, "maxPages": { "title": "Max pages", "minimum": 1, "maximum": 5000, "type": "integer", "description": "Upper bound on pages fetched per run. Pages beyond this limit are never treated as deleted, so lowering it will not wipe your index.", "default": 25 }, "onlyChanges": { "title": "Output only what changed since last run", "type": "boolean", "description": "On every run after the first, output only new and edited chunks (operation upsert) plus the ids to delete (operation delete). Turn off to also get the unchanged chunks again (full export). The first run for a site always outputs everything.", "default": true }, "removeMissingPages": { "title": "Delete chunks of pages that left the sitemap", "type": "boolean", "description": "When a page that was crawled before is gone from the sitemap, or returns 404/410, output delete rows for its chunk ids. Pages that only failed with a temporary error keep their state and are never deleted.", "default": true }, "resetState": { "title": "Reset state (treat as first run)", "type": "boolean", "description": "Forget the stored crawl for this site and output everything as new. Use after you rebuilt your vector DB from scratch.", "default": false }, "stateStoreName": { "title": "State store name", "type": "string", "description": "Named key-value store that remembers what each site looked like at the last run. Use a different name per project or client to keep their states separate.", "default": "website-to-rag-state" }, "outputUnit": { "title": "Output unit", "enum": [ "chunks", "pages" ], "type": "string", "description": "chunks: embedding-ready chunks split on headings with a token budget. pages: one clean markdown row per page (no chunking), still with change detection.", "default": "chunks" }, "outputFormats": { "title": "Extra export files", "type": "array", "description": "Besides the dataset, also write the changes to the key-value store as CHANGES.jsonl (one JSON object per line, ready for bulk loaders), CHANGES.json or CHANGES.md. DELETED_IDS.json is always written.", "items": { "type": "string", "enum": [ "jsonl", "json", "markdown" ], "enumTitles": [ "JSONL", "JSON", "Markdown" ] }, "default": [ "jsonl" ] }, "includePatterns": { "title": "Only URLs containing", "type": "array", "description": "Keep only URLs containing one of these substrings, e.g. /docs/, /guides/. Empty means all URLs.", "default": [], "items": { "type": "string" } }, "excludePatterns": { "title": "Skip URLs containing", "type": "array", "description": "Drop URLs containing any of these substrings, e.g. /tag/, /author/, ?page=.", "default": [], "items": { "type": "string" } }, "respectRobotsTxt": { "title": "Respect robots.txt", "type": "boolean", "description": "Skip pages that robots.txt disallows for all crawlers (User-agent: *) and read Sitemap: lines from it.", "default": true }, "useSitemap": { "title": "Use sitemap.xml", "type": "boolean", "description": "Prefer the sitemap (including sitemap indexes) for discovery. Falls back to following same-origin links up to 3 levels deep when there is none.", "default": true }, "targetTokens": { "title": "Target tokens per chunk", "minimum": 20, "maximum": 4000, "type": "integer", "description": "Preferred chunk size when a section has to be split. A whole section under the max is kept as one chunk. Token counts are estimates (about 4 characters per token), not a real tokenizer.", "default": 350 }, "maxTokens": { "title": "Max tokens per chunk", "minimum": 20, "maximum": 8000, "type": "integer", "description": "Hard cap per chunk, except a single code block or table bigger than the cap, which is kept whole rather than cut in half. Must be at least the target.", "default": 512 }, "overlapTokens": { "title": "Overlap between adjacent chunks (tokens)", "minimum": 0, "maximum": 500, "type": "integer", "description": "How much of the end of a chunk is repeated at the start of the next chunk of the same page, on a whole sentence or block. 0 disables overlap.", "default": 40 }, "minChunkTokens": { "title": "Minimum tokens to keep a chunk", "minimum": 0, "maximum": 200, "type": "integer", "description": "Chunks estimated below this size are dropped as too small to be useful, e.g. a lone heading with no text.", "default": 15 }, "dedupThreshold": { "title": "Near-duplicate similarity threshold (1-100)", "minimum": 1, "maximum": 100, "type": "integer", "description": "Chunks anywhere in the crawl whose shingle similarity is at or above this percentage are merged into one, which removes repeated boilerplate. 85 means 0.85. Higher means fewer merges.", "default": 85 }, "shingleSize": { "title": "Shingle size (words)", "minimum": 2, "maximum": 12, "type": "integer", "description": "Length of the word n-grams used to fingerprint chunks for near-duplicate detection. Smaller is noisier, larger is stricter.", "default": 5 }, "includeLinks": { "title": "Keep markdown links", "type": "boolean", "description": "Keep [text](url) links in the extracted markdown. Turn off for pure prose without link noise.", "default": true }, "includeDuplicateRows": { "title": "Include removed-duplicate rows", "type": "boolean", "description": "Output one row per near-duplicate chunk that was merged away (type duplicate), with the surviving chunk id and similarity. Off keeps the dataset a clean corpus.", "default": false }, "webhookUrl": { "title": "Alert webhook URL", "type": "string", "description": "Optional Slack, Discord or generic webhook. It receives a short message only when a re-run finds changes (never on the first run). Leave empty to disable.", "default": "" }, "concurrency": { "title": "Concurrency", "minimum": 1, "maximum": 20, "type": "integer", "description": "How many pages to fetch in parallel.", "default": 5 }, "proxyConfiguration": { "title": "Proxy", "type": "object", "description": "Optional Apify proxy for sites that block datacenter IPs. Off by default.", "default": { "useApifyProxy": false } } } }, "runsResponseSchema": { "type": "object", "properties": { "data": { "type": "object", "properties": { "id": { "type": "string" }, "actId": { "type": "string" }, "userId": { "type": "string" }, "startedAt": { "type": "string", "format": "date-time", "example": "2025-01-08T00:00:00.000Z" }, "finishedAt": { "type": "string", "format": "date-time", "example": "2025-01-08T00:00:00.000Z" }, "status": { "type": "string", "example": "READY" }, "meta": { "type": "object", "properties": { "origin": { "type": "string", "example": "API" }, "userAgent": { "type": "string" } } }, "stats": { "type": "object", "properties": { "inputBodyLen": { "type": "integer", "example": 2000 }, "rebootCount": { "type": "integer", "example": 0 }, "restartCount": { "type": "integer", "example": 0 }, "resurrectCount": { "type": "integer", "example": 0 }, "computeUnits": { "type": "integer", "example": 0 } } }, "options": { "type": "object", "properties": { "build": { "type": "string", "example": "latest" }, "timeoutSecs": { "type": "integer", "example": 300 }, "memoryMbytes": { "type": "integer", "example": 1024 }, "diskMbytes": { "type": "integer", "example": 2048 } } }, "buildId": { "type": "string" }, "defaultKeyValueStoreId": { "type": "string" }, "defaultDatasetId": { "type": "string" }, "defaultRequestQueueId": { "type": "string" }, "buildNumber": { "type": "string", "example": "1.0.0" }, "containerUrl": { "type": "string" }, "usage": { "type": "object", "properties": { "ACTOR_COMPUTE_UNITS": { "type": "integer", "example": 0 }, "DATASET_READS": { "type": "integer", "example": 0 }, "DATASET_WRITES": { "type": "integer", "example": 0 }, "KEY_VALUE_STORE_READS": { "type": "integer", "example": 0 }, "KEY_VALUE_STORE_WRITES": { "type": "integer", "example": 1 }, "KEY_VALUE_STORE_LISTS": { "type": "integer", "example": 0 }, "REQUEST_QUEUE_READS": { "type": "integer", "example": 0 }, "REQUEST_QUEUE_WRITES": { "type": "integer", "example": 0 }, "DATA_TRANSFER_INTERNAL_GBYTES": { "type": "integer", "example": 0 }, "DATA_TRANSFER_EXTERNAL_GBYTES": { "type": "integer", "example": 0 }, "PROXY_RESIDENTIAL_TRANSFER_GBYTES": { "type": "integer", "example": 0 }, "PROXY_SERPS": { "type": "integer", "example": 0 }, "PROXY_UNBLOCKER_UNITS": { "type": "integer", "example": 0 } } }, "usageTotalUsd": { "type": "number", "example": 0.00005 }, "usageUsd": { "type": "object", "properties": { "ACTOR_COMPUTE_UNITS": { "type": "integer", "example": 0 }, "DATASET_READS": { "type": "integer", "example": 0 }, "DATASET_WRITES": { "type": "integer", "example": 0 }, "KEY_VALUE_STORE_READS": { "type": "integer", "example": 0 }, "KEY_VALUE_STORE_WRITES": { "type": "number", "example": 0.00005 }, "KEY_VALUE_STORE_LISTS": { "type": "integer", "example": 0 }, "REQUEST_QUEUE_READS": { "type": "integer", "example": 0 }, "REQUEST_QUEUE_WRITES": { "type": "integer", "example": 0 }, "DATA_TRANSFER_INTERNAL_GBYTES": { "type": "integer", "example": 0 }, "DATA_TRANSFER_EXTERNAL_GBYTES": { "type": "integer", "example": 0 }, "PROXY_RESIDENTIAL_TRANSFER_GBYTES": { "type": "integer", "example": 0 }, "PROXY_SERPS": { "type": "integer", "example": 0 }, "PROXY_UNBLOCKER_UNITS": { "type": "integer", "example": 0 } } } } } } } } } }