{ "openapi": "3.0.1", "info": { "title": "RAG Incremental Reindexer - Only Changed Pages & Chunks", "description": "Tells you which of your RAG sources actually changed since the last run, and returns embedding-ready chunks only for the delta. Saves embedding costs by skipping unchanged pages.", "version": "0.1", "x-build-id": "3fichzI8CO9ipTzCN" }, "servers": [ { "url": "https://api.apify.com/v2" } ], "paths": { "/acts/dvorzhik~rag-incremental-reindexer/run-sync-get-dataset-items": { "post": { "operationId": "run-sync-get-dataset-items-dvorzhik-rag-incremental-reindexer", "x-openai-isConsequential": false, "summary": "Executes an Actor, waits for its completion, and returns Actor's dataset items in response.", "tags": [ "Run Actor" ], "requestBody": { "required": true, "content": { "application/json": { "schema": { "$ref": "#/components/schemas/inputSchema" } } } }, "parameters": [ { "name": "token", "in": "query", "required": true, "schema": { "type": "string" }, "description": "Enter your Apify token here" } ], "responses": { "200": { "description": "OK" } } } }, "/acts/dvorzhik~rag-incremental-reindexer/runs": { "post": { "operationId": "runs-sync-dvorzhik-rag-incremental-reindexer", "x-openai-isConsequential": false, "summary": "Executes an Actor and returns information about the initiated run in response.", "tags": [ "Run Actor" ], "requestBody": { "required": true, "content": { "application/json": { "schema": { "$ref": "#/components/schemas/inputSchema" } } } }, "parameters": [ { "name": "token", "in": "query", "required": true, "schema": { "type": "string" }, "description": "Enter your Apify token here" } ], "responses": { "200": { "description": "OK", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/runsResponseSchema" } } } } } } }, "/acts/dvorzhik~rag-incremental-reindexer/run-sync": { "post": { "operationId": "run-sync-dvorzhik-rag-incremental-reindexer", "x-openai-isConsequential": false, "summary": "Executes an Actor, waits for completion, and returns the OUTPUT from Key-value store in response.", "tags": [ "Run Actor" ], "requestBody": { "required": true, "content": { "application/json": { "schema": { "$ref": "#/components/schemas/inputSchema" } } } }, "parameters": [ { "name": "token", "in": "query", "required": true, "schema": { "type": "string" }, "description": "Enter your Apify token here" } ], "responses": { "200": { "description": "OK" } } } } }, "components": { "schemas": { "inputSchema": { "type": "object", "required": [ "startUrls" ], "properties": { "startUrls": { "title": "URLs to check", "type": "array", "description": "One or more pages to compare against your stored baseline. Only server-rendered / static HTML is supported - JavaScript-only pages cannot be read without a browser.", "items": { "type": "object", "required": [ "url" ], "properties": { "url": { "type": "string", "title": "URL of a web page", "format": "uri" } } } }, "maxPages": { "title": "Max pages", "minimum": 1, "type": "integer", "description": "Safety limit. The Actor stops after checking this many pages, which prevents accidental runaway costs on large URL lists.", "default": 500 }, "crawlerType": { "title": "Fetch engine", "enum": [ "cheerio", "jsdom" ], "type": "string", "description": "Cheerio is a plain HTTP client: cheapest and fastest, but returns nothing for sites that render text with JavaScript. jsdom can execute some scripts in-process but is noticeably slower and less reliable. Sites built as SPAs will need a browser-based Actor instead of this one.", "default": "cheerio" }, "proxyConfiguration": { "title": "Proxy configuration", "type": "object", "description": "Optional. Enable if the sites you check block requests from datacenter IPs.", "default": { "useApifyProxy": false } }, "stateStoreName": { "title": "State store name", "pattern": "^[a-zA-Z0-9][a-zA-Z0-9-]{0,61}$", "type": "string", "description": "Name of the named Key-Value Store that persists your baseline between runs. This store is retained indefinitely, so scheduled runs always have something to compare against. Use a different store name for each separate project or corpus.", "default": "rag-reindexer-baseline" }, "resetBaseline": { "title": "Reset baseline (re-index everything)", "type": "boolean", "description": "When enabled, the Actor discards the stored baseline and treats every page as changed. Use it on the very first run if you want a full corpus, or after a major content migration.", "default": false }, "compareMode": { "title": "What to compare", "enum": [ "text", "text-and-metadata" ], "type": "string", "description": "Comparing visible text instead of raw HTML is what makes the detection useful: raw HTML 'changes' on every page view because of ads, counters and nonces. Use 'text-and-metadata' if a title or description change should also trigger re-indexing.", "default": "text" }, "maskVolatile": { "title": "Mask volatile content", "type": "object", "description": "Patterns that are replaced with placeholders before hashing, so they never count as a real change.", "properties": { "dates": { "title": "Mask dates and timestamps", "type": "boolean", "description": "Replaces rendered dates (2026-01-31, 31/01/2026, 31 January 2026) with a placeholder. Prevents 'updated 5 minutes ago', publish timestamps and 'last modified' footers from triggering false changes.", "default": true }, "dateFormats": { "title": "Date formats to mask", "type": "array", "description": "Which date shapes to look for. 'iso' = 2026-01-31; 'slash' = 31/01/2026; 'dots' = 31.01.2026; 'long' = 31 January 2026. Narrow this if your content legitimately contains dates that should trigger a re-index.", "editor": "stringList", "items": { "type": "string" }, "default": [ "iso", "slash", "dots", "long" ] }, "viewCounters": { "title": "Mask view / reaction counters", "type": "boolean", "description": "Replaces view counts, likes, comment counters and similar numbers followed by counter words (views, likes, reads, просмотров, комментариев). This is the single biggest source of false positives on news and blog sites.", "default": true }, "extraPatterns": { "title": "Extra regex patterns to mask", "type": "array", "description": "Advanced. Additional JavaScript regular expressions (as strings, without slashes) whose matches are replaced with a placeholder before comparing. Use it for site-specific noise such as product stock counters or A/B experiment IDs.", "editor": "stringList", "items": { "type": "string" }, "default": [] }, "ignoreSelectors": { "title": "CSS selectors to remove before comparing", "type": "array", "description": "Advanced. Elements matching these CSS selectors are stripped from the page before its content is compared - handy for cookie banners, promo strips or a live chat widget.", "editor": "stringList", "items": { "type": "string" }, "default": [] } }, "default": { "dates": true, "dateFormats": [ "iso", "slash", "dots", "long" ], "viewCounters": true } }, "changeThresholdPercent": { "title": "Minimum change threshold (%)", "minimum": 0, "maximum": 100, "type": "number", "description": "A page is only reported as changed when at least this percentage of the visible text differs. Set to 0 to report any difference. Raising it (for example to 2) filters out trivial wording tweaks that are not worth re-embedding.", "default": 0 }, "emitChunks": { "title": "Emit embedding-ready chunks", "type": "boolean", "description": "Include a list of text chunks (with character offsets) for every changed page, ready to be embedded and upserted. Charges one 'chunk-exported' event per chunk.", "default": true }, "chunkSize": { "title": "Target chunk size (characters)", "minimum": 200, "maximum": 8000, "type": "integer", "description": "Approximate number of characters per chunk. Chunking tries to end on a paragraph or sentence boundary rather than cutting mid-word. Set the value your embedding model expects (around 2000 characters is comfortable for most models).", "default": 2000 }, "chunkOverlap": { "title": "Chunk overlap (characters)", "minimum": 0, "maximum": 1000, "type": "integer", "description": "How many characters each chunk repeats from the previous one. A little overlap helps retrieval quality by keeping sentences that straddle a boundary intact.", "default": 200 }, "emitCleanMarkdown": { "title": "Include clean Markdown", "type": "boolean", "description": "Also return the extracted article body as Markdown for changed pages. Useful when you rebuild your index from Markdown rather than from raw text.", "default": false }, "outputMode": { "title": "Which pages to output", "enum": [ "changedOnly", "all" ], "type": "string", "description": "By default the dataset contains only changed pages, which keeps the output small and directly usable as a re-indexing queue. Switch to 'all' when you want a record of every page checked.", "default": "changedOnly" }, "webhookUrl": { "title": "Completion webhook URL", "type": "string", "description": "Optional. When set, the Actor sends a small POST request here after the run finishes with the number of changed pages and the dataset ID, so your pipeline can trigger re-embedding automatically.", "default": "" } } }, "runsResponseSchema": { "type": "object", "properties": { "data": { "type": "object", "properties": { "id": { "type": "string" }, "actId": { "type": "string" }, "userId": { "type": "string" }, "startedAt": { "type": "string", "format": "date-time", "example": "2025-01-08T00:00:00.000Z" }, "finishedAt": { "type": "string", "format": "date-time", "example": "2025-01-08T00:00:00.000Z" }, "status": { "type": "string", "example": "READY" }, "meta": { "type": "object", "properties": { "origin": { "type": "string", "example": "API" }, "userAgent": { "type": "string" } } }, "stats": { "type": "object", "properties": { "inputBodyLen": { "type": "integer", "example": 2000 }, "rebootCount": { "type": "integer", "example": 0 }, "restartCount": { "type": "integer", "example": 0 }, "resurrectCount": { "type": "integer", "example": 0 }, "computeUnits": { "type": "integer", "example": 0 } } }, "options": { "type": "object", "properties": { "build": { "type": "string", "example": "latest" }, "timeoutSecs": { "type": "integer", "example": 300 }, "memoryMbytes": { "type": "integer", "example": 1024 }, "diskMbytes": { "type": "integer", "example": 2048 } } }, "buildId": { "type": "string" }, "defaultKeyValueStoreId": { "type": "string" }, "defaultDatasetId": { "type": "string" }, "defaultRequestQueueId": { "type": "string" }, "buildNumber": { "type": "string", "example": "1.0.0" }, "containerUrl": { "type": "string" }, "usage": { "type": "object", "properties": { "ACTOR_COMPUTE_UNITS": { "type": "integer", "example": 0 }, "DATASET_READS": { "type": "integer", "example": 0 }, "DATASET_WRITES": { "type": "integer", "example": 0 }, "KEY_VALUE_STORE_READS": { "type": "integer", "example": 0 }, "KEY_VALUE_STORE_WRITES": { "type": "integer", "example": 1 }, "KEY_VALUE_STORE_LISTS": { "type": "integer", "example": 0 }, "REQUEST_QUEUE_READS": { "type": "integer", "example": 0 }, "REQUEST_QUEUE_WRITES": { "type": "integer", "example": 0 }, "DATA_TRANSFER_INTERNAL_GBYTES": { "type": "integer", "example": 0 }, "DATA_TRANSFER_EXTERNAL_GBYTES": { "type": "integer", "example": 0 }, "PROXY_RESIDENTIAL_TRANSFER_GBYTES": { "type": "integer", "example": 0 }, "PROXY_SERPS": { "type": "integer", "example": 0 }, "PROXY_UNBLOCKER_UNITS": { "type": "integer", "example": 0 } } }, "usageTotalUsd": { "type": "number", "example": 0.00005 }, "usageUsd": { "type": "object", "properties": { "ACTOR_COMPUTE_UNITS": { "type": "integer", "example": 0 }, "DATASET_READS": { "type": "integer", "example": 0 }, "DATASET_WRITES": { "type": "integer", "example": 0 }, "KEY_VALUE_STORE_READS": { "type": "integer", "example": 0 }, "KEY_VALUE_STORE_WRITES": { "type": "number", "example": 0.00005 }, "KEY_VALUE_STORE_LISTS": { "type": "integer", "example": 0 }, "REQUEST_QUEUE_READS": { "type": "integer", "example": 0 }, "REQUEST_QUEUE_WRITES": { "type": "integer", "example": 0 }, "DATA_TRANSFER_INTERNAL_GBYTES": { "type": "integer", "example": 0 }, "DATA_TRANSFER_EXTERNAL_GBYTES": { "type": "integer", "example": 0 }, "PROXY_RESIDENTIAL_TRANSFER_GBYTES": { "type": "integer", "example": 0 }, "PROXY_SERPS": { "type": "integer", "example": 0 }, "PROXY_UNBLOCKER_UNITS": { "type": "integer", "example": 0 } } } } } } } } } }