{ "openapi": "3.0.1", "info": { "title": "AI-Ready Website Change Feed - Chunked Diffs for RAG Pipelines", "description": "Crawls a list of URLs or a sitemap on a schedule and emits only the content chunks that are new, changed, or removed since the previous run - token-windowed with stable chunk IDs and content hashes, ready for direct vector-database upsert/delete.", "version": "1.0", "x-build-id": "XofiG4VVu7jMSZRJ0" }, "servers": [ { "url": "https://api.apify.com/v2" } ], "paths": { "/acts/mibedk~website-diff-rag-feed/run-sync-get-dataset-items": { "post": { "operationId": "run-sync-get-dataset-items-mibedk-website-diff-rag-feed", "x-openai-isConsequential": false, "summary": "Executes an Actor, waits for its completion, and returns Actor's dataset items in response.", "tags": [ "Run Actor" ], "requestBody": { "required": true, "content": { "application/json": { "schema": { "$ref": "#/components/schemas/inputSchema" } } } }, "parameters": [ { "name": "token", "in": "query", "required": true, "schema": { "type": "string" }, "description": "Enter your Apify token here" } ], "responses": { "200": { "description": "OK" } } } }, "/acts/mibedk~website-diff-rag-feed/runs": { "post": { "operationId": "runs-sync-mibedk-website-diff-rag-feed", "x-openai-isConsequential": false, "summary": "Executes an Actor and returns information about the initiated run in response.", "tags": [ "Run Actor" ], "requestBody": { "required": true, "content": { "application/json": { "schema": { "$ref": "#/components/schemas/inputSchema" } } } }, "parameters": [ { "name": "token", "in": "query", "required": true, "schema": { "type": "string" }, "description": "Enter your Apify token here" } ], "responses": { "200": { "description": "OK", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/runsResponseSchema" } } } } } } }, "/acts/mibedk~website-diff-rag-feed/run-sync": { "post": { "operationId": "run-sync-mibedk-website-diff-rag-feed", "x-openai-isConsequential": false, "summary": "Executes an Actor, waits for completion, and returns the OUTPUT from Key-value store in response.", "tags": [ "Run Actor" ], "requestBody": { "required": true, "content": { "application/json": { "schema": { "$ref": "#/components/schemas/inputSchema" } } } }, "parameters": [ { "name": "token", "in": "query", "required": true, "schema": { "type": "string" }, "description": "Enter your Apify token here" } ], "responses": { "200": { "description": "OK" } } } } }, "components": { "schemas": { "inputSchema": { "type": "object", "properties": { "startUrls": { "title": "Start URLs", "type": "array", "description": "Pages to track. Every run re-fetches each URL here and diffs its content chunks against the previous run's stored hashes.", "default": [ { "url": "https://crawlee.dev/js/docs/quick-start" }, { "url": "https://crawlee.dev/js/docs/introduction" } ], "items": { "type": "object", "required": [ "url" ], "properties": { "url": { "type": "string", "title": "URL of a web page", "format": "uri" } } } }, "sitemapUrls": { "title": "Sitemap URLs (optional)", "type": "array", "description": "One or more sitemap.xml URLs. Every page URL found is added to the crawl list (one level of nested sitemap-index is followed), up to 'Max pages per run'.", "default": [], "items": { "type": "string" } }, "maxPagesPerRun": { "title": "Max pages per run", "minimum": 1, "maximum": 500, "type": "integer", "description": "Stop after fetching this many pages, combining Start URLs and sitemap-expanded URLs. Keeps a large sitemap from blowing the run budget.", "default": 20 }, "chunkTokens": { "title": "Chunk size (tokens)", "minimum": 50, "maximum": 2000, "type": "integer", "description": "Target size of each content chunk, measured in GPT-style (cl100k_base) tokens. Matches common embedding-model input windows.", "default": 500 }, "chunkOverlapTokens": { "title": "Chunk overlap (tokens)", "minimum": 0, "maximum": 500, "type": "integer", "description": "How many tokens consecutive chunks share, to avoid losing context at a chunk boundary. Must be smaller than 'Chunk size (tokens)'; it is automatically capped if not.", "default": 50 }, "forceFullRescan": { "title": "Force full rescan", "type": "boolean", "description": "Treat every chunk on every page as new, ignoring previously stored hashes. Use for the first backfill into a vector database, or after a chunking-config change. The stored state is still overwritten with this run's results.", "default": false }, "stateKeyValueStoreName": { "title": "State store name", "type": "string", "description": "Name of the named key-value store used to persist chunk hashes between runs, so this actor can diff against 'last time'. Keep this the same across scheduled runs of the same site to get real diffs. Use a different value to track a second site independently, or to force a clean baseline.", "default": "website-diff-rag-feed-state" }, "requestTimeoutSecs": { "title": "Request timeout (seconds)", "minimum": 5, "maximum": 120, "type": "integer", "description": "How long to wait for each page to respond before treating it as failed and retrying.", "default": 30 }, "maxRequestRetries": { "title": "Max retries per page", "minimum": 0, "maximum": 10, "type": "integer", "description": "How many times to retry a page after a failed fetch before giving up on it and marking its previously known chunks as removed.", "default": 3 }, "proxyConfiguration": { "title": "Proxy configuration", "type": "object", "description": "Optional. Most sites work fine with no proxy. Turn this on only if a target site blocks Apify's data-center IPs.", "default": { "useApifyProxy": false } } } }, "runsResponseSchema": { "type": "object", "properties": { "data": { "type": "object", "properties": { "id": { "type": "string" }, "actId": { "type": "string" }, "userId": { "type": "string" }, "startedAt": { "type": "string", "format": "date-time", "example": "2025-01-08T00:00:00.000Z" }, "finishedAt": { "type": "string", "format": "date-time", "example": "2025-01-08T00:00:00.000Z" }, "status": { "type": "string", "example": "READY" }, "meta": { "type": "object", "properties": { "origin": { "type": "string", "example": "API" }, "userAgent": { "type": "string" } } }, "stats": { "type": "object", "properties": { "inputBodyLen": { "type": "integer", "example": 2000 }, "rebootCount": { "type": "integer", "example": 0 }, "restartCount": { "type": "integer", "example": 0 }, "resurrectCount": { "type": "integer", "example": 0 }, "computeUnits": { "type": "integer", "example": 0 } } }, "options": { "type": "object", "properties": { "build": { "type": "string", "example": "latest" }, "timeoutSecs": { "type": "integer", "example": 300 }, "memoryMbytes": { "type": "integer", "example": 1024 }, "diskMbytes": { "type": "integer", "example": 2048 } } }, "buildId": { "type": "string" }, "defaultKeyValueStoreId": { "type": "string" }, "defaultDatasetId": { "type": "string" }, "defaultRequestQueueId": { "type": "string" }, "buildNumber": { "type": "string", "example": "1.0.0" }, "containerUrl": { "type": "string" }, "usage": { "type": "object", "properties": { "ACTOR_COMPUTE_UNITS": { "type": "integer", "example": 0 }, "DATASET_READS": { "type": "integer", "example": 0 }, "DATASET_WRITES": { "type": "integer", "example": 0 }, "KEY_VALUE_STORE_READS": { "type": "integer", "example": 0 }, "KEY_VALUE_STORE_WRITES": { "type": "integer", "example": 1 }, "KEY_VALUE_STORE_LISTS": { "type": "integer", "example": 0 }, "REQUEST_QUEUE_READS": { "type": "integer", "example": 0 }, "REQUEST_QUEUE_WRITES": { "type": "integer", "example": 0 }, "DATA_TRANSFER_INTERNAL_GBYTES": { "type": "integer", "example": 0 }, "DATA_TRANSFER_EXTERNAL_GBYTES": { "type": "integer", "example": 0 }, "PROXY_RESIDENTIAL_TRANSFER_GBYTES": { "type": "integer", "example": 0 }, "PROXY_SERPS": { "type": "integer", "example": 0 }, "PROXY_UNBLOCKER_UNITS": { "type": "integer", "example": 0 } } }, "usageTotalUsd": { "type": "number", "example": 0.00005 }, "usageUsd": { "type": "object", "properties": { "ACTOR_COMPUTE_UNITS": { "type": "integer", "example": 0 }, "DATASET_READS": { "type": "integer", "example": 0 }, "DATASET_WRITES": { "type": "integer", "example": 0 }, "KEY_VALUE_STORE_READS": { "type": "integer", "example": 0 }, "KEY_VALUE_STORE_WRITES": { "type": "number", "example": 0.00005 }, "KEY_VALUE_STORE_LISTS": { "type": "integer", "example": 0 }, "REQUEST_QUEUE_READS": { "type": "integer", "example": 0 }, "REQUEST_QUEUE_WRITES": { "type": "integer", "example": 0 }, "DATA_TRANSFER_INTERNAL_GBYTES": { "type": "integer", "example": 0 }, "DATA_TRANSFER_EXTERNAL_GBYTES": { "type": "integer", "example": 0 }, "PROXY_RESIDENTIAL_TRANSFER_GBYTES": { "type": "integer", "example": 0 }, "PROXY_SERPS": { "type": "integer", "example": 0 }, "PROXY_UNBLOCKER_UNITS": { "type": "integer", "example": 0 } } } } } } } } } }