{ "openapi": "3.0.1", "info": { "title": "RAG Sync Monitor: Changed Chunks Only", "description": "Incremental RAG sync - Keep your vector DB in sync with any docs site, help center or PDFs. Outputs only the chunks added, changed or removed since the last run, with stable IDs for upserts and deletes. Pinecone, Qdrant, Chroma, pgvector.", "version": "0.4", "x-build-id": "2sVAnFkBt4kHis9V4" }, "servers": [ { "url": "https://api.apify.com/v2" } ], "paths": { "/acts/cipherwise~rag-sync-monitor/run-sync-get-dataset-items": { "post": { "operationId": "run-sync-get-dataset-items-cipherwise-rag-sync-monitor", "x-openai-isConsequential": false, "summary": "Executes an Actor, waits for its completion, and returns Actor's dataset items in response.", "tags": [ "Run Actor" ], "requestBody": { "required": true, "content": { "application/json": { "schema": { "$ref": "#/components/schemas/inputSchema" } } } }, "parameters": [ { "name": "token", "in": "query", "required": true, "schema": { "type": "string" }, "description": "Enter your Apify token here" } ], "responses": { "200": { "description": "OK" } } } }, "/acts/cipherwise~rag-sync-monitor/runs": { "post": { "operationId": "runs-sync-cipherwise-rag-sync-monitor", "x-openai-isConsequential": false, "summary": "Executes an Actor and returns information about the initiated run in response.", "tags": [ "Run Actor" ], "requestBody": { "required": true, "content": { "application/json": { "schema": { "$ref": "#/components/schemas/inputSchema" } } } }, "parameters": [ { "name": "token", "in": "query", "required": true, "schema": { "type": "string" }, "description": "Enter your Apify token here" } ], "responses": { "200": { "description": "OK", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/runsResponseSchema" } } } } } } }, "/acts/cipherwise~rag-sync-monitor/run-sync": { "post": { "operationId": "run-sync-cipherwise-rag-sync-monitor", "x-openai-isConsequential": false, "summary": "Executes an Actor, waits for completion, and returns the OUTPUT from Key-value store in response.", "tags": [ "Run Actor" ], "requestBody": { "required": true, "content": { "application/json": { "schema": { "$ref": "#/components/schemas/inputSchema" } } } }, "parameters": [ { "name": "token", "in": "query", "required": true, "schema": { "type": "string" }, "description": "Enter your Apify token here" } ], "responses": { "200": { "description": "OK" } } } } }, "components": { "schemas": { "inputSchema": { "type": "object", "properties": { "startUrls": { "title": "Start URLs", "type": "array", "description": "Pages to start crawling from, e.g. the root of a docs site. Only pages under the URL prefixes are crawled.", "items": { "type": "object", "required": [ "url" ], "properties": { "url": { "type": "string", "title": "URL of a web page", "format": "uri" } } } }, "sitemapUrls": { "title": "Sitemap URLs", "type": "array", "description": "Optional sitemap.xml or sitemap index URLs. Pages listed in them are added to the crawl.", "items": { "type": "object", "required": [ "url" ], "properties": { "url": { "type": "string", "title": "URL of a web page", "format": "uri" } } } }, "useSitemaps": { "title": "Find sitemaps automatically", "type": "boolean", "description": "Also load sitemaps listed in each start host's robots.txt (or /sitemap.xml), so pages that aren't linked from navigation are found too.", "default": false }, "urlPrefixes": { "title": "URL prefixes (scope)", "type": "array", "description": "Only URLs starting with one of these are crawled. Defaults to the directory of each start URL.", "items": { "type": "string" } }, "excludeUrlPatterns": { "title": "Exclude URL patterns (regex)", "type": "array", "description": "Regular expressions; matching URLs are skipped, e.g. \"/changelog/\" or \"\\\\?page=\".", "items": { "type": "string" } }, "includeUrlPatterns": { "title": "Include URL patterns (regex)", "type": "array", "description": "If set, a URL must also match one of these regular expressions to be synced, e.g. \"/guides/\" or \"/api/v2/\".", "items": { "type": "string" } }, "crawlLinks": { "title": "Follow links", "type": "boolean", "description": "Discover new pages by following in-scope links. Turn off to sync only start URLs and sitemap pages.", "default": true }, "maxCrawlDepth": { "title": "Max crawl depth", "minimum": 0, "type": "integer", "description": "How many links away from the start URLs to follow. 0 = only the start URLs (and sitemap pages). Empty = no limit." }, "maxPages": { "title": "Max pages per run", "minimum": 1, "type": "integer", "description": "Upper limit on pages checked in one run. The form starts at 25 so a first try is quick and cheap; raise it for a full sync (the default when called via API is 200). Unchanged pages are cheap to check thanks to conditional requests.", "default": 200 }, "stateKey": { "title": "Sync state name", "type": "string", "description": "Name of the saved baseline that runs are compared against. Defaults to the first start URL. Use a different name per site or per vector index. Change it to start a fresh baseline." }, "emitBaseline": { "title": "Output all chunks on the first run", "type": "boolean", "description": "On the first run for a state name, output every chunk as an upsert. Turn off if the site is already indexed and you only want future changes.", "default": true }, "chunkSize": { "title": "Max chunk size (characters)", "minimum": 200, "type": "integer", "description": "Chunks follow headings and are packed by paragraph up to this size. Changing it re-chunks every page on the next run.", "default": 1500 }, "chunkOverlap": { "title": "Chunk overlap (characters)", "minimum": 0, "type": "integer", "description": "Characters repeated from the previous chunk within the same section.", "default": 150 }, "granularity": { "title": "Output granularity", "enum": [ "chunk", "page" ], "type": "string", "description": "Chunks: changed chunks are output. Pages: each changed page is output once as full Markdown, with a stable ID per URL.", "default": "chunk" }, "contentSelector": { "title": "Content CSS selector", "type": "string", "description": "Optional CSS selector for the main content, e.g. \"main .markdown\". Defaults to main, article or body." }, "removeSelectors": { "title": "Remove CSS selectors", "type": "array", "description": "Extra elements to strip before chunking, e.g. \".sidebar\", \".edit-this-page\". Nav, header, footer and scripts are always removed.", "items": { "type": "string" } }, "removeRepeatedText": { "title": "Remove text repeated across pages", "type": "boolean", "description": "Learns lines that appear on most pages of the site (menus, footers, banners, rotating sponsor lists) and removes them from chunks. Headings and code are never removed. When new lines are learned, the next run re-downloads every page once to apply the cleanup.", "default": false }, "renderJavaScript": { "title": "Render JavaScript", "enum": [ "never", "auto", "always" ], "type": "string", "description": "For sites that build their content in the browser (some help centers and single-page apps). Auto renders only pages with too little static text. Pages that need the browser are charged an extra 'page rendered' event. Use 1024 MB of run memory.", "default": "never" }, "dynamicContentWaitSecs": { "title": "Max wait for dynamic content (seconds)", "minimum": 1, "maximum": 60, "type": "integer", "description": "Browser only: how long to wait for the page to finish loading content.", "default": 10 }, "waitForSelector": { "title": "Wait for CSS selector", "type": "string", "description": "Browser only: wait until this element appears before extracting, e.g. \"article\"." }, "includeFiles": { "title": "Sync linked files", "type": "array", "description": "Also sync linked PDF and Word files found on the same host. PDF chunks are grouped by page (\"Page 3\") for citations. Scanned PDFs without a text layer produce no text.", "items": { "type": "string", "enum": [ "pdf", "docx" ], "enumTitles": [ "PDF", "Word (DOCX)" ] } }, "useCanonicalUrls": { "title": "Use canonical URLs", "type": "boolean", "description": "If a page declares another URL as canonical (rel=canonical), sync that URL instead, which removes mirrored and duplicate pages.", "default": true }, "keepUrlFragments": { "title": "Treat #/ routes as pages", "type": "boolean", "description": "For hash-routed apps (URLs like /#/guide). Plain #section anchors are always ignored. Usually combined with Render JavaScript.", "default": false }, "maxConcurrency": { "title": "Max concurrency", "minimum": 1, "maximum": 20, "type": "integer", "description": "Parallel requests. Keep it low to be polite to the site.", "default": 5 }, "proxyConfiguration": { "title": "Proxy configuration", "type": "object", "description": "Use Apify Proxy (or your own) for sites that block cloud IP addresses." }, "customHttpHeaders": { "title": "Custom HTTP headers", "type": "object", "description": "Extra headers sent with every request to the crawled hosts only, e.g. {\"Authorization\": \"Bearer ...\"} for docs behind a login. Stored encrypted." }, "cookies": { "title": "Cookies", "type": "array", "description": "Cookies sent to the crawled hosts only, e.g. [{\"name\": \"session\", \"value\": \"...\", \"domain\": \"docs.example.com\"}]. Stored encrypted." }, "forceRefetch": { "title": "Ignore ETag / Last-Modified", "type": "boolean", "description": "Re-download every page even if the server says it is unchanged. Output is still only real chunk changes.", "default": false }, "dryRun": { "title": "Dry run (don't save state)", "type": "boolean", "description": "Output what would change without updating the saved baseline, so the next run reports the same changes. Useful for testing a pipeline.", "default": false }, "respectRobotsTxt": { "title": "Respect robots.txt", "type": "boolean", "description": "Skip pages disallowed by the site's robots.txt.", "default": true } } }, "runsResponseSchema": { "type": "object", "properties": { "data": { "type": "object", "properties": { "id": { "type": "string" }, "actId": { "type": "string" }, "userId": { "type": "string" }, "startedAt": { "type": "string", "format": "date-time", "example": "2025-01-08T00:00:00.000Z" }, "finishedAt": { "type": "string", "format": "date-time", "example": "2025-01-08T00:00:00.000Z" }, "status": { "type": "string", "example": "READY" }, "meta": { "type": "object", "properties": { "origin": { "type": "string", "example": "API" }, "userAgent": { "type": "string" } } }, "stats": { "type": "object", "properties": { "inputBodyLen": { "type": "integer", "example": 2000 }, "rebootCount": { "type": "integer", "example": 0 }, "restartCount": { "type": "integer", "example": 0 }, "resurrectCount": { "type": "integer", "example": 0 }, "computeUnits": { "type": "integer", "example": 0 } } }, "options": { "type": "object", "properties": { "build": { "type": "string", "example": "latest" }, "timeoutSecs": { "type": "integer", "example": 300 }, "memoryMbytes": { "type": "integer", "example": 1024 }, "diskMbytes": { "type": "integer", "example": 2048 } } }, "buildId": { "type": "string" }, "defaultKeyValueStoreId": { "type": "string" }, "defaultDatasetId": { "type": "string" }, "defaultRequestQueueId": { "type": "string" }, "buildNumber": { "type": "string", "example": "1.0.0" }, "containerUrl": { "type": "string" }, "usage": { "type": "object", "properties": { "ACTOR_COMPUTE_UNITS": { "type": "integer", "example": 0 }, "DATASET_READS": { "type": "integer", "example": 0 }, "DATASET_WRITES": { "type": "integer", "example": 0 }, "KEY_VALUE_STORE_READS": { "type": "integer", "example": 0 }, "KEY_VALUE_STORE_WRITES": { "type": "integer", "example": 1 }, "KEY_VALUE_STORE_LISTS": { "type": "integer", "example": 0 }, "REQUEST_QUEUE_READS": { "type": "integer", "example": 0 }, "REQUEST_QUEUE_WRITES": { "type": "integer", "example": 0 }, "DATA_TRANSFER_INTERNAL_GBYTES": { "type": "integer", "example": 0 }, "DATA_TRANSFER_EXTERNAL_GBYTES": { "type": "integer", "example": 0 }, "PROXY_RESIDENTIAL_TRANSFER_GBYTES": { "type": "integer", "example": 0 }, "PROXY_SERPS": { "type": "integer", "example": 0 }, "PROXY_UNBLOCKER_UNITS": { "type": "integer", "example": 0 } } }, "usageTotalUsd": { "type": "number", "example": 0.00005 }, "usageUsd": { "type": "object", "properties": { "ACTOR_COMPUTE_UNITS": { "type": "integer", "example": 0 }, "DATASET_READS": { "type": "integer", "example": 0 }, "DATASET_WRITES": { "type": "integer", "example": 0 }, "KEY_VALUE_STORE_READS": { "type": "integer", "example": 0 }, "KEY_VALUE_STORE_WRITES": { "type": "number", "example": 0.00005 }, "KEY_VALUE_STORE_LISTS": { "type": "integer", "example": 0 }, "REQUEST_QUEUE_READS": { "type": "integer", "example": 0 }, "REQUEST_QUEUE_WRITES": { "type": "integer", "example": 0 }, "DATA_TRANSFER_INTERNAL_GBYTES": { "type": "integer", "example": 0 }, "DATA_TRANSFER_EXTERNAL_GBYTES": { "type": "integer", "example": 0 }, "PROXY_RESIDENTIAL_TRANSFER_GBYTES": { "type": "integer", "example": 0 }, "PROXY_SERPS": { "type": "integer", "example": 0 }, "PROXY_UNBLOCKER_UNITS": { "type": "integer", "example": 0 } } } } } } } } } }