{ "openapi": "3.0.1", "info": { "title": "RAG Post Processor - Text Cleaner & Chunker for LLM Pipelines", "description": "Clean and chunk scraped text into embedding-ready chunks, sized in tokens (tiktoken) rather than characters. Preserves headings, tables, code blocks and links. Every chunk carries its token count, heading path and content hash. Chain after any scraper. Billed per KB of text processed.", "version": "0.0", "x-build-id": "1BCjpZulBdhwc1G1h" }, "servers": [ { "url": "https://api.apify.com/v2" } ], "paths": { "/acts/jalicia~rag-post-processor/run-sync-get-dataset-items": { "post": { "operationId": "run-sync-get-dataset-items-jalicia-rag-post-processor", "x-openai-isConsequential": false, "summary": "Executes an Actor, waits for its completion, and returns Actor's dataset items in response.", "tags": [ "Run Actor" ], "requestBody": { "required": true, "content": { "application/json": { "schema": { "$ref": "#/components/schemas/inputSchema" } } } }, "parameters": [ { "name": "token", "in": "query", "required": true, "schema": { "type": "string" }, "description": "Enter your Apify token here" } ], "responses": { "200": { "description": "OK" } } } }, "/acts/jalicia~rag-post-processor/runs": { "post": { "operationId": "runs-sync-jalicia-rag-post-processor", "x-openai-isConsequential": false, "summary": "Executes an Actor and returns information about the initiated run in response.", "tags": [ "Run Actor" ], "requestBody": { "required": true, "content": { "application/json": { "schema": { "$ref": "#/components/schemas/inputSchema" } } } }, "parameters": [ { "name": "token", "in": "query", "required": true, "schema": { "type": "string" }, "description": "Enter your Apify token here" } ], "responses": { "200": { "description": "OK", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/runsResponseSchema" } } } } } } }, "/acts/jalicia~rag-post-processor/run-sync": { "post": { "operationId": "run-sync-jalicia-rag-post-processor", "x-openai-isConsequential": false, "summary": "Executes an Actor, waits for completion, and returns the OUTPUT from Key-value store in response.", "tags": [ "Run Actor" ], "requestBody": { "required": true, "content": { "application/json": { "schema": { "$ref": "#/components/schemas/inputSchema" } } } }, "parameters": [ { "name": "token", "in": "query", "required": true, "schema": { "type": "string" }, "description": "Enter your Apify token here" } ], "responses": { "200": { "description": "OK" } } } } }, "components": { "schemas": { "inputSchema": { "type": "object", "properties": { "datasetId": { "title": "Dataset ID (for chaining)", "type": "string", "description": "Apify dataset ID from a previous Actor run. Use this to chain directly after any scraper. All pages are read, not just the first." }, "data": { "title": "Input items", "type": "array", "description": "Array of objects from a previous scraper. Each needs a text field — markdown, text, content, body, html and around 25 other common names are recognised automatically.", "items": { "type": "object" } }, "text_field": { "title": "Text field name", "type": "string", "description": "Name the field to read instead of relying on automatic detection. Useful when your scraper uses a custom field name, or when an item has several text fields and you want a specific one. The field actually used is reported on every output row as source_field." }, "max_tokens": { "title": "Max tokens per chunk", "minimum": 16, "maximum": 8191, "type": "integer", "description": "Hard ceiling on tokens per chunk, measured with the encoding below. No emitted chunk ever exceeds this. 8191 is the input limit for OpenAI's text-embedding-3 models.", "default": 512 }, "overlap_tokens": { "title": "Overlap between chunks (tokens)", "minimum": 0, "maximum": 8191, "type": "integer", "description": "Tokens of trailing context repeated at the start of the next chunk. Only whole sentences are carried, so a chunk never begins mid-word. Clamped to 50% of max_tokens; 50% is fully supported. Overlap duplicates tokens, and the run log reports the resulting amplification so you can budget your embedding spend.", "default": 64 }, "min_tokens": { "title": "Minimum tokens per chunk", "minimum": 0, "type": "integer", "description": "Chunks below this are merged into the previous chunk rather than emitted as their own row. No text is ever dropped.", "default": 24 }, "encoding": { "title": "Tokenizer encoding", "enum": [ "cl100k_base", "o200k_base", "p50k_base", "r50k_base" ], "type": "string", "description": "Which tokenizer to measure with. Pick the one your embedding model uses. Note that text-embedding-3-small and -large use cl100k_base, not o200k_base, despite being newer models. The BPE tables are bundled into the image, so no download happens at run time.", "default": "cl100k_base" }, "split_on_heading_level": { "title": "Force a new chunk at heading level", "minimum": 0, "maximum": 6, "type": "integer", "description": "0 packs sections together to fill the budget. Set 1 to 6 to start a new chunk at every heading of that level or higher, so one section never bleeds into the next. Use 2 for typical documentation pages.", "default": 0 }, "include_heading_context": { "title": "Prefix chunks with their heading path", "type": "boolean", "description": "Prepend the breadcrumb (for example \"Widget SDK > Quick start\") to each chunk's text, so an isolated passage still carries the context a retriever needs. The path is also emitted separately as heading_path and section on every row.", "default": true }, "preserve_links": { "title": "Keep URLs and links", "type": "boolean", "description": "Keep URLs in the text and convert HTML anchors to markdown links, so retrieved passages can still be cited. Turn off only if you want URLs stripped entirely.", "default": true }, "drop_nav": { "title": "Drop navigation chrome", "type": "boolean", "description": "Remove nav and menu contents. Script, style, noscript, template, svg and comment contents are always removed.", "default": true } } }, "runsResponseSchema": { "type": "object", "properties": { "data": { "type": "object", "properties": { "id": { "type": "string" }, "actId": { "type": "string" }, "userId": { "type": "string" }, "startedAt": { "type": "string", "format": "date-time", "example": "2025-01-08T00:00:00.000Z" }, "finishedAt": { "type": "string", "format": "date-time", "example": "2025-01-08T00:00:00.000Z" }, "status": { "type": "string", "example": "READY" }, "meta": { "type": "object", "properties": { "origin": { "type": "string", "example": "API" }, "userAgent": { "type": "string" } } }, "stats": { "type": "object", "properties": { "inputBodyLen": { "type": "integer", "example": 2000 }, "rebootCount": { "type": "integer", "example": 0 }, "restartCount": { "type": "integer", "example": 0 }, "resurrectCount": { "type": "integer", "example": 0 }, "computeUnits": { "type": "integer", "example": 0 } } }, "options": { "type": "object", "properties": { "build": { "type": "string", "example": "latest" }, "timeoutSecs": { "type": "integer", "example": 300 }, "memoryMbytes": { "type": "integer", "example": 1024 }, "diskMbytes": { "type": "integer", "example": 2048 } } }, "buildId": { "type": "string" }, "defaultKeyValueStoreId": { "type": "string" }, "defaultDatasetId": { "type": "string" }, "defaultRequestQueueId": { "type": "string" }, "buildNumber": { "type": "string", "example": "1.0.0" }, "containerUrl": { "type": "string" }, "usage": { "type": "object", "properties": { "ACTOR_COMPUTE_UNITS": { "type": "integer", "example": 0 }, "DATASET_READS": { "type": "integer", "example": 0 }, "DATASET_WRITES": { "type": "integer", "example": 0 }, "KEY_VALUE_STORE_READS": { "type": "integer", "example": 0 }, "KEY_VALUE_STORE_WRITES": { "type": "integer", "example": 1 }, "KEY_VALUE_STORE_LISTS": { "type": "integer", "example": 0 }, "REQUEST_QUEUE_READS": { "type": "integer", "example": 0 }, "REQUEST_QUEUE_WRITES": { "type": "integer", "example": 0 }, "DATA_TRANSFER_INTERNAL_GBYTES": { "type": "integer", "example": 0 }, "DATA_TRANSFER_EXTERNAL_GBYTES": { "type": "integer", "example": 0 }, "PROXY_RESIDENTIAL_TRANSFER_GBYTES": { "type": "integer", "example": 0 }, "PROXY_SERPS": { "type": "integer", "example": 0 } } }, "usageTotalUsd": { "type": "number", "example": 0.00005 }, "usageUsd": { "type": "object", "properties": { "ACTOR_COMPUTE_UNITS": { "type": "integer", "example": 0 }, "DATASET_READS": { "type": "integer", "example": 0 }, "DATASET_WRITES": { "type": "integer", "example": 0 }, "KEY_VALUE_STORE_READS": { "type": "integer", "example": 0 }, "KEY_VALUE_STORE_WRITES": { "type": "number", "example": 0.00005 }, "KEY_VALUE_STORE_LISTS": { "type": "integer", "example": 0 }, "REQUEST_QUEUE_READS": { "type": "integer", "example": 0 }, "REQUEST_QUEUE_WRITES": { "type": "integer", "example": 0 }, "DATA_TRANSFER_INTERNAL_GBYTES": { "type": "integer", "example": 0 }, "DATA_TRANSFER_EXTERNAL_GBYTES": { "type": "integer", "example": 0 }, "PROXY_RESIDENTIAL_TRANSFER_GBYTES": { "type": "integer", "example": 0 }, "PROXY_SERPS": { "type": "integer", "example": 0 } } } } } } } } } }