{ "openapi": "3.0.1", "info": { "title": "Docs-to-RAG Pipeline Builder", "description": "Crawl any documentation site into clean, chunked, embedding-ready markdown. Free embeddings via your own Cloudflare Workers AI account, or on-device. No paid API keys.", "version": "1.0", "x-build-id": "viC1OUJLmLbQK79iv" }, "servers": [ { "url": "https://api.apify.com/v2" } ], "paths": { "/acts/optirefine~docs-to-rag-pipeline/run-sync-get-dataset-items": { "post": { "operationId": "run-sync-get-dataset-items-optirefine-docs-to-rag-pipeline", "x-openai-isConsequential": false, "summary": "Executes an Actor, waits for its completion, and returns Actor's dataset items in response.", "tags": [ "Run Actor" ], "requestBody": { "required": true, "content": { "application/json": { "schema": { "$ref": "#/components/schemas/inputSchema" } } } }, "parameters": [ { "name": "token", "in": "query", "required": true, "schema": { "type": "string" }, "description": "Enter your Apify token here" } ], "responses": { "200": { "description": "OK" } } } }, "/acts/optirefine~docs-to-rag-pipeline/runs": { "post": { "operationId": "runs-sync-optirefine-docs-to-rag-pipeline", "x-openai-isConsequential": false, "summary": "Executes an Actor and returns information about the initiated run in response.", "tags": [ "Run Actor" ], "requestBody": { "required": true, "content": { "application/json": { "schema": { "$ref": "#/components/schemas/inputSchema" } } } }, "parameters": [ { "name": "token", "in": "query", "required": true, "schema": { "type": "string" }, "description": "Enter your Apify token here" } ], "responses": { "200": { "description": "OK", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/runsResponseSchema" } } } } } } }, "/acts/optirefine~docs-to-rag-pipeline/run-sync": { "post": { "operationId": "run-sync-optirefine-docs-to-rag-pipeline", "x-openai-isConsequential": false, "summary": "Executes an Actor, waits for completion, and returns the OUTPUT from Key-value store in response.", "tags": [ "Run Actor" ], "requestBody": { "required": true, "content": { "application/json": { "schema": { "$ref": "#/components/schemas/inputSchema" } } } }, "parameters": [ { "name": "token", "in": "query", "required": true, "schema": { "type": "string" }, "description": "Enter your Apify token here" } ], "responses": { "200": { "description": "OK" } } } } }, "components": { "schemas": { "inputSchema": { "type": "object", "required": [ "startUrls" ], "properties": { "startUrls": { "title": "Start URLs", "type": "array", "description": "Documentation root URLs. The crawler follows links from here, and also reads the site's sitemap unless you turn that off.", "items": { "type": "object", "required": [ "url" ], "properties": { "url": { "type": "string", "title": "URL of a web page", "format": "uri" } } } }, "crawlScope": { "title": "Crawl scope", "enum": [ "same-hostname", "same-domain", "same-path-prefix", "any" ], "type": "string", "description": "Which links are considered in-scope. 'Same hostname' is the safe default. 'Same domain' also follows subdomains. 'Same path prefix' restricts to the start URL's directory, which is the cleanest way to grab only /docs.", "default": "same-hostname" }, "includeUrlGlobs": { "title": "Include URL globs", "type": "array", "description": "If set, only URLs matching one of these patterns are crawled. Supports *, **, ? and {a,b}. Example: **/docs/**", "default": [], "items": { "type": "object", "required": [ "glob" ], "properties": { "glob": { "type": "string", "title": "Glob of a web page" } } } }, "excludeUrlGlobs": { "title": "Exclude URL globs", "type": "array", "description": "URLs matching any of these are never crawled. Exclusions always beat inclusions. Example: **/blog/**", "default": [], "items": { "type": "object", "required": [ "glob" ], "properties": { "glob": { "type": "string", "title": "Glob of a web page" } } } }, "maxCrawlPages": { "title": "Max pages", "minimum": 1, "maximum": 100000, "type": "integer", "description": "Hard page budget for the run. The default keeps a first run quick; raise it to take a whole site. Reached limits are reported in the run summary.", "default": 50 }, "maxCrawlTimeSecs": { "title": "Max crawl time", "minimum": 0, "maximum": 86400, "type": "integer", "description": "Stop crawling after this many seconds and write out whatever has been collected, so a slow site produces a partial corpus instead of nothing. Leave at 0 to derive the budget from the run's own timeout, keeping a reserve for embedding and output.", "default": 0 }, "maxCrawlDepth": { "title": "Max link depth", "minimum": 0, "maximum": 100, "type": "integer", "description": "How many links deep to follow from the start URLs. Sitemap-discovered pages count as depth 1.", "default": 20 }, "respectRobotsTxt": { "title": "Respect robots.txt", "type": "boolean", "description": "Obey the site's robots.txt rules. Leave this on unless you own the site.", "default": true }, "useSitemaps": { "title": "Seed from sitemaps", "type": "boolean", "description": "Read sitemap.xml (discovered via robots.txt, then by convention) before crawling. This is what finds pages that no sidebar links to.", "default": true }, "maxConcurrency": { "title": "Max concurrency", "minimum": 1, "maximum": 200, "type": "integer", "description": "Parallel HTTP requests. The browser pass automatically runs at a third of this.", "default": 10 }, "requestTimeoutSecs": { "title": "Request timeout", "minimum": 5, "maximum": 600, "type": "integer", "description": "Per-request timeout.", "default": 60 }, "maxRequestRetries": { "title": "Max retries", "minimum": 0, "maximum": 10, "type": "integer", "description": "How many times to retry a failed request before giving up on it.", "default": 3 }, "ignoreQueryParams": { "title": "Ignore query parameters", "type": "array", "description": "Extra query parameters to strip when deciding whether two URLs are the same page. Common tracking parameters (utm_*, gclid, fbclid, ...) are always stripped.", "default": [], "items": { "type": "string" } }, "ignoreAllQueryParams": { "title": "Ignore all query parameters", "type": "boolean", "description": "Treat ?a=1 and ?a=2 as the same page. Useful for docs sites that append view state to every link.", "default": false }, "renderingMode": { "title": "Rendering mode", "enum": [ "auto", "http-only", "browser-only" ], "type": "string", "description": "Auto fetches over plain HTTP first and automatically re-fetches in a real browser any page that comes back empty or too thin. Use browser-only for fully client-rendered docs (Docsify, Redoc), http-only for maximum speed on static sites.", "default": "auto" }, "escalateBelowWords": { "title": "Escalate below word count", "minimum": 0, "maximum": 10000, "type": "integer", "description": "In auto mode, a page whose extracted content has fewer words than this is re-fetched in a browser.", "default": 60 }, "browserWaitMs": { "title": "Extra browser wait", "minimum": 0, "maximum": 60000, "type": "integer", "description": "Additional settle time after page load, for docs that hydrate late.", "default": 0 }, "browserWaitForSelector": { "title": "Wait for selector", "type": "string", "description": "CSS selector to wait for before extracting, in browser mode. Leave empty unless a site needs it." }, "mainContentSelector": { "title": "Main content selector", "type": "string", "description": "CSS selector for the article container. When set and non-empty it wins outright - use it when you know the site. Leave empty to let generator detection and content scoring choose." }, "excludeSelectors": { "title": "Exclude selectors", "type": "array", "description": "CSS selectors stripped from every page. Replaces the built-in list entirely - use 'Extra exclude selectors' to add to it instead. Admonitions and callouts are protected from removal even if a selector would match them.", "default": [ "nav", "header", "footer", "aside", "script", "style", "noscript", "iframe", "template", "form", "[role=\"navigation\"]", "[role=\"banner\"]", "[role=\"contentinfo\"]", "[role=\"search\"]", "[aria-hidden=\"true\"]", ".sr-only", ".visually-hidden", ".skip-link", ".skip-to-content", ".breadcrumb", ".breadcrumbs", ".pagination", ".edit-page", ".edit-this-page", ".feedback", ".cookie-banner", ".announcement", ".advertisement", ".carbon-ads", "#carbonads", ".headerlink", ".header-anchor", "a.hash-link", ".anchor-link", "a.anchorjs-link", ".copybtn", ".copy-button", "[data-copy-button]" ], "items": { "type": "string" } }, "extraExcludeSelectors": { "title": "Extra exclude selectors", "type": "array", "description": "Additional selectors to strip, on top of the defaults. This is usually what you want.", "default": [], "items": { "type": "string" } }, "useGeneratorProfiles": { "title": "Use documentation generator profiles", "type": "boolean", "description": "Detect MkDocs, Docusaurus, Sphinx, Starlight, VitePress, Nextra, Antora, GitBook, Mintlify, Docsify, Docsy, Just the Docs and others, and use their known content containers. A stale profile loses to the generic extractors rather than emptying the page.", "default": true }, "extractionFallback": { "title": "Fallback extractor", "enum": [ "defuddle", "heuristic", "none" ], "type": "string", "description": "What to fall back to when no profile matches. Defuddle is a readability-style extractor tuned for articles; heuristic scores containers by text and link density.", "default": "defuddle" }, "keepLinks": { "title": "Keep links", "type": "boolean", "description": "Keep inline links in the markdown. Turn off for the smallest possible corpus.", "default": true }, "keepImages": { "title": "Keep images", "type": "boolean", "description": "Keep image references (with alt text) in the markdown.", "default": true }, "keepTables": { "title": "Keep tables", "type": "boolean", "description": "Convert HTML tables to GitHub-flavoured markdown tables. Turn off to drop them entirely.", "default": true }, "absoluteUrls": { "title": "Absolute URLs", "type": "boolean", "description": "Rewrite every relative link and image to an absolute URL, so chunks stay usable away from the source page.", "default": true }, "chunkStrategy": { "title": "Chunk strategy", "enum": [ "heading-recursive", "heading", "page" ], "type": "string", "description": "Heading + recursive splits at headings and then splits oversized sections down to the chunk size. Heading only splits at headings and lets sections exceed the limit. Page emits one chunk per page.", "default": "heading-recursive" }, "chunkSize": { "title": "Chunk size", "minimum": 32, "maximum": 32000, "type": "integer", "description": "Maximum size of a chunk, in the unit below. 800 tokens suits most embedding models.", "default": 800 }, "chunkOverlap": { "title": "Chunk overlap", "minimum": 0, "maximum": 8000, "type": "integer", "description": "How much of the end of a chunk to repeat at the start of the next. Overlap is never taken from inside a code block.", "default": 100 }, "minChunkSize": { "title": "Minimum chunk size", "minimum": 0, "maximum": 8000, "type": "integer", "description": "Chunks smaller than this are merged into a neighbour. Prevents one-line orphan chunks that match on stray terms and answer nothing.", "default": 120 }, "chunkUnit": { "title": "Chunk unit", "enum": [ "token", "character" ], "type": "string", "description": "Measure chunks in real BPE tokens (offline, no API) or in characters.", "default": "token" }, "tokenizer": { "title": "Tokenizer", "enum": [ "cl100k_base", "o200k_base", "p50k_base", "r50k_base" ], "type": "string", "description": "BPE encoding used for token counts. cl100k_base matches most current embedding models; o200k_base matches the newest OpenAI models.", "default": "cl100k_base" }, "splitHeadingLevel": { "title": "Split at heading level", "minimum": 1, "maximum": 6, "type": "integer", "description": "Start a new chunk at headings up to this depth. Deeper headings stay inside their parent chunk, which keeps a section about one topic together.", "default": 3 }, "includeBreadcrumbInChunk": { "title": "Include breadcrumb in chunk text", "type": "boolean", "description": "Prefix each chunk with its heading path (Page > Section > Subsection). This is the single biggest retrieval-quality win in the whole pipeline.", "default": true }, "deduplicate": { "title": "Deduplicate", "type": "boolean", "description": "Drop duplicate and near-duplicate pages and chunks. Versioned and translated documentation repeats itself heavily.", "default": true }, "nearDuplicateThreshold": { "title": "Near-duplicate threshold", "minimum": 0.5, "maximum": 1, "type": "number", "description": "Similarity above which two chunks count as the same, from 0.5 to 1. The default is conservative on purpose: near-duplicate detection cannot distinguish a versioned mirror from a distinct section that shares boilerplate, and dropping real content is worse than keeping a duplicate. Lower it to 0.92 to collapse versioned mirrors, or raise it to 1 to drop only byte-identical chunks.", "default": 0.97 }, "embeddingProvider": { "title": "Embedding provider", "enum": [ "none", "local", "cloudflare-worker" ], "type": "string", "description": "Leave as None to output chunks only and embed them yourself.", "default": "none" }, "embeddingModel": { "title": "Embedding model", "type": "string", "description": "Defaults to Xenova/all-MiniLM-L6-v2 for local, @cf/baai/bge-m3 for the Worker." }, "workerUrl": { "title": "Cloudflare Worker URL", "type": "string", "description": "Base URL of your deployed Worker, for example https://docs-to-rag-worker.you.workers.dev. The actor calls /embed and /llm under it." }, "workerApiKey": { "title": "Worker API token", "type": "string", "description": "The API_TOKEN secret you set on the Worker. Sent as a bearer token." }, "embeddingEndpoint": { "title": "Custom embedding endpoint", "type": "string", "description": "Overrides {workerUrl}/embed. Point this at any endpoint that accepts { model, texts } and returns { vectors }." }, "embeddingBatchSize": { "title": "Embedding batch size", "minimum": 1, "maximum": 256, "type": "integer", "description": "Chunks per embedding request.", "default": 32 }, "embeddingConcurrency": { "title": "Embedding concurrency", "minimum": 1, "maximum": 16, "type": "integer", "description": "Parallel embedding requests. Keep this low on a free tier.", "default": 2 }, "contextualizeChunks": { "title": "Contextualise chunks", "type": "boolean", "description": "Requires a Worker URL. Measurably improves retrieval on chunks that use pronouns or bare parameter names.", "default": false }, "llmModel": { "title": "LLM model", "enum": [ "@cf/meta/llama-3.1-8b-instruct-fast", "@cf/meta/llama-3.2-3b-instruct", "@cf/meta/llama-3.2-1b-instruct", "@cf/ibm-granite/granite-4.0-h-micro" ], "type": "string", "description": "Workers AI text-generation model used for contextualisation.", "default": "@cf/meta/llama-3.1-8b-instruct-fast" }, "llmEndpoint": { "title": "Custom LLM endpoint", "type": "string", "description": "Overrides {workerUrl}/llm." }, "maxContextualizedChunks": { "title": "Max contextualised chunks", "minimum": 0, "maximum": 100000, "type": "integer", "description": "Hard cap on LLM calls, so a large crawl cannot exhaust a free daily allocation.", "default": 500 }, "outputMode": { "title": "Output mode", "enum": [ "chunks", "pages", "both" ], "type": "string", "description": "Chunks pushes one dataset item per chunk, ready for a vector store. Pages pushes one item per page with its full markdown. Both pushes both.", "default": "chunks" }, "saveMarkdownArtifacts": { "title": "Save corpus files", "type": "boolean", "description": "Write corpus.jsonl and corpus.md to the key-value store alongside the dataset.", "default": true }, "saveLlmsTxt": { "title": "Save llms.txt", "type": "boolean", "description": "Write llms.txt (an index of the crawled pages) and llms-full.txt (the whole corpus as one document) to the key-value store.", "default": true }, "includeMarkdownInPageItems": { "title": "Include markdown in page items", "type": "boolean", "description": "Include each page's full markdown in page-mode dataset items. Turn off for a lighter dataset.", "default": true }, "proxyConfiguration": { "title": "Proxy configuration", "type": "object", "description": "Optional. Documentation sites rarely need a proxy; use one if you are rate-limited." } } }, "runsResponseSchema": { "type": "object", "properties": { "data": { "type": "object", "properties": { "id": { "type": "string" }, "actId": { "type": "string" }, "userId": { "type": "string" }, "startedAt": { "type": "string", "format": "date-time", "example": "2025-01-08T00:00:00.000Z" }, "finishedAt": { "type": "string", "format": "date-time", "example": "2025-01-08T00:00:00.000Z" }, "status": { "type": "string", "example": "READY" }, "meta": { "type": "object", "properties": { "origin": { "type": "string", "example": "API" }, "userAgent": { "type": "string" } } }, "stats": { "type": "object", "properties": { "inputBodyLen": { "type": "integer", "example": 2000 }, "rebootCount": { "type": "integer", "example": 0 }, "restartCount": { "type": "integer", "example": 0 }, "resurrectCount": { "type": "integer", "example": 0 }, "computeUnits": { "type": "integer", "example": 0 } } }, "options": { "type": "object", "properties": { "build": { "type": "string", "example": "latest" }, "timeoutSecs": { "type": "integer", "example": 300 }, "memoryMbytes": { "type": "integer", "example": 1024 }, "diskMbytes": { "type": "integer", "example": 2048 } } }, "buildId": { "type": "string" }, "defaultKeyValueStoreId": { "type": "string" }, "defaultDatasetId": { "type": "string" }, "defaultRequestQueueId": { "type": "string" }, "buildNumber": { "type": "string", "example": "1.0.0" }, "containerUrl": { "type": "string" }, "usage": { "type": "object", "properties": { "ACTOR_COMPUTE_UNITS": { "type": "integer", "example": 0 }, "DATASET_READS": { "type": "integer", "example": 0 }, "DATASET_WRITES": { "type": "integer", "example": 0 }, "KEY_VALUE_STORE_READS": { "type": "integer", "example": 0 }, "KEY_VALUE_STORE_WRITES": { "type": "integer", "example": 1 }, "KEY_VALUE_STORE_LISTS": { "type": "integer", "example": 0 }, "REQUEST_QUEUE_READS": { "type": "integer", "example": 0 }, "REQUEST_QUEUE_WRITES": { "type": "integer", "example": 0 }, "DATA_TRANSFER_INTERNAL_GBYTES": { "type": "integer", "example": 0 }, "DATA_TRANSFER_EXTERNAL_GBYTES": { "type": "integer", "example": 0 }, "PROXY_RESIDENTIAL_TRANSFER_GBYTES": { "type": "integer", "example": 0 }, "PROXY_SERPS": { "type": "integer", "example": 0 } } }, "usageTotalUsd": { "type": "number", "example": 0.00005 }, "usageUsd": { "type": "object", "properties": { "ACTOR_COMPUTE_UNITS": { "type": "integer", "example": 0 }, "DATASET_READS": { "type": "integer", "example": 0 }, "DATASET_WRITES": { "type": "integer", "example": 0 }, "KEY_VALUE_STORE_READS": { "type": "integer", "example": 0 }, "KEY_VALUE_STORE_WRITES": { "type": "number", "example": 0.00005 }, "KEY_VALUE_STORE_LISTS": { "type": "integer", "example": 0 }, "REQUEST_QUEUE_READS": { "type": "integer", "example": 0 }, "REQUEST_QUEUE_WRITES": { "type": "integer", "example": 0 }, "DATA_TRANSFER_INTERNAL_GBYTES": { "type": "integer", "example": 0 }, "DATA_TRANSFER_EXTERNAL_GBYTES": { "type": "integer", "example": 0 }, "PROXY_RESIDENTIAL_TRANSFER_GBYTES": { "type": "integer", "example": 0 }, "PROXY_SERPS": { "type": "integer", "example": 0 } } } } } } } } } }