{ "openapi": "3.0.1", "info": { "title": "RAG Web Extractor — Chunked Content for AI Pipelines", "description": "Extract clean markdown from websites for RAG pipelines. Strip nav, ads, boilerplate. Preserve headings, links, images. Recursive crawling with depth control. Chunked output for embedding pipelines. Build AI knowledge bases.", "version": "1.0", "x-build-id": "Xxw5OJrxVbZA1jbP3" }, "servers": [ { "url": "https://api.apify.com/v2" } ], "paths": { "/acts/junipr~rag-web-extractor/run-sync-get-dataset-items": { "post": { "operationId": "run-sync-get-dataset-items-junipr-rag-web-extractor", "x-openai-isConsequential": false, "summary": "Executes an Actor, waits for its completion, and returns Actor's dataset items in response.", "tags": [ "Run Actor" ], "requestBody": { "required": true, "content": { "application/json": { "schema": { "$ref": "#/components/schemas/inputSchema" } } } }, "parameters": [ { "name": "token", "in": "query", "required": true, "schema": { "type": "string" }, "description": "Enter your Apify token here" } ], "responses": { "200": { "description": "OK" } } } }, "/acts/junipr~rag-web-extractor/runs": { "post": { "operationId": "runs-sync-junipr-rag-web-extractor", "x-openai-isConsequential": false, "summary": "Executes an Actor and returns information about the initiated run in response.", "tags": [ "Run Actor" ], "requestBody": { "required": true, "content": { "application/json": { "schema": { "$ref": "#/components/schemas/inputSchema" } } } }, "parameters": [ { "name": "token", "in": "query", "required": true, "schema": { "type": "string" }, "description": "Enter your Apify token here" } ], "responses": { "200": { "description": "OK", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/runsResponseSchema" } } } } } } }, "/acts/junipr~rag-web-extractor/run-sync": { "post": { "operationId": "run-sync-junipr-rag-web-extractor", "x-openai-isConsequential": false, "summary": "Executes an Actor, waits for completion, and returns the OUTPUT from Key-value store in response.", "tags": [ "Run Actor" ], "requestBody": { "required": true, "content": { "application/json": { "schema": { "$ref": "#/components/schemas/inputSchema" } } } }, "parameters": [ { "name": "token", "in": "query", "required": true, "schema": { "type": "string" }, "description": "Enter your Apify token here" } ], "responses": { "200": { "description": "OK" } } } } }, "components": { "schemas": { "inputSchema": { "type": "object", "properties": { "startUrls": { "title": "Start URLs", "type": "array", "description": "List of URLs to scrape. Only the URL field is required per entry.", "default": [ { "url": "https://crawlee.dev/docs/introduction" } ], "items": { "type": "object", "required": [ "url" ], "properties": { "url": { "type": "string", "title": "URL of a web page", "format": "uri" } } } }, "maxPages": { "title": "Max Pages", "minimum": 1, "maximum": 100000, "type": "integer", "description": "Maximum number of pages to scrape per run.", "default": 100 }, "maxDepth": { "title": "Max Crawl Depth", "minimum": 0, "maximum": 10, "type": "integer", "description": "Link-following depth from start URLs. 0 = only start URLs.", "default": 0 }, "outputFormats": { "title": "Output Formats", "type": "array", "description": "Output format(s) to generate. Multiple allowed. Valid values: markdown, plainText, structuredJson, html", "items": { "type": "string", "enum": [ "markdown", "plainText", "structuredJson", "html" ] }, "default": [ "markdown" ] }, "includeRawHtml": { "title": "Include Raw HTML", "type": "boolean", "description": "Include cleaned raw HTML in output alongside selected formats.", "default": false }, "enableChunking": { "title": "Enable Content Chunking", "type": "boolean", "description": "Split content into chunks for RAG pipelines.", "default": false }, "chunkSize": { "title": "Chunk Size (characters)", "minimum": 100, "maximum": 10000, "type": "integer", "description": "Target chunk size in characters.", "default": 1000 }, "chunkOverlap": { "title": "Chunk Overlap (characters)", "minimum": 0, "maximum": 5000, "type": "integer", "description": "Overlap between consecutive chunks in characters. Max is half of chunk size.", "default": 200 }, "chunkStrategy": { "title": "Chunking Strategy", "enum": [ "semantic", "fixed", "sentence" ], "type": "string", "description": "How to split content into chunks. Semantic splits on heading/paragraph boundaries. Fixed splits by exact character count. Sentence splits on sentence boundaries.", "default": "semantic" }, "renderJs": { "title": "Render JavaScript", "type": "boolean", "description": "Use headless browser for JavaScript rendering. Disable for static sites to save compute.", "default": true }, "waitForSelector": { "title": "Wait for Selector", "type": "string", "description": "CSS selector to wait for before extraction. Useful for SPAs where content loads dynamically." }, "waitForTimeout": { "title": "Wait Timeout (ms)", "minimum": 1000, "maximum": 60000, "type": "integer", "description": "Maximum milliseconds to wait for page load.", "default": 5000 }, "handleInfiniteScroll": { "title": "Handle Infinite Scroll", "type": "boolean", "description": "Scroll to bottom to load lazy content.", "default": false }, "maxScrolls": { "title": "Max Scroll Iterations", "minimum": 1, "maximum": 100, "type": "integer", "description": "Maximum scroll iterations for infinite scroll pages.", "default": 20 }, "handlePagination": { "title": "Handle Pagination", "type": "boolean", "description": "Follow pagination links automatically.", "default": false }, "paginationSelector": { "title": "Pagination Selector", "type": "string", "description": "CSS selector for 'next page' button/link. Auto-detected if not set." }, "paginationMaxPages": { "title": "Max Pagination Pages", "minimum": 1, "maximum": 100, "type": "integer", "description": "Maximum pagination pages to follow per start URL.", "default": 10 }, "removeSelectors": { "title": "Remove Selectors", "type": "array", "description": "CSS selectors for elements to remove before extraction (e.g., nav, footer, ads).", "items": { "type": "string" }, "default": [] }, "includeSelectors": { "title": "Include Selectors", "type": "array", "description": "If set, ONLY extract content from these selectors. Overrides removeSelectors.", "items": { "type": "string" }, "default": [] }, "removeNavigation": { "title": "Remove Navigation", "type": "boolean", "description": "Automatically remove