{
"openapi": "3.0.1",
"info": {
"title": "Web2RAG – Website to RAG Dataset",
"description": "Turn permitted public websites into clean, sentence-aware text chunks with source metadata, ready for embeddings, vector databases and AI assistants. Exports JSON, JSONL or Markdown.",
"version": "0.1",
"x-build-id": "awHNEgb44vqhZYZmY"
},
"servers": [
{
"url": "https://api.apify.com/v2"
}
],
"paths": {
"/acts/mdkhadikul~web2rag/run-sync-get-dataset-items": {
"post": {
"operationId": "run-sync-get-dataset-items-mdkhadikul-web2rag",
"x-openai-isConsequential": false,
"summary": "Executes an Actor, waits for its completion, and returns Actor's dataset items in response.",
"tags": [
"Run Actor"
],
"requestBody": {
"required": true,
"content": {
"application/json": {
"schema": {
"$ref": "#/components/schemas/inputSchema"
}
}
}
},
"parameters": [
{
"name": "token",
"in": "query",
"required": true,
"schema": {
"type": "string"
},
"description": "Enter your Apify token here"
}
],
"responses": {
"200": {
"description": "OK"
}
}
}
},
"/acts/mdkhadikul~web2rag/runs": {
"post": {
"operationId": "runs-sync-mdkhadikul-web2rag",
"x-openai-isConsequential": false,
"summary": "Executes an Actor and returns information about the initiated run in response.",
"tags": [
"Run Actor"
],
"requestBody": {
"required": true,
"content": {
"application/json": {
"schema": {
"$ref": "#/components/schemas/inputSchema"
}
}
}
},
"parameters": [
{
"name": "token",
"in": "query",
"required": true,
"schema": {
"type": "string"
},
"description": "Enter your Apify token here"
}
],
"responses": {
"200": {
"description": "OK",
"content": {
"application/json": {
"schema": {
"$ref": "#/components/schemas/runsResponseSchema"
}
}
}
}
}
}
},
"/acts/mdkhadikul~web2rag/run-sync": {
"post": {
"operationId": "run-sync-mdkhadikul-web2rag",
"x-openai-isConsequential": false,
"summary": "Executes an Actor, waits for completion, and returns the OUTPUT from Key-value store in response.",
"tags": [
"Run Actor"
],
"requestBody": {
"required": true,
"content": {
"application/json": {
"schema": {
"$ref": "#/components/schemas/inputSchema"
}
}
}
},
"parameters": [
{
"name": "token",
"in": "query",
"required": true,
"schema": {
"type": "string"
},
"description": "Enter your Apify token here"
}
],
"responses": {
"200": {
"description": "OK"
}
}
}
}
},
"components": {
"schemas": {
"inputSchema": {
"type": "object",
"required": [
"startUrl"
],
"properties": {
"startUrl": {
"title": "Start URL",
"pattern": "^https?://[^\\s/$.?#][^\\s]*$",
"maxLength": 2048,
"type": "string",
"description": "The public page where crawling starts, for example a documentation home page or a help center. Web2RAG follows links from this page. The address must start with http:// or https://. Local and private network addresses are blocked."
},
"maxPages": {
"title": "Maximum pages",
"minimum": 1,
"maximum": 5000,
"type": "integer",
"description": "Upper limit on the number of pages requested in this run. Start small (10–20) to check the results for a new site, then increase. Pages blocked by robots.txt do not count toward this limit.",
"default": 20
},
"maxDepth": {
"title": "Maximum crawl depth",
"minimum": 0,
"maximum": 20,
"type": "integer",
"description": "How many clicks away from the start URL to go. 0 = only the start page, 1 = the start page and the pages it links to, and so on. Pagination (\"next page\" links) is followed regardless of depth, up to 50 pages per listing.",
"default": 2
},
"outputFormat": {
"title": "Output format",
"enum": [
"jsonl",
"json",
"markdown"
],
"type": "string",
"description": "Format of the extra download file saved in the run's key-value store. Every chunk is always saved in the run's Dataset too, which can be exported as JSON, JSONL, CSV, Excel, XML or HTML.",
"default": "jsonl"
},
"chunkSize": {
"title": "Chunk size",
"minimum": 100,
"maximum": 10000,
"type": "integer",
"description": "Maximum length of each text chunk (one dataset item). Chunks end at sentence boundaries whenever possible. About 1000 characters (≈200–250 English words) suits most embedding models. Use smaller chunks for precise retrieval and larger ones for more context. You are billed per chunk, so larger chunks also cost less per page.",
"default": 1000
},
"chunkOverlap": {
"title": "Chunk overlap",
"minimum": 0,
"maximum": 5000,
"type": "integer",
"description": "How much text from the end of one chunk is repeated at the start of the next, so ideas that cross a boundary are not lost. Whole sentences are repeated when they fit. Must be at most half of the chunk size. Set to 0 to disable.",
"default": 150
},
"sameDomainOnly": {
"title": "Stay on the starting website",
"type": "boolean",
"description": "Only follow links on the start URL's website. www.example.com and example.com count as the same site; other subdomains such as blog.example.com do not. Turn this off only if you are allowed to collect the external websites it links to.",
"default": true
},
"respectRobotsTxt": {
"title": "Respect robots.txt",
"type": "boolean",
"description": "Skip pages that the website's robots.txt file disallows, and honor its Crawl-delay. Strongly recommended. Turn this off only for websites you own or have explicit permission to crawl. Pages marked noindex are always excluded.",
"default": true
},
"requestDelay": {
"title": "Delay between requests",
"minimum": 0,
"maximum": 60,
"type": "number",
"description": "Minimum wait between two requests to the same website. Higher values are gentler on the site but make the run take longer. If the site's robots.txt asks for a longer delay, that delay is used instead (up to 30 seconds).",
"default": 0.5
},
"requestTimeout": {
"title": "Request timeout",
"minimum": 1,
"maximum": 300,
"type": "integer",
"description": "How long to wait for a page to respond before giving up. Timeouts and temporary server errors are retried twice before the page is recorded as failed.",
"default": 15
}
}
},
"runsResponseSchema": {
"type": "object",
"properties": {
"data": {
"type": "object",
"properties": {
"id": {
"type": "string"
},
"actId": {
"type": "string"
},
"userId": {
"type": "string"
},
"startedAt": {
"type": "string",
"format": "date-time",
"example": "2025-01-08T00:00:00.000Z"
},
"finishedAt": {
"type": "string",
"format": "date-time",
"example": "2025-01-08T00:00:00.000Z"
},
"status": {
"type": "string",
"example": "READY"
},
"meta": {
"type": "object",
"properties": {
"origin": {
"type": "string",
"example": "API"
},
"userAgent": {
"type": "string"
}
}
},
"stats": {
"type": "object",
"properties": {
"inputBodyLen": {
"type": "integer",
"example": 2000
},
"rebootCount": {
"type": "integer",
"example": 0
},
"restartCount": {
"type": "integer",
"example": 0
},
"resurrectCount": {
"type": "integer",
"example": 0
},
"computeUnits": {
"type": "integer",
"example": 0
}
}
},
"options": {
"type": "object",
"properties": {
"build": {
"type": "string",
"example": "latest"
},
"timeoutSecs": {
"type": "integer",
"example": 300
},
"memoryMbytes": {
"type": "integer",
"example": 1024
},
"diskMbytes": {
"type": "integer",
"example": 2048
}
}
},
"buildId": {
"type": "string"
},
"defaultKeyValueStoreId": {
"type": "string"
},
"defaultDatasetId": {
"type": "string"
},
"defaultRequestQueueId": {
"type": "string"
},
"buildNumber": {
"type": "string",
"example": "1.0.0"
},
"containerUrl": {
"type": "string"
},
"usage": {
"type": "object",
"properties": {
"ACTOR_COMPUTE_UNITS": {
"type": "integer",
"example": 0
},
"DATASET_READS": {
"type": "integer",
"example": 0
},
"DATASET_WRITES": {
"type": "integer",
"example": 0
},
"KEY_VALUE_STORE_READS": {
"type": "integer",
"example": 0
},
"KEY_VALUE_STORE_WRITES": {
"type": "integer",
"example": 1
},
"KEY_VALUE_STORE_LISTS": {
"type": "integer",
"example": 0
},
"REQUEST_QUEUE_READS": {
"type": "integer",
"example": 0
},
"REQUEST_QUEUE_WRITES": {
"type": "integer",
"example": 0
},
"DATA_TRANSFER_INTERNAL_GBYTES": {
"type": "integer",
"example": 0
},
"DATA_TRANSFER_EXTERNAL_GBYTES": {
"type": "integer",
"example": 0
},
"PROXY_RESIDENTIAL_TRANSFER_GBYTES": {
"type": "integer",
"example": 0
},
"PROXY_SERPS": {
"type": "integer",
"example": 0
},
"PROXY_UNBLOCKER_UNITS": {
"type": "integer",
"example": 0
}
}
},
"usageTotalUsd": {
"type": "number",
"example": 0.00005
},
"usageUsd": {
"type": "object",
"properties": {
"ACTOR_COMPUTE_UNITS": {
"type": "integer",
"example": 0
},
"DATASET_READS": {
"type": "integer",
"example": 0
},
"DATASET_WRITES": {
"type": "integer",
"example": 0
},
"KEY_VALUE_STORE_READS": {
"type": "integer",
"example": 0
},
"KEY_VALUE_STORE_WRITES": {
"type": "number",
"example": 0.00005
},
"KEY_VALUE_STORE_LISTS": {
"type": "integer",
"example": 0
},
"REQUEST_QUEUE_READS": {
"type": "integer",
"example": 0
},
"REQUEST_QUEUE_WRITES": {
"type": "integer",
"example": 0
},
"DATA_TRANSFER_INTERNAL_GBYTES": {
"type": "integer",
"example": 0
},
"DATA_TRANSFER_EXTERNAL_GBYTES": {
"type": "integer",
"example": 0
},
"PROXY_RESIDENTIAL_TRANSFER_GBYTES": {
"type": "integer",
"example": 0
},
"PROXY_SERPS": {
"type": "integer",
"example": 0
},
"PROXY_UNBLOCKER_UNITS": {
"type": "integer",
"example": 0
}
}
}
}
}
}
}
}
}
}