{
  "openapi": "3.0.1",
  "info": {
    "title": "RAG Markdown Scraper",
    "description": "Turn any website into a clean Vector Database pipeline. Bypasses bot protections to extract pure GitHub-Flavored Markdown. Features built-in AI Vision OCR for images, Delta Crawling to save costs, Custom Regex, and direct Pinecone upserts for RAG agents.",
    "version": "0.0",
    "x-build-id": "w29IeIO2jVLufb3Eh"
  },
  "servers": [
    {
      "url": "https://api.apify.com/v2"
    }
  ],
  "paths": {
    "/acts/tillzero~rag-markdown-scraper/run-sync-get-dataset-items": {
      "post": {
        "operationId": "run-sync-get-dataset-items-tillzero-rag-markdown-scraper",
        "x-openai-isConsequential": false,
        "summary": "Executes an Actor, waits for its completion, and returns Actor's dataset items in response.",
        "tags": [
          "Run Actor"
        ],
        "requestBody": {
          "required": true,
          "content": {
            "application/json": {
              "schema": {
                "$ref": "#/components/schemas/inputSchema"
              }
            }
          }
        },
        "parameters": [
          {
            "name": "token",
            "in": "query",
            "required": true,
            "schema": {
              "type": "string"
            },
            "description": "Enter your Apify token here"
          }
        ],
        "responses": {
          "200": {
            "description": "OK"
          }
        }
      }
    },
    "/acts/tillzero~rag-markdown-scraper/runs": {
      "post": {
        "operationId": "runs-sync-tillzero-rag-markdown-scraper",
        "x-openai-isConsequential": false,
        "summary": "Executes an Actor and returns information about the initiated run in response.",
        "tags": [
          "Run Actor"
        ],
        "requestBody": {
          "required": true,
          "content": {
            "application/json": {
              "schema": {
                "$ref": "#/components/schemas/inputSchema"
              }
            }
          }
        },
        "parameters": [
          {
            "name": "token",
            "in": "query",
            "required": true,
            "schema": {
              "type": "string"
            },
            "description": "Enter your Apify token here"
          }
        ],
        "responses": {
          "200": {
            "description": "OK",
            "content": {
              "application/json": {
                "schema": {
                  "$ref": "#/components/schemas/runsResponseSchema"
                }
              }
            }
          }
        }
      }
    },
    "/acts/tillzero~rag-markdown-scraper/run-sync": {
      "post": {
        "operationId": "run-sync-tillzero-rag-markdown-scraper",
        "x-openai-isConsequential": false,
        "summary": "Executes an Actor, waits for completion, and returns the OUTPUT from Key-value store in response.",
        "tags": [
          "Run Actor"
        ],
        "requestBody": {
          "required": true,
          "content": {
            "application/json": {
              "schema": {
                "$ref": "#/components/schemas/inputSchema"
              }
            }
          }
        },
        "parameters": [
          {
            "name": "token",
            "in": "query",
            "required": true,
            "schema": {
              "type": "string"
            },
            "description": "Enter your Apify token here"
          }
        ],
        "responses": {
          "200": {
            "description": "OK"
          }
        }
      }
    }
  },
  "components": {
    "schemas": {
      "inputSchema": {
        "type": "object",
        "required": [
          "startUrls"
        ],
        "properties": {
          "startUrls": {
            "title": "Start URLs",
            "type": "array",
            "description": "The URLs you want to scrape for your RAG pipeline.",
            "items": {
              "type": "object",
              "required": [
                "url"
              ],
              "properties": {
                "url": {
                  "type": "string",
                  "title": "URL of a web page",
                  "format": "uri"
                }
              }
            }
          },
          "sitemapUrls": {
            "title": "Sitemap URLs (Ultimate Efficiency)",
            "type": "array",
            "description": "Provide links to sitemap.xml files. The crawler will instantly extract all pages and scrape them without clicking through the site.",
            "default": [],
            "items": {
              "type": "object",
              "required": [
                "url"
              ],
              "properties": {
                "url": {
                  "type": "string",
                  "title": "URL of a web page",
                  "format": "uri"
                }
              }
            }
          },
          "exportAsZip": {
            "title": "Export as ZIP (Markdown Files)",
            "type": "boolean",
            "description": "If enabled, the scraper will generate a physical .zip file containing perfectly formatted .md files for each page.",
            "default": false
          },
          "exportFailedUrls": {
            "title": "Export Audit Log (Failed URLs)",
            "type": "boolean",
            "description": "Generates a FAILED_URLS.json file containing all dead or failed links and the exact error reasons.",
            "default": true
          },
          "onlyScrapeUpdatedPages": {
            "title": "Delta Crawl (Only Updated Pages)",
            "type": "boolean",
            "description": "Saves state between runs. Only scrapes and outputs pages that are brand new or whose content has changed since the last run. Saves huge LLM costs.",
            "default": false
          },
          "webhookUrl": {
            "title": "Webhook URL",
            "type": "string",
            "description": "Send an automated POST request with run details (JSON and ZIP links) to Zapier, Make.com, or your backend when the run finishes.",
            "default": ""
          },
          "aiApiKey": {
            "title": "AI API Key",
            "type": "string",
            "description": "Your API key for OpenAI, Anthropic, OpenRouter, or Groq.",
            "default": ""
          },
          "aiApiBaseUrl": {
            "title": "AI API Base URL",
            "type": "string",
            "description": "Use https://api.openai.com/v1 for OpenAI, or https://openrouter.ai/api/v1 for Claude/Gemini.",
            "default": "https://api.openai.com/v1"
          },
          "aiModel": {
            "title": "AI Model",
            "type": "string",
            "description": "The model to use (e.g. gpt-4o-mini, claude-3-5-sonnet).",
            "default": "gpt-4o-mini"
          },
          "enableVision": {
            "title": "Enable AI Vision (OCR)",
            "type": "boolean",
            "description": "Warning: Can be expensive. Automatically sends all images to the Vision API to extract text and describe charts, injecting the result into the Markdown.",
            "default": false
          },
          "customAiPrompt": {
            "title": "Custom AI Prompt",
            "type": "string",
            "description": "Ask the AI to extract specific data from every page (e.g. 'Summarize the pricing model').",
            "default": ""
          },
          "pineconeApiKey": {
            "title": "Pinecone API Key (Zero-Code RAG)",
            "type": "string",
            "description": "The Actor will automatically upsert vector embeddings directly to this database.",
            "default": ""
          },
          "pineconeIndexHost": {
            "title": "Pinecone Index Host (Zero-Code RAG)",
            "type": "string",
            "description": "E.g. index-1234.svc.pinecone.io. Requires Pinecone API key.",
            "default": ""
          },
          "customRegexExtractors": {
            "title": "Custom Regex Extractors",
            "type": "array",
            "description": "Array of objects like [{\"name\": \"emails\", \"regex\": \"[a-z]+@[a-z]+\\\\.[a-z]+\"}] to scrape specific data points from the Markdown.",
            "default": []
          },
          "injectFrontmatter": {
            "title": "Inject YAML Frontmatter",
            "type": "boolean",
            "description": "Embeds Title, URL, and Author as a YAML block at the top of the Markdown. Perfect for Obsidian/Notion users.",
            "default": false
          },
          "initialCookies": {
            "title": "Initial Cookies (Login Support)",
            "type": "array",
            "description": "Paste a JSON array of session cookies to scrape private pages behind a login wall.",
            "default": []
          },
          "useBrowser": {
            "title": "Use Browser (JS Rendering)",
            "type": "boolean",
            "description": "Enable this to scrape React/Next.js or sites requiring JavaScript. It is slower and costs more CU, but guarantees perfect rendering.",
            "default": false
          },
          "scrollToBottom": {
            "title": "Auto-Scroll to Bottom (Browser Only)",
            "type": "boolean",
            "description": "Automatically scrolls to the bottom of the page to trigger lazy-loaded text.",
            "default": false
          },
          "waitForSelector": {
            "title": "Wait For Selector (Browser Only)",
            "type": "string",
            "description": "A CSS selector (e.g. '.main-content') to wait for before scraping. Ensures lazy-loaded text is captured.",
            "default": ""
          },
          "clickSelector": {
            "title": "Click Selector (Browser Only)",
            "type": "string",
            "description": "A CSS selector (e.g. '#accept-cookies') to click before scraping. Useful for closing popups/overlays.",
            "default": ""
          },
          "chunkSize": {
            "title": "Chunk Size",
            "type": "integer",
            "description": "Automatically split the text into chunks of this size. Leave empty to output the full page as one string.",
            "default": 1000
          },
          "chunkOverlap": {
            "title": "Chunk Overlap",
            "type": "integer",
            "description": "How many characters should overlap between chunks to preserve context.",
            "default": 200
          },
          "mustIncludeUrls": {
            "title": "Whitelist URLs (Must include)",
            "type": "array",
            "description": "Only crawl URLs that contain at least one of these strings (e.g. ['/docs/', '/help/']). Leave empty to crawl everything.",
            "default": [],
            "items": {
              "type": "object",
              "required": [
                "url"
              ],
              "properties": {
                "url": {
                  "type": "string",
                  "title": "URL of a web page",
                  "format": "uri"
                }
              }
            }
          },
          "excludeUrls": {
            "title": "Exclude URLs",
            "type": "array",
            "description": "URL patterns to skip (e.g. ['https://example.com/login/*']).",
            "default": [],
            "items": {
              "type": "object",
              "required": [
                "url"
              ],
              "properties": {
                "url": {
                  "type": "string",
                  "title": "URL of a web page",
                  "format": "uri"
                }
              }
            }
          },
          "proxyConfiguration": {
            "title": "Proxy configuration",
            "type": "object",
            "description": "Select proxies to be used by the crawler to avoid getting blocked."
          },
          "maxPagesPerCrawl": {
            "title": "Max pages to crawl",
            "type": "integer",
            "description": "Set a hard limit to manage your Compute Unit (CU) costs.",
            "default": 100
          },
          "maxCrawlDepth": {
            "title": "Max Crawl Depth",
            "type": "integer",
            "description": "How deep into the website to crawl. 0 = only Start URLs, 1 = one click away, etc.",
            "default": 20
          },
          "maxRequestRetries": {
            "title": "Max Request Retries",
            "type": "integer",
            "description": "How many times to retry a failed/dead link. Set to 1 for fast-fail and cost savings.",
            "default": 1
          },
          "maxConcurrency": {
            "title": "Max Concurrency",
            "type": "integer",
            "description": "Lower concurrency uses less RAM (saves costs), higher is faster.",
            "default": 10
          },
          "stripQueryParameters": {
            "title": "Strip Query Parameters",
            "type": "boolean",
            "description": "Dramatically saves costs by preventing infinite loops on URLs like /page?sort=asc",
            "default": true
          },
          "enableDeduplication": {
            "title": "Enable Content Deduplication",
            "type": "boolean",
            "description": "If enabled, pages with the exact same text content will be skipped (saves RAG tokens).",
            "default": true
          },
          "removeSelectors": {
            "title": "Remove Elements (CSS Selectors)",
            "type": "string",
            "description": "Comma-separated CSS selectors to strip out of the page before converting to Markdown.",
            "default": "nav, footer, aside, script, style, noscript, iframe, svg, .cookie-banner, .ads"
          }
        }
      },
      "runsResponseSchema": {
        "type": "object",
        "properties": {
          "data": {
            "type": "object",
            "properties": {
              "id": {
                "type": "string"
              },
              "actId": {
                "type": "string"
              },
              "userId": {
                "type": "string"
              },
              "startedAt": {
                "type": "string",
                "format": "date-time",
                "example": "2025-01-08T00:00:00.000Z"
              },
              "finishedAt": {
                "type": "string",
                "format": "date-time",
                "example": "2025-01-08T00:00:00.000Z"
              },
              "status": {
                "type": "string",
                "example": "READY"
              },
              "meta": {
                "type": "object",
                "properties": {
                  "origin": {
                    "type": "string",
                    "example": "API"
                  },
                  "userAgent": {
                    "type": "string"
                  }
                }
              },
              "stats": {
                "type": "object",
                "properties": {
                  "inputBodyLen": {
                    "type": "integer",
                    "example": 2000
                  },
                  "rebootCount": {
                    "type": "integer",
                    "example": 0
                  },
                  "restartCount": {
                    "type": "integer",
                    "example": 0
                  },
                  "resurrectCount": {
                    "type": "integer",
                    "example": 0
                  },
                  "computeUnits": {
                    "type": "integer",
                    "example": 0
                  }
                }
              },
              "options": {
                "type": "object",
                "properties": {
                  "build": {
                    "type": "string",
                    "example": "latest"
                  },
                  "timeoutSecs": {
                    "type": "integer",
                    "example": 300
                  },
                  "memoryMbytes": {
                    "type": "integer",
                    "example": 1024
                  },
                  "diskMbytes": {
                    "type": "integer",
                    "example": 2048
                  }
                }
              },
              "buildId": {
                "type": "string"
              },
              "defaultKeyValueStoreId": {
                "type": "string"
              },
              "defaultDatasetId": {
                "type": "string"
              },
              "defaultRequestQueueId": {
                "type": "string"
              },
              "buildNumber": {
                "type": "string",
                "example": "1.0.0"
              },
              "containerUrl": {
                "type": "string"
              },
              "usage": {
                "type": "object",
                "properties": {
                  "ACTOR_COMPUTE_UNITS": {
                    "type": "integer",
                    "example": 0
                  },
                  "DATASET_READS": {
                    "type": "integer",
                    "example": 0
                  },
                  "DATASET_WRITES": {
                    "type": "integer",
                    "example": 0
                  },
                  "KEY_VALUE_STORE_READS": {
                    "type": "integer",
                    "example": 0
                  },
                  "KEY_VALUE_STORE_WRITES": {
                    "type": "integer",
                    "example": 1
                  },
                  "KEY_VALUE_STORE_LISTS": {
                    "type": "integer",
                    "example": 0
                  },
                  "REQUEST_QUEUE_READS": {
                    "type": "integer",
                    "example": 0
                  },
                  "REQUEST_QUEUE_WRITES": {
                    "type": "integer",
                    "example": 0
                  },
                  "DATA_TRANSFER_INTERNAL_GBYTES": {
                    "type": "integer",
                    "example": 0
                  },
                  "DATA_TRANSFER_EXTERNAL_GBYTES": {
                    "type": "integer",
                    "example": 0
                  },
                  "PROXY_RESIDENTIAL_TRANSFER_GBYTES": {
                    "type": "integer",
                    "example": 0
                  },
                  "PROXY_SERPS": {
                    "type": "integer",
                    "example": 0
                  }
                }
              },
              "usageTotalUsd": {
                "type": "number",
                "example": 0.00005
              },
              "usageUsd": {
                "type": "object",
                "properties": {
                  "ACTOR_COMPUTE_UNITS": {
                    "type": "integer",
                    "example": 0
                  },
                  "DATASET_READS": {
                    "type": "integer",
                    "example": 0
                  },
                  "DATASET_WRITES": {
                    "type": "integer",
                    "example": 0
                  },
                  "KEY_VALUE_STORE_READS": {
                    "type": "integer",
                    "example": 0
                  },
                  "KEY_VALUE_STORE_WRITES": {
                    "type": "number",
                    "example": 0.00005
                  },
                  "KEY_VALUE_STORE_LISTS": {
                    "type": "integer",
                    "example": 0
                  },
                  "REQUEST_QUEUE_READS": {
                    "type": "integer",
                    "example": 0
                  },
                  "REQUEST_QUEUE_WRITES": {
                    "type": "integer",
                    "example": 0
                  },
                  "DATA_TRANSFER_INTERNAL_GBYTES": {
                    "type": "integer",
                    "example": 0
                  },
                  "DATA_TRANSFER_EXTERNAL_GBYTES": {
                    "type": "integer",
                    "example": 0
                  },
                  "PROXY_RESIDENTIAL_TRANSFER_GBYTES": {
                    "type": "integer",
                    "example": 0
                  },
                  "PROXY_SERPS": {
                    "type": "integer",
                    "example": 0
                  }
                }
              }
            }
          }
        }
      }
    }
  }
}