{
  "openapi": "3.0.1",
  "info": {
    "title": "Website to Knowledge Base (RAG / Chatbot Ready)",
    "description": "Crawl a website and produce clean Markdown, semantic chunks, FAQs and an llms.txt for chatbot and RAG development.",
    "version": "2.0",
    "x-build-id": "F7vQYctj5QaxnqVNM"
  },
  "servers": [
    {
      "url": "https://api.apify.com/v2"
    }
  ],
  "paths": {
    "/acts/codepal~website-to-knowledge-base/run-sync-get-dataset-items": {
      "post": {
        "operationId": "run-sync-get-dataset-items-codepal-website-to-knowledge-base",
        "x-openai-isConsequential": false,
        "summary": "Executes an Actor, waits for its completion, and returns Actor's dataset items in response.",
        "tags": [
          "Run Actor"
        ],
        "requestBody": {
          "required": true,
          "content": {
            "application/json": {
              "schema": {
                "$ref": "#/components/schemas/inputSchema"
              }
            }
          }
        },
        "parameters": [
          {
            "name": "token",
            "in": "query",
            "required": true,
            "schema": {
              "type": "string"
            },
            "description": "Enter your Apify token here"
          }
        ],
        "responses": {
          "200": {
            "description": "OK"
          }
        }
      }
    },
    "/acts/codepal~website-to-knowledge-base/runs": {
      "post": {
        "operationId": "runs-sync-codepal-website-to-knowledge-base",
        "x-openai-isConsequential": false,
        "summary": "Executes an Actor and returns information about the initiated run in response.",
        "tags": [
          "Run Actor"
        ],
        "requestBody": {
          "required": true,
          "content": {
            "application/json": {
              "schema": {
                "$ref": "#/components/schemas/inputSchema"
              }
            }
          }
        },
        "parameters": [
          {
            "name": "token",
            "in": "query",
            "required": true,
            "schema": {
              "type": "string"
            },
            "description": "Enter your Apify token here"
          }
        ],
        "responses": {
          "200": {
            "description": "OK",
            "content": {
              "application/json": {
                "schema": {
                  "$ref": "#/components/schemas/runsResponseSchema"
                }
              }
            }
          }
        }
      }
    },
    "/acts/codepal~website-to-knowledge-base/run-sync": {
      "post": {
        "operationId": "run-sync-codepal-website-to-knowledge-base",
        "x-openai-isConsequential": false,
        "summary": "Executes an Actor, waits for completion, and returns the OUTPUT from Key-value store in response.",
        "tags": [
          "Run Actor"
        ],
        "requestBody": {
          "required": true,
          "content": {
            "application/json": {
              "schema": {
                "$ref": "#/components/schemas/inputSchema"
              }
            }
          }
        },
        "parameters": [
          {
            "name": "token",
            "in": "query",
            "required": true,
            "schema": {
              "type": "string"
            },
            "description": "Enter your Apify token here"
          }
        ],
        "responses": {
          "200": {
            "description": "OK"
          }
        }
      }
    }
  },
  "components": {
    "schemas": {
      "inputSchema": {
        "type": "object",
        "required": [
          "startUrls"
        ],
        "properties": {
          "startUrls": {
            "title": "Start URLs",
            "type": "array",
            "description": "Where the crawl begins: a homepage, docs root or help center. You can also paste a sitemap URL (e.g. https://example.com/sitemap.xml) to crawl exactly the pages it lists.",
            "items": {
              "type": "object",
              "required": [
                "url"
              ],
              "properties": {
                "url": {
                  "type": "string",
                  "title": "URL of a web page",
                  "format": "uri"
                }
              }
            }
          },
          "maxPages": {
            "title": "Max pages",
            "minimum": 1,
            "maximum": 100000,
            "type": "integer",
            "description": "Hard limit on pages fetched. You only pay for pages saved to the knowledge base (thin, duplicate and blocked pages are free).",
            "default": 100
          },
          "renderingType": {
            "title": "Rendering",
            "enum": [
              "auto",
              "http",
              "browser"
            ],
            "type": "string",
            "description": "<b>Auto</b> (recommended) fetches pages over fast HTTP and only opens a headless browser for pages that need JavaScript or block plain requests. <b>HTTP only</b> is fastest and cheapest. <b>Browser</b> renders every page.",
            "default": "auto"
          },
          "crawlScope": {
            "title": "Crawl scope",
            "enum": [
              "same-hostname",
              "same-domain",
              "all"
            ],
            "type": "string",
            "description": "Which links are followed. www/non-www and redirects are handled automatically.",
            "default": "same-hostname"
          },
          "maxDepth": {
            "title": "Max crawl depth",
            "minimum": 0,
            "maximum": 30,
            "type": "integer",
            "description": "How many clicks away from the start URLs to go. 0 = only the start URLs.",
            "default": 3
          },
          "includePdfs": {
            "title": "Include PDF documents",
            "type": "boolean",
            "description": "Download linked PDFs (manuals, policies, brochures, price lists) and add their text to the knowledge base.",
            "default": false
          },
          "chunkSize": {
            "title": "Chunk size (tokens)",
            "minimum": 100,
            "maximum": 4000,
            "type": "integer",
            "description": "Target size of each RAG chunk. 300–800 suits most embedding models. Small sections are merged instead of becoming fragment chunks.",
            "default": 500
          },
          "chunkOverlap": {
            "title": "Chunk overlap (tokens)",
            "minimum": 0,
            "maximum": 1000,
            "type": "integer",
            "description": "Context carried between consecutive chunks of the same section. Overlap is made of whole sentences, never cut words.",
            "default": 60
          },
          "chunkLinks": {
            "title": "Links inside chunks",
            "enum": [
              "text",
              "keep"
            ],
            "type": "string",
            "description": "<b>Text</b> keeps anchor text in the chunk and moves URLs into the chunk's <code>links</code> field — cleaner embeddings and fewer tokens. <b>Keep</b> leaves Markdown links in the text.",
            "default": "text"
          },
          "includeFaqChunks": {
            "title": "Add FAQ chunks",
            "type": "boolean",
            "description": "Add each Q&A pair found on the site as its own chunk. Question-shaped chunks match user questions very well.",
            "default": true
          },
          "datasetOutput": {
            "title": "Default dataset contains",
            "enum": [
              "pages",
              "chunks"
            ],
            "type": "string",
            "description": "Choose <b>Chunks</b> when you connect this Actor to a vector-database integration (Pinecone, Qdrant, Weaviate, Milvus, Chroma, Supabase…), which reads the run's default dataset. The other type is always saved too (see OUTPUT).",
            "default": "pages"
          },
          "incrementalMode": {
            "title": "Incremental mode (only new & changed pages)",
            "type": "boolean",
            "description": "Remembers page fingerprints between runs. Unchanged pages are skipped and <b>not charged</b>; a CHANGES record lists new, changed and removed pages. Ideal for scheduled runs that keep a chatbot up to date.",
            "default": false
          },
          "incrementalStateKey": {
            "title": "Incremental state name",
            "type": "string",
            "description": "Optional. Runs with the same name share history. Defaults to the start URL's domain."
          },
          "skipUnchangedPages": {
            "title": "Leave unchanged pages out of the results",
            "type": "boolean",
            "description": "On: results contain only new and changed pages (delta upload). Off: everything is output, with a changeStatus field on each page.",
            "default": true
          },
          "useSitemap": {
            "title": "Use sitemap.xml",
            "type": "boolean",
            "description": "Also queue URLs from the site's sitemaps (found via robots.txt) for better coverage of pages that aren't linked.",
            "default": true
          },
          "maxSitemapUrls": {
            "title": "Max sitemap URLs",
            "minimum": 0,
            "type": "integer",
            "description": "Large sitemaps are sampled evenly down to this many URLs so one section (e.g. products) can't dominate.",
            "default": 300
          },
          "includeUrlGlobs": {
            "title": "Include URL patterns",
            "type": "array",
            "description": "Only follow URLs matching these globs, e.g. <code>https://example.com/docs/**</code>. Empty = everything in scope.",
            "default": [],
            "items": {
              "type": "string"
            }
          },
          "excludeUrlGlobs": {
            "title": "Exclude URL patterns",
            "type": "array",
            "description": "Never crawl URLs matching these globs, e.g. <code>**/tag/**</code>, <code>**/page/*</code>.",
            "default": [],
            "items": {
              "type": "string"
            }
          },
          "respectRobotsTxt": {
            "title": "Respect robots.txt",
            "type": "boolean",
            "description": "Skip URLs the site disallows for crawlers. Turn off only for sites you own or have permission to crawl.",
            "default": true
          },
          "skipUtilityPages": {
            "title": "Skip utility pages",
            "type": "boolean",
            "description": "Skip cart, checkout, login, account, wishlist and on-site search result pages.",
            "default": true
          },
          "allowedLanguages": {
            "title": "Languages to keep",
            "type": "array",
            "description": "Optional. Keep only pages whose language starts with one of these codes, e.g. <code>en</code>, <code>fr</code>. Pages without a declared language are kept.",
            "default": [],
            "items": {
              "type": "string"
            }
          },
          "ignoreQueryParams": {
            "title": "Ignore query parameters",
            "type": "array",
            "description": "Removed from URLs so the same page isn't crawled many times (utm_* is always removed).",
            "default": [
              "sort",
              "ref",
              "source",
              "fbclid",
              "gclid",
              "tag",
              "tab"
            ],
            "items": {
              "type": "string"
            }
          },
          "urlLimits": {
            "title": "Per-section page limits",
            "type": "array",
            "description": "Advanced. Cap any URL section, e.g. <code>[{\"pattern\":\"**/blog/**\",\"maxPages\":30}]</code>.",
            "default": []
          },
          "maxProductPages": {
            "title": "Max product pages",
            "minimum": 0,
            "type": "integer",
            "description": "Caps individual product pages so help, policy and category pages are not crowded out on large stores.",
            "default": 100
          },
          "followLinksFromProducts": {
            "title": "Follow links found on product pages",
            "type": "boolean",
            "description": "Off by default: product pages mostly link to other products.",
            "default": false
          },
          "contentSelector": {
            "title": "Main content CSS selector",
            "type": "string",
            "description": "Optional. Force the content area (e.g. <code>article.post</code>). Empty = auto-detect."
          },
          "removeSelectors": {
            "title": "Extra selectors to remove",
            "type": "array",
            "description": "Additional CSS selectors to strip (popups, related-posts widgets…).",
            "default": [],
            "items": {
              "type": "string"
            }
          },
          "minWords": {
            "title": "Minimum words per page",
            "minimum": 0,
            "type": "integer",
            "description": "Thinner pages are skipped (and not charged).",
            "default": 20
          },
          "removeDuplicateBlocks": {
            "title": "Remove repeated blocks across pages",
            "type": "boolean",
            "description": "A paragraph that repeats on many pages (promo strips, shared disclaimers, product cards) is kept in the chunks only once.",
            "default": true
          },
          "skipNearDuplicates": {
            "title": "Skip near-duplicate pages",
            "type": "boolean",
            "description": "Skip pages that are almost identical to one already saved (print versions, tracking variants, mirrored pages).",
            "default": true
          },
          "enableAIEnrichment": {
            "title": "AI enrichment (Claude)",
            "type": "boolean",
            "description": "Adds per-page summaries, key facts, topics, entities and FAQs, plus a site overview and a ready-to-use chatbot system prompt. Uses your own Anthropic API key (you pay Anthropic directly at their prices).",
            "default": false
          },
          "anthropicApiKey": {
            "title": "Anthropic API key",
            "type": "string",
            "description": "Required only when AI enrichment is on. Stored encrypted."
          },
          "aiModel": {
            "title": "Claude model",
            "type": "string",
            "description": "Model ID, e.g. <code>claude-haiku-5-5</code> (fast and cheapest, recommended) or <code>claude-sonnet-5-5</code> (higher quality).",
            "default": "claude-haiku-5-5"
          },
          "aiMaxPages": {
            "title": "Max pages to enrich",
            "minimum": 1,
            "type": "integer",
            "description": "Cost control. Help, policy and info pages are prioritized; products get at most half of this budget.",
            "default": 100
          },
          "aiConcurrency": {
            "title": "Parallel AI requests",
            "minimum": 1,
            "maximum": 20,
            "type": "integer",
            "description": "Lower it if your Anthropic account hits rate limits.",
            "default": 4
          },
          "dynamicContentWaitSecs": {
            "title": "Wait for dynamic content (seconds)",
            "minimum": 0,
            "maximum": 60,
            "type": "integer",
            "description": "Browser only. Max time to wait for the network to go idle before reading the page.",
            "default": 5
          },
          "waitForSelector": {
            "title": "Wait for selector",
            "type": "string",
            "description": "Browser only. Optional CSS selector to wait for before extracting (e.g. <code>.article-body</code>)."
          },
          "expandClickableElements": {
            "title": "Expand accordions & tabs",
            "type": "boolean",
            "description": "Browser only. Opens collapsed sections (<code>[aria-expanded=false]</code>, <code>&lt;details&gt;</code>) so hidden FAQ answers are captured.",
            "default": true
          },
          "maxScrollHeightPixels": {
            "title": "Max scroll height (px)",
            "minimum": 0,
            "type": "integer",
            "description": "Browser only. Scrolls to load lazy content. 0 disables scrolling.",
            "default": 5000
          },
          "proxyConfiguration": {
            "title": "Proxy",
            "type": "object",
            "description": "Datacenter proxy is enough for most sites. Use residential only for sites that block crawlers.",
            "default": {
              "useApifyProxy": true
            }
          },
          "initialCookies": {
            "title": "Cookies",
            "type": "array",
            "description": "Crawl pages behind a login (only sites you are allowed to access). JSON list like <code>[{\"name\":\"session\",\"value\":\"…\",\"domain\":\".example.com\"}]</code>.",
            "default": []
          },
          "customHttpHeaders": {
            "title": "HTTP headers",
            "type": "object",
            "description": "Extra headers for every request, e.g. <code>{\"Authorization\": \"Bearer …\"}</code>.",
            "default": {}
          },
          "maxConcurrency": {
            "title": "Max concurrency",
            "minimum": 1,
            "maximum": 200,
            "type": "integer",
            "description": "Max parallel requests (scales automatically with available memory). Browser pages use at most 20.",
            "default": 50
          },
          "maxRequestsPerMinute": {
            "title": "Max requests per minute",
            "minimum": 0,
            "type": "integer",
            "description": "Be polite to small sites. 0 = no limit.",
            "default": 0
          },
          "requestTimeoutSecs": {
            "title": "Request timeout (seconds)",
            "minimum": 5,
            "maximum": 300,
            "type": "integer",
            "description": "Max time to load one page before retrying.",
            "default": 30
          },
          "maxRequestRetries": {
            "title": "Max retries",
            "minimum": 0,
            "maximum": 10,
            "type": "integer",
            "description": "Retries on network errors and blocks (with a new session/IP each time).",
            "default": 3
          },
          "ignoreHttpsErrors": {
            "title": "Ignore HTTPS certificate errors",
            "type": "boolean",
            "description": "Accept sites with invalid or self-signed certificates (HTTP mode).",
            "default": false
          }
        }
      },
      "runsResponseSchema": {
        "type": "object",
        "properties": {
          "data": {
            "type": "object",
            "properties": {
              "id": {
                "type": "string"
              },
              "actId": {
                "type": "string"
              },
              "userId": {
                "type": "string"
              },
              "startedAt": {
                "type": "string",
                "format": "date-time",
                "example": "2025-01-08T00:00:00.000Z"
              },
              "finishedAt": {
                "type": "string",
                "format": "date-time",
                "example": "2025-01-08T00:00:00.000Z"
              },
              "status": {
                "type": "string",
                "example": "READY"
              },
              "meta": {
                "type": "object",
                "properties": {
                  "origin": {
                    "type": "string",
                    "example": "API"
                  },
                  "userAgent": {
                    "type": "string"
                  }
                }
              },
              "stats": {
                "type": "object",
                "properties": {
                  "inputBodyLen": {
                    "type": "integer",
                    "example": 2000
                  },
                  "rebootCount": {
                    "type": "integer",
                    "example": 0
                  },
                  "restartCount": {
                    "type": "integer",
                    "example": 0
                  },
                  "resurrectCount": {
                    "type": "integer",
                    "example": 0
                  },
                  "computeUnits": {
                    "type": "integer",
                    "example": 0
                  }
                }
              },
              "options": {
                "type": "object",
                "properties": {
                  "build": {
                    "type": "string",
                    "example": "latest"
                  },
                  "timeoutSecs": {
                    "type": "integer",
                    "example": 300
                  },
                  "memoryMbytes": {
                    "type": "integer",
                    "example": 1024
                  },
                  "diskMbytes": {
                    "type": "integer",
                    "example": 2048
                  }
                }
              },
              "buildId": {
                "type": "string"
              },
              "defaultKeyValueStoreId": {
                "type": "string"
              },
              "defaultDatasetId": {
                "type": "string"
              },
              "defaultRequestQueueId": {
                "type": "string"
              },
              "buildNumber": {
                "type": "string",
                "example": "1.0.0"
              },
              "containerUrl": {
                "type": "string"
              },
              "usage": {
                "type": "object",
                "properties": {
                  "ACTOR_COMPUTE_UNITS": {
                    "type": "integer",
                    "example": 0
                  },
                  "DATASET_READS": {
                    "type": "integer",
                    "example": 0
                  },
                  "DATASET_WRITES": {
                    "type": "integer",
                    "example": 0
                  },
                  "KEY_VALUE_STORE_READS": {
                    "type": "integer",
                    "example": 0
                  },
                  "KEY_VALUE_STORE_WRITES": {
                    "type": "integer",
                    "example": 1
                  },
                  "KEY_VALUE_STORE_LISTS": {
                    "type": "integer",
                    "example": 0
                  },
                  "REQUEST_QUEUE_READS": {
                    "type": "integer",
                    "example": 0
                  },
                  "REQUEST_QUEUE_WRITES": {
                    "type": "integer",
                    "example": 0
                  },
                  "DATA_TRANSFER_INTERNAL_GBYTES": {
                    "type": "integer",
                    "example": 0
                  },
                  "DATA_TRANSFER_EXTERNAL_GBYTES": {
                    "type": "integer",
                    "example": 0
                  },
                  "PROXY_RESIDENTIAL_TRANSFER_GBYTES": {
                    "type": "integer",
                    "example": 0
                  },
                  "PROXY_SERPS": {
                    "type": "integer",
                    "example": 0
                  },
                  "PROXY_UNBLOCKER_UNITS": {
                    "type": "integer",
                    "example": 0
                  }
                }
              },
              "usageTotalUsd": {
                "type": "number",
                "example": 0.00005
              },
              "usageUsd": {
                "type": "object",
                "properties": {
                  "ACTOR_COMPUTE_UNITS": {
                    "type": "integer",
                    "example": 0
                  },
                  "DATASET_READS": {
                    "type": "integer",
                    "example": 0
                  },
                  "DATASET_WRITES": {
                    "type": "integer",
                    "example": 0
                  },
                  "KEY_VALUE_STORE_READS": {
                    "type": "integer",
                    "example": 0
                  },
                  "KEY_VALUE_STORE_WRITES": {
                    "type": "number",
                    "example": 0.00005
                  },
                  "KEY_VALUE_STORE_LISTS": {
                    "type": "integer",
                    "example": 0
                  },
                  "REQUEST_QUEUE_READS": {
                    "type": "integer",
                    "example": 0
                  },
                  "REQUEST_QUEUE_WRITES": {
                    "type": "integer",
                    "example": 0
                  },
                  "DATA_TRANSFER_INTERNAL_GBYTES": {
                    "type": "integer",
                    "example": 0
                  },
                  "DATA_TRANSFER_EXTERNAL_GBYTES": {
                    "type": "integer",
                    "example": 0
                  },
                  "PROXY_RESIDENTIAL_TRANSFER_GBYTES": {
                    "type": "integer",
                    "example": 0
                  },
                  "PROXY_SERPS": {
                    "type": "integer",
                    "example": 0
                  },
                  "PROXY_UNBLOCKER_UNITS": {
                    "type": "integer",
                    "example": 0
                  }
                }
              }
            }
          }
        }
      }
    }
  }
}