{
  "openapi": "3.0.1",
  "info": {
    "title": "Website to Markdown Crawler for RAG and LLMs",
    "description": "Crawl a website from start URLs and get each page as clean Markdown for RAG, LLM and vector database pipelines: title, description, word count and optional chunks. Follows robots.txt, stays on the site, depth and page limits, no browser. Empty input = free demo.",
    "version": "0.1",
    "x-build-id": "08kRZbiKcUfap65rI"
  },
  "servers": [
    {
      "url": "https://api.apify.com/v2"
    }
  ],
  "paths": {
    "/acts/ucchiii~website-markdown-crawler/run-sync-get-dataset-items": {
      "post": {
        "operationId": "run-sync-get-dataset-items-ucchiii-website-markdown-crawler",
        "x-openai-isConsequential": false,
        "summary": "Executes an Actor, waits for its completion, and returns Actor's dataset items in response.",
        "tags": [
          "Run Actor"
        ],
        "requestBody": {
          "required": true,
          "content": {
            "application/json": {
              "schema": {
                "$ref": "#/components/schemas/inputSchema"
              }
            }
          }
        },
        "parameters": [
          {
            "name": "token",
            "in": "query",
            "required": true,
            "schema": {
              "type": "string"
            },
            "description": "Enter your Apify token here"
          }
        ],
        "responses": {
          "200": {
            "description": "OK"
          }
        }
      }
    },
    "/acts/ucchiii~website-markdown-crawler/runs": {
      "post": {
        "operationId": "runs-sync-ucchiii-website-markdown-crawler",
        "x-openai-isConsequential": false,
        "summary": "Executes an Actor and returns information about the initiated run in response.",
        "tags": [
          "Run Actor"
        ],
        "requestBody": {
          "required": true,
          "content": {
            "application/json": {
              "schema": {
                "$ref": "#/components/schemas/inputSchema"
              }
            }
          }
        },
        "parameters": [
          {
            "name": "token",
            "in": "query",
            "required": true,
            "schema": {
              "type": "string"
            },
            "description": "Enter your Apify token here"
          }
        ],
        "responses": {
          "200": {
            "description": "OK",
            "content": {
              "application/json": {
                "schema": {
                  "$ref": "#/components/schemas/runsResponseSchema"
                }
              }
            }
          }
        }
      }
    },
    "/acts/ucchiii~website-markdown-crawler/run-sync": {
      "post": {
        "operationId": "run-sync-ucchiii-website-markdown-crawler",
        "x-openai-isConsequential": false,
        "summary": "Executes an Actor, waits for completion, and returns the OUTPUT from Key-value store in response.",
        "tags": [
          "Run Actor"
        ],
        "requestBody": {
          "required": true,
          "content": {
            "application/json": {
              "schema": {
                "$ref": "#/components/schemas/inputSchema"
              }
            }
          }
        },
        "parameters": [
          {
            "name": "token",
            "in": "query",
            "required": true,
            "schema": {
              "type": "string"
            },
            "description": "Enter your Apify token here"
          }
        ],
        "responses": {
          "200": {
            "description": "OK"
          }
        }
      }
    }
  },
  "components": {
    "schemas": {
      "inputSchema": {
        "type": "object",
        "properties": {
          "startUrls": {
            "title": "Start URLs",
            "type": "array",
            "description": "Where the crawl starts: full URLs with http:// or https://. The crawler follows links from these pages and stays on the same site (see Scope). A sitemap URL (for example https://example.com/sitemap.xml) is read as a list of pages. Up to 100 start URLs. Leave empty for a free demo (not charged).",
            "items": {
              "type": "object",
              "required": [
                "url"
              ],
              "properties": {
                "url": {
                  "type": "string",
                  "title": "URL of a web page",
                  "format": "uri"
                }
              }
            }
          },
          "maxPages": {
            "title": "Max pages",
            "minimum": 1,
            "maximum": 5000,
            "type": "integer",
            "description": "The most pages fetched in this run, from 1 to 5,000 (default 100). The crawl stops when it is reached. Pages skipped because of robots.txt are not counted and not charged.",
            "default": 100
          },
          "maxDepth": {
            "title": "Max link depth",
            "minimum": 0,
            "maximum": 20,
            "type": "integer",
            "description": "How many clicks away from a start URL the crawler goes, from 0 to 20 (default 5). 0 = only the start URLs (and sitemap pages); 1 = also the pages they link to.",
            "default": 5
          },
          "scope": {
            "title": "Scope",
            "enum": [
              "folder",
              "host",
              "domain"
            ],
            "type": "string",
            "description": "Which links are followed. Same folder: https://example.com/docs/intro crawls only pages under https://example.com/docs/. Whole host: every page on example.com. Host and subdomains: also blog.example.com, docs.example.com and so on. Links to other websites are never followed.",
            "default": "folder"
          },
          "includeUrlPatterns": {
            "title": "Only URLs matching",
            "type": "array",
            "description": "Optional. Only pages whose URL matches one of these patterns are fetched (start URLs always are). ** matches anything, * matches anything except /; a pattern without * matches when the URL contains it. Example: https://example.com/docs/** or /blog/. Pages that do not match are not fetched, so their links are not followed either.",
            "items": {
              "type": "string"
            }
          },
          "excludeUrlPatterns": {
            "title": "Skip URLs matching",
            "type": "array",
            "description": "Optional. Pages whose URL matches one of these patterns are not fetched. Same pattern rules as above. Example: **/tag/** or /login or **?page=*.",
            "items": {
              "type": "string"
            }
          },
          "useSitemaps": {
            "title": "Read sitemaps",
            "type": "boolean",
            "description": "Also read the site's sitemaps (the Sitemap lines of robots.txt, else /sitemap.xml) and crawl the pages listed there that are inside the scope and patterns. Useful for pages no link points to.",
            "default": false
          },
          "mainContentOnly": {
            "title": "Main content only",
            "type": "boolean",
            "description": "Remove menus, headers, footers, sidebars, cookie banners, share buttons and ad-like blocks, and keep the main or article element when the page has one. Scripts and styles are always removed.",
            "default": true
          },
          "keepImages": {
            "title": "Keep images",
            "type": "boolean",
            "description": "Keep images as ![alt](absolute URL) in the Markdown. Off by default, which is usually best for LLM text. Images are never downloaded.",
            "default": false
          },
          "maxCharsPerPage": {
            "title": "Max characters per page",
            "minimum": 0,
            "maximum": 1000000,
            "type": "integer",
            "description": "Cut each page's Markdown at this length, at a paragraph boundary where possible (the row gets truncated: true). 0 = no limit.",
            "default": 0
          },
          "splitIntoChunks": {
            "title": "Split into chunks for RAG",
            "type": "boolean",
            "description": "Add a chunks array to each row: the Markdown split by headings into pieces of about Chunk size characters, with headingPath, charCount and approxTokens. Code blocks are not cut in the middle. Chunks are not charged separately.",
            "default": false
          },
          "chunkSize": {
            "title": "Chunk size (characters)",
            "minimum": 200,
            "maximum": 20000,
            "type": "integer",
            "description": "Target size of each chunk in characters, from 200 to 20,000 (default 1,500, roughly 375 tokens). Used only when Split into chunks is on.",
            "default": 1500
          },
          "chunkOverlap": {
            "title": "Chunk overlap (characters)",
            "minimum": 0,
            "maximum": 5000,
            "type": "integer",
            "description": "Characters repeated from the end of one chunk at the start of the next, to keep context (default 200, at most half the chunk size). Used only when Split into chunks is on.",
            "default": 200
          },
          "concurrency": {
            "title": "Concurrency",
            "minimum": 1,
            "maximum": 20,
            "type": "integer",
            "description": "How many pages are processed in parallel, from 1 to 20 (default 5). Whatever you set, one host gets at most 2 connections at a time and at least 0.5 s between requests (longer if its robots.txt asks for a Crawl-delay).",
            "default": 5
          },
          "timeoutSecs": {
            "title": "Timeout (seconds)",
            "minimum": 1,
            "maximum": 60,
            "type": "integer",
            "description": "Timeout for each request, in seconds, from 1 to 60 (default 20). A page that keeps sending slowly is stopped after 3 times this value.",
            "default": 20
          },
          "userAgent": {
            "title": "User-Agent",
            "type": "string",
            "description": "Optional custom User-Agent header. The default names this Actor. robots.txt rules are always read for the name website-markdown-crawler and for *."
          }
        }
      },
      "runsResponseSchema": {
        "type": "object",
        "properties": {
          "data": {
            "type": "object",
            "properties": {
              "id": {
                "type": "string"
              },
              "actId": {
                "type": "string"
              },
              "userId": {
                "type": "string"
              },
              "startedAt": {
                "type": "string",
                "format": "date-time",
                "example": "2025-01-08T00:00:00.000Z"
              },
              "finishedAt": {
                "type": "string",
                "format": "date-time",
                "example": "2025-01-08T00:00:00.000Z"
              },
              "status": {
                "type": "string",
                "example": "READY"
              },
              "meta": {
                "type": "object",
                "properties": {
                  "origin": {
                    "type": "string",
                    "example": "API"
                  },
                  "userAgent": {
                    "type": "string"
                  }
                }
              },
              "stats": {
                "type": "object",
                "properties": {
                  "inputBodyLen": {
                    "type": "integer",
                    "example": 2000
                  },
                  "rebootCount": {
                    "type": "integer",
                    "example": 0
                  },
                  "restartCount": {
                    "type": "integer",
                    "example": 0
                  },
                  "resurrectCount": {
                    "type": "integer",
                    "example": 0
                  },
                  "computeUnits": {
                    "type": "integer",
                    "example": 0
                  }
                }
              },
              "options": {
                "type": "object",
                "properties": {
                  "build": {
                    "type": "string",
                    "example": "latest"
                  },
                  "timeoutSecs": {
                    "type": "integer",
                    "example": 300
                  },
                  "memoryMbytes": {
                    "type": "integer",
                    "example": 1024
                  },
                  "diskMbytes": {
                    "type": "integer",
                    "example": 2048
                  }
                }
              },
              "buildId": {
                "type": "string"
              },
              "defaultKeyValueStoreId": {
                "type": "string"
              },
              "defaultDatasetId": {
                "type": "string"
              },
              "defaultRequestQueueId": {
                "type": "string"
              },
              "buildNumber": {
                "type": "string",
                "example": "1.0.0"
              },
              "containerUrl": {
                "type": "string"
              },
              "usage": {
                "type": "object",
                "properties": {
                  "ACTOR_COMPUTE_UNITS": {
                    "type": "integer",
                    "example": 0
                  },
                  "DATASET_READS": {
                    "type": "integer",
                    "example": 0
                  },
                  "DATASET_WRITES": {
                    "type": "integer",
                    "example": 0
                  },
                  "KEY_VALUE_STORE_READS": {
                    "type": "integer",
                    "example": 0
                  },
                  "KEY_VALUE_STORE_WRITES": {
                    "type": "integer",
                    "example": 1
                  },
                  "KEY_VALUE_STORE_LISTS": {
                    "type": "integer",
                    "example": 0
                  },
                  "REQUEST_QUEUE_READS": {
                    "type": "integer",
                    "example": 0
                  },
                  "REQUEST_QUEUE_WRITES": {
                    "type": "integer",
                    "example": 0
                  },
                  "DATA_TRANSFER_INTERNAL_GBYTES": {
                    "type": "integer",
                    "example": 0
                  },
                  "DATA_TRANSFER_EXTERNAL_GBYTES": {
                    "type": "integer",
                    "example": 0
                  },
                  "PROXY_RESIDENTIAL_TRANSFER_GBYTES": {
                    "type": "integer",
                    "example": 0
                  },
                  "PROXY_SERPS": {
                    "type": "integer",
                    "example": 0
                  },
                  "PROXY_UNBLOCKER_UNITS": {
                    "type": "integer",
                    "example": 0
                  }
                }
              },
              "usageTotalUsd": {
                "type": "number",
                "example": 0.00005
              },
              "usageUsd": {
                "type": "object",
                "properties": {
                  "ACTOR_COMPUTE_UNITS": {
                    "type": "integer",
                    "example": 0
                  },
                  "DATASET_READS": {
                    "type": "integer",
                    "example": 0
                  },
                  "DATASET_WRITES": {
                    "type": "integer",
                    "example": 0
                  },
                  "KEY_VALUE_STORE_READS": {
                    "type": "integer",
                    "example": 0
                  },
                  "KEY_VALUE_STORE_WRITES": {
                    "type": "number",
                    "example": 0.00005
                  },
                  "KEY_VALUE_STORE_LISTS": {
                    "type": "integer",
                    "example": 0
                  },
                  "REQUEST_QUEUE_READS": {
                    "type": "integer",
                    "example": 0
                  },
                  "REQUEST_QUEUE_WRITES": {
                    "type": "integer",
                    "example": 0
                  },
                  "DATA_TRANSFER_INTERNAL_GBYTES": {
                    "type": "integer",
                    "example": 0
                  },
                  "DATA_TRANSFER_EXTERNAL_GBYTES": {
                    "type": "integer",
                    "example": 0
                  },
                  "PROXY_RESIDENTIAL_TRANSFER_GBYTES": {
                    "type": "integer",
                    "example": 0
                  },
                  "PROXY_SERPS": {
                    "type": "integer",
                    "example": 0
                  },
                  "PROXY_UNBLOCKER_UNITS": {
                    "type": "integer",
                    "example": 0
                  }
                }
              }
            }
          }
        }
      }
    }
  }
}