{
  "openapi": "3.0.1",
  "info": {
    "title": "Docs to RAG Crawler: Site to Markdown | Johto Data",
    "description": "Crawl a docs site into token-budgeted Markdown for RAG. llms.txt-first. Re-crawl incrementally: pass last hashes and unchanged or duplicate pages are free. $2.50 per 1,000 crawled pages.",
    "version": "0.1",
    "x-build-id": "2R6Um2g0ubnYez4XP"
  },
  "servers": [
    {
      "url": "https://api.apify.com/v2"
    }
  ],
  "paths": {
    "/acts/johto.data~site-markdown-crawler/run-sync-get-dataset-items": {
      "post": {
        "operationId": "run-sync-get-dataset-items-johto.data-site-markdown-crawler",
        "x-openai-isConsequential": false,
        "summary": "Executes an Actor, waits for its completion, and returns Actor's dataset items in response.",
        "tags": [
          "Run Actor"
        ],
        "requestBody": {
          "required": true,
          "content": {
            "application/json": {
              "schema": {
                "$ref": "#/components/schemas/inputSchema"
              }
            }
          }
        },
        "parameters": [
          {
            "name": "token",
            "in": "query",
            "required": true,
            "schema": {
              "type": "string"
            },
            "description": "Enter your Apify token here"
          }
        ],
        "responses": {
          "200": {
            "description": "OK"
          }
        }
      }
    },
    "/acts/johto.data~site-markdown-crawler/runs": {
      "post": {
        "operationId": "runs-sync-johto.data-site-markdown-crawler",
        "x-openai-isConsequential": false,
        "summary": "Executes an Actor and returns information about the initiated run in response.",
        "tags": [
          "Run Actor"
        ],
        "requestBody": {
          "required": true,
          "content": {
            "application/json": {
              "schema": {
                "$ref": "#/components/schemas/inputSchema"
              }
            }
          }
        },
        "parameters": [
          {
            "name": "token",
            "in": "query",
            "required": true,
            "schema": {
              "type": "string"
            },
            "description": "Enter your Apify token here"
          }
        ],
        "responses": {
          "200": {
            "description": "OK",
            "content": {
              "application/json": {
                "schema": {
                  "$ref": "#/components/schemas/runsResponseSchema"
                }
              }
            }
          }
        }
      }
    },
    "/acts/johto.data~site-markdown-crawler/run-sync": {
      "post": {
        "operationId": "run-sync-johto.data-site-markdown-crawler",
        "x-openai-isConsequential": false,
        "summary": "Executes an Actor, waits for completion, and returns the OUTPUT from Key-value store in response.",
        "tags": [
          "Run Actor"
        ],
        "requestBody": {
          "required": true,
          "content": {
            "application/json": {
              "schema": {
                "$ref": "#/components/schemas/inputSchema"
              }
            }
          }
        },
        "parameters": [
          {
            "name": "token",
            "in": "query",
            "required": true,
            "schema": {
              "type": "string"
            },
            "description": "Enter your Apify token here"
          }
        ],
        "responses": {
          "200": {
            "description": "OK"
          }
        }
      }
    }
  },
  "components": {
    "schemas": {
      "inputSchema": {
        "type": "object",
        "properties": {
          "mode": {
            "title": "Mode",
            "enum": [
              "crawl",
              "single-page"
            ],
            "type": "string",
            "description": "crawl = follow links from startUrls. single-page = read exactly the URLs in `urls` or `query`, one row each, with the same output fields as llm-web-reader. Single-page mode is also used automatically when `urls` or `query` is set and `startUrls` is empty.",
            "default": "crawl"
          },
          "query": {
            "title": "Page URLs (space-separated)",
            "type": "string",
            "description": "The web page URL to read, e.g. `https://docs.apify.com/platform/actors`. Same field name as apify/rag-web-browser. Several URLs separated by spaces are fetched in parallel; commas are kept as part of the URL. Keyword search is NOT supported: search with your own tool, then pass the URLs here. Use when: you have one page URL (or a few, space-separated). Don't use when: you only have keywords (search first) or the page needs JavaScript to render. You are responsible for the URLs you supply and for following the target site's terms."
          },
          "urls": {
            "title": "More URLs",
            "type": "array",
            "description": "Optional list of page URLs to read in the same call (batch). Each item is one URL, used as given. Each successful page is one `single-page` event ($0.0002 per page, 60% under the $0.0005 per page that same-output rivals charge). Use when: you read a batch of URLs in one run (one start fee per batch). You are responsible for the URLs you supply and for following the target site's terms.",
            "items": {
              "type": "string"
            }
          },
          "outputFormats": {
            "title": "Output formats",
            "type": "array",
            "description": "Formats to return for each page. `markdown` (default) is best for LLMs. Same values as apify/rag-web-browser.",
            "items": {
              "type": "string",
              "enum": [
                "markdown",
                "text",
                "html"
              ]
            },
            "default": [
              "markdown"
            ]
          },
          "includeLinks": {
            "title": "Include links",
            "type": "boolean",
            "description": "Also return the page's absolute links (up to 500) in `links`, for agents that browse onward. Use when: your agent browses onward from this page.",
            "default": false
          },
          "maxUrls": {
            "title": "Max URLs per run",
            "minimum": 1,
            "maximum": 1000,
            "type": "integer",
            "description": "Safety cap on how many URLs one run reads in single-page mode. Use it to cap spend: each page read costs $0.0002 (event `single-page`).",
            "default": 100
          },
          "startUrls": {
            "title": "Start URLs",
            "type": "array",
            "description": "Where to start, e.g. `[{\"url\": \"https://docs.apify.com/academy\"}]`. Only pages whose URL starts with a start URL are crawled (same rule as apify/website-content-crawler), unless you set includeUrlGlobs. Use when: you want a whole docs site, help center or blog. Don't use when: you have one known page (use johto.data/llm-web-reader) or the site needs JavaScript to render. You are responsible for the URLs you supply and for following the target site's terms.",
            "items": {
              "type": "object",
              "required": [
                "url"
              ],
              "properties": {
                "url": {
                  "type": "string",
                  "title": "URL of a web page",
                  "format": "uri"
                }
              }
            }
          },
          "maxCrawlPages": {
            "title": "Max pages",
            "minimum": 1,
            "maximum": 100000,
            "type": "integer",
            "description": "Maximum pages to save and charge. Small default (20 pages = $0.009) so agent test calls stay cheap; raise it for full sites.",
            "default": 20
          },
          "maxCrawlDepth": {
            "title": "Max depth",
            "minimum": 0,
            "maximum": 20,
            "type": "integer",
            "description": "Link depth from the start URLs (0 = only the start URLs).",
            "default": 2
          },
          "maxTokens": {
            "title": "Token budget per page",
            "minimum": 50,
            "maximum": 200000,
            "type": "integer",
            "description": "Cut each page to about this many tokens (chars/4 estimate), e.g. `1500` for embedding-sized pages. Overrides maxContentChars. Use when: pages go into embeddings or a context window."
          },
          "knownHashes": {
            "title": "Known content hashes (incremental re-crawl)",
            "type": "object",
            "description": "Map of URL to the `stats.contentHash` from your last crawl. Pages whose content is unchanged are skipped and NOT charged; their links are still followed. Re-crawl a docs site on a schedule and pay only for pages that changed or are new. Use when: you re-crawl a site and only want to pay for changed pages."
          },
          "matchHashesAnyUrl": {
            "title": "Match hashes on any URL",
            "type": "boolean",
            "description": "Also skip a page if its hash matches any known hash, even under a different URL (moved pages).",
            "default": false
          },
          "includeUrlGlobs": {
            "title": "Include URL globs",
            "type": "array",
            "description": "Optional. Only crawl URLs matching these globs, e.g. `[{\"glob\": \"https://example.com/docs/**\"}]`. Overrides the start-URL prefix rule. Use when: you only want one section of a site.",
            "items": {
              "type": "object",
              "required": [
                "glob"
              ],
              "properties": {
                "glob": {
                  "type": "string",
                  "title": "Glob of a web page"
                }
              }
            }
          },
          "excludeUrlGlobs": {
            "title": "Exclude URL globs",
            "type": "array",
            "description": "Skip URLs matching these globs, e.g. `[{\"glob\": \"**/changelog/**\"}]`.",
            "items": {
              "type": "object",
              "required": [
                "glob"
              ],
              "properties": {
                "glob": {
                  "type": "string",
                  "title": "Glob of a web page"
                }
              }
            }
          },
          "useLlmsTxt": {
            "title": "Use llms.txt",
            "type": "boolean",
            "description": "If the site publishes /llms.txt, enqueue the pages it links (fast, curated coverage of docs sites). Use when: the site has /llms.txt (fast, curated docs coverage).",
            "default": true
          },
          "useSitemaps": {
            "title": "Use sitemap.xml",
            "type": "boolean",
            "description": "Also enqueue in-scope URLs from /sitemap.xml.",
            "default": false
          },
          "saveMarkdown": {
            "title": "Save Markdown",
            "type": "boolean",
            "description": "Return `markdown` per page (recommended for LLMs). `text` is always returned.",
            "default": true
          },
          "saveHtml": {
            "title": "Save HTML",
            "type": "boolean",
            "description": "Also return cleaned body HTML in `html`.",
            "default": false
          },
          "maxContentChars": {
            "title": "Max characters per page",
            "minimum": 0,
            "type": "integer",
            "description": "Cut each page to this many characters to keep payloads small. 0 = no limit.",
            "default": 30000
          },
          "htmlTransformer": {
            "title": "Content extraction",
            "enum": [
              "readableText",
              "none"
            ],
            "type": "string",
            "description": "`readableText` keeps main content only; `none` converts the whole body after removing `removeElementsCssSelector`.",
            "default": "readableText"
          },
          "removeElementsCssSelector": {
            "title": "Remove elements (CSS)",
            "type": "string",
            "description": "CSS selector for elements removed before conversion.",
            "default": "nav, footer, script, style, noscript, svg, iframe, form, img[src^='data:'], [role=\"alert\"], [role=\"banner\"], [role=\"dialog\"], [role=\"alertdialog\"], [aria-modal=\"true\"], .cookie, #cookie, [class*=\"cookie-banner\"], [id*=\"cookie-banner\"]"
          },
          "keepUrlFragments": {
            "title": "Keep URL fragments",
            "type": "boolean",
            "description": "Treat `#fragment` URLs as separate pages.",
            "default": false
          },
          "maxConcurrency": {
            "title": "Max concurrency",
            "minimum": 1,
            "maximum": 30,
            "type": "integer",
            "description": "Parallel requests.",
            "default": 10
          },
          "requestTimeoutSecs": {
            "title": "Request timeout (s)",
            "minimum": 5,
            "maximum": 300,
            "type": "integer",
            "description": "Per-page timeout.",
            "default": 30
          },
          "maxResults": {
            "title": "Max results",
            "minimum": 1,
            "type": "integer",
            "description": "Optional extra cap on saved pages (same name as apify/website-content-crawler)."
          },
          "crawlerType": {
            "title": "Crawler type (compatibility)",
            "enum": [
              "cheerio",
              "playwright:adaptive",
              "playwright:firefox",
              "playwright:chrome",
              "jsdom"
            ],
            "type": "string",
            "description": "Accepted for apify/website-content-crawler compatibility. This Actor always uses fast raw HTTP (no JavaScript rendering).",
            "default": "cheerio"
          },
          "proxyConfiguration": {
            "title": "Proxy",
            "type": "object",
            "description": "Optional. Off by default. Use when: a site blocks direct requests. Leave off otherwise.",
            "default": {
              "useApifyProxy": false
            }
          },
          "debugMode": {
            "title": "Debug mode",
            "type": "boolean",
            "description": "Keep Open Graph and JSON-LD in `metadata`.",
            "default": false
          }
        }
      },
      "runsResponseSchema": {
        "type": "object",
        "properties": {
          "data": {
            "type": "object",
            "properties": {
              "id": {
                "type": "string"
              },
              "actId": {
                "type": "string"
              },
              "userId": {
                "type": "string"
              },
              "startedAt": {
                "type": "string",
                "format": "date-time",
                "example": "2025-01-08T00:00:00.000Z"
              },
              "finishedAt": {
                "type": "string",
                "format": "date-time",
                "example": "2025-01-08T00:00:00.000Z"
              },
              "status": {
                "type": "string",
                "example": "READY"
              },
              "meta": {
                "type": "object",
                "properties": {
                  "origin": {
                    "type": "string",
                    "example": "API"
                  },
                  "userAgent": {
                    "type": "string"
                  }
                }
              },
              "stats": {
                "type": "object",
                "properties": {
                  "inputBodyLen": {
                    "type": "integer",
                    "example": 2000
                  },
                  "rebootCount": {
                    "type": "integer",
                    "example": 0
                  },
                  "restartCount": {
                    "type": "integer",
                    "example": 0
                  },
                  "resurrectCount": {
                    "type": "integer",
                    "example": 0
                  },
                  "computeUnits": {
                    "type": "integer",
                    "example": 0
                  }
                }
              },
              "options": {
                "type": "object",
                "properties": {
                  "build": {
                    "type": "string",
                    "example": "latest"
                  },
                  "timeoutSecs": {
                    "type": "integer",
                    "example": 300
                  },
                  "memoryMbytes": {
                    "type": "integer",
                    "example": 1024
                  },
                  "diskMbytes": {
                    "type": "integer",
                    "example": 2048
                  }
                }
              },
              "buildId": {
                "type": "string"
              },
              "defaultKeyValueStoreId": {
                "type": "string"
              },
              "defaultDatasetId": {
                "type": "string"
              },
              "defaultRequestQueueId": {
                "type": "string"
              },
              "buildNumber": {
                "type": "string",
                "example": "1.0.0"
              },
              "containerUrl": {
                "type": "string"
              },
              "usage": {
                "type": "object",
                "properties": {
                  "ACTOR_COMPUTE_UNITS": {
                    "type": "integer",
                    "example": 0
                  },
                  "DATASET_READS": {
                    "type": "integer",
                    "example": 0
                  },
                  "DATASET_WRITES": {
                    "type": "integer",
                    "example": 0
                  },
                  "KEY_VALUE_STORE_READS": {
                    "type": "integer",
                    "example": 0
                  },
                  "KEY_VALUE_STORE_WRITES": {
                    "type": "integer",
                    "example": 1
                  },
                  "KEY_VALUE_STORE_LISTS": {
                    "type": "integer",
                    "example": 0
                  },
                  "REQUEST_QUEUE_READS": {
                    "type": "integer",
                    "example": 0
                  },
                  "REQUEST_QUEUE_WRITES": {
                    "type": "integer",
                    "example": 0
                  },
                  "DATA_TRANSFER_INTERNAL_GBYTES": {
                    "type": "integer",
                    "example": 0
                  },
                  "DATA_TRANSFER_EXTERNAL_GBYTES": {
                    "type": "integer",
                    "example": 0
                  },
                  "PROXY_RESIDENTIAL_TRANSFER_GBYTES": {
                    "type": "integer",
                    "example": 0
                  },
                  "PROXY_SERPS": {
                    "type": "integer",
                    "example": 0
                  },
                  "PROXY_UNBLOCKER_UNITS": {
                    "type": "integer",
                    "example": 0
                  }
                }
              },
              "usageTotalUsd": {
                "type": "number",
                "example": 0.00005
              },
              "usageUsd": {
                "type": "object",
                "properties": {
                  "ACTOR_COMPUTE_UNITS": {
                    "type": "integer",
                    "example": 0
                  },
                  "DATASET_READS": {
                    "type": "integer",
                    "example": 0
                  },
                  "DATASET_WRITES": {
                    "type": "integer",
                    "example": 0
                  },
                  "KEY_VALUE_STORE_READS": {
                    "type": "integer",
                    "example": 0
                  },
                  "KEY_VALUE_STORE_WRITES": {
                    "type": "number",
                    "example": 0.00005
                  },
                  "KEY_VALUE_STORE_LISTS": {
                    "type": "integer",
                    "example": 0
                  },
                  "REQUEST_QUEUE_READS": {
                    "type": "integer",
                    "example": 0
                  },
                  "REQUEST_QUEUE_WRITES": {
                    "type": "integer",
                    "example": 0
                  },
                  "DATA_TRANSFER_INTERNAL_GBYTES": {
                    "type": "integer",
                    "example": 0
                  },
                  "DATA_TRANSFER_EXTERNAL_GBYTES": {
                    "type": "integer",
                    "example": 0
                  },
                  "PROXY_RESIDENTIAL_TRANSFER_GBYTES": {
                    "type": "integer",
                    "example": 0
                  },
                  "PROXY_SERPS": {
                    "type": "integer",
                    "example": 0
                  },
                  "PROXY_UNBLOCKER_UNITS": {
                    "type": "integer",
                    "example": 0
                  }
                }
              }
            }
          }
        }
      }
    }
  }
}