{
  "openapi": "3.0.1",
  "info": {
    "title": "Web to Markdown & llms.txt Generator for LLMs",
    "description": "Convert public web pages to clean Markdown for LLMs, RAG and AI agents, with optional chunks, and generate llms.txt / llms-full.txt for a website from its sitemap. No browser; robots.txt and AI opt-outs respected.",
    "version": "1.0",
    "x-build-id": "SP3ySuAq0fi9x3rbC"
  },
  "servers": [
    {
      "url": "https://api.apify.com/v2"
    }
  ],
  "paths": {
    "/acts/thriving_adder~web-to-markdown-llms-txt/run-sync-get-dataset-items": {
      "post": {
        "operationId": "run-sync-get-dataset-items-thriving_adder-web-to-markdown-llms-txt",
        "x-openai-isConsequential": false,
        "summary": "Executes an Actor, waits for its completion, and returns Actor's dataset items in response.",
        "tags": [
          "Run Actor"
        ],
        "requestBody": {
          "required": true,
          "content": {
            "application/json": {
              "schema": {
                "$ref": "#/components/schemas/inputSchema"
              }
            }
          }
        },
        "parameters": [
          {
            "name": "token",
            "in": "query",
            "required": true,
            "schema": {
              "type": "string"
            },
            "description": "Enter your Apify token here"
          }
        ],
        "responses": {
          "200": {
            "description": "OK"
          }
        }
      }
    },
    "/acts/thriving_adder~web-to-markdown-llms-txt/runs": {
      "post": {
        "operationId": "runs-sync-thriving_adder-web-to-markdown-llms-txt",
        "x-openai-isConsequential": false,
        "summary": "Executes an Actor and returns information about the initiated run in response.",
        "tags": [
          "Run Actor"
        ],
        "requestBody": {
          "required": true,
          "content": {
            "application/json": {
              "schema": {
                "$ref": "#/components/schemas/inputSchema"
              }
            }
          }
        },
        "parameters": [
          {
            "name": "token",
            "in": "query",
            "required": true,
            "schema": {
              "type": "string"
            },
            "description": "Enter your Apify token here"
          }
        ],
        "responses": {
          "200": {
            "description": "OK",
            "content": {
              "application/json": {
                "schema": {
                  "$ref": "#/components/schemas/runsResponseSchema"
                }
              }
            }
          }
        }
      }
    },
    "/acts/thriving_adder~web-to-markdown-llms-txt/run-sync": {
      "post": {
        "operationId": "run-sync-thriving_adder-web-to-markdown-llms-txt",
        "x-openai-isConsequential": false,
        "summary": "Executes an Actor, waits for completion, and returns the OUTPUT from Key-value store in response.",
        "tags": [
          "Run Actor"
        ],
        "requestBody": {
          "required": true,
          "content": {
            "application/json": {
              "schema": {
                "$ref": "#/components/schemas/inputSchema"
              }
            }
          }
        },
        "parameters": [
          {
            "name": "token",
            "in": "query",
            "required": true,
            "schema": {
              "type": "string"
            },
            "description": "Enter your Apify token here"
          }
        ],
        "responses": {
          "200": {
            "description": "OK"
          }
        }
      }
    }
  },
  "components": {
    "schemas": {
      "inputSchema": {
        "type": "object",
        "properties": {
          "urls": {
            "title": "Pages to convert",
            "type": "array",
            "description": "Public http(s) pages to convert to Markdown (max 50,000 per run). Each item is {\"url\": \"https://...\"}; plain strings and URLs without a scheme are accepted. Charged per converted page (page-converted). Localhost/private-network URLs are rejected and not charged.",
            "items": {
              "type": "object",
              "required": [
                "url"
              ],
              "properties": {
                "url": {
                  "type": "string",
                  "title": "URL of a web page",
                  "format": "uri"
                }
              }
            }
          },
          "llmsTxtSites": {
            "title": "Sites for llms.txt",
            "type": "array",
            "description": "Site home pages (e.g. https://docs.example.com) to generate an llms.txt for. Pages are found in the site's sitemap (or home-page links), each page's title and description are read, and the file is grouped into sections by URL path. Charged once per generated file (llms-txt-generated).",
            "items": {
              "type": "string"
            }
          },
          "maxPagesPerSite": {
            "title": "Max pages per llms.txt",
            "minimum": 1,
            "maximum": 500,
            "type": "integer",
            "description": "How many pages to read and list per site (1-500). The most important (shallowest) URLs are picked first. Included in the llms.txt price.",
            "default": 25
          },
          "generateLlmsFullTxt": {
            "title": "Also generate llms-full.txt",
            "type": "boolean",
            "description": "Also build llms-full.txt (the full Markdown of every listed page in one file). Each included page is then also returned as a page row and charged as page-converted.",
            "default": false
          },
          "includeUrlPattern": {
            "title": "Include URL pattern (sites)",
            "type": "string",
            "description": "Optional regular expression: only sitemap URLs matching it are considered for llms.txt (e.g. /docs/)."
          },
          "excludeUrlPattern": {
            "title": "Exclude URL pattern (sites)",
            "type": "string",
            "description": "Optional regular expression: sitemap URLs matching it are ignored (e.g. /(tag|author|page)/)."
          },
          "contentMode": {
            "title": "Content",
            "enum": [
              "main",
              "full"
            ],
            "type": "string",
            "description": "main: find the main content (main/article/role=main) and drop navigation, headers, footers, sidebars and cookie/consent banners. full: convert the whole <body> (scripts and forms still removed).",
            "default": "main"
          },
          "includeLinks": {
            "title": "Keep links",
            "type": "boolean",
            "description": "Keep hyperlinks as [text](absolute URL). Off: plain text only (fewer tokens).",
            "default": true
          },
          "includeImages": {
            "title": "Keep images",
            "type": "boolean",
            "description": "Keep images as ![alt](absolute URL). Off by default to save tokens.",
            "default": false
          },
          "includeLinkList": {
            "title": "Return link list",
            "type": "boolean",
            "description": "Add a links array (absolute URLs found in the content, max 500) to every page row, e.g. for an agent that wants to follow links.",
            "default": false
          },
          "chunkTokens": {
            "title": "RAG chunk size (tokens)",
            "minimum": 0,
            "maximum": 16000,
            "type": "integer",
            "description": "If > 0, also split each page's Markdown into chunks of about this many tokens on heading/paragraph boundaries (chunks array with heading path). 0 = no chunks.",
            "default": 0
          },
          "chunkOverlapTokens": {
            "title": "Chunk overlap (tokens)",
            "minimum": 0,
            "maximum": 2000,
            "type": "integer",
            "description": "Tokens repeated from the end of one chunk at the start of the next.",
            "default": 0
          },
          "maxMarkdownChars": {
            "title": "Max Markdown characters",
            "minimum": 1000,
            "maximum": 2000000,
            "type": "integer",
            "description": "Markdown longer than this is cut and flagged truncated=true.",
            "default": 200000
          },
          "redactContactDetails": {
            "title": "Remove e-mails and phone numbers",
            "type": "boolean",
            "description": "Remove e-mail addresses, international phone numbers and mailto:/tel: links from the Markdown (privacy by default). Each row reports contactDetailsRedacted.",
            "default": true
          },
          "respectRobotsTxt": {
            "title": "Respect robots.txt",
            "type": "boolean",
            "description": "Skip URLs that robots.txt disallows for our user agent (WebToMarkdownBot) and honour Crawl-delay (capped at 10 s). Skipped URLs are free.",
            "default": true
          },
          "respectAiOptOuts": {
            "title": "Respect AI opt-outs",
            "type": "boolean",
            "description": "Also skip pages whose robots.txt blocks common AI crawlers (GPTBot, CCBot, Google-Extended, ClaudeBot, PerplexityBot...) or that send noai in X-Robots-Tag or meta robots. Recommended: the site owner has asked not to be used for AI. Skipped pages are free.",
            "default": true
          },
          "maxConcurrency": {
            "title": "Max concurrency",
            "minimum": 1,
            "maximum": 20,
            "type": "integer",
            "description": "Parallel page requests (1-20). At most 2 requests run at the same time against any one host.",
            "default": 5
          },
          "requestTimeoutSecs": {
            "title": "Request timeout (s)",
            "minimum": 3,
            "maximum": 120,
            "type": "integer",
            "description": "Timeout per HTTP request, in seconds.",
            "default": 20
          },
          "maxRetries": {
            "title": "Retries",
            "minimum": 0,
            "maximum": 3,
            "type": "integer",
            "description": "Retries for timeouts, network errors and HTTP 408/429/5xx (with backoff; Retry-After honoured up to 30 s). Retries are free.",
            "default": 1
          },
          "userAgent": {
            "title": "User agent",
            "type": "string",
            "description": "User-Agent header sent to sites. The default identifies this Actor honestly so site owners can recognise and control it in robots.txt.",
            "default": "Mozilla/5.0 (compatible; WebToMarkdownBot/1.0; +https://apify.com/thriving_adder/web-to-markdown-llms-txt)"
          }
        }
      },
      "runsResponseSchema": {
        "type": "object",
        "properties": {
          "data": {
            "type": "object",
            "properties": {
              "id": {
                "type": "string"
              },
              "actId": {
                "type": "string"
              },
              "userId": {
                "type": "string"
              },
              "startedAt": {
                "type": "string",
                "format": "date-time",
                "example": "2025-01-08T00:00:00.000Z"
              },
              "finishedAt": {
                "type": "string",
                "format": "date-time",
                "example": "2025-01-08T00:00:00.000Z"
              },
              "status": {
                "type": "string",
                "example": "READY"
              },
              "meta": {
                "type": "object",
                "properties": {
                  "origin": {
                    "type": "string",
                    "example": "API"
                  },
                  "userAgent": {
                    "type": "string"
                  }
                }
              },
              "stats": {
                "type": "object",
                "properties": {
                  "inputBodyLen": {
                    "type": "integer",
                    "example": 2000
                  },
                  "rebootCount": {
                    "type": "integer",
                    "example": 0
                  },
                  "restartCount": {
                    "type": "integer",
                    "example": 0
                  },
                  "resurrectCount": {
                    "type": "integer",
                    "example": 0
                  },
                  "computeUnits": {
                    "type": "integer",
                    "example": 0
                  }
                }
              },
              "options": {
                "type": "object",
                "properties": {
                  "build": {
                    "type": "string",
                    "example": "latest"
                  },
                  "timeoutSecs": {
                    "type": "integer",
                    "example": 300
                  },
                  "memoryMbytes": {
                    "type": "integer",
                    "example": 1024
                  },
                  "diskMbytes": {
                    "type": "integer",
                    "example": 2048
                  }
                }
              },
              "buildId": {
                "type": "string"
              },
              "defaultKeyValueStoreId": {
                "type": "string"
              },
              "defaultDatasetId": {
                "type": "string"
              },
              "defaultRequestQueueId": {
                "type": "string"
              },
              "buildNumber": {
                "type": "string",
                "example": "1.0.0"
              },
              "containerUrl": {
                "type": "string"
              },
              "usage": {
                "type": "object",
                "properties": {
                  "ACTOR_COMPUTE_UNITS": {
                    "type": "integer",
                    "example": 0
                  },
                  "DATASET_READS": {
                    "type": "integer",
                    "example": 0
                  },
                  "DATASET_WRITES": {
                    "type": "integer",
                    "example": 0
                  },
                  "KEY_VALUE_STORE_READS": {
                    "type": "integer",
                    "example": 0
                  },
                  "KEY_VALUE_STORE_WRITES": {
                    "type": "integer",
                    "example": 1
                  },
                  "KEY_VALUE_STORE_LISTS": {
                    "type": "integer",
                    "example": 0
                  },
                  "REQUEST_QUEUE_READS": {
                    "type": "integer",
                    "example": 0
                  },
                  "REQUEST_QUEUE_WRITES": {
                    "type": "integer",
                    "example": 0
                  },
                  "DATA_TRANSFER_INTERNAL_GBYTES": {
                    "type": "integer",
                    "example": 0
                  },
                  "DATA_TRANSFER_EXTERNAL_GBYTES": {
                    "type": "integer",
                    "example": 0
                  },
                  "PROXY_RESIDENTIAL_TRANSFER_GBYTES": {
                    "type": "integer",
                    "example": 0
                  },
                  "PROXY_SERPS": {
                    "type": "integer",
                    "example": 0
                  },
                  "PROXY_UNBLOCKER_UNITS": {
                    "type": "integer",
                    "example": 0
                  }
                }
              },
              "usageTotalUsd": {
                "type": "number",
                "example": 0.00005
              },
              "usageUsd": {
                "type": "object",
                "properties": {
                  "ACTOR_COMPUTE_UNITS": {
                    "type": "integer",
                    "example": 0
                  },
                  "DATASET_READS": {
                    "type": "integer",
                    "example": 0
                  },
                  "DATASET_WRITES": {
                    "type": "integer",
                    "example": 0
                  },
                  "KEY_VALUE_STORE_READS": {
                    "type": "integer",
                    "example": 0
                  },
                  "KEY_VALUE_STORE_WRITES": {
                    "type": "number",
                    "example": 0.00005
                  },
                  "KEY_VALUE_STORE_LISTS": {
                    "type": "integer",
                    "example": 0
                  },
                  "REQUEST_QUEUE_READS": {
                    "type": "integer",
                    "example": 0
                  },
                  "REQUEST_QUEUE_WRITES": {
                    "type": "integer",
                    "example": 0
                  },
                  "DATA_TRANSFER_INTERNAL_GBYTES": {
                    "type": "integer",
                    "example": 0
                  },
                  "DATA_TRANSFER_EXTERNAL_GBYTES": {
                    "type": "integer",
                    "example": 0
                  },
                  "PROXY_RESIDENTIAL_TRANSFER_GBYTES": {
                    "type": "integer",
                    "example": 0
                  },
                  "PROXY_SERPS": {
                    "type": "integer",
                    "example": 0
                  },
                  "PROXY_UNBLOCKER_UNITS": {
                    "type": "integer",
                    "example": 0
                  }
                }
              }
            }
          }
        }
      }
    }
  }
}