{
  "openapi": "3.0.1",
  "info": {
    "title": "Sitemap & Robots.txt Explorer with AI Crawlers",
    "description": "List every URL in a website's sitemaps with last changed dates, and see which AI crawlers its robots.txt blocks. Works with many sites at once. No login needed.",
    "version": "0.1",
    "x-build-id": "jSfopkgdBphSPdmpD"
  },
  "servers": [
    {
      "url": "https://api.apify.com/v2"
    }
  ],
  "paths": {
    "/acts/opendatadesk~sitemap-robots-explorer/run-sync-get-dataset-items": {
      "post": {
        "operationId": "run-sync-get-dataset-items-opendatadesk-sitemap-robots-explorer",
        "x-openai-isConsequential": false,
        "summary": "Executes an Actor, waits for its completion, and returns Actor's dataset items in response.",
        "tags": [
          "Run Actor"
        ],
        "requestBody": {
          "required": true,
          "content": {
            "application/json": {
              "schema": {
                "$ref": "#/components/schemas/inputSchema"
              }
            }
          }
        },
        "parameters": [
          {
            "name": "token",
            "in": "query",
            "required": true,
            "schema": {
              "type": "string"
            },
            "description": "Enter your Apify token here"
          }
        ],
        "responses": {
          "200": {
            "description": "OK"
          }
        }
      }
    },
    "/acts/opendatadesk~sitemap-robots-explorer/runs": {
      "post": {
        "operationId": "runs-sync-opendatadesk-sitemap-robots-explorer",
        "x-openai-isConsequential": false,
        "summary": "Executes an Actor and returns information about the initiated run in response.",
        "tags": [
          "Run Actor"
        ],
        "requestBody": {
          "required": true,
          "content": {
            "application/json": {
              "schema": {
                "$ref": "#/components/schemas/inputSchema"
              }
            }
          }
        },
        "parameters": [
          {
            "name": "token",
            "in": "query",
            "required": true,
            "schema": {
              "type": "string"
            },
            "description": "Enter your Apify token here"
          }
        ],
        "responses": {
          "200": {
            "description": "OK",
            "content": {
              "application/json": {
                "schema": {
                  "$ref": "#/components/schemas/runsResponseSchema"
                }
              }
            }
          }
        }
      }
    },
    "/acts/opendatadesk~sitemap-robots-explorer/run-sync": {
      "post": {
        "operationId": "run-sync-opendatadesk-sitemap-robots-explorer",
        "x-openai-isConsequential": false,
        "summary": "Executes an Actor, waits for completion, and returns the OUTPUT from Key-value store in response.",
        "tags": [
          "Run Actor"
        ],
        "requestBody": {
          "required": true,
          "content": {
            "application/json": {
              "schema": {
                "$ref": "#/components/schemas/inputSchema"
              }
            }
          }
        },
        "parameters": [
          {
            "name": "token",
            "in": "query",
            "required": true,
            "schema": {
              "type": "string"
            },
            "description": "Enter your Apify token here"
          }
        ],
        "responses": {
          "200": {
            "description": "OK"
          }
        }
      }
    }
  },
  "components": {
    "schemas": {
      "inputSchema": {
        "type": "object",
        "required": [
          "sites"
        ],
        "properties": {
          "sites": {
            "title": "Websites",
            "type": "array",
            "description": "Domains or links, one per line. For example example.com or https://example.com. You can also paste a direct link to a sitemap file, like https://example.com/sitemap.xml.",
            "items": {
              "type": "string"
            }
          },
          "discoverSitemaps": {
            "title": "Find sitemaps automatically",
            "type": "boolean",
            "description": "Reads the sitemap addresses listed in robots.txt. If there are none, it tries the usual places such as /sitemap.xml and /sitemap_index.xml. Turn this off if you only want to read the sitemap links you pasted.",
            "default": true
          },
          "includeRobotsReport": {
            "title": "Include robots.txt and AI crawler report",
            "type": "boolean",
            "description": "Adds one report per website: which sitemaps are declared, the crawl delay, whether llms.txt exists, and whether AI crawlers such as GPTBot and ClaudeBot are blocked. Saved in the key-value store under ROBOTS_REPORT. Billed once per website.",
            "default": true
          },
          "maxUrlsPerSite": {
            "title": "Max URLs per website",
            "minimum": 0,
            "type": "integer",
            "description": "Stops reading a website after this many URLs. Use 0 for no limit.",
            "default": 5000
          },
          "maxItems": {
            "title": "Max URLs in total",
            "minimum": 0,
            "type": "integer",
            "description": "The run stops after this many URLs across all websites. Use 0 for no limit.",
            "default": 10000
          },
          "includePatterns": {
            "title": "Only URLs matching (any of)",
            "type": "array",
            "description": "Regular expressions. A URL is kept only if it matches at least one of them. Example: /blog/ keeps only blog pages. Leave empty to keep everything.",
            "default": [],
            "items": {
              "type": "string"
            }
          },
          "excludePatterns": {
            "title": "Skip URLs matching (any of)",
            "type": "array",
            "description": "Regular expressions. A URL is dropped if it matches any of them. Example: \\.(pdf|jpg)$ or /tag/.",
            "default": [],
            "items": {
              "type": "string"
            }
          },
          "modifiedAfter": {
            "title": "Only URLs changed after",
            "type": "string",
            "description": "Date like 2026-01-31. Keeps only URLs whose sitemap entry says they changed on or after this date. Sitemaps are optional about dates, so see the next setting for URLs without one.",
            "default": ""
          },
          "keepUrlsWithoutLastmod": {
            "title": "Keep URLs without a date",
            "type": "boolean",
            "description": "Only matters when you set a date above. When on, URLs that have no date in the sitemap are kept. When off, they are dropped.",
            "default": true
          },
          "maxSitemapDepth": {
            "title": "How deep to follow sitemap indexes",
            "minimum": 1,
            "maximum": 10,
            "type": "integer",
            "description": "A sitemap can point to other sitemaps. This is how many levels the Actor follows.",
            "default": 5
          },
          "maxSitemapFilesPerSite": {
            "title": "Max sitemap files per website",
            "minimum": 1,
            "type": "integer",
            "description": "A safety limit for very large websites with thousands of sitemap files.",
            "default": 500
          },
          "proxyConfiguration": {
            "title": "Proxy (optional)",
            "type": "object",
            "description": "Most websites do not need a proxy for sitemaps. Turn on Apify Proxy only if a website blocks your requests.",
            "default": {}
          }
        }
      },
      "runsResponseSchema": {
        "type": "object",
        "properties": {
          "data": {
            "type": "object",
            "properties": {
              "id": {
                "type": "string"
              },
              "actId": {
                "type": "string"
              },
              "userId": {
                "type": "string"
              },
              "startedAt": {
                "type": "string",
                "format": "date-time",
                "example": "2025-01-08T00:00:00.000Z"
              },
              "finishedAt": {
                "type": "string",
                "format": "date-time",
                "example": "2025-01-08T00:00:00.000Z"
              },
              "status": {
                "type": "string",
                "example": "READY"
              },
              "meta": {
                "type": "object",
                "properties": {
                  "origin": {
                    "type": "string",
                    "example": "API"
                  },
                  "userAgent": {
                    "type": "string"
                  }
                }
              },
              "stats": {
                "type": "object",
                "properties": {
                  "inputBodyLen": {
                    "type": "integer",
                    "example": 2000
                  },
                  "rebootCount": {
                    "type": "integer",
                    "example": 0
                  },
                  "restartCount": {
                    "type": "integer",
                    "example": 0
                  },
                  "resurrectCount": {
                    "type": "integer",
                    "example": 0
                  },
                  "computeUnits": {
                    "type": "integer",
                    "example": 0
                  }
                }
              },
              "options": {
                "type": "object",
                "properties": {
                  "build": {
                    "type": "string",
                    "example": "latest"
                  },
                  "timeoutSecs": {
                    "type": "integer",
                    "example": 300
                  },
                  "memoryMbytes": {
                    "type": "integer",
                    "example": 1024
                  },
                  "diskMbytes": {
                    "type": "integer",
                    "example": 2048
                  }
                }
              },
              "buildId": {
                "type": "string"
              },
              "defaultKeyValueStoreId": {
                "type": "string"
              },
              "defaultDatasetId": {
                "type": "string"
              },
              "defaultRequestQueueId": {
                "type": "string"
              },
              "buildNumber": {
                "type": "string",
                "example": "1.0.0"
              },
              "containerUrl": {
                "type": "string"
              },
              "usage": {
                "type": "object",
                "properties": {
                  "ACTOR_COMPUTE_UNITS": {
                    "type": "integer",
                    "example": 0
                  },
                  "DATASET_READS": {
                    "type": "integer",
                    "example": 0
                  },
                  "DATASET_WRITES": {
                    "type": "integer",
                    "example": 0
                  },
                  "KEY_VALUE_STORE_READS": {
                    "type": "integer",
                    "example": 0
                  },
                  "KEY_VALUE_STORE_WRITES": {
                    "type": "integer",
                    "example": 1
                  },
                  "KEY_VALUE_STORE_LISTS": {
                    "type": "integer",
                    "example": 0
                  },
                  "REQUEST_QUEUE_READS": {
                    "type": "integer",
                    "example": 0
                  },
                  "REQUEST_QUEUE_WRITES": {
                    "type": "integer",
                    "example": 0
                  },
                  "DATA_TRANSFER_INTERNAL_GBYTES": {
                    "type": "integer",
                    "example": 0
                  },
                  "DATA_TRANSFER_EXTERNAL_GBYTES": {
                    "type": "integer",
                    "example": 0
                  },
                  "PROXY_RESIDENTIAL_TRANSFER_GBYTES": {
                    "type": "integer",
                    "example": 0
                  },
                  "PROXY_SERPS": {
                    "type": "integer",
                    "example": 0
                  },
                  "PROXY_UNBLOCKER_UNITS": {
                    "type": "integer",
                    "example": 0
                  }
                }
              },
              "usageTotalUsd": {
                "type": "number",
                "example": 0.00005
              },
              "usageUsd": {
                "type": "object",
                "properties": {
                  "ACTOR_COMPUTE_UNITS": {
                    "type": "integer",
                    "example": 0
                  },
                  "DATASET_READS": {
                    "type": "integer",
                    "example": 0
                  },
                  "DATASET_WRITES": {
                    "type": "integer",
                    "example": 0
                  },
                  "KEY_VALUE_STORE_READS": {
                    "type": "integer",
                    "example": 0
                  },
                  "KEY_VALUE_STORE_WRITES": {
                    "type": "number",
                    "example": 0.00005
                  },
                  "KEY_VALUE_STORE_LISTS": {
                    "type": "integer",
                    "example": 0
                  },
                  "REQUEST_QUEUE_READS": {
                    "type": "integer",
                    "example": 0
                  },
                  "REQUEST_QUEUE_WRITES": {
                    "type": "integer",
                    "example": 0
                  },
                  "DATA_TRANSFER_INTERNAL_GBYTES": {
                    "type": "integer",
                    "example": 0
                  },
                  "DATA_TRANSFER_EXTERNAL_GBYTES": {
                    "type": "integer",
                    "example": 0
                  },
                  "PROXY_RESIDENTIAL_TRANSFER_GBYTES": {
                    "type": "integer",
                    "example": 0
                  },
                  "PROXY_SERPS": {
                    "type": "integer",
                    "example": 0
                  },
                  "PROXY_UNBLOCKER_UNITS": {
                    "type": "integer",
                    "example": 0
                  }
                }
              }
            }
          }
        }
      }
    }
  }
}