{
  "openapi": "3.0.1",
  "info": {
    "title": "Sitemap URL Extractor with Change Tracking",
    "description": "Extracts every URL from a site's sitemaps (indexes, gzip, text and feed formats included) and reports which URLs were added, removed or re-dated since the previous run.",
    "version": "0.1",
    "x-build-id": "5yFXRh8wWEHuTTguq"
  },
  "servers": [
    {
      "url": "https://api.apify.com/v2"
    }
  ],
  "paths": {
    "/acts/opendatalabs~sitemap-change-tracker/run-sync-get-dataset-items": {
      "post": {
        "operationId": "run-sync-get-dataset-items-opendatalabs-sitemap-change-tracker",
        "x-openai-isConsequential": false,
        "summary": "Executes an Actor, waits for its completion, and returns Actor's dataset items in response.",
        "tags": [
          "Run Actor"
        ],
        "requestBody": {
          "required": true,
          "content": {
            "application/json": {
              "schema": {
                "$ref": "#/components/schemas/inputSchema"
              }
            }
          }
        },
        "parameters": [
          {
            "name": "token",
            "in": "query",
            "required": true,
            "schema": {
              "type": "string"
            },
            "description": "Enter your Apify token here"
          }
        ],
        "responses": {
          "200": {
            "description": "OK"
          }
        }
      }
    },
    "/acts/opendatalabs~sitemap-change-tracker/runs": {
      "post": {
        "operationId": "runs-sync-opendatalabs-sitemap-change-tracker",
        "x-openai-isConsequential": false,
        "summary": "Executes an Actor and returns information about the initiated run in response.",
        "tags": [
          "Run Actor"
        ],
        "requestBody": {
          "required": true,
          "content": {
            "application/json": {
              "schema": {
                "$ref": "#/components/schemas/inputSchema"
              }
            }
          }
        },
        "parameters": [
          {
            "name": "token",
            "in": "query",
            "required": true,
            "schema": {
              "type": "string"
            },
            "description": "Enter your Apify token here"
          }
        ],
        "responses": {
          "200": {
            "description": "OK",
            "content": {
              "application/json": {
                "schema": {
                  "$ref": "#/components/schemas/runsResponseSchema"
                }
              }
            }
          }
        }
      }
    },
    "/acts/opendatalabs~sitemap-change-tracker/run-sync": {
      "post": {
        "operationId": "run-sync-opendatalabs-sitemap-change-tracker",
        "x-openai-isConsequential": false,
        "summary": "Executes an Actor, waits for completion, and returns the OUTPUT from Key-value store in response.",
        "tags": [
          "Run Actor"
        ],
        "requestBody": {
          "required": true,
          "content": {
            "application/json": {
              "schema": {
                "$ref": "#/components/schemas/inputSchema"
              }
            }
          }
        },
        "parameters": [
          {
            "name": "token",
            "in": "query",
            "required": true,
            "schema": {
              "type": "string"
            },
            "description": "Enter your Apify token here"
          }
        ],
        "responses": {
          "200": {
            "description": "OK"
          }
        }
      }
    }
  },
  "components": {
    "schemas": {
      "inputSchema": {
        "type": "object",
        "properties": {
          "targets": {
            "title": "Domains or sitemap URLs",
            "minItems": 1,
            "maxItems": 1000,
            "type": "array",
            "description": "What to extract. Give a bare domain (example.com) and the Actor finds the sitemaps through robots.txt, falling back to common paths such as /sitemap.xml and /wp-sitemap.xml. Give a direct sitemap URL and it is used as-is. Sitemap indexes, .xml.gz files, plain-text URL lists and RSS/Atom feeds are all handled. Leave empty to run a small demo on crawlee.dev (100 URLs).",
            "items": {
              "type": "string"
            }
          },
          "mode": {
            "title": "Run mode",
            "enum": [
              "snapshot",
              "changes"
            ],
            "type": "string",
            "description": "Snapshot returns every URL found. Change tracking compares this run against the previous run's baseline for the same target and labels each URL added, changed, removed or unchanged. The first run in change mode has no baseline, so everything is reported as added and the baseline is written for next time.",
            "default": "snapshot"
          },
          "outputChangeTypes": {
            "title": "Change types to output",
            "type": "array",
            "description": "Only used in change-tracking mode. Leave the default to get just the differences, which is what a scheduled monitor usually wants. Add 'unchanged' to get a full snapshot with change labels attached.",
            "items": {
              "type": "string",
              "enum": [
                "added",
                "changed",
                "removed",
                "unchanged"
              ],
              "enumTitles": [
                "Added - URL is new since the last run",
                "Changed - lastmod moved",
                "Removed - URL is no longer listed",
                "Unchanged - still listed, same lastmod"
              ]
            },
            "default": [
              "added",
              "changed",
              "removed"
            ]
          },
          "stateStoreName": {
            "title": "Baseline key-value store name",
            "type": "string",
            "description": "Named key-value store that holds the previous run's URL set. A named store survives between runs, unlike the default store, which is what makes change tracking work. Use one store name per monitoring job; separate targets are kept under separate keys inside it.",
            "default": "sitemap-change-tracker-state"
          },
          "maxSitemapFailurePercentForBaselineUpdate": {
            "title": "Max sitemap failure rate for updating the baseline",
            "minimum": 0,
            "maximum": 100,
            "type": "integer",
            "description": "Safety valve for change tracking. If more than this percentage of a target's sitemaps fail to load, the baseline is left untouched, so a bad day at the origin server does not make the next run report the whole site as 'added'. A run that extracts zero URLs never overwrites the baseline, regardless of this setting.",
            "default": 25
          },
          "checkUrlStatus": {
            "title": "Check each URL's HTTP status",
            "type": "boolean",
            "description": "Sends a cheap HEAD request per URL (falling back to GET when a server rejects HEAD) and adds statusCode, statusOk and redirectedTo to every row. Useful for finding sitemap entries that 404. Off by default because it multiplies the number of requests.",
            "default": false
          },
          "statusCheckLimit": {
            "title": "Max URLs to status-check per target",
            "minimum": 1,
            "maximum": 50000,
            "type": "integer",
            "description": "Caps the HEAD checks so a 300,000-URL sitemap cannot turn into a 300,000-request run by accident.",
            "default": 500
          },
          "onlyUrlsModifiedSince": {
            "title": "Only URLs modified on or after",
            "type": "string",
            "description": "YYYY-MM-DD. Drops URLs whose lastmod is older than this date. URLs with no lastmod at all are kept, because a missing date is not evidence that a page is old."
          },
          "maxUrlsPerTarget": {
            "title": "Max URLs per target",
            "minimum": 1,
            "maximum": 5000000,
            "type": "integer",
            "description": "Hard ceiling on URLs collected per domain. Sitemaps are walked breadth-first, so hitting this limit still returns a spread across the whole site rather than everything from one section. The default of 1,000 caps a run at about $0.25 per target; raise it for a full inventory.",
            "default": 1000
          },
          "maxSitemapsPerTarget": {
            "title": "Max sitemap files per target",
            "minimum": 1,
            "maximum": 50000,
            "type": "integer",
            "description": "Hard ceiling on sitemap documents fetched per domain. Large e-commerce sites routinely publish hundreds of child sitemaps behind one index.",
            "default": 500
          },
          "maxSitemapDepth": {
            "title": "Max sitemap index depth",
            "minimum": 0,
            "maximum": 20,
            "type": "integer",
            "description": "How many levels of nested sitemap index to follow. 0 means only the sitemaps given or discovered directly.",
            "default": 5
          },
          "includeAlternates": {
            "title": "Include hreflang alternates in change mode",
            "type": "boolean",
            "description": "Adds the xhtml:link hreflang alternates to each row. Snapshot mode always includes them when present; this switch controls the (usually much larger) change-mode rows.",
            "default": false
          },
          "respectRobotsTxt": {
            "title": "Respect robots.txt",
            "type": "boolean",
            "description": "Checks the sitemap path against robots.txt before fetching it. Leave this on. Sitemaps are published for crawlers, but a few sites still disallow specific sitemap paths.",
            "default": true
          },
          "discoverViaCommonPaths": {
            "title": "Probe common sitemap paths",
            "type": "boolean",
            "description": "When robots.txt declares no Sitemap: directive, try /sitemap.xml, /sitemap_index.xml, /wp-sitemap.xml and nine other well-known locations. Turn this off to rely on robots.txt only.",
            "default": true
          },
          "requestsPerSecond": {
            "title": "Requests per second",
            "minimum": 1,
            "maximum": 50,
            "type": "integer",
            "description": "Global rate ceiling across all targets. Lower this for small or slow origin servers.",
            "default": 5
          },
          "concurrency": {
            "title": "Concurrent requests",
            "minimum": 1,
            "maximum": 50,
            "type": "integer",
            "description": "How many sitemap or status requests are in flight at once. The rate limit above still applies.",
            "default": 10
          },
          "maxRetries": {
            "title": "Attempts per request",
            "minimum": 1,
            "maximum": 10,
            "type": "integer",
            "description": "Total attempts, including the first. Retries use exponential backoff with jitter and honour Retry-After. Only transient failures (timeouts, 5xx, 429) are retried; a 404 fails immediately instead of burning four timeouts.",
            "default": 4
          },
          "requestTimeoutSecs": {
            "title": "Request timeout (seconds)",
            "minimum": 5,
            "maximum": 300,
            "type": "integer",
            "description": "How long to wait for a single sitemap or status request before giving up on that attempt. Large gzipped sitemap indexes on slow origins can legitimately take 30 seconds or more, so lowering this below the default will start costing you URLs.",
            "default": 60
          },
          "userAgentContact": {
            "title": "Contact string for the User-Agent",
            "type": "string",
            "description": "Appended to the User-Agent header, for example a URL or an email address, so site owners can see who is crawling them. Optional here; the Actor sends a descriptive User-Agent either way."
          }
        }
      },
      "runsResponseSchema": {
        "type": "object",
        "properties": {
          "data": {
            "type": "object",
            "properties": {
              "id": {
                "type": "string"
              },
              "actId": {
                "type": "string"
              },
              "userId": {
                "type": "string"
              },
              "startedAt": {
                "type": "string",
                "format": "date-time",
                "example": "2025-01-08T00:00:00.000Z"
              },
              "finishedAt": {
                "type": "string",
                "format": "date-time",
                "example": "2025-01-08T00:00:00.000Z"
              },
              "status": {
                "type": "string",
                "example": "READY"
              },
              "meta": {
                "type": "object",
                "properties": {
                  "origin": {
                    "type": "string",
                    "example": "API"
                  },
                  "userAgent": {
                    "type": "string"
                  }
                }
              },
              "stats": {
                "type": "object",
                "properties": {
                  "inputBodyLen": {
                    "type": "integer",
                    "example": 2000
                  },
                  "rebootCount": {
                    "type": "integer",
                    "example": 0
                  },
                  "restartCount": {
                    "type": "integer",
                    "example": 0
                  },
                  "resurrectCount": {
                    "type": "integer",
                    "example": 0
                  },
                  "computeUnits": {
                    "type": "integer",
                    "example": 0
                  }
                }
              },
              "options": {
                "type": "object",
                "properties": {
                  "build": {
                    "type": "string",
                    "example": "latest"
                  },
                  "timeoutSecs": {
                    "type": "integer",
                    "example": 300
                  },
                  "memoryMbytes": {
                    "type": "integer",
                    "example": 1024
                  },
                  "diskMbytes": {
                    "type": "integer",
                    "example": 2048
                  }
                }
              },
              "buildId": {
                "type": "string"
              },
              "defaultKeyValueStoreId": {
                "type": "string"
              },
              "defaultDatasetId": {
                "type": "string"
              },
              "defaultRequestQueueId": {
                "type": "string"
              },
              "buildNumber": {
                "type": "string",
                "example": "1.0.0"
              },
              "containerUrl": {
                "type": "string"
              },
              "usage": {
                "type": "object",
                "properties": {
                  "ACTOR_COMPUTE_UNITS": {
                    "type": "integer",
                    "example": 0
                  },
                  "DATASET_READS": {
                    "type": "integer",
                    "example": 0
                  },
                  "DATASET_WRITES": {
                    "type": "integer",
                    "example": 0
                  },
                  "KEY_VALUE_STORE_READS": {
                    "type": "integer",
                    "example": 0
                  },
                  "KEY_VALUE_STORE_WRITES": {
                    "type": "integer",
                    "example": 1
                  },
                  "KEY_VALUE_STORE_LISTS": {
                    "type": "integer",
                    "example": 0
                  },
                  "REQUEST_QUEUE_READS": {
                    "type": "integer",
                    "example": 0
                  },
                  "REQUEST_QUEUE_WRITES": {
                    "type": "integer",
                    "example": 0
                  },
                  "DATA_TRANSFER_INTERNAL_GBYTES": {
                    "type": "integer",
                    "example": 0
                  },
                  "DATA_TRANSFER_EXTERNAL_GBYTES": {
                    "type": "integer",
                    "example": 0
                  },
                  "PROXY_RESIDENTIAL_TRANSFER_GBYTES": {
                    "type": "integer",
                    "example": 0
                  },
                  "PROXY_SERPS": {
                    "type": "integer",
                    "example": 0
                  },
                  "PROXY_UNBLOCKER_UNITS": {
                    "type": "integer",
                    "example": 0
                  }
                }
              },
              "usageTotalUsd": {
                "type": "number",
                "example": 0.00005
              },
              "usageUsd": {
                "type": "object",
                "properties": {
                  "ACTOR_COMPUTE_UNITS": {
                    "type": "integer",
                    "example": 0
                  },
                  "DATASET_READS": {
                    "type": "integer",
                    "example": 0
                  },
                  "DATASET_WRITES": {
                    "type": "integer",
                    "example": 0
                  },
                  "KEY_VALUE_STORE_READS": {
                    "type": "integer",
                    "example": 0
                  },
                  "KEY_VALUE_STORE_WRITES": {
                    "type": "number",
                    "example": 0.00005
                  },
                  "KEY_VALUE_STORE_LISTS": {
                    "type": "integer",
                    "example": 0
                  },
                  "REQUEST_QUEUE_READS": {
                    "type": "integer",
                    "example": 0
                  },
                  "REQUEST_QUEUE_WRITES": {
                    "type": "integer",
                    "example": 0
                  },
                  "DATA_TRANSFER_INTERNAL_GBYTES": {
                    "type": "integer",
                    "example": 0
                  },
                  "DATA_TRANSFER_EXTERNAL_GBYTES": {
                    "type": "integer",
                    "example": 0
                  },
                  "PROXY_RESIDENTIAL_TRANSFER_GBYTES": {
                    "type": "integer",
                    "example": 0
                  },
                  "PROXY_SERPS": {
                    "type": "integer",
                    "example": 0
                  },
                  "PROXY_UNBLOCKER_UNITS": {
                    "type": "integer",
                    "example": 0
                  }
                }
              }
            }
          }
        }
      }
    }
  }
}