{
  "openapi": "3.0.1",
  "info": {
    "title": "Sitemap Extractor & Change Monitor - All URLs, New Pages",
    "description": "Extract every URL from any website's sitemaps with lastmod, changefreq & priority - and monitor which pages were added, removed or updated.",
    "version": "0.1",
    "x-build-id": "vqna9KgMHAEk2SQmd"
  },
  "servers": [
    {
      "url": "https://api.apify.com/v2"
    }
  ],
  "paths": {
    "/acts/prevailing_glow~sitemap-extractor-monitor/run-sync-get-dataset-items": {
      "post": {
        "operationId": "run-sync-get-dataset-items-prevailing_glow-sitemap-extractor-monitor",
        "x-openai-isConsequential": false,
        "summary": "Executes an Actor, waits for its completion, and returns Actor's dataset items in response.",
        "tags": [
          "Run Actor"
        ],
        "requestBody": {
          "required": true,
          "content": {
            "application/json": {
              "schema": {
                "$ref": "#/components/schemas/inputSchema"
              }
            }
          }
        },
        "parameters": [
          {
            "name": "token",
            "in": "query",
            "required": true,
            "schema": {
              "type": "string"
            },
            "description": "Enter your Apify token here"
          }
        ],
        "responses": {
          "200": {
            "description": "OK"
          }
        }
      }
    },
    "/acts/prevailing_glow~sitemap-extractor-monitor/runs": {
      "post": {
        "operationId": "runs-sync-prevailing_glow-sitemap-extractor-monitor",
        "x-openai-isConsequential": false,
        "summary": "Executes an Actor and returns information about the initiated run in response.",
        "tags": [
          "Run Actor"
        ],
        "requestBody": {
          "required": true,
          "content": {
            "application/json": {
              "schema": {
                "$ref": "#/components/schemas/inputSchema"
              }
            }
          }
        },
        "parameters": [
          {
            "name": "token",
            "in": "query",
            "required": true,
            "schema": {
              "type": "string"
            },
            "description": "Enter your Apify token here"
          }
        ],
        "responses": {
          "200": {
            "description": "OK",
            "content": {
              "application/json": {
                "schema": {
                  "$ref": "#/components/schemas/runsResponseSchema"
                }
              }
            }
          }
        }
      }
    },
    "/acts/prevailing_glow~sitemap-extractor-monitor/run-sync": {
      "post": {
        "operationId": "run-sync-prevailing_glow-sitemap-extractor-monitor",
        "x-openai-isConsequential": false,
        "summary": "Executes an Actor, waits for completion, and returns the OUTPUT from Key-value store in response.",
        "tags": [
          "Run Actor"
        ],
        "requestBody": {
          "required": true,
          "content": {
            "application/json": {
              "schema": {
                "$ref": "#/components/schemas/inputSchema"
              }
            }
          }
        },
        "parameters": [
          {
            "name": "token",
            "in": "query",
            "required": true,
            "schema": {
              "type": "string"
            },
            "description": "Enter your Apify token here"
          }
        ],
        "responses": {
          "200": {
            "description": "OK"
          }
        }
      }
    }
  },
  "components": {
    "schemas": {
      "inputSchema": {
        "type": "object",
        "required": [
          "startUrls"
        ],
        "properties": {
          "startUrls": {
            "title": "Websites or sitemap URLs",
            "minItems": 1,
            "type": "array",
            "description": "Website homepages (e.g. <code>https://example.com</code> or just <code>example.com</code>) - sitemaps are discovered automatically from robots.txt and common locations. You can also paste sitemap or sitemap-index URLs directly (<code>.xml</code>, <code>.xml.gz</code>, <code>.txt</code>, RSS/Atom feeds).",
            "items": {
              "type": "object",
              "required": [
                "url"
              ],
              "properties": {
                "url": {
                  "type": "string",
                  "title": "URL of a web page",
                  "format": "uri"
                }
              }
            }
          },
          "monitorMode": {
            "title": "Monitor mode (compare with previous run)",
            "type": "boolean",
            "description": "Saves the URL list of each site in a named key-value store and, on the next run, outputs only what changed: <code>added</code>, <code>removed</code> and <code>lastmodChanged</code> URLs. The very first run creates the baseline.",
            "default": false
          },
          "monitorOutput": {
            "title": "Monitor output",
            "enum": [
              "changes",
              "all"
            ],
            "type": "string",
            "description": "<b>Changes only</b> outputs added / removed / updated URLs. <b>All URLs</b> also outputs unchanged URLs (each row has a <code>changeType</code>).",
            "default": "changes"
          },
          "outputOnFirstRun": {
            "title": "Output all URLs on the first (baseline) run",
            "type": "boolean",
            "description": "When there is no previous state yet, output every URL with <code>changeType: \"initial\"</code>. Turn off to create the baseline silently (cheaper).",
            "default": true
          },
          "monitorStoreName": {
            "title": "State store name",
            "pattern": "^[a-zA-Z0-9-]{1,63}$",
            "type": "string",
            "description": "Name of the named key-value store that keeps the previous URL lists (one record per site). Use different names to run independent monitors of the same site.",
            "default": "sitemap-extractor-monitor-state"
          },
          "urlIncludePatterns": {
            "title": "Only include URLs matching (regex)",
            "type": "array",
            "description": "Keep only page URLs matching at least one of these regular expressions (case-insensitive), e.g. <code>/blog/</code> or <code>^https://example\\.com/products/</code>.",
            "items": {
              "type": "string"
            }
          },
          "urlExcludePatterns": {
            "title": "Exclude URLs matching (regex)",
            "type": "array",
            "description": "Drop page URLs matching any of these regular expressions, e.g. <code>/tag/</code> or <code>\\?page=</code>.",
            "items": {
              "type": "string"
            }
          },
          "sitemapExcludePatterns": {
            "title": "Skip sitemaps matching (regex)",
            "type": "array",
            "description": "Don't download child sitemaps whose URL matches, e.g. <code>/sitemaps/20(0|1)\\d</code> to skip old archive years on news sites, or <code>video</code>.",
            "items": {
              "type": "string"
            }
          },
          "modifiedSince": {
            "title": "Only URLs modified since",
            "pattern": "^(\\d{4})-(0[1-9]|1[0-2])-(0[1-9]|[12]\\d|3[01])(T[0-2]\\d:[0-5]\\d(:[0-5]\\d)?(\\.\\d+)?Z?)?$|^(\\d+)\\s*(minute|hour|day|week|month|year)s?$",
            "type": "string",
            "description": "Output only URLs whose <code>lastmod</code> is on/after this date (UTC). Absolute (<code>2024-05-01</code>) or relative (<code>7 days</code>, <code>2 weeks</code>, <code>24 hours</code>). URLs without lastmod are skipped when this is set."
          },
          "maxResultsPerSite": {
            "title": "Max results per site (cost cap)",
            "minimum": 0,
            "type": "integer",
            "description": "Maximum number of result rows (billed URLs) output per site in one run. Protects you from surprise costs on sites that list millions of URLs. Set 0 for no limit. In extract mode the site stops when the cap is reached. In monitor mode the whole sitemap is still scanned (so removals are detected and the state stays complete); changes beyond the cap are not output in this run and are reported on the next run instead, and on the first run the full baseline is saved even if only the first N URLs are output.",
            "default": 100000
          },
          "maxUrlsPerSite": {
            "title": "Max URLs scanned per site",
            "minimum": 0,
            "type": "integer",
            "description": "Stop reading a site after this many unique URLs (0 = no limit). Unlike the results cap, this also limits monitor mode: a truncated scan cannot detect removed pages. Use it to bound run time and memory on huge sites.",
            "default": 0
          },
          "maxSitemapsPerSite": {
            "title": "Max sitemap files per site",
            "minimum": 1,
            "type": "integer",
            "description": "Safety limit on how many sitemap files are downloaded per site. Large news sites can list several thousand monthly/daily sitemaps.",
            "default": 10000
          },
          "maxSitemapDepth": {
            "title": "Max sitemap index depth",
            "minimum": 1,
            "maximum": 50,
            "type": "integer",
            "description": "How many levels of nested sitemap indexes to follow.",
            "default": 10
          },
          "respectRobotsTxt": {
            "title": "Respect robots.txt",
            "type": "boolean",
            "description": "Skip sitemap files disallowed by robots.txt and honour Crawl-delay (capped by <i>Max crawl delay</i>). Follows RFC 9309: if robots.txt is unreachable (5xx / network error) the host is treated as disallowed.",
            "default": true
          },
          "probeCommonLocations": {
            "title": "Probe common sitemap locations",
            "type": "boolean",
            "description": "Besides robots.txt, also try <code>/sitemap.xml</code> and <code>/sitemap_index.xml</code> (and, if nothing is found, <code>/sitemap-index.xml</code>, <code>/sitemap.xml.gz</code>, <code>/wp-sitemap.xml</code>, <code>/sitemaps.xml</code>, <code>/sitemap/sitemap.xml</code>, <code>/sitemap.txt</code>).",
            "default": true
          },
          "maxConcurrency": {
            "title": "Max concurrent requests (total)",
            "minimum": 1,
            "maximum": 100,
            "type": "integer",
            "description": "Maximum parallel downloads across all sites.",
            "default": 10
          },
          "maxRequestsPerHost": {
            "title": "Max concurrent requests per host",
            "minimum": 1,
            "maximum": 20,
            "type": "integer",
            "description": "Keep this low to be polite to the target server.",
            "default": 2
          },
          "minDelayBetweenRequestsMs": {
            "title": "Min delay between requests to one host (ms)",
            "minimum": 0,
            "maximum": 60000,
            "type": "integer",
            "description": "Minimum gap between request starts on the same host. Automatically raised to the site's robots.txt Crawl-delay.",
            "default": 100
          },
          "maxCrawlDelaySecs": {
            "title": "Max crawl delay (seconds)",
            "minimum": 0,
            "maximum": 120,
            "type": "integer",
            "description": "Upper bound for a robots.txt Crawl-delay (some sites ask for 30-60 s, which would make big sitemap sets take hours).",
            "default": 10
          },
          "requestTimeoutSecs": {
            "title": "Request timeout (seconds)",
            "minimum": 5,
            "maximum": 600,
            "type": "integer",
            "description": "Max wait for response headers and max idle time between body chunks.",
            "default": 60
          },
          "maxRetries": {
            "title": "Max retries per sitemap",
            "minimum": 0,
            "maximum": 10,
            "type": "integer",
            "description": "Retries on network errors, timeouts, truncated downloads, HTTP 408/429/5xx (exponential backoff with jitter, honours Retry-After).",
            "default": 4
          },
          "maxSitemapSizeMb": {
            "title": "Max size of one sitemap file (MB, uncompressed)",
            "minimum": 1,
            "maximum": 2048,
            "type": "integer",
            "description": "Files are streamed, so memory use does not grow with file size; this only guards against endless responses. The sitemap protocol limit is 50 MB.",
            "default": 300
          },
          "userAgent": {
            "title": "User-Agent",
            "type": "string",
            "description": "User-Agent header (also used to match robots.txt groups). Leave empty for the default <code>Mozilla/5.0 (compatible; SitemapMonitor/1.0)</code>."
          },
          "proxyConfiguration": {
            "title": "Proxy configuration",
            "type": "object",
            "description": "Optional. Sitemaps are usually not protected, so no proxy is needed by default. Use Apify Proxy or your own proxies if a site blocks datacenter IPs.",
            "default": {
              "useApifyProxy": false
            }
          }
        }
      },
      "runsResponseSchema": {
        "type": "object",
        "properties": {
          "data": {
            "type": "object",
            "properties": {
              "id": {
                "type": "string"
              },
              "actId": {
                "type": "string"
              },
              "userId": {
                "type": "string"
              },
              "startedAt": {
                "type": "string",
                "format": "date-time",
                "example": "2025-01-08T00:00:00.000Z"
              },
              "finishedAt": {
                "type": "string",
                "format": "date-time",
                "example": "2025-01-08T00:00:00.000Z"
              },
              "status": {
                "type": "string",
                "example": "READY"
              },
              "meta": {
                "type": "object",
                "properties": {
                  "origin": {
                    "type": "string",
                    "example": "API"
                  },
                  "userAgent": {
                    "type": "string"
                  }
                }
              },
              "stats": {
                "type": "object",
                "properties": {
                  "inputBodyLen": {
                    "type": "integer",
                    "example": 2000
                  },
                  "rebootCount": {
                    "type": "integer",
                    "example": 0
                  },
                  "restartCount": {
                    "type": "integer",
                    "example": 0
                  },
                  "resurrectCount": {
                    "type": "integer",
                    "example": 0
                  },
                  "computeUnits": {
                    "type": "integer",
                    "example": 0
                  }
                }
              },
              "options": {
                "type": "object",
                "properties": {
                  "build": {
                    "type": "string",
                    "example": "latest"
                  },
                  "timeoutSecs": {
                    "type": "integer",
                    "example": 300
                  },
                  "memoryMbytes": {
                    "type": "integer",
                    "example": 1024
                  },
                  "diskMbytes": {
                    "type": "integer",
                    "example": 2048
                  }
                }
              },
              "buildId": {
                "type": "string"
              },
              "defaultKeyValueStoreId": {
                "type": "string"
              },
              "defaultDatasetId": {
                "type": "string"
              },
              "defaultRequestQueueId": {
                "type": "string"
              },
              "buildNumber": {
                "type": "string",
                "example": "1.0.0"
              },
              "containerUrl": {
                "type": "string"
              },
              "usage": {
                "type": "object",
                "properties": {
                  "ACTOR_COMPUTE_UNITS": {
                    "type": "integer",
                    "example": 0
                  },
                  "DATASET_READS": {
                    "type": "integer",
                    "example": 0
                  },
                  "DATASET_WRITES": {
                    "type": "integer",
                    "example": 0
                  },
                  "KEY_VALUE_STORE_READS": {
                    "type": "integer",
                    "example": 0
                  },
                  "KEY_VALUE_STORE_WRITES": {
                    "type": "integer",
                    "example": 1
                  },
                  "KEY_VALUE_STORE_LISTS": {
                    "type": "integer",
                    "example": 0
                  },
                  "REQUEST_QUEUE_READS": {
                    "type": "integer",
                    "example": 0
                  },
                  "REQUEST_QUEUE_WRITES": {
                    "type": "integer",
                    "example": 0
                  },
                  "DATA_TRANSFER_INTERNAL_GBYTES": {
                    "type": "integer",
                    "example": 0
                  },
                  "DATA_TRANSFER_EXTERNAL_GBYTES": {
                    "type": "integer",
                    "example": 0
                  },
                  "PROXY_RESIDENTIAL_TRANSFER_GBYTES": {
                    "type": "integer",
                    "example": 0
                  },
                  "PROXY_SERPS": {
                    "type": "integer",
                    "example": 0
                  },
                  "PROXY_UNBLOCKER_UNITS": {
                    "type": "integer",
                    "example": 0
                  }
                }
              },
              "usageTotalUsd": {
                "type": "number",
                "example": 0.00005
              },
              "usageUsd": {
                "type": "object",
                "properties": {
                  "ACTOR_COMPUTE_UNITS": {
                    "type": "integer",
                    "example": 0
                  },
                  "DATASET_READS": {
                    "type": "integer",
                    "example": 0
                  },
                  "DATASET_WRITES": {
                    "type": "integer",
                    "example": 0
                  },
                  "KEY_VALUE_STORE_READS": {
                    "type": "integer",
                    "example": 0
                  },
                  "KEY_VALUE_STORE_WRITES": {
                    "type": "number",
                    "example": 0.00005
                  },
                  "KEY_VALUE_STORE_LISTS": {
                    "type": "integer",
                    "example": 0
                  },
                  "REQUEST_QUEUE_READS": {
                    "type": "integer",
                    "example": 0
                  },
                  "REQUEST_QUEUE_WRITES": {
                    "type": "integer",
                    "example": 0
                  },
                  "DATA_TRANSFER_INTERNAL_GBYTES": {
                    "type": "integer",
                    "example": 0
                  },
                  "DATA_TRANSFER_EXTERNAL_GBYTES": {
                    "type": "integer",
                    "example": 0
                  },
                  "PROXY_RESIDENTIAL_TRANSFER_GBYTES": {
                    "type": "integer",
                    "example": 0
                  },
                  "PROXY_SERPS": {
                    "type": "integer",
                    "example": 0
                  },
                  "PROXY_UNBLOCKER_UNITS": {
                    "type": "integer",
                    "example": 0
                  }
                }
              }
            }
          }
        }
      }
    }
  }
}