{ "openapi": "3.0.1", "info": { "title": "Sitemap URL Extractor: Every URL, lastmod & New Page Monitor", "description": "Get every URL a website lists in its sitemaps: robots.txt discovery, nested sitemap indexes, .xml.gz, plain-text and RSS/Atom sitemaps, with lastmod, changefreq, priority, images, news, video and hreflang. Filters, and only new pages since the last run for monitoring.", "version": "0.0", "x-build-id": "djpBagzTwXTqADSdG" }, "servers": [ { "url": "https://api.apify.com/v2" } ], "paths": { "/acts/charlesmarkette~sitemap-url-extractor-new-page-monitor/run-sync-get-dataset-items": { "post": { "operationId": "run-sync-get-dataset-items-charlesmarkette-sitemap-url-extractor-new-page-monitor", "x-openai-isConsequential": false, "summary": "Executes an Actor, waits for its completion, and returns Actor's dataset items in response.", "tags": [ "Run Actor" ], "requestBody": { "required": true, "content": { "application/json": { "schema": { "$ref": "#/components/schemas/inputSchema" } } } }, "parameters": [ { "name": "token", "in": "query", "required": true, "schema": { "type": "string" }, "description": "Enter your Apify token here" } ], "responses": { "200": { "description": "OK" } } } }, "/acts/charlesmarkette~sitemap-url-extractor-new-page-monitor/runs": { "post": { "operationId": "runs-sync-charlesmarkette-sitemap-url-extractor-new-page-monitor", "x-openai-isConsequential": false, "summary": "Executes an Actor and returns information about the initiated run in response.", "tags": [ "Run Actor" ], "requestBody": { "required": true, "content": { "application/json": { "schema": { "$ref": "#/components/schemas/inputSchema" } } } }, "parameters": [ { "name": "token", "in": "query", "required": true, "schema": { "type": "string" }, "description": "Enter your Apify token here" } ], "responses": { "200": { "description": "OK", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/runsResponseSchema" } } } } } } }, "/acts/charlesmarkette~sitemap-url-extractor-new-page-monitor/run-sync": { "post": { "operationId": "run-sync-charlesmarkette-sitemap-url-extractor-new-page-monitor", "x-openai-isConsequential": false, "summary": "Executes an Actor, waits for completion, and returns the OUTPUT from Key-value store in response.", "tags": [ "Run Actor" ], "requestBody": { "required": true, "content": { "application/json": { "schema": { "$ref": "#/components/schemas/inputSchema" } } } }, "parameters": [ { "name": "token", "in": "query", "required": true, "schema": { "type": "string" }, "description": "Enter your Apify token here" } ], "responses": { "200": { "description": "OK" } } } } }, "components": { "schemas": { "inputSchema": { "type": "object", "properties": { "startUrls": { "title": "Start URLs", "type": "array", "description": "One per line. A website (https://example.com: its sitemaps are found from robots.txt, or at /sitemap.xml, /sitemap_index.xml or /sitemap.xml.gz), a sitemap URL (.xml, .xml.gz, .txt, or an RSS/Atom feed used as a sitemap), or a robots.txt URL. Sitemap index files are followed.", "items": { "type": "string" } }, "datasetId": { "title": "Dataset of websites", "type": "string", "description": "Optional. A dataset from another run (company lists, leads, search results…). For each item, the first field below that holds a website or sitemap URL is read, and the item's fields are added to each of its URL rows. Use {{resource.defaultDatasetId}} in an integration." }, "urlFields": { "title": "URL fields", "type": "array", "description": "Item fields to take the website or sitemap URL from, in order of preference. Dotted paths work (company.website). Default: url, website, sitemapUrl, domain.", "items": { "type": "string" } }, "keepItemFields": { "title": "Keep the item's original fields", "type": "boolean", "description": "On: each URL row is the original item plus the sitemap columns (the sitemap's url wins over an item field of the same name). Off: only the sitemap columns.", "default": true }, "includePatterns": { "title": "Include URLs matching", "type": "array", "description": "Optional. Keep only URLs that match at least one of these. A glob like */blog/* (matched against the whole URL, ignoring case), plain text like /products/ (the URL contains it, ignoring case), or a regular expression starting with re: (for example re:/20(25|26)/).", "items": { "type": "string" } }, "excludePatterns": { "title": "Exclude URLs matching", "type": "array", "description": "Optional. Leave out URLs that match any of these (same pattern rules as above), for example *.pdf or /tag/.", "items": { "type": "string" } }, "lastmodSince": { "title": "Changed since", "type": "string", "description": "Optional. Keep only URLs whose sitemap lastmod is on or after this: a date like 2026-10-01 or 2026-10-01T08:00:00Z (UTC), or a period like '7 days', '24 hours', '2 weeks'. URLs without a lastmod are left out when this is set. lastmod is the site's own claim." }, "sameDomainOnly": { "title": "Only URLs on the input's domain", "type": "boolean", "description": "Leave out URLs on other registered domains than the input's (or the domain it redirects to). Subdomains count as the same domain: blog.example.com is kept for example.com.", "default": true }, "onlyNew": { "title": "Only new URLs", "type": "boolean", "description": "Output only URLs that weren't in this site's sitemaps on an earlier run with this setting. The first run outputs every URL (turn on the next option to skip them). Memory: up to 1,000,000 URLs per site, in the key-value store 'sitemap-extractor-state'.", "default": false }, "skipExistingOnFirstRun": { "title": "Skip existing URLs on the first run", "type": "boolean", "description": "With 'Only new URLs': the first run for a site only remembers its current URLs and outputs nothing (and costs nothing per URL), so later runs output only pages added after you started monitoring.", "default": false }, "stateKey": { "title": "Memory name", "type": "string", "description": "Optional. Runs with the same memory name share their 'only new' memory. By default each saved task has its own memory, and changing the filters starts a fresh one. Letters, digits, '-', '_' and '.'." }, "maxSitemaps": { "title": "Max sitemaps per site", "minimum": 1, "maximum": 100000, "type": "integer", "description": "Stop reading a site's sitemaps after this many files (sitemap indexes count). Sitemap indexes are followed up to 5 levels deep.", "default": 500 }, "concurrency": { "title": "Parallel sites", "minimum": 1, "maximum": 10, "type": "integer", "description": "How many sites to read at the same time (1–10). Each host still gets at most 2 requests at a time.", "default": 3 }, "maxResults": { "title": "Max output rows", "minimum": 1, "type": "integer", "description": "Optional cap on output rows for the whole run (URL rows plus any free error or warning rows). Prefilled at 100 so a first try costs about $0.06; clear it to get every URL." } } }, "runsResponseSchema": { "type": "object", "properties": { "data": { "type": "object", "properties": { "id": { "type": "string" }, "actId": { "type": "string" }, "userId": { "type": "string" }, "startedAt": { "type": "string", "format": "date-time", "example": "2025-01-08T00:00:00.000Z" }, "finishedAt": { "type": "string", "format": "date-time", "example": "2025-01-08T00:00:00.000Z" }, "status": { "type": "string", "example": "READY" }, "meta": { "type": "object", "properties": { "origin": { "type": "string", "example": "API" }, "userAgent": { "type": "string" } } }, "stats": { "type": "object", "properties": { "inputBodyLen": { "type": "integer", "example": 2000 }, "rebootCount": { "type": "integer", "example": 0 }, "restartCount": { "type": "integer", "example": 0 }, "resurrectCount": { "type": "integer", "example": 0 }, "computeUnits": { "type": "integer", "example": 0 } } }, "options": { "type": "object", "properties": { "build": { "type": "string", "example": "latest" }, "timeoutSecs": { "type": "integer", "example": 300 }, "memoryMbytes": { "type": "integer", "example": 1024 }, "diskMbytes": { "type": "integer", "example": 2048 } } }, "buildId": { "type": "string" }, "defaultKeyValueStoreId": { "type": "string" }, "defaultDatasetId": { "type": "string" }, "defaultRequestQueueId": { "type": "string" }, "buildNumber": { "type": "string", "example": "1.0.0" }, "containerUrl": { "type": "string" }, "usage": { "type": "object", "properties": { "ACTOR_COMPUTE_UNITS": { "type": "integer", "example": 0 }, "DATASET_READS": { "type": "integer", "example": 0 }, "DATASET_WRITES": { "type": "integer", "example": 0 }, "KEY_VALUE_STORE_READS": { "type": "integer", "example": 0 }, "KEY_VALUE_STORE_WRITES": { "type": "integer", "example": 1 }, "KEY_VALUE_STORE_LISTS": { "type": "integer", "example": 0 }, "REQUEST_QUEUE_READS": { "type": "integer", "example": 0 }, "REQUEST_QUEUE_WRITES": { "type": "integer", "example": 0 }, "DATA_TRANSFER_INTERNAL_GBYTES": { "type": "integer", "example": 0 }, "DATA_TRANSFER_EXTERNAL_GBYTES": { "type": "integer", "example": 0 }, "PROXY_RESIDENTIAL_TRANSFER_GBYTES": { "type": "integer", "example": 0 }, "PROXY_SERPS": { "type": "integer", "example": 0 }, "PROXY_UNBLOCKER_UNITS": { "type": "integer", "example": 0 } } }, "usageTotalUsd": { "type": "number", "example": 0.00005 }, "usageUsd": { "type": "object", "properties": { "ACTOR_COMPUTE_UNITS": { "type": "integer", "example": 0 }, "DATASET_READS": { "type": "integer", "example": 0 }, "DATASET_WRITES": { "type": "integer", "example": 0 }, "KEY_VALUE_STORE_READS": { "type": "integer", "example": 0 }, "KEY_VALUE_STORE_WRITES": { "type": "number", "example": 0.00005 }, "KEY_VALUE_STORE_LISTS": { "type": "integer", "example": 0 }, "REQUEST_QUEUE_READS": { "type": "integer", "example": 0 }, "REQUEST_QUEUE_WRITES": { "type": "integer", "example": 0 }, "DATA_TRANSFER_INTERNAL_GBYTES": { "type": "integer", "example": 0 }, "DATA_TRANSFER_EXTERNAL_GBYTES": { "type": "integer", "example": 0 }, "PROXY_RESIDENTIAL_TRANSFER_GBYTES": { "type": "integer", "example": 0 }, "PROXY_SERPS": { "type": "integer", "example": 0 }, "PROXY_UNBLOCKER_UNITS": { "type": "integer", "example": 0 } } } } } } } } } }