{ "openapi": "3.0.1", "info": { "title": "Sitemap URL Extractor: Every Page with Last-Modified Dates", "description": "Get every page a site lists in its sitemaps, with last-modified dates, from robots.txt through sitemap indexes and .gz files. Track new pages of competitors with Only new rows. robots.txt obeyed, no personal data, pay only per URL delivered.", "version": "1.0", "x-build-id": "CthEFgmW0PmYHg55J" }, "servers": [ { "url": "https://api.apify.com/v2" } ], "paths": { "/acts/rowcraft~sitemap-url-extractor/run-sync-get-dataset-items": { "post": { "operationId": "run-sync-get-dataset-items-rowcraft-sitemap-url-extractor", "x-openai-isConsequential": false, "summary": "Executes an Actor, waits for its completion, and returns Actor's dataset items in response.", "tags": [ "Run Actor" ], "requestBody": { "required": true, "content": { "application/json": { "schema": { "$ref": "#/components/schemas/inputSchema" } } } }, "parameters": [ { "name": "token", "in": "query", "required": true, "schema": { "type": "string" }, "description": "Enter your Apify token here" } ], "responses": { "200": { "description": "OK" } } } }, "/acts/rowcraft~sitemap-url-extractor/runs": { "post": { "operationId": "runs-sync-rowcraft-sitemap-url-extractor", "x-openai-isConsequential": false, "summary": "Executes an Actor and returns information about the initiated run in response.", "tags": [ "Run Actor" ], "requestBody": { "required": true, "content": { "application/json": { "schema": { "$ref": "#/components/schemas/inputSchema" } } } }, "parameters": [ { "name": "token", "in": "query", "required": true, "schema": { "type": "string" }, "description": "Enter your Apify token here" } ], "responses": { "200": { "description": "OK", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/runsResponseSchema" } } } } } } }, "/acts/rowcraft~sitemap-url-extractor/run-sync": { "post": { "operationId": "run-sync-rowcraft-sitemap-url-extractor", "x-openai-isConsequential": false, "summary": "Executes an Actor, waits for completion, and returns the OUTPUT from Key-value store in response.", "tags": [ "Run Actor" ], "requestBody": { "required": true, "content": { "application/json": { "schema": { "$ref": "#/components/schemas/inputSchema" } } } }, "parameters": [ { "name": "token", "in": "query", "required": true, "schema": { "type": "string" }, "description": "Enter your Apify token here" } ], "responses": { "200": { "description": "OK" } } } } }, "components": { "schemas": { "inputSchema": { "type": "object", "required": [ "startUrls" ], "properties": { "startUrls": { "title": "Domains, robots.txt or sitemap URLs", "type": "array", "description": "One per line. A domain (example.com) finds the sitemaps through robots.txt; a robots.txt address (https://example.com/robots.txt) reads the sitemaps it lists; a sitemap address (https://example.com/sitemap.xml or .xml.gz) is read directly. Duplicates and invalid entries are skipped and never charged.", "items": { "type": "string" } }, "discover": { "title": "Find sitemaps through robots.txt", "type": "boolean", "description": "For a domain or a robots.txt address: read the Sitemap lines of robots.txt, and when there are none try /sitemap.xml, /sitemap_index.xml and /wp-sitemap.xml. Turn off to try only /sitemap.xml.", "default": true }, "maxItems": { "title": "Maximum URLs delivered", "minimum": 0, "type": "integer", "description": "Stop after this many URLs are delivered. The default is a safe start; raise it, or use 0, to get everything. Set a Maximum total charge in the run options for a hard cost cap.", "default": 1000 }, "urlIncludes": { "title": "Keep only URLs that contain", "type": "array", "description": "Text that must appear in the path or query of the URL, for example /blog/ or /products/. Start a line with re: for a regular expression, for example re:/20(25|26)/. A URL is kept when it matches any line. Leave empty to keep all.", "items": { "type": "string" } }, "urlExcludes": { "title": "Drop URLs that contain", "type": "array", "description": "Text, or re: and a regular expression, matched in the path or query. A URL that matches any line is dropped and not charged, for example /tag/ or ?replytocom.", "items": { "type": "string" } }, "lastmodAfter": { "title": "Changed after", "type": "string", "description": "Keep only URLs whose last-modified date is on or after this day: YYYY-MM-DD, or a relative value such as 7 days or 2 weeks. URLs without a date are dropped, and sitemap files of an index whose own date is older are not opened." }, "maxUrlsPerSite": { "title": "Maximum URLs per site", "minimum": 0, "type": "integer", "description": "Stop reading a site after this many URLs (one entry of the list). 0 = no limit.", "default": 0 }, "maxSitemapFiles": { "title": "Maximum sitemap files per site", "minimum": 1, "maximum": 100000, "type": "integer", "description": "Stop after this many sitemap files of one site (an index can name thousands). Files inside indexes are followed up to 5 levels deep.", "default": 500 }, "countOnly": { "title": "Count only (free check)", "type": "boolean", "description": "Only count how many rows match your filters and estimate the cost at your price tier. Nothing is delivered and nothing is charged apart from the start event. Turn it off to get the data.", "default": false }, "onlyNew": { "title": "Only new rows since the last run", "type": "boolean", "description": "Remember the rows already delivered to you (in a key-value store in your Apify account) and deliver only rows that appeared since the last run with the same state key. The first run delivers everything that matches; later runs cost only the new rows. Ideal for schedules.", "default": false }, "stateName": { "title": "State name", "type": "string", "description": "Name of the memory used by \"Only new rows\". Use a different name for each schedule or search (for example `stockholm-python`) so they do not share one memory.", "default": "default" }, "notifyUrl": { "title": "Notify this URL when the run ends", "type": "string", "description": "A webhook address that receives one JSON POST with the number of new rows and links to the dataset: a Slack or Discord incoming webhook, a Make, n8n or Zapier catch hook, or your own endpoint. The message is also in the `text` (Slack) and `content` (Discord) fields." }, "notifyWhenEmpty": { "title": "Notify also when there are no new rows", "type": "boolean", "description": "By default the notification is sent only when at least one row was delivered.", "default": false }, "requestIntervalMs": { "title": "Delay between requests (ms)", "minimum": 250, "maximum": 10000, "type": "integer", "description": "Default 1000 ms (one request per second to the same host). Every sitemap file is one request, plus one robots.txt per host. Minimum 250.", "default": 1000 } } }, "runsResponseSchema": { "type": "object", "properties": { "data": { "type": "object", "properties": { "id": { "type": "string" }, "actId": { "type": "string" }, "userId": { "type": "string" }, "startedAt": { "type": "string", "format": "date-time", "example": "2025-01-08T00:00:00.000Z" }, "finishedAt": { "type": "string", "format": "date-time", "example": "2025-01-08T00:00:00.000Z" }, "status": { "type": "string", "example": "READY" }, "meta": { "type": "object", "properties": { "origin": { "type": "string", "example": "API" }, "userAgent": { "type": "string" } } }, "stats": { "type": "object", "properties": { "inputBodyLen": { "type": "integer", "example": 2000 }, "rebootCount": { "type": "integer", "example": 0 }, "restartCount": { "type": "integer", "example": 0 }, "resurrectCount": { "type": "integer", "example": 0 }, "computeUnits": { "type": "integer", "example": 0 } } }, "options": { "type": "object", "properties": { "build": { "type": "string", "example": "latest" }, "timeoutSecs": { "type": "integer", "example": 300 }, "memoryMbytes": { "type": "integer", "example": 1024 }, "diskMbytes": { "type": "integer", "example": 2048 } } }, "buildId": { "type": "string" }, "defaultKeyValueStoreId": { "type": "string" }, "defaultDatasetId": { "type": "string" }, "defaultRequestQueueId": { "type": "string" }, "buildNumber": { "type": "string", "example": "1.0.0" }, "containerUrl": { "type": "string" }, "usage": { "type": "object", "properties": { "ACTOR_COMPUTE_UNITS": { "type": "integer", "example": 0 }, "DATASET_READS": { "type": "integer", "example": 0 }, "DATASET_WRITES": { "type": "integer", "example": 0 }, "KEY_VALUE_STORE_READS": { "type": "integer", "example": 0 }, "KEY_VALUE_STORE_WRITES": { "type": "integer", "example": 1 }, "KEY_VALUE_STORE_LISTS": { "type": "integer", "example": 0 }, "REQUEST_QUEUE_READS": { "type": "integer", "example": 0 }, "REQUEST_QUEUE_WRITES": { "type": "integer", "example": 0 }, "DATA_TRANSFER_INTERNAL_GBYTES": { "type": "integer", "example": 0 }, "DATA_TRANSFER_EXTERNAL_GBYTES": { "type": "integer", "example": 0 }, "PROXY_RESIDENTIAL_TRANSFER_GBYTES": { "type": "integer", "example": 0 }, "PROXY_SERPS": { "type": "integer", "example": 0 }, "PROXY_UNBLOCKER_UNITS": { "type": "integer", "example": 0 } } }, "usageTotalUsd": { "type": "number", "example": 0.00005 }, "usageUsd": { "type": "object", "properties": { "ACTOR_COMPUTE_UNITS": { "type": "integer", "example": 0 }, "DATASET_READS": { "type": "integer", "example": 0 }, "DATASET_WRITES": { "type": "integer", "example": 0 }, "KEY_VALUE_STORE_READS": { "type": "integer", "example": 0 }, "KEY_VALUE_STORE_WRITES": { "type": "number", "example": 0.00005 }, "KEY_VALUE_STORE_LISTS": { "type": "integer", "example": 0 }, "REQUEST_QUEUE_READS": { "type": "integer", "example": 0 }, "REQUEST_QUEUE_WRITES": { "type": "integer", "example": 0 }, "DATA_TRANSFER_INTERNAL_GBYTES": { "type": "integer", "example": 0 }, "DATA_TRANSFER_EXTERNAL_GBYTES": { "type": "integer", "example": 0 }, "PROXY_RESIDENTIAL_TRANSFER_GBYTES": { "type": "integer", "example": 0 }, "PROXY_SERPS": { "type": "integer", "example": 0 }, "PROXY_UNBLOCKER_UNITS": { "type": "integer", "example": 0 } } } } } } } } } }