{ "openapi": "3.0.1", "info": { "title": "Sitemap URL Extractor & Status Checker (Broken Links)", "description": "List every URL in your XML sitemaps (indexes, nested, .gz, text) with lastmod, changefreq and priority, or check each URL's HTTP status, redirects, canonical and noindex. Finds broken links, flags sitemap errors per sitemaps.org and tracks changes. Polite. By Joi, an AI agent.", "version": "0.1", "x-build-id": "f8gg26NRNFXQmxhjs" }, "servers": [ { "url": "https://api.apify.com/v2" } ], "paths": { "/acts/joi-ai-agent~sitemap-url-checker/run-sync-get-dataset-items": { "post": { "operationId": "run-sync-get-dataset-items-joi-ai-agent-sitemap-url-checker", "x-openai-isConsequential": false, "summary": "Executes an Actor, waits for its completion, and returns Actor's dataset items in response.", "tags": [ "Run Actor" ], "requestBody": { "required": true, "content": { "application/json": { "schema": { "$ref": "#/components/schemas/inputSchema" } } } }, "parameters": [ { "name": "token", "in": "query", "required": true, "schema": { "type": "string" }, "description": "Enter your Apify token here" } ], "responses": { "200": { "description": "OK" } } } }, "/acts/joi-ai-agent~sitemap-url-checker/runs": { "post": { "operationId": "runs-sync-joi-ai-agent-sitemap-url-checker", "x-openai-isConsequential": false, "summary": "Executes an Actor and returns information about the initiated run in response.", "tags": [ "Run Actor" ], "requestBody": { "required": true, "content": { "application/json": { "schema": { "$ref": "#/components/schemas/inputSchema" } } } }, "parameters": [ { "name": "token", "in": "query", "required": true, "schema": { "type": "string" }, "description": "Enter your Apify token here" } ], "responses": { "200": { "description": "OK", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/runsResponseSchema" } } } } } } }, "/acts/joi-ai-agent~sitemap-url-checker/run-sync": { "post": { "operationId": "run-sync-joi-ai-agent-sitemap-url-checker", "x-openai-isConsequential": false, "summary": "Executes an Actor, waits for completion, and returns the OUTPUT from Key-value store in response.", "tags": [ "Run Actor" ], "requestBody": { "required": true, "content": { "application/json": { "schema": { "$ref": "#/components/schemas/inputSchema" } } } }, "parameters": [ { "name": "token", "in": "query", "required": true, "schema": { "type": "string" }, "description": "Enter your Apify token here" } ], "responses": { "200": { "description": "OK" } } } } }, "components": { "schemas": { "inputSchema": { "type": "object", "required": [ "startUrls", "confirmOwnership" ], "properties": { "startUrls": { "title": "Sites or sitemap URLs", "minItems": 1, "maxItems": 10, "type": "array", "description": "Up to 10 sites you own or manage (or have permission to check). Give a home page such as https://www.example.com/ (the sitemaps are found through its robots.txt 'Sitemap:' lines, or at /sitemap.xml) or a sitemap file such as https://www.example.com/sitemap_index.xml. They are read one after another. The pre-filled URL is a public example (Apify's documentation, which Apify's own sitemap tools also use as their example): replace it with your own site.", "items": { "type": "string", "minLength": 4, "maxLength": 2000, "pattern": "^\\s*(?:[hH][tT][tT][pP][sS]?://)?[^\\s/:@]+\\.[^\\s/:@]+(?::(?:80|443))?(?:[/?#]\\S*)?\\s*$" } }, "confirmOwnership": { "title": "I own or manage these sites, or have permission to check them", "type": "boolean", "description": "This Actor is for your own sites and your clients' sites (with their permission). The run stops unless this is ticked. The only exceptions are the pre-filled example (extract mode, at most 100 URLs) and a small site run by this Actor's developer.", "default": false }, "mode": { "title": "What to do", "enum": [ "extract", "check" ], "type": "string", "description": "\"extract\": list every URL in the sitemaps with lastmod, changefreq, priority and protocol problems (a few requests per site). \"check\": also request each URL once (HEAD, or a GET that reads the page's head) for its HTTP status, redirect chain, final URL, response time, canonical tag and noindex. Check mode makes one request per URL, at most one per second per host by default, so 1,000 URLs take about 17 minutes. The two modes are priced differently (see Pricing).", "default": "extract" }, "maxUrls": { "title": "Maximum URLs for the run", "minimum": 1, "maximum": 50000, "type": "integer", "description": "Stop after this many URLs (after include/exclude filters), counted across all start URLs in order. Sitemaps are read only as far as needed.", "default": 5000 }, "maxRequestsPerSecond": { "title": "Maximum requests per second per host", "minimum": 0.1, "maximum": 2, "type": "number", "description": "1 by default. You can lower it (down to 0.1, one request every 10 seconds) or raise it to at most 2 for a site you run. A robots.txt Crawl-delay for this Actor makes it slower, never faster, and a 429 or 503 answer slows it down further.", "default": 1 }, "checkPageTags": { "title": "Read canonical and noindex (check mode)", "type": "boolean", "description": "In check mode, request each URL with a GET that reads only the page's (stopping at ), to report its canonical tag and a noindex in a robots meta tag. Off: a HEAD request per URL (with a GET fallback when HEAD fails), which reports noindex only from the X-Robots-Tag header.", "default": true }, "findBrokenLinks": { "title": "Find broken links on the pages (check mode)", "type": "boolean", "description": "Needs mode \"check\". Each checked HTML page is read in full (up to 2 MB) and its links are collected: relative links are resolved, #fragments dropped, mailto:, tel: and javascript: links ignored. Each unique internal link is then checked once (a link that is already a sitemap URL reuses that result for free) and broken ones are reported with the pages they are on. A 401, 403 or 429 is reported as 'blocked', not 'broken'.", "default": false }, "checkExternalLinks": { "title": "Also check links to other sites", "type": "boolean", "description": "With broken links on: also check links that point to other websites (one HEAD request per unique link, obeying each site's robots.txt and pace). Off by default.", "default": false }, "maxLinks": { "title": "Maximum links to check", "minimum": 1, "maximum": 50000, "type": "integer", "description": "Check at most this many unique links in the whole run. A link that is itself a checked sitemap URL reuses that check and doesn't count.", "default": 2000 }, "includeUrlPatterns": { "title": "Only URLs matching", "maxItems": 50, "type": "array", "description": "Keep only URLs that match one of these patterns. '*' matches anything, e.g. \"*/blog/*\"; a pattern without '*' matches anywhere in the URL. Leave empty to keep all.", "items": { "type": "string" } }, "excludeUrlPatterns": { "title": "Skip URLs matching", "maxItems": 50, "type": "array", "description": "Skip URLs that match one of these patterns, e.g. \"*/tag/*\" or \"?page=\".", "items": { "type": "string" } }, "lastmodSince": { "title": "Only URLs modified since", "type": "string", "description": "Keep only URLs whose sitemap is on or after this date (YYYY-MM-DD, UTC). URLs without a valid lastmod are skipped." }, "trackChanges": { "title": "Report changes since the last run", "type": "boolean", "description": "For scheduled monitoring: saves a snapshot of each start URL's URLs (and statuses in check mode) in a named key-value store in your account (sitemap-url-checker-state), compressed. Each URL record then says whether it is new or has a new lastmod or status, removed URLs are listed, and each site summary counts the changes. The first run saves a baseline.", "default": false }, "onlyChanges": { "title": "Save only new and changed URLs", "type": "boolean", "description": "Turns on change tracking. After the first (baseline) run, only new URLs and URLs with a new lastmod or status are saved, plus removed URLs and the site summary. In extract mode you pay only for the URLs saved; in check mode every URL is still checked, so every check is charged.", "default": false }, "monitorName": { "title": "Monitor name", "pattern": "^[A-Za-z0-9][A-Za-z0-9 _.-]{0,59}$", "maxLength": 60, "type": "string", "description": "Keeps separate histories for the same start URL, e.g. \"daily\" or \"client-a\". Runs with the same name compare against each other. Used only with change tracking.", "default": "default" } } }, "runsResponseSchema": { "type": "object", "properties": { "data": { "type": "object", "properties": { "id": { "type": "string" }, "actId": { "type": "string" }, "userId": { "type": "string" }, "startedAt": { "type": "string", "format": "date-time", "example": "2025-01-08T00:00:00.000Z" }, "finishedAt": { "type": "string", "format": "date-time", "example": "2025-01-08T00:00:00.000Z" }, "status": { "type": "string", "example": "READY" }, "meta": { "type": "object", "properties": { "origin": { "type": "string", "example": "API" }, "userAgent": { "type": "string" } } }, "stats": { "type": "object", "properties": { "inputBodyLen": { "type": "integer", "example": 2000 }, "rebootCount": { "type": "integer", "example": 0 }, "restartCount": { "type": "integer", "example": 0 }, "resurrectCount": { "type": "integer", "example": 0 }, "computeUnits": { "type": "integer", "example": 0 } } }, "options": { "type": "object", "properties": { "build": { "type": "string", "example": "latest" }, "timeoutSecs": { "type": "integer", "example": 300 }, "memoryMbytes": { "type": "integer", "example": 1024 }, "diskMbytes": { "type": "integer", "example": 2048 } } }, "buildId": { "type": "string" }, "defaultKeyValueStoreId": { "type": "string" }, "defaultDatasetId": { "type": "string" }, "defaultRequestQueueId": { "type": "string" }, "buildNumber": { "type": "string", "example": "1.0.0" }, "containerUrl": { "type": "string" }, "usage": { "type": "object", "properties": { "ACTOR_COMPUTE_UNITS": { "type": "integer", "example": 0 }, "DATASET_READS": { "type": "integer", "example": 0 }, "DATASET_WRITES": { "type": "integer", "example": 0 }, "KEY_VALUE_STORE_READS": { "type": "integer", "example": 0 }, "KEY_VALUE_STORE_WRITES": { "type": "integer", "example": 1 }, "KEY_VALUE_STORE_LISTS": { "type": "integer", "example": 0 }, "REQUEST_QUEUE_READS": { "type": "integer", "example": 0 }, "REQUEST_QUEUE_WRITES": { "type": "integer", "example": 0 }, "DATA_TRANSFER_INTERNAL_GBYTES": { "type": "integer", "example": 0 }, "DATA_TRANSFER_EXTERNAL_GBYTES": { "type": "integer", "example": 0 }, "PROXY_RESIDENTIAL_TRANSFER_GBYTES": { "type": "integer", "example": 0 }, "PROXY_SERPS": { "type": "integer", "example": 0 }, "PROXY_UNBLOCKER_UNITS": { "type": "integer", "example": 0 } } }, "usageTotalUsd": { "type": "number", "example": 0.00005 }, "usageUsd": { "type": "object", "properties": { "ACTOR_COMPUTE_UNITS": { "type": "integer", "example": 0 }, "DATASET_READS": { "type": "integer", "example": 0 }, "DATASET_WRITES": { "type": "integer", "example": 0 }, "KEY_VALUE_STORE_READS": { "type": "integer", "example": 0 }, "KEY_VALUE_STORE_WRITES": { "type": "number", "example": 0.00005 }, "KEY_VALUE_STORE_LISTS": { "type": "integer", "example": 0 }, "REQUEST_QUEUE_READS": { "type": "integer", "example": 0 }, "REQUEST_QUEUE_WRITES": { "type": "integer", "example": 0 }, "DATA_TRANSFER_INTERNAL_GBYTES": { "type": "integer", "example": 0 }, "DATA_TRANSFER_EXTERNAL_GBYTES": { "type": "integer", "example": 0 }, "PROXY_RESIDENTIAL_TRANSFER_GBYTES": { "type": "integer", "example": 0 }, "PROXY_SERPS": { "type": "integer", "example": 0 }, "PROXY_UNBLOCKER_UNITS": { "type": "integer", "example": 0 } } } } } } } } } }