{
  "openapi": "3.0.1",
  "info": {
    "title": "AI Crawler & Content-Signal Permission Delta Monitor",
    "description": "Watches any domain and reports the instant its AI-crawler permissions change: a robots.txt Allow/Disallow flip for GPTBot, ClaudeBot and 15 other tracked tokens, a Cloudflare Content-Signal flip, or llms.txt appearing/changing. Pay-per-event: billed only for what changed, with a free baseline audit.",
    "version": "1.0",
    "x-build-id": "rkWahJmh3T0mAifbS"
  },
  "servers": [
    {
      "url": "https://api.apify.com/v2"
    }
  ],
  "paths": {
    "/acts/stefano_seggio~ai-crawler-content-signal-permission-monitor/run-sync-get-dataset-items": {
      "post": {
        "operationId": "run-sync-get-dataset-items-stefano_seggio-ai-crawler-content-signal-permission-monitor",
        "x-openai-isConsequential": false,
        "summary": "Executes an Actor, waits for its completion, and returns Actor's dataset items in response.",
        "tags": [
          "Run Actor"
        ],
        "requestBody": {
          "required": true,
          "content": {
            "application/json": {
              "schema": {
                "$ref": "#/components/schemas/inputSchema"
              }
            }
          }
        },
        "parameters": [
          {
            "name": "token",
            "in": "query",
            "required": true,
            "schema": {
              "type": "string"
            },
            "description": "Enter your Apify token here"
          }
        ],
        "responses": {
          "200": {
            "description": "OK"
          }
        }
      }
    },
    "/acts/stefano_seggio~ai-crawler-content-signal-permission-monitor/runs": {
      "post": {
        "operationId": "runs-sync-stefano_seggio-ai-crawler-content-signal-permission-monitor",
        "x-openai-isConsequential": false,
        "summary": "Executes an Actor and returns information about the initiated run in response.",
        "tags": [
          "Run Actor"
        ],
        "requestBody": {
          "required": true,
          "content": {
            "application/json": {
              "schema": {
                "$ref": "#/components/schemas/inputSchema"
              }
            }
          }
        },
        "parameters": [
          {
            "name": "token",
            "in": "query",
            "required": true,
            "schema": {
              "type": "string"
            },
            "description": "Enter your Apify token here"
          }
        ],
        "responses": {
          "200": {
            "description": "OK",
            "content": {
              "application/json": {
                "schema": {
                  "$ref": "#/components/schemas/runsResponseSchema"
                }
              }
            }
          }
        }
      }
    },
    "/acts/stefano_seggio~ai-crawler-content-signal-permission-monitor/run-sync": {
      "post": {
        "operationId": "run-sync-stefano_seggio-ai-crawler-content-signal-permission-monitor",
        "x-openai-isConsequential": false,
        "summary": "Executes an Actor, waits for completion, and returns the OUTPUT from Key-value store in response.",
        "tags": [
          "Run Actor"
        ],
        "requestBody": {
          "required": true,
          "content": {
            "application/json": {
              "schema": {
                "$ref": "#/components/schemas/inputSchema"
              }
            }
          }
        },
        "parameters": [
          {
            "name": "token",
            "in": "query",
            "required": true,
            "schema": {
              "type": "string"
            },
            "description": "Enter your Apify token here"
          }
        ],
        "responses": {
          "200": {
            "description": "OK"
          }
        }
      }
    }
  },
  "components": {
    "schemas": {
      "inputSchema": {
        "type": "object",
        "required": [
          "domains"
        ],
        "properties": {
          "domains": {
            "title": "Domains to watch",
            "type": "array",
            "description": "Bare hostnames to monitor - no protocol, no path, no trailing slash. Example: \"cloudflare.com\", \"openai.com\". Each domain is checked at its own https://{domain}/robots.txt (and, if enabled, /llms.txt and /llms-full.txt). Mix your own domains with competitor/vendor domains freely - each gets fully independent delta state.",
            "default": [],
            "items": {
              "type": "string"
            }
          },
          "trackedBots": {
            "title": "AI-crawler user-agent tokens to track",
            "type": "array",
            "description": "The robots.txt User-agent tokens whose Allow/Disallow directives are watched for changes over time. Defaults to 18 currently-documented AI-crawler tokens spanning OpenAI, Anthropic, Google, Apple, Perplexity, Common Crawl, ByteDance, Amazon, Meta, Diffbot, and Cohere. Add a newly-published crawler token, or narrow this to one vendor family, by editing this list.",
            "default": [
              "GPTBot",
              "ChatGPT-User",
              "OAI-SearchBot",
              "ClaudeBot",
              "Claude-User",
              "Claude-SearchBot",
              "Google-Extended",
              "Applebot-Extended",
              "PerplexityBot",
              "Perplexity-User",
              "CCBot",
              "Bytespider",
              "Amazonbot",
              "Meta-ExternalAgent",
              "Meta-ExternalFetcher",
              "Diffbot",
              "cohere-ai",
              "omgilibot"
            ],
            "items": {
              "type": "string"
            }
          },
          "checkContentSignals": {
            "title": "Check Cloudflare Content-Signal directives",
            "type": "boolean",
            "description": "Parses any \"Content-Signal: search=..., ai-input=..., ai-train=...\" line found in robots.txt (Cloudflare's proposed extension to the standard, formalized as the IETF draft draft-romm-aipref-contentsignals) and tracks all three category values for change over time. Turn this off only if you exclusively care about classic robots.txt Allow/Disallow directives.",
            "default": true
          },
          "checkLlmsTxt": {
            "title": "Check /llms.txt and /llms-full.txt",
            "type": "boolean",
            "description": "Fetches both well-known paths (llmstxt.org's proposed convention) and tracks presence/absence plus a content hash, so an added, removed, or edited llms.txt is reported as a CHANGED event. llms.txt is an informal community proposal, not a ratified standard - this Actor reports its presence and content as a signal, never as a compliance guarantee.",
            "default": true
          },
          "onlyNew": {
            "title": "Only deliver changed events on repeat runs",
            "type": "boolean",
            "description": "When true (default), a repeat run of an already-baselined domain only delivers ALLOWED, DISALLOWED, and CHANGED events - the free NO_CHANGE row is suppressed when nothing differs. A domain's first-ever check always delivers a free BASELINE_SNAPSHOT regardless of this setting, since that is the point of adding a domain to the watchlist. Set to false to also see an explicit \"nothing changed\" row on every run.",
            "default": true
          },
          "maxDomainsPerRun": {
            "title": "Max domains this run",
            "minimum": 1,
            "maximum": 30,
            "type": "integer",
            "description": "Hard cap on how many domains from the list are checked (and, where applicable, charged) in a single run. Domains beyond this cap are simply checked on the next run - nothing about their state is lost or marked as seen. Capped at 30 so this Actor's own worst case fits inside its 300-second default run timeout with real margin: with default concurrency (15) and default retry settings (requestTimeoutSecs=15, maxRetries=4), one domain's robots.txt/llms.txt/llms-full.txt fetches (run concurrently) can together take up to ~91s if every one of them needs its full 5-attempt retry budget, so a full run of up to 30 domains (2 concurrency waves) has a worst case of ~182s - about 61% of the 300s timeout. Raising this above 30 without also raising this Actor's run timeout (via the API's \"timeout\" run option or a Console/schedule override, roughly +91s per extra 15 domains) risks the run being killed mid-walk on a bad day; that is safe (no state is lost or double-committed) but wastes the retry compute already spent on the domains that did not finish.",
            "default": 30
          },
          "concurrency": {
            "title": "Max simultaneous in-flight requests",
            "minimum": 1,
            "maximum": 50,
            "type": "integer",
            "description": "Caps the total number of in-flight HTTP requests across the whole run. Each domain fans out to up to three independent resource fetches (robots.txt, llms.txt, llms-full.txt), so this bounds total concurrent load across every domain being checked, not per-domain concurrency. Raise cautiously - most of the wall-clock cost of a large watchlist run is waiting on slow or unresponsive third-party domains, not this Actor's own compute.",
            "default": 15
          },
          "requestTimeoutSecs": {
            "title": "Per-request timeout (seconds)",
            "minimum": 5,
            "maximum": 60,
            "type": "integer",
            "description": "Aborts a single robots.txt/llms.txt/llms-full.txt request that takes longer than this, so one slow or unresponsive domain never stalls the whole run. Applies per individual attempt, not per domain overall.",
            "default": 15
          },
          "maxRetries": {
            "title": "Max retry attempts per request",
            "minimum": 0,
            "maximum": 8,
            "type": "integer",
            "description": "Retry attempts (exponential backoff with jitter, capped at 15 seconds between attempts) before a single request is given up on as failed. Applies only to retryable outcomes (429 and 5xx responses, or a request timeout) - any other non-2xx, non-404 response is treated as final immediately and does not consume retry budget. A 404 is never a failure: it is parsed as a valid \"resource absent\" signal.",
            "default": 4
          },
          "deltaStateName": {
            "title": "Delta state name",
            "type": "string",
            "description": "Names this run's persistent delta-state store. Use a distinct name per independent schedule (for example \"own-sites\" versus \"competitor-watchlist\") so their baselines never cross-contaminate each other.",
            "default": "default"
          },
          "resetState": {
            "title": "Reset state",
            "type": "boolean",
            "description": "Wipes all remembered state for every domain under this deltaStateName before the run starts, so every domain is re-baselined from scratch on this run (delivered as free BASELINE_SNAPSHOT records, subject to onlyNew's usual free-baseline behavior).",
            "default": false
          }
        }
      },
      "runsResponseSchema": {
        "type": "object",
        "properties": {
          "data": {
            "type": "object",
            "properties": {
              "id": {
                "type": "string"
              },
              "actId": {
                "type": "string"
              },
              "userId": {
                "type": "string"
              },
              "startedAt": {
                "type": "string",
                "format": "date-time",
                "example": "2025-01-08T00:00:00.000Z"
              },
              "finishedAt": {
                "type": "string",
                "format": "date-time",
                "example": "2025-01-08T00:00:00.000Z"
              },
              "status": {
                "type": "string",
                "example": "READY"
              },
              "meta": {
                "type": "object",
                "properties": {
                  "origin": {
                    "type": "string",
                    "example": "API"
                  },
                  "userAgent": {
                    "type": "string"
                  }
                }
              },
              "stats": {
                "type": "object",
                "properties": {
                  "inputBodyLen": {
                    "type": "integer",
                    "example": 2000
                  },
                  "rebootCount": {
                    "type": "integer",
                    "example": 0
                  },
                  "restartCount": {
                    "type": "integer",
                    "example": 0
                  },
                  "resurrectCount": {
                    "type": "integer",
                    "example": 0
                  },
                  "computeUnits": {
                    "type": "integer",
                    "example": 0
                  }
                }
              },
              "options": {
                "type": "object",
                "properties": {
                  "build": {
                    "type": "string",
                    "example": "latest"
                  },
                  "timeoutSecs": {
                    "type": "integer",
                    "example": 300
                  },
                  "memoryMbytes": {
                    "type": "integer",
                    "example": 1024
                  },
                  "diskMbytes": {
                    "type": "integer",
                    "example": 2048
                  }
                }
              },
              "buildId": {
                "type": "string"
              },
              "defaultKeyValueStoreId": {
                "type": "string"
              },
              "defaultDatasetId": {
                "type": "string"
              },
              "defaultRequestQueueId": {
                "type": "string"
              },
              "buildNumber": {
                "type": "string",
                "example": "1.0.0"
              },
              "containerUrl": {
                "type": "string"
              },
              "usage": {
                "type": "object",
                "properties": {
                  "ACTOR_COMPUTE_UNITS": {
                    "type": "integer",
                    "example": 0
                  },
                  "DATASET_READS": {
                    "type": "integer",
                    "example": 0
                  },
                  "DATASET_WRITES": {
                    "type": "integer",
                    "example": 0
                  },
                  "KEY_VALUE_STORE_READS": {
                    "type": "integer",
                    "example": 0
                  },
                  "KEY_VALUE_STORE_WRITES": {
                    "type": "integer",
                    "example": 1
                  },
                  "KEY_VALUE_STORE_LISTS": {
                    "type": "integer",
                    "example": 0
                  },
                  "REQUEST_QUEUE_READS": {
                    "type": "integer",
                    "example": 0
                  },
                  "REQUEST_QUEUE_WRITES": {
                    "type": "integer",
                    "example": 0
                  },
                  "DATA_TRANSFER_INTERNAL_GBYTES": {
                    "type": "integer",
                    "example": 0
                  },
                  "DATA_TRANSFER_EXTERNAL_GBYTES": {
                    "type": "integer",
                    "example": 0
                  },
                  "PROXY_RESIDENTIAL_TRANSFER_GBYTES": {
                    "type": "integer",
                    "example": 0
                  },
                  "PROXY_SERPS": {
                    "type": "integer",
                    "example": 0
                  }
                }
              },
              "usageTotalUsd": {
                "type": "number",
                "example": 0.00005
              },
              "usageUsd": {
                "type": "object",
                "properties": {
                  "ACTOR_COMPUTE_UNITS": {
                    "type": "integer",
                    "example": 0
                  },
                  "DATASET_READS": {
                    "type": "integer",
                    "example": 0
                  },
                  "DATASET_WRITES": {
                    "type": "integer",
                    "example": 0
                  },
                  "KEY_VALUE_STORE_READS": {
                    "type": "integer",
                    "example": 0
                  },
                  "KEY_VALUE_STORE_WRITES": {
                    "type": "number",
                    "example": 0.00005
                  },
                  "KEY_VALUE_STORE_LISTS": {
                    "type": "integer",
                    "example": 0
                  },
                  "REQUEST_QUEUE_READS": {
                    "type": "integer",
                    "example": 0
                  },
                  "REQUEST_QUEUE_WRITES": {
                    "type": "integer",
                    "example": 0
                  },
                  "DATA_TRANSFER_INTERNAL_GBYTES": {
                    "type": "integer",
                    "example": 0
                  },
                  "DATA_TRANSFER_EXTERNAL_GBYTES": {
                    "type": "integer",
                    "example": 0
                  },
                  "PROXY_RESIDENTIAL_TRANSFER_GBYTES": {
                    "type": "integer",
                    "example": 0
                  },
                  "PROXY_SERPS": {
                    "type": "integer",
                    "example": 0
                  }
                }
              }
            }
          }
        }
      }
    }
  }
}