{ "openapi": "3.0.1", "info": { "title": "Katana Web Crawler (ProjectDiscovery)", "description": "Crawl websites with ProjectDiscovery Katana and stream results as structured JSONL into an Apify dataset.", "version": "0.1", "x-build-id": "bdAJGCRUcrkFLPkvm" }, "servers": [ { "url": "https://api.apify.com/v2" } ], "paths": { "/acts/rl1987~pd-katana/run-sync-get-dataset-items": { "post": { "operationId": "run-sync-get-dataset-items-rl1987-pd-katana", "x-openai-isConsequential": false, "summary": "Executes an Actor, waits for its completion, and returns Actor's dataset items in response.", "tags": [ "Run Actor" ], "requestBody": { "required": true, "content": { "application/json": { "schema": { "$ref": "#/components/schemas/inputSchema" } } } }, "parameters": [ { "name": "token", "in": "query", "required": true, "schema": { "type": "string" }, "description": "Enter your Apify token here" } ], "responses": { "200": { "description": "OK" } } } }, "/acts/rl1987~pd-katana/runs": { "post": { "operationId": "runs-sync-rl1987-pd-katana", "x-openai-isConsequential": false, "summary": "Executes an Actor and returns information about the initiated run in response.", "tags": [ "Run Actor" ], "requestBody": { "required": true, "content": { "application/json": { "schema": { "$ref": "#/components/schemas/inputSchema" } } } }, "parameters": [ { "name": "token", "in": "query", "required": true, "schema": { "type": "string" }, "description": "Enter your Apify token here" } ], "responses": { "200": { "description": "OK", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/runsResponseSchema" } } } } } } }, "/acts/rl1987~pd-katana/run-sync": { "post": { "operationId": "run-sync-rl1987-pd-katana", "x-openai-isConsequential": false, "summary": "Executes an Actor, waits for completion, and returns the OUTPUT from Key-value store in response.", "tags": [ "Run Actor" ], "requestBody": { "required": true, "content": { "application/json": { "schema": { "$ref": "#/components/schemas/inputSchema" } } } }, "parameters": [ { "name": "token", "in": "query", "required": true, "schema": { "type": "string" }, "description": "Enter your Apify token here" } ], "responses": { "200": { "description": "OK" } } } } }, "components": { "schemas": { "inputSchema": { "type": "object", "required": [ "urls" ], "properties": { "urls": { "title": "Target URLs", "type": "array", "description": "One or more target URLs / domains to crawl. Each entry is passed to Katana as a seed (-u).", "items": { "type": "string" } }, "exclude": { "title": "Exclude hosts", "type": "array", "description": "Exclude hosts matching a filter: 'cdn', 'private-ips', a CIDR, an IP, or a regex (Katana -exclude).", "items": { "type": "string" } }, "depth": { "title": "Crawl depth", "minimum": 1, "maximum": 100, "type": "integer", "description": "Maximum depth to crawl (Katana -depth).", "default": 3 }, "crawlDuration": { "title": "Max crawl duration (seconds)", "minimum": 0, "type": "integer", "description": "Maximum time to crawl per target in seconds (Katana -crawl-duration). 0 = unlimited.", "default": 0 }, "strategy": { "title": "Visit strategy", "enum": [ "depth-first", "breadth-first" ], "type": "string", "description": "Traversal order (Katana -strategy).", "default": "depth-first" }, "jsCrawl": { "title": "JavaScript crawling", "type": "boolean", "description": "Parse & crawl endpoints inside JavaScript files (Katana -js-crawl).", "default": false }, "jsluice": { "title": "Jsluice parsing", "type": "boolean", "description": "Enable jsluice parsing in JavaScript files. Memory intensive (Katana -jsluice).", "default": false }, "knownFiles": { "title": "Crawl known files", "enum": [ "none", "all", "robotstxt", "sitemapxml" ], "type": "string", "description": "Crawl known files such as robots.txt and sitemap.xml (requires depth >= 3).", "default": "none" }, "automaticFormFill": { "title": "Automatic form fill", "type": "boolean", "description": "Automatically fill and submit forms while crawling. Experimental (Katana -automatic-form-fill).", "default": false }, "formExtraction": { "title": "Extract forms", "type": "boolean", "description": "Extract form, input, textarea & select elements into output (Katana -form-extraction).", "default": false }, "techDetect": { "title": "Technology detection", "type": "boolean", "description": "Enable technology detection on crawled responses (Katana -tech-detect).", "default": false }, "knowledgeBase": { "title": "Knowledge base classification", "type": "boolean", "description": "Enable knowledge base classification (Katana -knowledge-base).", "default": false }, "pathClimb": { "title": "Path climb", "type": "boolean", "description": "Auto-crawl parent paths of discovered URLs (Katana -path-climb).", "default": false }, "ignoreQueryParams": { "title": "Ignore query params", "type": "boolean", "description": "Ignore crawling the same path with different query-param values (Katana -ignore-query-params).", "default": false }, "filterSimilar": { "title": "Filter similar URLs", "type": "boolean", "description": "Filter crawling of similar-looking URLs, e.g. /users/123 and /users/456 (Katana -filter-similar).", "default": false }, "filterSimilarThreshold": { "title": "Similar filter threshold", "minimum": 0, "type": "integer", "description": "Distinct values before a path position is treated as a parameter (Katana -filter-similar-threshold). 0 = default.", "default": 0 }, "disableRedirects": { "title": "Disable redirects", "type": "boolean", "description": "Do not follow HTTP redirects (Katana -disable-redirects).", "default": false }, "tlsImpersonate": { "title": "TLS impersonation (JA3)", "type": "boolean", "description": "Enable experimental client-hello (JA3) TLS randomization (Katana -tls-impersonate).", "default": false }, "retry": { "title": "Retries", "minimum": 0, "type": "integer", "description": "Number of times to retry a failed request (Katana -retry). 0 = Katana default.", "default": 0 }, "timeout": { "title": "Request timeout (seconds)", "minimum": 1, "type": "integer", "description": "Time to wait for a request in seconds (Katana -timeout).", "default": 10 }, "timeStable": { "title": "Page stable wait (seconds)", "minimum": 0, "type": "integer", "description": "Time to wait until the page is stable, headless mode (Katana -time-stable). 0 = default.", "default": 0 }, "maxResponseSize": { "title": "Max response size (bytes)", "minimum": 0, "type": "integer", "description": "Maximum response size to read (Katana -max-response-size). 0 = Katana default (4 MB).", "default": 0 }, "headers": { "title": "Custom headers", "type": "array", "description": "Custom headers/cookies for every request, in 'Name: value' format (Katana -headers).", "items": { "type": "string" } }, "resolvers": { "title": "Custom DNS resolvers", "type": "array", "description": "Custom DNS resolvers, e.g. 1.1.1.1, 8.8.8.8 (Katana -resolvers).", "items": { "type": "string" } }, "headless": { "title": "Headless mode", "type": "boolean", "description": "Enable headless Chromium crawling. Slower but handles JS-heavy sites (Katana -headless).", "default": false }, "hybrid": { "title": "Hybrid crawling", "type": "boolean", "description": "Enable headless hybrid crawling combining standard and headless (Katana -hybrid).", "default": false }, "xhrExtraction": { "title": "XHR extraction", "type": "boolean", "description": "Extract XHR request URL & method into output (Katana -xhr-extraction).", "default": false }, "noIncognito": { "title": "No incognito", "type": "boolean", "description": "Start headless Chrome without incognito mode (Katana -no-incognito).", "default": false }, "headlessOptions": { "title": "Headless Chrome options", "type": "array", "description": "Additional flags passed to headless Chrome (Katana -headless-options).", "items": { "type": "string" } }, "chromeWsUrl": { "title": "Remote Chrome WS URL", "type": "string", "description": "Connect to a Chrome instance launched elsewhere via its debugger WebSocket URL (Katana -chrome-ws-url)." }, "captchaSolverProvider": { "title": "Captcha solver provider", "type": "string", "description": "Captcha solver provider, e.g. capsolver (Katana -captcha-solver-provider)." }, "captchaSolverKey": { "title": "Captcha solver API key", "type": "string", "description": "API key for the captcha solver provider (Katana -captcha-solver-key)." }, "fieldScope": { "title": "Field scope", "type": "string", "description": "Pre-defined scope (dn, rdn, fqdn) or a custom regex (Katana -field-scope).", "default": "rdn" }, "crawlScope": { "title": "In-scope regexes", "type": "array", "description": "Regexes of URLs that are in scope and should be followed (Katana -crawl-scope).", "items": { "type": "string" } }, "crawlOutScope": { "title": "Out-of-scope regexes", "type": "array", "description": "Regexes of URLs that are out of scope and should not be followed (Katana -crawl-out-scope).", "items": { "type": "string" } }, "noScope": { "title": "Disable scope", "type": "boolean", "description": "Disable the host-based default scope, crawling everything reachable (Katana -no-scope).", "default": false }, "displayOutScope": { "title": "Display out-of-scope", "type": "boolean", "description": "Display external endpoints found during scoped crawling (Katana -display-out-scope).", "default": false }, "matchRegex": { "title": "Match output regexes", "type": "array", "description": "Only output URLs matching these regexes (Katana -match-regex).", "items": { "type": "string" } }, "filterRegex": { "title": "Filter output regexes", "type": "array", "description": "Drop output URLs matching these regexes (Katana -filter-regex).", "items": { "type": "string" } }, "extensionMatch": { "title": "Match extensions", "type": "array", "description": "Only output URLs with these extensions, e.g. php, html, js (Katana -extension-match).", "items": { "type": "string" } }, "extensionFilter": { "title": "Filter extensions", "type": "array", "description": "Drop output URLs with these extensions, e.g. png, css (Katana -extension-filter).", "items": { "type": "string" } }, "noDefaultExtFilter": { "title": "No default extension filter", "type": "boolean", "description": "Remove the built-in default extension filter list (Katana -no-default-ext-filter).", "default": false }, "matchCondition": { "title": "Match condition (DSL)", "type": "string", "description": "Match responses with a DSL-based condition (Katana -match-condition)." }, "filterCondition": { "title": "Filter condition (DSL)", "type": "string", "description": "Filter responses with a DSL-based condition (Katana -filter-condition)." }, "filterPageType": { "title": "Filter page types", "type": "array", "description": "Filter responses by page type: error, captcha, parked (Katana -filter-page-type).", "items": { "type": "string" } }, "disableUniqueFilter": { "title": "Disable unique filter", "type": "boolean", "description": "Disable duplicate-content filtering (Katana -disable-unique-filter).", "default": false }, "concurrency": { "title": "Concurrency", "minimum": 1, "type": "integer", "description": "Number of concurrent fetchers (Katana -concurrency).", "default": 10 }, "parallelism": { "title": "Parallelism", "minimum": 1, "type": "integer", "description": "Number of inputs processed in parallel (Katana -parallelism).", "default": 10 }, "rateLimit": { "title": "Rate limit (req/sec)", "minimum": 1, "type": "integer", "description": "Maximum requests to send per second (Katana -rate-limit).", "default": 150 }, "rateLimitMinute": { "title": "Rate limit (req/min)", "minimum": 0, "type": "integer", "description": "Maximum requests to send per minute (Katana -rate-limit-minute). 0 = disabled.", "default": 0 }, "delay": { "title": "Request delay (seconds)", "minimum": 0, "type": "integer", "description": "Delay between each request in seconds (Katana -delay). 0 = none.", "default": 0 }, "proxyConfiguration": { "title": "Proxy configuration", "type": "object", "description": "Proxy to route Katana's requests through. Use Apify Proxy (datacenter or residential), or supply your own proxy URLs. Leave 'Use Apify Proxy' off and empty for no proxy.", "default": { "useApifyProxy": false } }, "proxyUrl": { "title": "Proxy URL (manual override)", "type": "string", "description": "HTTP/SOCKS5 proxy, e.g. http://user:pass@host:port (Katana -proxy). Takes precedence over Proxy configuration when set. Leave empty to use Proxy configuration instead." }, "flattenOutput": { "title": "Flatten output", "type": "boolean", "description": "Flatten each record's nested request/response objects into dotted top-level fields (e.g. response.status_code) so every field shows as its own dataset column. Disable to keep the raw nested Katana JSON.", "default": true }, "omitBody": { "title": "Omit response body", "type": "boolean", "description": "Omit the response body from JSONL output to reduce dataset size (Katana -omit-body).", "default": true }, "omitRaw": { "title": "Omit raw request/response", "type": "boolean", "description": "Omit raw request/response dumps from JSONL output (Katana -omit-raw).", "default": true }, "storeField": { "title": "Store field", "type": "string", "description": "Additional per-host field to store, e.g. url,path,fqdn,rdn (Katana -store-field)." }, "excludeOutputFields": { "title": "Exclude output fields", "type": "array", "description": "Fields to exclude from the JSONL output records (Katana -exclude-output-fields).", "items": { "type": "string" } }, "verbose": { "title": "Verbose logs", "type": "boolean", "description": "Enable verbose Katana logging (Katana -verbose). Logs go to the run log, not the dataset.", "default": false }, "extraArgs": { "title": "Extra Katana arguments", "type": "array", "description": "Additional raw Katana CLI flags for anything not exposed above, passed verbatim, e.g. [\"-time-stable\", \"2\"].", "items": { "type": "string" } } } }, "runsResponseSchema": { "type": "object", "properties": { "data": { "type": "object", "properties": { "id": { "type": "string" }, "actId": { "type": "string" }, "userId": { "type": "string" }, "startedAt": { "type": "string", "format": "date-time", "example": "2025-01-08T00:00:00.000Z" }, "finishedAt": { "type": "string", "format": "date-time", "example": "2025-01-08T00:00:00.000Z" }, "status": { "type": "string", "example": "READY" }, "meta": { "type": "object", "properties": { "origin": { "type": "string", "example": "API" }, "userAgent": { "type": "string" } } }, "stats": { "type": "object", "properties": { "inputBodyLen": { "type": "integer", "example": 2000 }, "rebootCount": { "type": "integer", "example": 0 }, "restartCount": { "type": "integer", "example": 0 }, "resurrectCount": { "type": "integer", "example": 0 }, "computeUnits": { "type": "integer", "example": 0 } } }, "options": { "type": "object", "properties": { "build": { "type": "string", "example": "latest" }, "timeoutSecs": { "type": "integer", "example": 300 }, "memoryMbytes": { "type": "integer", "example": 1024 }, "diskMbytes": { "type": "integer", "example": 2048 } } }, "buildId": { "type": "string" }, "defaultKeyValueStoreId": { "type": "string" }, "defaultDatasetId": { "type": "string" }, "defaultRequestQueueId": { "type": "string" }, "buildNumber": { "type": "string", "example": "1.0.0" }, "containerUrl": { "type": "string" }, "usage": { "type": "object", "properties": { "ACTOR_COMPUTE_UNITS": { "type": "integer", "example": 0 }, "DATASET_READS": { "type": "integer", "example": 0 }, "DATASET_WRITES": { "type": "integer", "example": 0 }, "KEY_VALUE_STORE_READS": { "type": "integer", "example": 0 }, "KEY_VALUE_STORE_WRITES": { "type": "integer", "example": 1 }, "KEY_VALUE_STORE_LISTS": { "type": "integer", "example": 0 }, "REQUEST_QUEUE_READS": { "type": "integer", "example": 0 }, "REQUEST_QUEUE_WRITES": { "type": "integer", "example": 0 }, "DATA_TRANSFER_INTERNAL_GBYTES": { "type": "integer", "example": 0 }, "DATA_TRANSFER_EXTERNAL_GBYTES": { "type": "integer", "example": 0 }, "PROXY_RESIDENTIAL_TRANSFER_GBYTES": { "type": "integer", "example": 0 }, "PROXY_SERPS": { "type": "integer", "example": 0 } } }, "usageTotalUsd": { "type": "number", "example": 0.00005 }, "usageUsd": { "type": "object", "properties": { "ACTOR_COMPUTE_UNITS": { "type": "integer", "example": 0 }, "DATASET_READS": { "type": "integer", "example": 0 }, "DATASET_WRITES": { "type": "integer", "example": 0 }, "KEY_VALUE_STORE_READS": { "type": "integer", "example": 0 }, "KEY_VALUE_STORE_WRITES": { "type": "number", "example": 0.00005 }, "KEY_VALUE_STORE_LISTS": { "type": "integer", "example": 0 }, "REQUEST_QUEUE_READS": { "type": "integer", "example": 0 }, "REQUEST_QUEUE_WRITES": { "type": "integer", "example": 0 }, "DATA_TRANSFER_INTERNAL_GBYTES": { "type": "integer", "example": 0 }, "DATA_TRANSFER_EXTERNAL_GBYTES": { "type": "integer", "example": 0 }, "PROXY_RESIDENTIAL_TRANSFER_GBYTES": { "type": "integer", "example": 0 }, "PROXY_SERPS": { "type": "integer", "example": 0 } } } } } } } } } }