{ "openapi": "3.0.1", "info": { "title": "Universal Open Data Portal Fetcher (CKAN + Socrata)", "description": "Search dataset catalogs and fetch rows from thousands of government open data portals running CKAN or Socrata, via their official APIs. Auto-detects the portal, pages through results, and supports incremental runs.", "version": "1.0", "x-build-id": "w5fhhgQEoanyXwJQe" }, "servers": [ { "url": "https://api.apify.com/v2" } ], "paths": { "/acts/wdurham~open-data-portal-fetcher/run-sync-get-dataset-items": { "post": { "operationId": "run-sync-get-dataset-items-wdurham-open-data-portal-fetcher", "x-openai-isConsequential": false, "summary": "Executes an Actor, waits for its completion, and returns Actor's dataset items in response.", "tags": [ "Run Actor" ], "requestBody": { "required": true, "content": { "application/json": { "schema": { "$ref": "#/components/schemas/inputSchema" } } } }, "parameters": [ { "name": "token", "in": "query", "required": true, "schema": { "type": "string" }, "description": "Enter your Apify token here" } ], "responses": { "200": { "description": "OK" } } } }, "/acts/wdurham~open-data-portal-fetcher/runs": { "post": { "operationId": "runs-sync-wdurham-open-data-portal-fetcher", "x-openai-isConsequential": false, "summary": "Executes an Actor and returns information about the initiated run in response.", "tags": [ "Run Actor" ], "requestBody": { "required": true, "content": { "application/json": { "schema": { "$ref": "#/components/schemas/inputSchema" } } } }, "parameters": [ { "name": "token", "in": "query", "required": true, "schema": { "type": "string" }, "description": "Enter your Apify token here" } ], "responses": { "200": { "description": "OK", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/runsResponseSchema" } } } } } } }, "/acts/wdurham~open-data-portal-fetcher/run-sync": { "post": { "operationId": "run-sync-wdurham-open-data-portal-fetcher", "x-openai-isConsequential": false, "summary": "Executes an Actor, waits for completion, and returns the OUTPUT from Key-value store in response.", "tags": [ "Run Actor" ], "requestBody": { "required": true, "content": { "application/json": { "schema": { "$ref": "#/components/schemas/inputSchema" } } } }, "parameters": [ { "name": "token", "in": "query", "required": true, "schema": { "type": "string" }, "description": "Enter your Apify token here" } ], "responses": { "200": { "description": "OK" } } } } }, "components": { "schemas": { "inputSchema": { "type": "object", "required": [ "portalUrl" ], "properties": { "portalUrl": { "title": "Portal URL", "type": "string", "description": "Base URL of a CKAN or Socrata portal, e.g. https://data.gov.uk, https://data.boston.gov, https://data.cityofnewyork.us, https://data.cityofchicago.org. Any page URL on the portal also works; API/UI suffixes are stripped." }, "portalType": { "title": "Portal type", "enum": [ "auto", "ckan", "socrata" ], "type": "string", "description": "Auto-detect by probing the CKAN action API and the Socrata catalog API, or force a type.", "default": "auto" }, "mode": { "title": "Mode", "enum": [ "search", "fetch" ], "type": "string", "description": "search = one item per dataset (metadata). fetch = data rows of one dataset/resource.", "default": "search" }, "query": { "title": "Search query", "type": "string", "description": "Full-text query for search mode (CKAN q / Socrata Discovery q)." }, "maxItems": { "title": "Max datasets", "minimum": 1, "maximum": 100000, "type": "integer", "description": "Maximum number of datasets to return in search mode.", "default": 100 }, "ckanFq": { "title": "CKAN filter query (fq)", "type": "string", "description": "Solr filter for CKAN package_search, e.g. organization:city-of-boston or res_format:CSV." }, "ckanSort": { "title": "CKAN sort", "type": "string", "description": "Sort for package_search (default 'metadata_modified desc') or for datastore_search in fetch mode (e.g. '_id desc')." }, "socrataSearchParams": { "title": "Extra Socrata Discovery API params", "type": "object", "description": "Extra query parameters for the Socrata Discovery API, e.g. {\"categories\": \"Buildings\"} or {\"tags\": \"permits\"}." }, "datasetId": { "title": "Dataset ID", "type": "string", "description": "fetch mode: CKAN package name/id (e.g. approved-building-permits) or Socrata 4x4 id (e.g. ydr8-5enu)." }, "resourceId": { "title": "CKAN resource ID", "type": "string", "description": "CKAN only: specific resource id or name. If empty, the first DataStore resource is used, else the first CSV/TSV/JSON/GeoJSON/XLSX file." }, "preferredFormat": { "title": "Preferred file format (CKAN)", "enum": [ "CSV", "TSV", "JSON", "GEOJSON", "XLSX" ], "type": "string", "description": "CKAN only: prefer a resource of this format when no resourceId is given." }, "forceDownload": { "title": "Download file instead of DataStore", "type": "boolean", "description": "CKAN only: download the resource file even if the DataStore API is available.", "default": false }, "maxRows": { "title": "Max rows", "minimum": 1, "maximum": 10000000, "type": "integer", "description": "Maximum number of rows written in fetch mode.", "default": 1000 }, "pageSize": { "title": "Page size", "minimum": 1, "maximum": 50000, "type": "integer", "description": "Rows per API request (SODA $limit / DataStore limit).", "default": 1000 }, "soqlSelect": { "title": "SoQL $select", "type": "string", "description": "Socrata only, e.g. permit_, issue_date, total_fee." }, "soqlWhere": { "title": "SoQL $where", "type": "string", "description": "Socrata only, e.g. issue_date > '2026-01-01' AND permit_type = 'PERMIT - NEW CONSTRUCTION'." }, "soqlOrder": { "title": "SoQL $order", "type": "string", "description": "Socrata only, e.g. issue_date DESC. Defaults to :id for stable paging." }, "soqlQ": { "title": "SoQL $q (full text)", "type": "string", "description": "Socrata only: full-text search within rows." }, "datastoreQ": { "title": "CKAN DataStore q", "type": "string", "description": "CKAN only: full-text search within DataStore rows." }, "filters": { "title": "Exact-match filters (CKAN)", "type": "object", "description": "CKAN only: {\"column\": \"value\"} exact filters (passed to datastore_search, or applied after download for files)." }, "socrataAppToken": { "title": "Socrata app token", "type": "string", "description": "Optional Socrata app token for higher rate limits (sent as X-App-Token)." }, "incremental": { "title": "Incremental (only new/changed)", "type": "boolean", "description": "Remember what was seen in a named key-value store and only emit new/changed datasets (search) or rows (fetch) on later runs.", "default": false }, "incrementalField": { "title": "Incremental field", "type": "string", "description": "fetch mode: monotonically increasing column. Socrata default ':updated_at' (system field); CKAN DataStore default '_id'. CKAN file resources use the resource last_modified instead." }, "stateStoreName": { "title": "State store name", "type": "string", "description": "Named key-value store that keeps incremental state across runs.", "default": "open-data-portal-state" }, "resetState": { "title": "Reset state", "type": "boolean", "description": "Ignore stored state for this query/dataset and start over (state is then overwritten).", "default": false }, "requestTimeoutSecs": { "title": "Request timeout (s)", "minimum": 5, "maximum": 600, "type": "integer", "description": "Per-request timeout.", "default": 60 }, "maxRetries": { "title": "Max retries", "minimum": 0, "maximum": 10, "type": "integer", "description": "Retries for timeouts, 429 and 5xx (exponential backoff, honours Retry-After).", "default": 4 }, "maxDownloadMb": { "title": "Max download size (MB)", "minimum": 1, "maximum": 2000, "type": "integer", "description": "Size cap for CKAN file downloads.", "default": 50 } } }, "runsResponseSchema": { "type": "object", "properties": { "data": { "type": "object", "properties": { "id": { "type": "string" }, "actId": { "type": "string" }, "userId": { "type": "string" }, "startedAt": { "type": "string", "format": "date-time", "example": "2025-01-08T00:00:00.000Z" }, "finishedAt": { "type": "string", "format": "date-time", "example": "2025-01-08T00:00:00.000Z" }, "status": { "type": "string", "example": "READY" }, "meta": { "type": "object", "properties": { "origin": { "type": "string", "example": "API" }, "userAgent": { "type": "string" } } }, "stats": { "type": "object", "properties": { "inputBodyLen": { "type": "integer", "example": 2000 }, "rebootCount": { "type": "integer", "example": 0 }, "restartCount": { "type": "integer", "example": 0 }, "resurrectCount": { "type": "integer", "example": 0 }, "computeUnits": { "type": "integer", "example": 0 } } }, "options": { "type": "object", "properties": { "build": { "type": "string", "example": "latest" }, "timeoutSecs": { "type": "integer", "example": 300 }, "memoryMbytes": { "type": "integer", "example": 1024 }, "diskMbytes": { "type": "integer", "example": 2048 } } }, "buildId": { "type": "string" }, "defaultKeyValueStoreId": { "type": "string" }, "defaultDatasetId": { "type": "string" }, "defaultRequestQueueId": { "type": "string" }, "buildNumber": { "type": "string", "example": "1.0.0" }, "containerUrl": { "type": "string" }, "usage": { "type": "object", "properties": { "ACTOR_COMPUTE_UNITS": { "type": "integer", "example": 0 }, "DATASET_READS": { "type": "integer", "example": 0 }, "DATASET_WRITES": { "type": "integer", "example": 0 }, "KEY_VALUE_STORE_READS": { "type": "integer", "example": 0 }, "KEY_VALUE_STORE_WRITES": { "type": "integer", "example": 1 }, "KEY_VALUE_STORE_LISTS": { "type": "integer", "example": 0 }, "REQUEST_QUEUE_READS": { "type": "integer", "example": 0 }, "REQUEST_QUEUE_WRITES": { "type": "integer", "example": 0 }, "DATA_TRANSFER_INTERNAL_GBYTES": { "type": "integer", "example": 0 }, "DATA_TRANSFER_EXTERNAL_GBYTES": { "type": "integer", "example": 0 }, "PROXY_RESIDENTIAL_TRANSFER_GBYTES": { "type": "integer", "example": 0 }, "PROXY_SERPS": { "type": "integer", "example": 0 }, "PROXY_UNBLOCKER_UNITS": { "type": "integer", "example": 0 } } }, "usageTotalUsd": { "type": "number", "example": 0.00005 }, "usageUsd": { "type": "object", "properties": { "ACTOR_COMPUTE_UNITS": { "type": "integer", "example": 0 }, "DATASET_READS": { "type": "integer", "example": 0 }, "DATASET_WRITES": { "type": "integer", "example": 0 }, "KEY_VALUE_STORE_READS": { "type": "integer", "example": 0 }, "KEY_VALUE_STORE_WRITES": { "type": "number", "example": 0.00005 }, "KEY_VALUE_STORE_LISTS": { "type": "integer", "example": 0 }, "REQUEST_QUEUE_READS": { "type": "integer", "example": 0 }, "REQUEST_QUEUE_WRITES": { "type": "integer", "example": 0 }, "DATA_TRANSFER_INTERNAL_GBYTES": { "type": "integer", "example": 0 }, "DATA_TRANSFER_EXTERNAL_GBYTES": { "type": "integer", "example": 0 }, "PROXY_RESIDENTIAL_TRANSFER_GBYTES": { "type": "integer", "example": 0 }, "PROXY_SERPS": { "type": "integer", "example": 0 }, "PROXY_UNBLOCKER_UNITS": { "type": "integer", "example": 0 } } } } } } } } } }