{ "openapi": "3.0.1", "info": { "title": "GovData Germany Scraper - CKAN Open Data Portal", "description": "Scrape Germany's national open data portal: 156,000+ Datensätze with full DCAT-AP.de metadata, distributions, publishers, EU data themes, High-Value Dataset flags, licences and live link checks.", "version": "0.1", "x-build-id": "ueU880K39RPe2lkWo" }, "servers": [ { "url": "https://api.apify.com/v2" } ], "paths": { "/acts/parseforge~govdata-germany-scraper/run-sync-get-dataset-items": { "post": { "operationId": "run-sync-get-dataset-items-parseforge-govdata-germany-scraper", "x-openai-isConsequential": false, "summary": "Executes an Actor, waits for its completion, and returns Actor's dataset items in response.", "tags": [ "Run Actor" ], "requestBody": { "required": true, "content": { "application/json": { "schema": { "$ref": "#/components/schemas/inputSchema" } } } }, "parameters": [ { "name": "token", "in": "query", "required": true, "schema": { "type": "string" }, "description": "Enter your Apify token here" } ], "responses": { "200": { "description": "OK" } } } }, "/acts/parseforge~govdata-germany-scraper/runs": { "post": { "operationId": "runs-sync-parseforge-govdata-germany-scraper", "x-openai-isConsequential": false, "summary": "Executes an Actor and returns information about the initiated run in response.", "tags": [ "Run Actor" ], "requestBody": { "required": true, "content": { "application/json": { "schema": { "$ref": "#/components/schemas/inputSchema" } } } }, "parameters": [ { "name": "token", "in": "query", "required": true, "schema": { "type": "string" }, "description": "Enter your Apify token here" } ], "responses": { "200": { "description": "OK", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/runsResponseSchema" } } } } } } }, "/acts/parseforge~govdata-germany-scraper/run-sync": { "post": { "operationId": "run-sync-parseforge-govdata-germany-scraper", "x-openai-isConsequential": false, "summary": "Executes an Actor, waits for completion, and returns the OUTPUT from Key-value store in response.", "tags": [ "Run Actor" ], "requestBody": { "required": true, "content": { "application/json": { "schema": { "$ref": "#/components/schemas/inputSchema" } } } }, "parameters": [ { "name": "token", "in": "query", "required": true, "schema": { "type": "string" }, "description": "Enter your Apify token here" } ], "responses": { "200": { "description": "OK" } } } } }, "components": { "schemas": { "inputSchema": { "type": "object", "required": [ "dataset" ], "properties": { "dataset": { "title": "Dataset", "enum": [ "datasets", "resources", "contacts", "organizations", "categories", "tags", "formats", "licenses" ], "type": "string", "description": "Which surface of the GovData catalogue to return. 'Datasets' is one row per dataset with its distributions nested; 'Distributions' flattens to one row per file or service; 'Contact points' deduplicates the publishing bodies behind them. The remaining four are directories and each cost a single request.", "default": "datasets" }, "searchQuery": { "title": "Search query", "type": "string", "description": "Full-text search across titles, descriptions, tags and publishers, exactly as the portal's own search box works. German stemming applies, so 'Wasser' also matches 'Gewässer'. Leave empty for the whole catalogue." }, "titleContains": { "title": "Title contains", "type": "string", "description": "Restrict the search to dataset titles. Combined with Search query using AND." }, "descriptionContains": { "title": "Description contains", "type": "string", "description": "Restrict the search to dataset descriptions. Combined with Search query using AND." }, "organizations": { "title": "Organizations", "uniqueItems": true, "type": "array", "description": "Publishing organization slugs, e.g. berlin-open-data, open-data-bayern, transparenzportal-hamburg, gdi-de. Run the 'Organizations' dataset once to get the full list of 76 with their dataset counts. Multiple values are ORed.", "items": { "type": "string" } }, "categories": { "title": "Categories (EU data themes)", "uniqueItems": true, "type": "array", "description": "The 13 EU data themes. Multiple values are ORed.", "items": { "type": "string", "enum": [ "agri", "educ", "ener", "envi", "econ", "gove", "heal", "intr", "just", "regi", "soci", "tech", "tran" ], "enumTitles": [ "agri — Agriculture, fisheries, forestry, food", "educ — Education, culture and sport", "ener — Energy", "envi — Environment", "econ — Economy and finance", "gove — Government and public sector", "heal — Health", "intr — International issues", "just — Justice and public safety", "regi — Regions and cities", "soci — Population and society", "tech — Science and technology", "tran — Transport" ] } }, "tags": { "title": "Tags", "uniqueItems": true, "type": "array", "description": "Exact keyword tags, e.g. opendata, bauleitplanung, vermessung, haushalt. Case-sensitive and matched exactly — run the 'Tags' dataset to see what exists and how many datasets each covers. Multiple values are ORed.", "items": { "type": "string" } }, "formats": { "title": "Distribution formats", "uniqueItems": true, "type": "array", "description": "Keep only datasets that publish at least one distribution in these formats, e.g. CSV, JSON, GEOJSON, XLSX, SHP, WFS, PARQUET. Each name is expanded to every spelling the catalogue actually uses — the 403 raw values include full EU file-type URIs, bare names and free text, and filtering on the bare name alone would miss 88% of the CSV datasets.", "items": { "type": "string" } }, "excludeFormats": { "title": "Exclude formats", "uniqueItems": true, "type": "array", "description": "Drop any dataset that publishes one of these formats. Applied after fetching, on the normalised short name (CSV, PDF, WMS, …).", "items": { "type": "string" } }, "datasetTypes": { "title": "Dataset type", "uniqueItems": true, "type": "array", "description": "dct:type as DCAT-AP.de records it. Populated on about a quarter of the catalogue; leave empty unless you specifically need it.", "items": { "type": "string", "enum": [ "dataset", "series", "collection" ], "enumTitles": [ "Dataset", "Series", "Collection" ] } }, "hvdOnly": { "title": "High-Value Datasets only", "type": "boolean", "description": "Keep only datasets flagged as EU High-Value Datasets under Implementing Regulation (EU) 2023/138 — the categories member states must publish openly and free of charge. 10,514 of the catalogue qualify.", "default": false }, "hvdCategories": { "title": "High-Value Dataset categories", "uniqueItems": true, "type": "array", "description": "Narrow to specific HVD categories. Overrides the checkbox above. Measured counts: earth-environment 9,419 · geospatial 603 · mobility 313 · statistics 217 · meteorological 81 · companies 5.", "items": { "type": "string", "enum": [ "geospatial", "earth-environment", "meteorological", "statistics", "companies", "mobility" ], "enumTitles": [ "Geospatial", "Earth observation and environment", "Meteorological", "Statistics", "Companies and company ownership", "Mobility" ] } }, "administrativeLevels": { "title": "Administrative level", "uniqueItems": true, "type": "array", "description": "Which tier of German government the dataset covers, from the DCAT-AP.de political geocoding vocabulary. Measured: municipality 24,910 · state 19,986 · district 4,089 · federal 3,789.", "items": { "type": "string", "enum": [ "federal", "state", "district", "municipality" ], "enumTitles": [ "Bund — federal", "Land — state", "Kreis — administrative district", "Gemeinde — municipality" ] } }, "regionalKeys": { "title": "Regional keys (AGS)", "uniqueItems": true, "type": "array", "description": "Amtlicher Gemeindeschlüssel codes, e.g. 130030000000 for Rostock or 11 for Berlin as a state. Matches the dcat-ap.de regionalKey and stateKey URIs. Multiple values are ORed.", "items": { "type": "string" } }, "language": { "title": "Metadata language", "enum": [ "", "de", "en" ], "type": "string", "description": "The declared language of the dataset record. German covers 93,655 records; English only 289, so this is mostly useful for excluding non-German material.", "default": "" }, "updateFrequencies": { "title": "Update frequency", "uniqueItems": true, "type": "array", "description": "How often the publisher says the data is refreshed. Two vocabularies are in use across the catalogue for the same idea, and each option matches both.", "items": { "type": "string", "enum": [ "continuous", "daily", "weekly", "monthly", "quarterly", "annual", "irregular", "as-needed", "never", "other" ], "enumTitles": [ "Continuous", "Daily", "Weekly", "Monthly", "Quarterly", "Annual", "Irregular", "As needed", "Never", "Other or unknown" ] } }, "publisherName": { "title": "Publisher name", "type": "string", "description": "Exact publisher name as DCAT-AP.de records it, e.g. 'Ministerium für Bildung'. This is the actual publishing body, which is finer-grained than the CKAN organization — 76 organizations front many thousands of publishers." }, "sourcePortal": { "title": "Source portal contains", "type": "string", "description": "Substring of the regional portal GovData harvested the record from, e.g. berlin, hamburg, rlp. GovData is a harvesting catalogue and this is the trail back to the original." }, "onlyPublicAccess": { "title": "Machine-readable PUBLIC access rights only", "type": "boolean", "description": "Keep only records that declare the EU access-right value PUBLIC (28,951 datasets). Most publishers put free German prose in this field instead of the controlled value, so this narrows hard — leave it off unless you need the machine-readable declaration.", "default": false }, "withSpatialCoverage": { "title": "Only datasets with a geometry", "type": "boolean", "description": "Keep only records carrying a spatial extent, so every row gets a bounding box and centroid. 72,621 datasets qualify.", "default": false }, "minResources": { "title": "Minimum distributions", "minimum": 0, "type": "integer", "description": "Drop datasets with fewer than this many distributions. Set to 1 to exclude the 239 metadata-only records." }, "modifiedAfter": { "title": "Metadata modified after", "type": "string", "description": "ISO date or datetime, e.g. 2026-08-01. Together with the sort order, this is how you run the Actor as a change monitor." }, "modifiedBefore": { "title": "Metadata modified before", "type": "string", "description": "ISO date or datetime." }, "createdAfter": { "title": "First published after", "type": "string", "description": "ISO date or datetime. Filters on when the record first appeared on GovData, which is how you find genuinely new datasets rather than re-harvested ones." }, "createdBefore": { "title": "First published before", "type": "string", "description": "ISO date or datetime." }, "temporalFrom": { "title": "Data covers time from", "type": "string", "description": "ISO date. Filters on the period the DATA covers, not on when the record was edited. A dataset matches when its coverage window overlaps yours." }, "temporalTo": { "title": "Data covers time to", "type": "string", "description": "ISO date. See above." }, "licenses": { "title": "Licences", "uniqueItems": true, "type": "array", "description": "Keep only datasets whose distributions carry one of these licences — substring match against the licence URI, e.g. dl-by-de, cc-by, cc-zero, official-work. Applied after fetching, because the licence lives on the distribution: package-level licence is empty on 99.2% of the catalogue.", "items": { "type": "string" } }, "searchText": { "title": "Row must contain", "type": "string", "description": "Plain substring filter over the finished row's title, description, publisher, tags and place name. Runs after fetching and before any optional block is bought, so filtering never costs you a link check." }, "filterQuery": { "title": "Filter query (advanced)", "type": "string", "description": "Raw Solr fq appended to everything above, e.g. extras_conforms_to:* or res_url:*wfs*. Careful: an unknown field name returns zero rows rather than an error." }, "checkResourceLinks": { "title": "Check that distribution links are alive", "type": "boolean", "description": "HEAD every distribution URL and attach status, content type, byte size, redirect target and latency. GovData harvests metadata from ~100 regional portals and never re-verifies the links: measured 2026-08-27 across the full age range of the catalogue, 5.3% of distributions were dead 404s with nothing in the metadata to warn you. Billed per URL checked.", "default": false }, "previewResourceColumns": { "title": "Preview distribution columns", "type": "boolean", "description": "Read the first 64 KB of each CSV, TSV, JSON, GeoJSON or XML distribution with a range request and return its actual column names, delimiter, encoding and a sample row. The catalogue tells you a dataset is 'a CSV'; it never tells you what is inside. Billed per distribution previewed.", "default": false }, "onlyLiveResources": { "title": "Drop rows whose links are dead", "type": "boolean", "description": "Requires the link check above. Discards datasets where every distribution failed, or distributions that failed individually. Dropped rows are not billed.", "default": false }, "includeOrganizationProfile": { "title": "Profile each organization", "type": "boolean", "description": "Only for the 'Organizations' dataset. One extra request per organization returning what it actually publishes: its themes, its top formats and its top keywords, each with counts. Billed per organization profiled.", "default": false }, "includeRawExtras": { "title": "Include raw DCAT-AP.de extras", "type": "boolean", "description": "Attach the untouched CKAN extras key/value map alongside the parsed fields. Free — the data is already in the response — but it roughly doubles the row size.", "default": false }, "sortBy": { "title": "Sort by", "enum": [ "modified-desc", "modified-asc", "created-desc", "created-asc", "relevance", "title-asc", "name-asc", "id-asc" ], "type": "string", "description": "Only sort fields verified to actually change the order are offered. An unknown Solr sort field does not error on this API, it silently reverts to relevance.", "default": "modified-desc" }, "directoryLimit": { "title": "Directory rows", "minimum": 1, "type": "integer", "description": "How many rows to return for the Tags and Formats directories. The tag vocabulary has 97,863 entries and the long tail is mostly one-off keywords.", "default": 500 }, "maxItems": { "title": "Max items", "minimum": 1, "type": "integer", "description": "Hard ceiling on rows written, and therefore on cost. Leave empty for no ceiling." }, "maxPages": { "title": "Max catalogue pages", "minimum": 1, "type": "integer", "description": "Each page is 500 datasets and bills one catalogue-page event. 20 pages covers 10,000 datasets.", "default": 20 }, "concurrency": { "title": "Concurrency", "minimum": 1, "maximum": 12, "type": "integer", "description": "Parallel requests, 1 to 12. Measured on a bare datacenter IP: 8 workers moved 525 dataset rows a second with zero failures and no throttling. Above 8 the per-request overhead starts to dominate.", "default": 5 }, "timeoutSecs": { "title": "Stop after (seconds)", "minimum": 20, "type": "integer", "description": "Soft wall-clock budget. The run stops issuing requests and flushes what it has. The platform's own run timeout still applies and whichever is nearer wins." }, "proxyConfiguration": { "title": "Proxy", "type": "object", "description": "Off by default and not needed: 84 requests at up to 16 concurrent workers ran clean from a bare datacenter IP with no throttling. Turn it on only if you are sweeping the whole catalogue repeatedly on a schedule.", "default": { "useApifyProxy": false } } } }, "runsResponseSchema": { "type": "object", "properties": { "data": { "type": "object", "properties": { "id": { "type": "string" }, "actId": { "type": "string" }, "userId": { "type": "string" }, "startedAt": { "type": "string", "format": "date-time", "example": "2025-01-08T00:00:00.000Z" }, "finishedAt": { "type": "string", "format": "date-time", "example": "2025-01-08T00:00:00.000Z" }, "status": { "type": "string", "example": "READY" }, "meta": { "type": "object", "properties": { "origin": { "type": "string", "example": "API" }, "userAgent": { "type": "string" } } }, "stats": { "type": "object", "properties": { "inputBodyLen": { "type": "integer", "example": 2000 }, "rebootCount": { "type": "integer", "example": 0 }, "restartCount": { "type": "integer", "example": 0 }, "resurrectCount": { "type": "integer", "example": 0 }, "computeUnits": { "type": "integer", "example": 0 } } }, "options": { "type": "object", "properties": { "build": { "type": "string", "example": "latest" }, "timeoutSecs": { "type": "integer", "example": 300 }, "memoryMbytes": { "type": "integer", "example": 1024 }, "diskMbytes": { "type": "integer", "example": 2048 } } }, "buildId": { "type": "string" }, "defaultKeyValueStoreId": { "type": "string" }, "defaultDatasetId": { "type": "string" }, "defaultRequestQueueId": { "type": "string" }, "buildNumber": { "type": "string", "example": "1.0.0" }, "containerUrl": { "type": "string" }, "usage": { "type": "object", "properties": { "ACTOR_COMPUTE_UNITS": { "type": "integer", "example": 0 }, "DATASET_READS": { "type": "integer", "example": 0 }, "DATASET_WRITES": { "type": "integer", "example": 0 }, "KEY_VALUE_STORE_READS": { "type": "integer", "example": 0 }, "KEY_VALUE_STORE_WRITES": { "type": "integer", "example": 1 }, "KEY_VALUE_STORE_LISTS": { "type": "integer", "example": 0 }, "REQUEST_QUEUE_READS": { "type": "integer", "example": 0 }, "REQUEST_QUEUE_WRITES": { "type": "integer", "example": 0 }, "DATA_TRANSFER_INTERNAL_GBYTES": { "type": "integer", "example": 0 }, "DATA_TRANSFER_EXTERNAL_GBYTES": { "type": "integer", "example": 0 }, "PROXY_RESIDENTIAL_TRANSFER_GBYTES": { "type": "integer", "example": 0 }, "PROXY_SERPS": { "type": "integer", "example": 0 } } }, "usageTotalUsd": { "type": "number", "example": 0.00005 }, "usageUsd": { "type": "object", "properties": { "ACTOR_COMPUTE_UNITS": { "type": "integer", "example": 0 }, "DATASET_READS": { "type": "integer", "example": 0 }, "DATASET_WRITES": { "type": "integer", "example": 0 }, "KEY_VALUE_STORE_READS": { "type": "integer", "example": 0 }, "KEY_VALUE_STORE_WRITES": { "type": "number", "example": 0.00005 }, "KEY_VALUE_STORE_LISTS": { "type": "integer", "example": 0 }, "REQUEST_QUEUE_READS": { "type": "integer", "example": 0 }, "REQUEST_QUEUE_WRITES": { "type": "integer", "example": 0 }, "DATA_TRANSFER_INTERNAL_GBYTES": { "type": "integer", "example": 0 }, "DATA_TRANSFER_EXTERNAL_GBYTES": { "type": "integer", "example": 0 }, "PROXY_RESIDENTIAL_TRANSFER_GBYTES": { "type": "integer", "example": 0 }, "PROXY_SERPS": { "type": "integer", "example": 0 } } } } } } } } } }