{ "openapi": "3.0.1", "info": { "title": "Dataset Toolkit — Dedupe, Merge, Clean, Diff", "description": "Post-process any Apify dataset: remove duplicates, merge multiple datasets, clean and normalize fields, or diff two crawls. Outputs a new dataset plus a summary record.", "version": "0.1", "x-build-id": "NCnb6ttttmJkJmZB8" }, "servers": [ { "url": "https://api.apify.com/v2" } ], "paths": { "/acts/glueworks~dataset-toolkit/run-sync-get-dataset-items": { "post": { "operationId": "run-sync-get-dataset-items-glueworks-dataset-toolkit", "x-openai-isConsequential": false, "summary": "Executes an Actor, waits for its completion, and returns Actor's dataset items in response.", "tags": [ "Run Actor" ], "requestBody": { "required": true, "content": { "application/json": { "schema": { "$ref": "#/components/schemas/inputSchema" } } } }, "parameters": [ { "name": "token", "in": "query", "required": true, "schema": { "type": "string" }, "description": "Enter your Apify token here" } ], "responses": { "200": { "description": "OK" } } } }, "/acts/glueworks~dataset-toolkit/runs": { "post": { "operationId": "runs-sync-glueworks-dataset-toolkit", "x-openai-isConsequential": false, "summary": "Executes an Actor and returns information about the initiated run in response.", "tags": [ "Run Actor" ], "requestBody": { "required": true, "content": { "application/json": { "schema": { "$ref": "#/components/schemas/inputSchema" } } } }, "parameters": [ { "name": "token", "in": "query", "required": true, "schema": { "type": "string" }, "description": "Enter your Apify token here" } ], "responses": { "200": { "description": "OK", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/runsResponseSchema" } } } } } } }, "/acts/glueworks~dataset-toolkit/run-sync": { "post": { "operationId": "run-sync-glueworks-dataset-toolkit", "x-openai-isConsequential": false, "summary": "Executes an Actor, waits for completion, and returns the OUTPUT from Key-value store in response.", "tags": [ "Run Actor" ], "requestBody": { "required": true, "content": { "application/json": { "schema": { "$ref": "#/components/schemas/inputSchema" } } } }, "parameters": [ { "name": "token", "in": "query", "required": true, "schema": { "type": "string" }, "description": "Enter your Apify token here" } ], "responses": { "200": { "description": "OK" } } } } }, "components": { "schemas": { "inputSchema": { "type": "object", "properties": { "operation": { "title": "Operation", "enum": [ "dedupe", "merge", "clean", "diff" ], "type": "string", "description": "What to do with the input dataset(s). dedupe = remove duplicate records (see dedupeKeys, keepStrategy); merge = concatenate 1-10 datasets into one (see fieldMapping, addSourceField); clean = apply an ordered list of cleaning rules to every record (see cleanRules); diff = compare exactly 2 datasets and output added/removed/changed records (see diffKeys). Default: clean.", "default": "clean" }, "datasetIds": { "title": "Dataset IDs", "maxItems": 10, "type": "array", "description": "1–10 Apify dataset IDs (e.g. \"aBcDeFgHiJkLmNoP\", typically the defaultDatasetId of a previous Actor run) or named datasets to read. For diff, provide exactly 2: the OLD dataset first, the NEW one second. Leave EMPTY to run in demo mode on built-in sample data (useful to preview the output format). Source datasets are never modified.", "default": [], "items": { "type": "string" } }, "outputDatasetName": { "title": "Output dataset name", "type": "string", "description": "Optional name for the output (named) dataset. If empty, results go to the run's default dataset. Note: pushing to an existing named dataset appends to it.", "default": "" }, "dedupeKeys": { "title": "Dedupe: key fields", "type": "array", "description": "Only used when operation=dedupe. Field names whose combination defines a duplicate, e.g. [\"sku\", \"seller.name\"] (dot-notation supported for nested fields). Leave empty to deduplicate on the ENTIRE record. Matching is type-sensitive: 1 and \"1\" are different keys.", "default": [], "items": { "type": "string" } }, "keepStrategy": { "title": "Dedupe: keep strategy", "enum": [ "first", "last" ], "type": "string", "description": "Only used when operation=dedupe. Which occurrence of a duplicate to keep: \"first\" (default, streams with minimal memory) or \"last\" (keeps the freshest data, buffers one record per unique key). Use \"last\" when later records are more up to date.", "default": "first" }, "fieldMapping": { "title": "Merge: field mapping", "type": "object", "description": "Only used when operation=merge. Optional rename map applied while merging. Flat form {\"oldField\": \"newField\"} applies to all datasets; nested form {\"\": {\"old\": \"new\"}} applies per dataset. Dot-notation supported on both sides, e.g. {\"seller.name\": \"sellerName\"}. Example: {\"datasetA\": {\"cost\": \"price\"}, \"datasetB\": {\"productTitle\": \"title\"}}." }, "addSourceField": { "title": "Merge: add source field", "type": "string", "description": "Only used when operation=merge. If set to a field name (e.g. \"source\"), each merged record gets this field containing the ID of the dataset it came from. Leave empty to disable.", "default": "" }, "cleanRules": { "title": "Clean: rules", "type": "array", "description": "Only used when operation=clean. Ordered array of rule objects, applied to every record in sequence. Supported types: {\"type\": \"trim\"} (strip whitespace, optional fields array), {\"type\": \"removeEmpty\"} (drop null/\"\" fields; 0 and false are kept), {\"type\": \"convert\", \"field\": \"price\", \"to\": \"number\"} (e.g. \"$1,299.00\" -> 1299; unconvertible values pass through unchanged), {\"type\": \"pick\", \"fields\": [...]} (whitelist), {\"type\": \"omit\", \"fields\": [...]} (blacklist), {\"type\": \"regexReplace\", \"pattern\": \"<[^>]+>\", \"replacement\": \"\", \"field\": \"description\"} (regex substitution; omit field to apply to all string fields), {\"type\": \"rename\", \"from\": \"seller.name\", \"to\": \"sellerName\"}. Dot-notation supported. See README for full examples.", "default": [ { "type": "trim" }, { "type": "removeEmpty" } ] }, "diffKeys": { "title": "Diff: key fields", "type": "array", "description": "Only used (and required) when operation=diff. Field names identifying the same logical record across the two datasets, e.g. [\"url\"] or [\"sku\", \"seller.name\"] (dot-notation supported). Keys should be unique within each dataset; matching is type-sensitive. Default: [\"id\"].", "default": [ "id" ], "items": { "type": "string" } } } }, "runsResponseSchema": { "type": "object", "properties": { "data": { "type": "object", "properties": { "id": { "type": "string" }, "actId": { "type": "string" }, "userId": { "type": "string" }, "startedAt": { "type": "string", "format": "date-time", "example": "2025-01-08T00:00:00.000Z" }, "finishedAt": { "type": "string", "format": "date-time", "example": "2025-01-08T00:00:00.000Z" }, "status": { "type": "string", "example": "READY" }, "meta": { "type": "object", "properties": { "origin": { "type": "string", "example": "API" }, "userAgent": { "type": "string" } } }, "stats": { "type": "object", "properties": { "inputBodyLen": { "type": "integer", "example": 2000 }, "rebootCount": { "type": "integer", "example": 0 }, "restartCount": { "type": "integer", "example": 0 }, "resurrectCount": { "type": "integer", "example": 0 }, "computeUnits": { "type": "integer", "example": 0 } } }, "options": { "type": "object", "properties": { "build": { "type": "string", "example": "latest" }, "timeoutSecs": { "type": "integer", "example": 300 }, "memoryMbytes": { "type": "integer", "example": 1024 }, "diskMbytes": { "type": "integer", "example": 2048 } } }, "buildId": { "type": "string" }, "defaultKeyValueStoreId": { "type": "string" }, "defaultDatasetId": { "type": "string" }, "defaultRequestQueueId": { "type": "string" }, "buildNumber": { "type": "string", "example": "1.0.0" }, "containerUrl": { "type": "string" }, "usage": { "type": "object", "properties": { "ACTOR_COMPUTE_UNITS": { "type": "integer", "example": 0 }, "DATASET_READS": { "type": "integer", "example": 0 }, "DATASET_WRITES": { "type": "integer", "example": 0 }, "KEY_VALUE_STORE_READS": { "type": "integer", "example": 0 }, "KEY_VALUE_STORE_WRITES": { "type": "integer", "example": 1 }, "KEY_VALUE_STORE_LISTS": { "type": "integer", "example": 0 }, "REQUEST_QUEUE_READS": { "type": "integer", "example": 0 }, "REQUEST_QUEUE_WRITES": { "type": "integer", "example": 0 }, "DATA_TRANSFER_INTERNAL_GBYTES": { "type": "integer", "example": 0 }, "DATA_TRANSFER_EXTERNAL_GBYTES": { "type": "integer", "example": 0 }, "PROXY_RESIDENTIAL_TRANSFER_GBYTES": { "type": "integer", "example": 0 }, "PROXY_SERPS": { "type": "integer", "example": 0 } } }, "usageTotalUsd": { "type": "number", "example": 0.00005 }, "usageUsd": { "type": "object", "properties": { "ACTOR_COMPUTE_UNITS": { "type": "integer", "example": 0 }, "DATASET_READS": { "type": "integer", "example": 0 }, "DATASET_WRITES": { "type": "integer", "example": 0 }, "KEY_VALUE_STORE_READS": { "type": "integer", "example": 0 }, "KEY_VALUE_STORE_WRITES": { "type": "number", "example": 0.00005 }, "KEY_VALUE_STORE_LISTS": { "type": "integer", "example": 0 }, "REQUEST_QUEUE_READS": { "type": "integer", "example": 0 }, "REQUEST_QUEUE_WRITES": { "type": "integer", "example": 0 }, "DATA_TRANSFER_INTERNAL_GBYTES": { "type": "integer", "example": 0 }, "DATA_TRANSFER_EXTERNAL_GBYTES": { "type": "integer", "example": 0 }, "PROXY_RESIDENTIAL_TRANSFER_GBYTES": { "type": "integer", "example": 0 }, "PROXY_SERPS": { "type": "integer", "example": 0 } } } } } } } } } }