{ "openapi": "3.0.1", "info": { "title": "Dataset Diff, Dedupe & Merge - Change Detection", "description": "Compare today's scrape with the last run and get only new, changed and removed rows, with previous values. Also dedupes and merges Apify datasets, CSV and JSON by key fields.", "version": "0.1", "x-build-id": "Ynf0L7jUqsRdZ8GoB" }, "servers": [ { "url": "https://api.apify.com/v2" } ], "paths": { "/acts/noisy_jubilee~dataset-diff-dedupe/run-sync-get-dataset-items": { "post": { "operationId": "run-sync-get-dataset-items-noisy_jubilee-dataset-diff-dedupe", "x-openai-isConsequential": false, "summary": "Executes an Actor, waits for its completion, and returns Actor's dataset items in response.", "tags": [ "Run Actor" ], "requestBody": { "required": true, "content": { "application/json": { "schema": { "$ref": "#/components/schemas/inputSchema" } } } }, "parameters": [ { "name": "token", "in": "query", "required": true, "schema": { "type": "string" }, "description": "Enter your Apify token here" } ], "responses": { "200": { "description": "OK" } } } }, "/acts/noisy_jubilee~dataset-diff-dedupe/runs": { "post": { "operationId": "runs-sync-noisy_jubilee-dataset-diff-dedupe", "x-openai-isConsequential": false, "summary": "Executes an Actor and returns information about the initiated run in response.", "tags": [ "Run Actor" ], "requestBody": { "required": true, "content": { "application/json": { "schema": { "$ref": "#/components/schemas/inputSchema" } } } }, "parameters": [ { "name": "token", "in": "query", "required": true, "schema": { "type": "string" }, "description": "Enter your Apify token here" } ], "responses": { "200": { "description": "OK", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/runsResponseSchema" } } } } } } }, "/acts/noisy_jubilee~dataset-diff-dedupe/run-sync": { "post": { "operationId": "run-sync-noisy_jubilee-dataset-diff-dedupe", "x-openai-isConsequential": false, "summary": "Executes an Actor, waits for completion, and returns the OUTPUT from Key-value store in response.", "tags": [ "Run Actor" ], "requestBody": { "required": true, "content": { "application/json": { "schema": { "$ref": "#/components/schemas/inputSchema" } } } }, "parameters": [ { "name": "token", "in": "query", "required": true, "schema": { "type": "string" }, "description": "Enter your Apify token here" } ], "responses": { "200": { "description": "OK" } } } } }, "components": { "schemas": { "inputSchema": { "type": "object", "properties": { "mode": { "title": "Mode", "enum": [ "diff", "dedupe", "merge" ], "type": "string", "description": "diff = new/changed/removed rows vs a baseline or the previous run; dedupe = remove duplicates; merge = combine all sources and keep the latest row per key.", "default": "diff" }, "datasetIds": { "title": "Datasets (current data)", "type": "array", "description": "Apify datasets to read, e.g. the output of your scraper's latest run. Only the datasets you select are shared with this Actor (read-only)." }, "fileUrls": { "title": "File URLs (current data)", "type": "array", "description": "CSV, TSV, JSON or JSON Lines files you have the right to process.", "items": { "type": "string" } }, "items": { "title": "Inline rows (current data)", "type": "array", "description": "JSON array of objects." }, "keyFields": { "title": "Key fields", "type": "array", "description": "Fields that identify a row (dot paths allowed, e.g. 'id' or 'product.sku'). Required for diff; for dedupe, empty = whole row must match.", "items": { "type": "string" } }, "compareFields": { "title": "Compare only these fields", "type": "array", "description": "Diff mode: a row counts as changed only if one of these fields changed. Empty = all fields except ignored ones.", "items": { "type": "string" } }, "ignoreFields": { "title": "Ignore fields", "type": "array", "description": "Top-level fields that never count as a change, e.g. scrapedAt, crawledAt, position.", "items": { "type": "string" } }, "trimStrings": { "title": "Normalise whitespace", "type": "boolean", "description": "Treat ' Foo bar ' and 'Foo bar' as equal.", "default": true }, "caseInsensitiveKeys": { "title": "Case-insensitive keys", "type": "boolean", "description": "Match key values ignoring case.", "default": false }, "keep": { "title": "Which duplicate to keep", "enum": [ "first", "last" ], "type": "string", "description": "Dedupe/merge: keep the first or last occurrence (merge defaults to last = newest source)." }, "snapshotName": { "title": "Snapshot name (compare with previous run)", "type": "string", "description": "Diff mode without a baseline: compare with the data saved under this name by the previous run, then save the current data. The first run only saves the baseline (free). Stored in your named key-value store 'dataset-diff-snapshots'." }, "updateSnapshot": { "title": "Update snapshot after the run", "type": "boolean", "description": "Save the current data as the new snapshot.", "default": true }, "firstRunOutputsAll": { "title": "First run: output every row as new", "type": "boolean", "description": "By default the first snapshot run just saves the baseline. Enable to output (and pay for) every row as new.", "default": false }, "baselineDatasetIds": { "title": "Baseline datasets", "type": "array", "description": "Explicit baseline (e.g. the previous run's dataset). Overrides the snapshot." }, "baselineFileUrls": { "title": "Baseline file URLs", "type": "array", "description": "CSV/JSON/JSONL baseline files.", "items": { "type": "string" } }, "baselineItems": { "title": "Baseline inline rows", "type": "array", "description": "JSON array of objects." }, "outputChangeTypes": { "title": "Change types to output", "type": "array", "description": "Which changes to output (and pay for).", "items": { "type": "string", "enum": [ "new", "changed", "removed" ], "enumTitles": [ "New", "Changed", "Removed" ] }, "default": [ "new", "changed", "removed" ] }, "includeUnchanged": { "title": "Also output unchanged rows", "type": "boolean", "description": "Unchanged rows are output for free, marked _change=unchanged.", "default": false }, "maxRows": { "title": "Max rows per source", "minimum": 1, "type": "integer", "description": "Safety cap on rows read.", "default": 1000000 }, "maxFileSizeMb": { "title": "Max file size (MB)", "minimum": 1, "maximum": 1000, "type": "integer", "description": "Per file URL.", "default": 100 }, "respectRobotsTxt": { "title": "Respect robots.txt", "type": "boolean", "description": "Skip file URLs disallowed by the host's robots.txt.", "default": true } } }, "runsResponseSchema": { "type": "object", "properties": { "data": { "type": "object", "properties": { "id": { "type": "string" }, "actId": { "type": "string" }, "userId": { "type": "string" }, "startedAt": { "type": "string", "format": "date-time", "example": "2025-01-08T00:00:00.000Z" }, "finishedAt": { "type": "string", "format": "date-time", "example": "2025-01-08T00:00:00.000Z" }, "status": { "type": "string", "example": "READY" }, "meta": { "type": "object", "properties": { "origin": { "type": "string", "example": "API" }, "userAgent": { "type": "string" } } }, "stats": { "type": "object", "properties": { "inputBodyLen": { "type": "integer", "example": 2000 }, "rebootCount": { "type": "integer", "example": 0 }, "restartCount": { "type": "integer", "example": 0 }, "resurrectCount": { "type": "integer", "example": 0 }, "computeUnits": { "type": "integer", "example": 0 } } }, "options": { "type": "object", "properties": { "build": { "type": "string", "example": "latest" }, "timeoutSecs": { "type": "integer", "example": 300 }, "memoryMbytes": { "type": "integer", "example": 1024 }, "diskMbytes": { "type": "integer", "example": 2048 } } }, "buildId": { "type": "string" }, "defaultKeyValueStoreId": { "type": "string" }, "defaultDatasetId": { "type": "string" }, "defaultRequestQueueId": { "type": "string" }, "buildNumber": { "type": "string", "example": "1.0.0" }, "containerUrl": { "type": "string" }, "usage": { "type": "object", "properties": { "ACTOR_COMPUTE_UNITS": { "type": "integer", "example": 0 }, "DATASET_READS": { "type": "integer", "example": 0 }, "DATASET_WRITES": { "type": "integer", "example": 0 }, "KEY_VALUE_STORE_READS": { "type": "integer", "example": 0 }, "KEY_VALUE_STORE_WRITES": { "type": "integer", "example": 1 }, "KEY_VALUE_STORE_LISTS": { "type": "integer", "example": 0 }, "REQUEST_QUEUE_READS": { "type": "integer", "example": 0 }, "REQUEST_QUEUE_WRITES": { "type": "integer", "example": 0 }, "DATA_TRANSFER_INTERNAL_GBYTES": { "type": "integer", "example": 0 }, "DATA_TRANSFER_EXTERNAL_GBYTES": { "type": "integer", "example": 0 }, "PROXY_RESIDENTIAL_TRANSFER_GBYTES": { "type": "integer", "example": 0 }, "PROXY_SERPS": { "type": "integer", "example": 0 }, "PROXY_UNBLOCKER_UNITS": { "type": "integer", "example": 0 } } }, "usageTotalUsd": { "type": "number", "example": 0.00005 }, "usageUsd": { "type": "object", "properties": { "ACTOR_COMPUTE_UNITS": { "type": "integer", "example": 0 }, "DATASET_READS": { "type": "integer", "example": 0 }, "DATASET_WRITES": { "type": "integer", "example": 0 }, "KEY_VALUE_STORE_READS": { "type": "integer", "example": 0 }, "KEY_VALUE_STORE_WRITES": { "type": "number", "example": 0.00005 }, "KEY_VALUE_STORE_LISTS": { "type": "integer", "example": 0 }, "REQUEST_QUEUE_READS": { "type": "integer", "example": 0 }, "REQUEST_QUEUE_WRITES": { "type": "integer", "example": 0 }, "DATA_TRANSFER_INTERNAL_GBYTES": { "type": "integer", "example": 0 }, "DATA_TRANSFER_EXTERNAL_GBYTES": { "type": "integer", "example": 0 }, "PROXY_RESIDENTIAL_TRANSFER_GBYTES": { "type": "integer", "example": 0 }, "PROXY_SERPS": { "type": "integer", "example": 0 }, "PROXY_UNBLOCKER_UNITS": { "type": "integer", "example": 0 } } } } } } } } } }