{
  "openapi": "3.0.1",
  "info": {
    "title": "Merge & Dedupe Datasets: Combine Runs, Remove Duplicates",
    "description": "Merge Apify datasets from many runs, plus CSV, JSON or Excel files, into one table and remove duplicates by any field. Ignore case, spaces and URL tracking; keep the first, last or most complete row; get only rows new since earlier runs, or keep a master list. Runs with limited permissions.",
    "version": "0.1",
    "x-build-id": "1NgI4LotkqXsh83Gk"
  },
  "servers": [
    {
      "url": "https://api.apify.com/v2"
    }
  ],
  "paths": {
    "/acts/humble-echidna~dataset-merge/run-sync-get-dataset-items": {
      "post": {
        "operationId": "run-sync-get-dataset-items-humble-echidna-dataset-merge",
        "x-openai-isConsequential": false,
        "summary": "Executes an Actor, waits for its completion, and returns Actor's dataset items in response.",
        "tags": [
          "Run Actor"
        ],
        "requestBody": {
          "required": true,
          "content": {
            "application/json": {
              "schema": {
                "$ref": "#/components/schemas/inputSchema"
              }
            }
          }
        },
        "parameters": [
          {
            "name": "token",
            "in": "query",
            "required": true,
            "schema": {
              "type": "string"
            },
            "description": "Enter your Apify token here"
          }
        ],
        "responses": {
          "200": {
            "description": "OK"
          }
        }
      }
    },
    "/acts/humble-echidna~dataset-merge/runs": {
      "post": {
        "operationId": "runs-sync-humble-echidna-dataset-merge",
        "x-openai-isConsequential": false,
        "summary": "Executes an Actor and returns information about the initiated run in response.",
        "tags": [
          "Run Actor"
        ],
        "requestBody": {
          "required": true,
          "content": {
            "application/json": {
              "schema": {
                "$ref": "#/components/schemas/inputSchema"
              }
            }
          }
        },
        "parameters": [
          {
            "name": "token",
            "in": "query",
            "required": true,
            "schema": {
              "type": "string"
            },
            "description": "Enter your Apify token here"
          }
        ],
        "responses": {
          "200": {
            "description": "OK",
            "content": {
              "application/json": {
                "schema": {
                  "$ref": "#/components/schemas/runsResponseSchema"
                }
              }
            }
          }
        }
      }
    },
    "/acts/humble-echidna~dataset-merge/run-sync": {
      "post": {
        "operationId": "run-sync-humble-echidna-dataset-merge",
        "x-openai-isConsequential": false,
        "summary": "Executes an Actor, waits for completion, and returns the OUTPUT from Key-value store in response.",
        "tags": [
          "Run Actor"
        ],
        "requestBody": {
          "required": true,
          "content": {
            "application/json": {
              "schema": {
                "$ref": "#/components/schemas/inputSchema"
              }
            }
          }
        },
        "parameters": [
          {
            "name": "token",
            "in": "query",
            "required": true,
            "schema": {
              "type": "string"
            },
            "description": "Enter your Apify token here"
          }
        ],
        "responses": {
          "200": {
            "description": "OK"
          }
        }
      }
    }
  },
  "components": {
    "schemas": {
      "inputSchema": {
        "type": "object",
        "properties": {
          "datasetIds": {
            "title": "Apify datasets to merge",
            "type": "array",
            "description": "The datasets to combine, read in this order: ids or names of your own Apify datasets, e.g. the runs of a scraper (or pick them in the form). Up to 200 sources. Read-only access to exactly these, nothing else in your account. In an integration use `{{resource.defaultDatasetId}}`: Apify lets the run read only datasets named here. Reading runs by actor isn't possible under limited permissions, so list the datasets, or use masterDataset."
          },
          "fileUrls": {
            "title": "Or: file URLs (CSV, JSON, JSONL or Excel)",
            "type": "array",
            "description": "Direct links to CSV/TSV, JSON (a list of objects), JSON Lines or Excel (.xlsx, first sheet) files on public web addresses (http/https, ports 80 and 443), up to 200 MB each; read after the datasets. The default is the US Treasury's exchange rates on 2025-12-31 and 2024-12-31, the example's data; it's ignored as soon as a dataset is given.",
            "default": [
              "https://api.fiscaldata.treasury.gov/services/api/fiscal_service/v1/accounting/od/rates_of_exchange?format=csv&filter=record_date:eq:2025-12-31&page[size]=300",
              "https://api.fiscaldata.treasury.gov/services/api/fiscal_service/v1/accounting/od/rates_of_exchange?format=csv&filter=record_date:eq:2024-12-31&page[size]=300"
            ],
            "items": {
              "type": "string"
            }
          },
          "dedupeBy": {
            "title": "Dedupe by (fields that make a row unique)",
            "type": "array",
            "description": "Fields whose combined values identify a row, one per line or comma-separated, e.g. `email` or `name, city`. Dots reach nested fields (`address.postalCode`, `offers.0.url`). Names are case-sensitive. Empty, or `*`: only rows identical in every field are duplicates. A row with all these fields empty is always kept. The prefill is the example's key.",
            "items": {
              "type": "string"
            }
          },
          "normalize": {
            "title": "Ignore these differences when comparing",
            "type": "array",
            "description": "How key values are compared, any of: `trim` (the default: spaces at the ends and repeated spaces), `lowercase` (case), `accents` (é = e), `url` (web addresses compared without scheme, www, default port, fragment, trailing slash and tracking parameters such as utm_* and gclid), `digits` (only digits count, for phone numbers). The output keeps the original values.",
            "items": {
              "type": "string",
              "enum": [
                "trim",
                "lowercase",
                "accents",
                "url",
                "digits"
              ],
              "enumTitles": [
                "Extra spaces",
                "Upper/lower case",
                "Accents (é = e)",
                "URL details (https, www, trailing slash, utm_ and other tracking parameters)",
                "Everything but digits (phone numbers)"
              ]
            },
            "default": [
              "trim"
            ]
          },
          "keep": {
            "title": "Which duplicate to keep",
            "enum": [
              "first",
              "last",
              "mostComplete",
              "merge"
            ],
            "type": "string",
            "description": "`first` (the default): the first row with the key, written as it's read, so memory holds only keys. `last`: the last one. `mostComplete`: the one with the most filled-in fields (a tie keeps the earlier). `merge`: the first row with its empty fields filled in from its duplicates. The last three hold one row per key in memory and write at the end.",
            "default": "first"
          },
          "onlyNew": {
            "title": "Only rows not returned by earlier runs",
            "type": "boolean",
            "description": "Default false. When true, keys this actor returned in earlier runs are remembered in a key-value store in your account and skipped, free. For a scraper's integration this means each run gives only rows never seen before. The memory belongs to memoryName, else the saved task, else the scraper that wrote the first dataset, and starts afresh if dedupeBy or normalize change.",
            "default": false
          },
          "memoryName": {
            "title": "Memory name",
            "type": "string",
            "description": "Optional name for what onlyNew remembers, so several tasks or scrapers can share one memory, e.g. `all-leads`. Letters, digits and hyphens, at most 63 characters. Default: none, the memory follows the task or scraper."
          },
          "masterDataset": {
            "title": "Master dataset (growing list of every unique row)",
            "type": "string",
            "description": "Optional name of a dataset to keep a running list in, e.g. `all-leads`: created on first use, new unique rows are added on every run, and rows it already holds are skipped (this turns on onlyNew). Letters, digits and hyphens, at most 63 characters. Rows also go to the run's own dataset. If your account has that name from elsewhere, `humble-echidna-<name>` is used."
          },
          "addSourceField": {
            "title": "Add a field saying where each row came from",
            "type": "boolean",
            "description": "Default false. When true, each row gets `mergeSource`: `dataset:<id>` or the file's address it was read from (for `merge`, the first row's source).",
            "default": false
          },
          "exportFormat": {
            "title": "Also save as a file",
            "enum": [
              "none",
              "csv",
              "jsonl"
            ],
            "type": "string",
            "description": "`none` (the default), `csv` or `jsonl`: also save the output as `OUTPUT.csv` or `OUTPUT.jsonl` in the run's key-value store, up to 100 MB, free of charge. In the CSV, text a spreadsheet would run as a formula gets an apostrophe in front.",
            "default": "none"
          },
          "maxResults": {
            "title": "Max rows per run",
            "minimum": 1,
            "type": "integer",
            "description": "Stop after writing this many rows, e.g. 100. Minimum 1; leave empty (the default) for no limit. The run also stops cleanly at the maximum cost per run you set in the run options, whichever comes first. With onlyNew, rows not written are returned by the next run."
          }
        }
      },
      "runsResponseSchema": {
        "type": "object",
        "properties": {
          "data": {
            "type": "object",
            "properties": {
              "id": {
                "type": "string"
              },
              "actId": {
                "type": "string"
              },
              "userId": {
                "type": "string"
              },
              "startedAt": {
                "type": "string",
                "format": "date-time",
                "example": "2025-01-08T00:00:00.000Z"
              },
              "finishedAt": {
                "type": "string",
                "format": "date-time",
                "example": "2025-01-08T00:00:00.000Z"
              },
              "status": {
                "type": "string",
                "example": "READY"
              },
              "meta": {
                "type": "object",
                "properties": {
                  "origin": {
                    "type": "string",
                    "example": "API"
                  },
                  "userAgent": {
                    "type": "string"
                  }
                }
              },
              "stats": {
                "type": "object",
                "properties": {
                  "inputBodyLen": {
                    "type": "integer",
                    "example": 2000
                  },
                  "rebootCount": {
                    "type": "integer",
                    "example": 0
                  },
                  "restartCount": {
                    "type": "integer",
                    "example": 0
                  },
                  "resurrectCount": {
                    "type": "integer",
                    "example": 0
                  },
                  "computeUnits": {
                    "type": "integer",
                    "example": 0
                  }
                }
              },
              "options": {
                "type": "object",
                "properties": {
                  "build": {
                    "type": "string",
                    "example": "latest"
                  },
                  "timeoutSecs": {
                    "type": "integer",
                    "example": 300
                  },
                  "memoryMbytes": {
                    "type": "integer",
                    "example": 1024
                  },
                  "diskMbytes": {
                    "type": "integer",
                    "example": 2048
                  }
                }
              },
              "buildId": {
                "type": "string"
              },
              "defaultKeyValueStoreId": {
                "type": "string"
              },
              "defaultDatasetId": {
                "type": "string"
              },
              "defaultRequestQueueId": {
                "type": "string"
              },
              "buildNumber": {
                "type": "string",
                "example": "1.0.0"
              },
              "containerUrl": {
                "type": "string"
              },
              "usage": {
                "type": "object",
                "properties": {
                  "ACTOR_COMPUTE_UNITS": {
                    "type": "integer",
                    "example": 0
                  },
                  "DATASET_READS": {
                    "type": "integer",
                    "example": 0
                  },
                  "DATASET_WRITES": {
                    "type": "integer",
                    "example": 0
                  },
                  "KEY_VALUE_STORE_READS": {
                    "type": "integer",
                    "example": 0
                  },
                  "KEY_VALUE_STORE_WRITES": {
                    "type": "integer",
                    "example": 1
                  },
                  "KEY_VALUE_STORE_LISTS": {
                    "type": "integer",
                    "example": 0
                  },
                  "REQUEST_QUEUE_READS": {
                    "type": "integer",
                    "example": 0
                  },
                  "REQUEST_QUEUE_WRITES": {
                    "type": "integer",
                    "example": 0
                  },
                  "DATA_TRANSFER_INTERNAL_GBYTES": {
                    "type": "integer",
                    "example": 0
                  },
                  "DATA_TRANSFER_EXTERNAL_GBYTES": {
                    "type": "integer",
                    "example": 0
                  },
                  "PROXY_RESIDENTIAL_TRANSFER_GBYTES": {
                    "type": "integer",
                    "example": 0
                  },
                  "PROXY_SERPS": {
                    "type": "integer",
                    "example": 0
                  },
                  "PROXY_UNBLOCKER_UNITS": {
                    "type": "integer",
                    "example": 0
                  }
                }
              },
              "usageTotalUsd": {
                "type": "number",
                "example": 0.00005
              },
              "usageUsd": {
                "type": "object",
                "properties": {
                  "ACTOR_COMPUTE_UNITS": {
                    "type": "integer",
                    "example": 0
                  },
                  "DATASET_READS": {
                    "type": "integer",
                    "example": 0
                  },
                  "DATASET_WRITES": {
                    "type": "integer",
                    "example": 0
                  },
                  "KEY_VALUE_STORE_READS": {
                    "type": "integer",
                    "example": 0
                  },
                  "KEY_VALUE_STORE_WRITES": {
                    "type": "number",
                    "example": 0.00005
                  },
                  "KEY_VALUE_STORE_LISTS": {
                    "type": "integer",
                    "example": 0
                  },
                  "REQUEST_QUEUE_READS": {
                    "type": "integer",
                    "example": 0
                  },
                  "REQUEST_QUEUE_WRITES": {
                    "type": "integer",
                    "example": 0
                  },
                  "DATA_TRANSFER_INTERNAL_GBYTES": {
                    "type": "integer",
                    "example": 0
                  },
                  "DATA_TRANSFER_EXTERNAL_GBYTES": {
                    "type": "integer",
                    "example": 0
                  },
                  "PROXY_RESIDENTIAL_TRANSFER_GBYTES": {
                    "type": "integer",
                    "example": 0
                  },
                  "PROXY_SERPS": {
                    "type": "integer",
                    "example": 0
                  },
                  "PROXY_UNBLOCKER_UNITS": {
                    "type": "integer",
                    "example": 0
                  }
                }
              }
            }
          }
        }
      }
    }
  }
}