{
  "openapi": "3.0.1",
  "info": {
    "title": "Merge & Deduplicate Datasets (works with limited permissions)",
    "description": "Merge several Apify datasets (and public JSON files) into one and remove duplicates by any fields, with case/whitespace/URL normalization, keep first or last, only-new mode and field projection. Requests read-only access only to the datasets you pick.",
    "version": "0.0",
    "x-build-id": "y708gd20BaNfwvwJW"
  },
  "servers": [
    {
      "url": "https://api.apify.com/v2"
    }
  ],
  "paths": {
    "/acts/metal_spinney~dataset-merge-dedup/run-sync-get-dataset-items": {
      "post": {
        "operationId": "run-sync-get-dataset-items-metal_spinney-dataset-merge-dedup",
        "x-openai-isConsequential": false,
        "summary": "Executes an Actor, waits for its completion, and returns Actor's dataset items in response.",
        "tags": [
          "Run Actor"
        ],
        "requestBody": {
          "required": true,
          "content": {
            "application/json": {
              "schema": {
                "$ref": "#/components/schemas/inputSchema"
              }
            }
          }
        },
        "parameters": [
          {
            "name": "token",
            "in": "query",
            "required": true,
            "schema": {
              "type": "string"
            },
            "description": "Enter your Apify token here"
          }
        ],
        "responses": {
          "200": {
            "description": "OK"
          }
        }
      }
    },
    "/acts/metal_spinney~dataset-merge-dedup/runs": {
      "post": {
        "operationId": "runs-sync-metal_spinney-dataset-merge-dedup",
        "x-openai-isConsequential": false,
        "summary": "Executes an Actor and returns information about the initiated run in response.",
        "tags": [
          "Run Actor"
        ],
        "requestBody": {
          "required": true,
          "content": {
            "application/json": {
              "schema": {
                "$ref": "#/components/schemas/inputSchema"
              }
            }
          }
        },
        "parameters": [
          {
            "name": "token",
            "in": "query",
            "required": true,
            "schema": {
              "type": "string"
            },
            "description": "Enter your Apify token here"
          }
        ],
        "responses": {
          "200": {
            "description": "OK",
            "content": {
              "application/json": {
                "schema": {
                  "$ref": "#/components/schemas/runsResponseSchema"
                }
              }
            }
          }
        }
      }
    },
    "/acts/metal_spinney~dataset-merge-dedup/run-sync": {
      "post": {
        "operationId": "run-sync-metal_spinney-dataset-merge-dedup",
        "x-openai-isConsequential": false,
        "summary": "Executes an Actor, waits for completion, and returns the OUTPUT from Key-value store in response.",
        "tags": [
          "Run Actor"
        ],
        "requestBody": {
          "required": true,
          "content": {
            "application/json": {
              "schema": {
                "$ref": "#/components/schemas/inputSchema"
              }
            }
          }
        },
        "parameters": [
          {
            "name": "token",
            "in": "query",
            "required": true,
            "schema": {
              "type": "string"
            },
            "description": "Enter your Apify token here"
          }
        ],
        "responses": {
          "200": {
            "description": "OK"
          }
        }
      }
    }
  },
  "components": {
    "schemas": {
      "inputSchema": {
        "type": "object",
        "properties": {
          "datasetIds": {
            "title": "Datasets to merge",
            "type": "array",
            "description": "Pick one or more datasets (from your runs or saved datasets). Via API pass dataset IDs (or 'username~dataset-name'). Read-only access is requested only for the datasets you select, so this works under limited permissions and on accounts with restricted resource access."
          },
          "jsonUrls": {
            "title": "Public JSON URLs (optional)",
            "type": "array",
            "description": "Also merge any public JSON / JSON Lines file: a JSON array of objects, an object with an 'items' or 'data' array, or JSONL. Useful for public Apify dataset export links or files from other tools.",
            "items": {
              "type": "string"
            }
          },
          "dedupBy": {
            "title": "Deduplicate by",
            "enum": [
              "fields",
              "fullItem",
              "none"
            ],
            "type": "string",
            "description": "fields = items with the same values in the chosen fields are duplicates; fullItem = only exact duplicate items; none = just merge.",
            "default": "fields"
          },
          "dedupFields": {
            "title": "Dedup fields",
            "type": "array",
            "description": "Field names whose combined values must be unique, e.g. 'url', or 'name' + 'address'. Nested paths with dots work ('location.city', 'images.0'). Objects/arrays are compared by value.",
            "items": {
              "type": "string"
            }
          },
          "caseInsensitive": {
            "title": "Ignore letter case",
            "type": "boolean",
            "description": "'ACME Inc' and 'acme inc' count as the same value.",
            "default": false
          },
          "trimWhitespace": {
            "title": "Ignore extra whitespace",
            "type": "boolean",
            "description": "Trim and collapse whitespace in text values before comparing.",
            "default": true
          },
          "normalizeUrls": {
            "title": "Normalize URLs",
            "type": "boolean",
            "description": "Treat http/https, www., trailing slash, #fragment and utm_/fbclid/gclid parameters as the same URL.",
            "default": false
          },
          "nullAsUnique": {
            "title": "Missing values are unique",
            "type": "boolean",
            "description": "If on, items missing a dedup field (null/empty) are never treated as duplicates. If off, all items with the field missing count as one value.",
            "default": false
          },
          "keep": {
            "title": "Which duplicate to keep",
            "enum": [
              "first",
              "last"
            ],
            "type": "string",
            "description": "first = first occurrence in source order (streaming, low memory); last = last occurrence, e.g. the newest run if you list datasets oldest to newest (holds unique items in memory).",
            "default": "first"
          },
          "output": {
            "title": "What to output",
            "enum": [
              "unique",
              "duplicates",
              "summary"
            ],
            "type": "string",
            "description": "unique = merged, deduplicated items; duplicates = only the removed duplicates (for auditing); summary = no items, only duplicate counts in the SUMMARY record (costs nothing per item).",
            "default": "unique"
          },
          "excludeItemsInDatasetIds": {
            "title": "Only new: exclude items already in these datasets",
            "type": "array",
            "description": "Items whose dedup key already exists in these datasets (e.g. last week's merged output) are dropped, so you get only new items. Read-only access."
          },
          "outputFields": {
            "title": "Output only these fields",
            "type": "array",
            "description": "Field projection. Leave empty to keep whole items. Nested paths ('address.city') are output as flat 'address.city' keys.",
            "items": {
              "type": "string"
            }
          },
          "omitFields": {
            "title": "Remove these fields",
            "type": "array",
            "description": "Fields to drop from output items (nested paths allowed).",
            "items": {
              "type": "string"
            }
          },
          "addSourceField": {
            "title": "Add source field",
            "type": "boolean",
            "description": "Add '__source' with the dataset ID / URL each item came from.",
            "default": false
          },
          "cleanItems": {
            "title": "Clean items",
            "type": "boolean",
            "description": "Skip empty items and hidden '#' fields (Apify 'clean' format) when reading datasets.",
            "default": true
          },
          "maxItems": {
            "title": "Max output items",
            "minimum": 0,
            "type": "integer",
            "description": "Hard cap on output items (and cost). 0 = no limit.",
            "default": 0
          }
        }
      },
      "runsResponseSchema": {
        "type": "object",
        "properties": {
          "data": {
            "type": "object",
            "properties": {
              "id": {
                "type": "string"
              },
              "actId": {
                "type": "string"
              },
              "userId": {
                "type": "string"
              },
              "startedAt": {
                "type": "string",
                "format": "date-time",
                "example": "2025-01-08T00:00:00.000Z"
              },
              "finishedAt": {
                "type": "string",
                "format": "date-time",
                "example": "2025-01-08T00:00:00.000Z"
              },
              "status": {
                "type": "string",
                "example": "READY"
              },
              "meta": {
                "type": "object",
                "properties": {
                  "origin": {
                    "type": "string",
                    "example": "API"
                  },
                  "userAgent": {
                    "type": "string"
                  }
                }
              },
              "stats": {
                "type": "object",
                "properties": {
                  "inputBodyLen": {
                    "type": "integer",
                    "example": 2000
                  },
                  "rebootCount": {
                    "type": "integer",
                    "example": 0
                  },
                  "restartCount": {
                    "type": "integer",
                    "example": 0
                  },
                  "resurrectCount": {
                    "type": "integer",
                    "example": 0
                  },
                  "computeUnits": {
                    "type": "integer",
                    "example": 0
                  }
                }
              },
              "options": {
                "type": "object",
                "properties": {
                  "build": {
                    "type": "string",
                    "example": "latest"
                  },
                  "timeoutSecs": {
                    "type": "integer",
                    "example": 300
                  },
                  "memoryMbytes": {
                    "type": "integer",
                    "example": 1024
                  },
                  "diskMbytes": {
                    "type": "integer",
                    "example": 2048
                  }
                }
              },
              "buildId": {
                "type": "string"
              },
              "defaultKeyValueStoreId": {
                "type": "string"
              },
              "defaultDatasetId": {
                "type": "string"
              },
              "defaultRequestQueueId": {
                "type": "string"
              },
              "buildNumber": {
                "type": "string",
                "example": "1.0.0"
              },
              "containerUrl": {
                "type": "string"
              },
              "usage": {
                "type": "object",
                "properties": {
                  "ACTOR_COMPUTE_UNITS": {
                    "type": "integer",
                    "example": 0
                  },
                  "DATASET_READS": {
                    "type": "integer",
                    "example": 0
                  },
                  "DATASET_WRITES": {
                    "type": "integer",
                    "example": 0
                  },
                  "KEY_VALUE_STORE_READS": {
                    "type": "integer",
                    "example": 0
                  },
                  "KEY_VALUE_STORE_WRITES": {
                    "type": "integer",
                    "example": 1
                  },
                  "KEY_VALUE_STORE_LISTS": {
                    "type": "integer",
                    "example": 0
                  },
                  "REQUEST_QUEUE_READS": {
                    "type": "integer",
                    "example": 0
                  },
                  "REQUEST_QUEUE_WRITES": {
                    "type": "integer",
                    "example": 0
                  },
                  "DATA_TRANSFER_INTERNAL_GBYTES": {
                    "type": "integer",
                    "example": 0
                  },
                  "DATA_TRANSFER_EXTERNAL_GBYTES": {
                    "type": "integer",
                    "example": 0
                  },
                  "PROXY_RESIDENTIAL_TRANSFER_GBYTES": {
                    "type": "integer",
                    "example": 0
                  },
                  "PROXY_SERPS": {
                    "type": "integer",
                    "example": 0
                  },
                  "PROXY_UNBLOCKER_UNITS": {
                    "type": "integer",
                    "example": 0
                  }
                }
              },
              "usageTotalUsd": {
                "type": "number",
                "example": 0.00005
              },
              "usageUsd": {
                "type": "object",
                "properties": {
                  "ACTOR_COMPUTE_UNITS": {
                    "type": "integer",
                    "example": 0
                  },
                  "DATASET_READS": {
                    "type": "integer",
                    "example": 0
                  },
                  "DATASET_WRITES": {
                    "type": "integer",
                    "example": 0
                  },
                  "KEY_VALUE_STORE_READS": {
                    "type": "integer",
                    "example": 0
                  },
                  "KEY_VALUE_STORE_WRITES": {
                    "type": "number",
                    "example": 0.00005
                  },
                  "KEY_VALUE_STORE_LISTS": {
                    "type": "integer",
                    "example": 0
                  },
                  "REQUEST_QUEUE_READS": {
                    "type": "integer",
                    "example": 0
                  },
                  "REQUEST_QUEUE_WRITES": {
                    "type": "integer",
                    "example": 0
                  },
                  "DATA_TRANSFER_INTERNAL_GBYTES": {
                    "type": "integer",
                    "example": 0
                  },
                  "DATA_TRANSFER_EXTERNAL_GBYTES": {
                    "type": "integer",
                    "example": 0
                  },
                  "PROXY_RESIDENTIAL_TRANSFER_GBYTES": {
                    "type": "integer",
                    "example": 0
                  },
                  "PROXY_SERPS": {
                    "type": "integer",
                    "example": 0
                  },
                  "PROXY_UNBLOCKER_UNITS": {
                    "type": "integer",
                    "example": 0
                  }
                }
              }
            }
          }
        }
      }
    }
  }
}