"""Preview or save the private tasks from a Contact weekly MX planner export.""" import argparse from datetime import datetime, timezone from decimal import Decimal import json import os from pathlib import Path import re import sys from urllib.error import HTTPError from urllib.parse import urlsplit from urllib.request import HTTPRedirectHandler, Request, build_opener API = "https://api.apify.com/v2" ACTOR = "bHzVpEmytvDNJk48m" TASKS = "/actor-tasks" ACCOUNT = r"[a-zA-Z0-9][a-zA-Z0-9_-]{0,63}" TASK_NAME = r"[a-z0-9][a-z0-9-]{0,63}" RATES = { "starter": ("BRONZE", Decimal("0.008")), "scale": ("SILVER", Decimal("0.007")), "business": ("GOLD", Decimal("0.006")), "base": ("FREE", Decimal("0.01")), } INPUT_FLAGS = { "maxPagesPerSite": 3, "includeSubpages": True, "detectTechStack": False, "validatePhones": False, "verifyEmails": True, "deepEmailVerification": False, "enableAiAnalysis": False, "exportCrm": True, "crmRequireMx": True, "crmMxReservePerSite": True, "proxyConfiguration": {"useApifyProxy": True}, } OPTIONS = {"build": "latest", "memoryMbytes": 256, "timeoutSecs": 3600} EVENTS = ("contact-extracted", "verified-email-checked") MAX_PLAN_BYTES = 20_000_000 class PlanError(ValueError): pass class ApiError(RuntimeError): def __init__(self, status=None): super().__init__(f"Apify returned HTTP {status}." if status else "Apify response could not be verified.") self.status = status def money(value): if type(value) not in (int, float): raise PlanError("Charge caps must be numbers.") amount = Decimal(str(value)) if not amount.is_finite() or amount <= 0: raise PlanError("Charge caps must be positive.") return amount def object_fields(value, names): if not isinstance(value, dict) or value.keys() != set(names): raise PlanError("Choose the original contact-mx-task-plan.json export.") def read_plan(path, prefix): with path.open("rb") as stream: raw = stream.read(MAX_PLAN_BYTES + 1) if len(raw) > MAX_PLAN_BYTES: raise PlanError("The task plan exceeds 20 MB.") plan = json.loads(raw) object_fields(plan, ("version", "tier", "fullRefreshEventCeilingUsd", "manifestFile", "tasks")) if type(plan["version"]) is not int or plan["version"] != 1 or plan["manifestFile"] != "contact-refresh-manifest.json": raise PlanError("This installer accepts version 1 of the weekly MX task plan.") if not isinstance(plan["tier"], str) or plan["tier"] not in RATES: raise PlanError("Choose a supported pricing tier in the planner.") _, rate = RATES[plan["tier"]] if prefix and not re.fullmatch(r"[a-z0-9]+(?:-[a-z0-9]+)*", prefix): raise PlanError("Use lowercase letters, digits and hyphens in the task prefix.") requests, sites, total = [], set(), Decimal(0) if not isinstance(plan["tasks"], list) or not plan["tasks"]: raise PlanError("The export contains no task requests.") for number, entry in enumerate(plan["tasks"], 1): object_fields(entry, ("filename", "request")) batch = f"contact-refresh-{number:03}" request = entry["request"] object_fields(request, ("actId", "name", "input", "options")) if (entry["filename"], request["actId"], request["name"]) != (batch + ".json", ACTOR, batch + "-mx"): raise PlanError("Actor identity or batch order differs from the planner export.") inputs, options = request["input"], request["options"] object_fields(inputs, (*INPUT_FLAGS, "urls", "maxWebsites", "maxEmailsToVerify")) object_fields(options, (*OPTIONS, "maxTotalChargeUsd")) if any(type(inputs[key]) is not type(value) or inputs[key] != value for key, value in INPUT_FLAGS.items()): raise PlanError("Checked-only MX settings differ from the planner export.") if inputs["proxyConfiguration"]["useApifyProxy"] is not True: raise PlanError("The proxy setting differs from the planner export.") if any(type(options[key]) is not type(value) or options[key] != value for key, value in OPTIONS.items()): raise PlanError("Run options differ from the planner export.") urls = inputs["urls"] if not isinstance(urls, list) or not 1 <= len(urls) <= 200: raise PlanError("Each batch must contain 1–200 website URLs.") if any(type(inputs[key]) is not int or inputs[key] != len(urls) for key in ("maxWebsites", "maxEmailsToVerify")): raise PlanError("Each batch needs one MX attempt per planned website.") for value in urls: if not isinstance(value, str): raise PlanError("The export contains an invalid website URL.") url = urlsplit(value) if url.scheme not in ("https", "http") or not url.hostname or url.username is not None or url.query or url.fragment: raise PlanError("The export contains an invalid website URL.") site = url.hostname.lower().rstrip(".").removeprefix("www.") if site in sites: raise PlanError("A website appears in more than one input slot.") sites.add(site) cap = money(options["maxTotalChargeUsd"]) if cap != 2 * len(urls) * rate: raise PlanError("An event cap differs from the selected tier and batch size.") total += cap name = f"{prefix}-{request['name']}" if prefix else request["name"] if not re.fullmatch(TASK_NAME, name): raise PlanError("The task name is too long. Use a shorter prefix.") requests.append(request | {"name": name}) if money(plan["fullRefreshEventCeilingUsd"]) != total: raise PlanError("The summed caps differ from the export's weekly event ceiling.") return plan["tier"], requests, total, len(sites) class Direct(HTTPRedirectHandler): def redirect_request(self, *_args, **_kwargs): return None class Apify: def __init__(self, token): self.token = token self.opener = build_opener(Direct()) def request(self, path, body=None): if body is not None: if path != TASKS: raise ValueError("Unexpected write route.") elif path not in ("/users/me", "/acts/" + ACTOR) and not re.fullmatch(rf"{TASKS}/{ACCOUNT}~{TASK_NAME}", path): raise ValueError("Unexpected read route.") url = API + path request = Request(url, data=None if body is None else json.dumps(body, allow_nan=False).encode(), headers={"Authorization": "Bearer " + self.token, "Accept": "application/json", "Content-Type": "application/json"}) try: with self.opener.open(request, timeout=20) as response: if response.url != url or response.status != (200 if body is None else 201): raise ApiError(response.status) raw = response.read(4_000_001) if len(raw) > 4_000_000: raise ApiError() return json.loads(raw)["data"] except HTTPError as error: if error.code == 404 and body is None and path.startswith(TASKS + "/"): return None raise ApiError(error.code) from None except (OSError, ValueError, KeyError) as error: raise ApiError() from None def check_prices(actor, tier, account_tier, now): if (actor.get("id"), actor.get("username"), actor.get("name"), actor.get("isPublic")) != (ACTOR, "harvestlab", "contact-extractor", True): raise PlanError("The current public Contact Actor could not be verified.") effective = [] for item in actor["pricingInfos"]: if item.get("startedAt"): start = datetime.fromisoformat(item["startedAt"].replace("Z", "+00:00")) if start <= now: effective.append((start, item)) if not effective: raise PlanError("The Actor's current pricing could not be verified.") _, price = max(effective, key=lambda pair: pair[0]) tier_name, rate = RATES[tier] if price["pricingModel"] != "PAY_PER_EVENT" or price.get("isPPEPlatformUsagePaidByUser"): raise PlanError("The Actor's pricing model has changed. Review the current Store pricing.") events = price["pricingPerEvent"]["actorChargeEvents"] if any(name.startswith("apify-") for name in events) or any( money(events[name]["eventTieredPricingUsd"][tier_name]["tieredEventPriceUsd"]) != rate for name in EVENTS ): raise PlanError("The Actor's event prices have changed. Review the current Store pricing.") try: account_site_cost = sum( money(events[name]["eventTieredPricingUsd"][account_tier]["tieredEventPriceUsd"]) for name in EVENTS ) except (KeyError, TypeError, PlanError): raise PlanError("The current Contact rates for the destination account's tier could not be verified.") from None if account_site_cost > 2 * rate: raise PlanError("The selected caps are too low for the destination account. Re-export the plan at its pricing tier.") def check_task(saved, planned, owner): options = saved.get("options") or {} if (saved.get("userId"), saved.get("actId"), saved.get("name"), saved.get("isPublic")) != ( owner, ACTOR, planned["name"], False ) or saved.get("input") != planned["input"] or any(options.get(key) != value for key, value in planned["options"].items()): raise PlanError(f"Task {planned['name']} has different settings or visibility. Review it or choose a new prefix.") if options.get("maxItems") is not None or options.get("restartOnError") is True or options.get("forcePermissionLevel") not in (None, "LIMITED_PERMISSIONS"): raise PlanError(f"Task {planned['name']} has extra run options. Review it or choose a new prefix.") if not isinstance(saved.get("id"), str): raise PlanError(f"The ID for task {planned['name']} could not be verified.") return saved["id"] def save_tasks(api, tier, requests, now, report, *, expected_account): owner = api.request("/users/me") if owner.get("username") != expected_account or not isinstance(owner.get("id"), str): raise PlanError("APIFY_TOKEN does not identify the requested account.") plan = owner.get("plan") account_tier = plan.get("tier") if isinstance(plan, dict) else None if not isinstance(account_tier, str) or not account_tier: raise PlanError("The destination account's Apify tier could not be verified. Use a token for that account.") check_prices(api.request("/acts/" + ACTOR), tier, account_tier, now) def read_task(name): return api.request(f"{TASKS}/{owner['username']}~{name}") existing = {} for planned in requests: saved = read_task(planned["name"]) if saved is not None: existing[planned["name"]] = check_task(saved, planned, owner["id"]) for planned in requests: name = planned["name"] if name in existing: report(name, existing[name], "existing") continue try: created = api.request(TASKS, planned) except ApiError: created = None saved = read_task(name) if saved is None: raise PlanError(f"Task {name} could not be confirmed. Check Console before running the installer again.") task_id = check_task(saved, planned, owner["id"]) if created is not None and created.get("id") != task_id: raise PlanError(f"Task {name} returned inconsistent IDs. Review it in Console.") report(name, task_id, "created" if created is not None else "reconciled") def main(): parser = argparse.ArgumentParser(description=__doc__) parser.add_argument("plan", type=Path, help="The extracted contact-mx-task-plan.json file") parser.add_argument("--prefix", default="", help="Optional name prefix for a distinct website list") parser.add_argument("--expected-account", help="The Apify username where the tasks should be saved") parser.add_argument("--apply", action="store_true", help="Save tasks in the account identified by APIFY_TOKEN") args = parser.parse_args() if args.apply and (not args.expected_account or not re.fullmatch(ACCOUNT, args.expected_account)): parser.error("--apply requires the destination username in --expected-account") try: tier, requests, total, sites = read_plan(args.plan, args.prefix) print(f"{len(requests)} tasks, {sites} sites, ${total:f} maximum in Actor events per refresh ({tier}).", flush=True) if not args.apply: for item in requests: print(f"{item['name']} ${item['options']['maxTotalChargeUsd']:.3f}") return 0 token = os.environ.get("APIFY_TOKEN", "").strip() if not token: raise PlanError("Set APIFY_TOKEN for the account where you want to save the tasks.") print(f"Saving private tasks in {args.expected_account}.", flush=True) save_tasks(Apify(token), tier, requests, datetime.now(timezone.utc), lambda name, task_id, status: print(f"{name} {task_id} {status}", flush=True), expected_account=args.expected_account) except (PlanError, ApiError) as error: print(error, file=sys.stderr) return 2 except (OSError, ValueError, TypeError, KeyError, AttributeError): print("The plan or Apify response could not be read. Check the saved task results before rerunning.", file=sys.stderr) return 2 return 0 if __name__ == "__main__": sys.exit(main())