"""Build the CSVs behind the blog post from our own repos.

Run from the dh-datasets-agent repo root, with these clones next to it:
  ../postal-codes  (read at origin/main; fetch first)
  ../dji           (after `make all`, for build/divisions/divisions.csv)

    python3 blog/postal-codes-are-the-easy-part/src/prepare_data.py

Writes into blog/postal-codes-are-the-easy-part/data/. Standard library only.
"""

import csv
import re
import subprocess
from collections import Counter
from pathlib import Path

HERE = Path(__file__).resolve().parent
POST = HERE.parent
REPO = POST.parent.parent
DATA = POST / "data"
POSTAL = REPO.parent / "postal-codes"
DIVISIONS = REPO.parent / "dji" / "build" / "divisions" / "divisions.csv"
TRIAGE = REPO / "research" / "2102" / "postcodes-not-available.csv"

# Countries with postcodes and an open source we could add but haven't yet
# (research/2102/spike-codes-and-coverage.md, section B, category 2).
OPEN_NOT_YET_ADDED = {"am", "bl", "kr", "mf", "pw", "py", "sh", "tw"}

# Countries whose postal-codes file is a locality or admin-division directory
# with an empty postal_code column: their datapackage descriptions say the
# country has no postal-code system.
NO_POSTCODE_DIRECTORY = {"ao", "bj", "cd", "ci", "cw", "gy", "hk", "rw", "sc", "sl", "sr"}


def git_show(path: str) -> str:
    return subprocess.run(
        ["git", "-C", str(POSTAL), "show", f"origin/main:{path}"],
        check=True, capture_output=True, text=True,
    ).stdout


def postal_rows() -> dict[str, int]:
    """Size in bytes of each country's file on postal-codes main (header-only files are
    under 200 bytes). Some datapackages have no `rows` field, so size is the test."""
    names = subprocess.run(
        ["git", "-C", str(POSTAL), "ls-tree", "--name-only", "origin/main:datasets"],
        check=True, capture_output=True, text=True,
    ).stdout.split()
    rows = {}
    for cc in names:
        y = git_show(f"datasets/{cc}/datapackage.yaml")
        res = y.split("\nresources:", 1)[-1]
        m = re.search(r"^\s+bytes:\s*(\d+)", res, re.M)
        size = int(m.group(1)) if m else 0
        if not m and "premium_source_key:" in res:
            size = 201  # premium product without recorded stats (e.g. KW): it exists
        r = re.search(r"^\s+rows:\s*(\d+)", res, re.M)
        rows[cc] = (int(r.group(1)) if r else -1) if size > 200 else 0
    return rows


def main() -> None:
    DATA.mkdir(exist_ok=True)
    with open(DIVISIONS, encoding="utf-8") as f:
        divisions = list(csv.DictReader(f))
    countries = {r["code"]: r["name_en"] for r in divisions if r["level"] == "0"}
    level1 = Counter(r["country_code"] for r in divisions if r["level"] == "1")

    rows = postal_rows()
    triage = {}
    with open(TRIAGE, encoding="utf-8") as f:
        for r in csv.DictReader(f):
            triage[r["cc"]] = r["status"]

    out = []
    for cc, name in sorted(countries.items()):
        lc = cc.lower()
        if lc in NO_POSTCODE_DIRECTORY:
            status = "no_postcode_system"
        elif rows.get(lc, 0) != 0:
            status = "postal_codes_published"
        elif lc in OPEN_NOT_YET_ADDED:
            status = "open_source_not_yet_added"
        elif triage.get(lc) == "no_postcode_system":
            status = "no_postcode_system"
        elif triage.get(lc) == "postcodes_exist_no_open_source":
            status = "closed_or_paid_source_only"
        else:
            raise SystemExit(f"{cc}: no status")
        out.append({
            "country_code": cc, "country": name, "postal_status": status,
            "postal_code_rows": max(rows.get(lc, 0), 0) if lc not in NO_POSTCODE_DIRECTORY else 0, "level1_divisions": level1.get(cc, 0),
        })

    with open(DATA / "countries.csv", "w", newline="", encoding="utf-8") as f:
        w = csv.DictWriter(f, fieldnames=list(out[0]), lineterminator="\n")
        w.writeheader()
        w.writerows(out)
    print(Counter(r["postal_status"] for r in out))
    print("countries", len(out), "level1", sum(level1.values()),
          "with level1", sum(1 for r in out if r["level1_divisions"]),
          "postal rows (where recorded)", sum(r["postal_code_rows"] for r in out))


if __name__ == "__main__":
    main()
