#!/usr/bin/env python3
"""Build the LogShip import catalog for the demo merchant "VONROC" (VONROC B.V., Zwolle NL — tools, garden,
home & water products, https://vonroc.de — Magento 2) from the storefront's public GraphQL endpoint
`POST /graphql` (header `Store: de`, answers unauthenticated; ~1 060 simple products, 10 per page — bigger pages answer 503).

Output: catalog.json — one entry per shop product for import-merchant-catalog.sh (+ `parentValue` = key of
its master article when several SKUs share a stem, e.g. AP501AC_BEIGE / AP501AC_WHITE → AP501AC) and
masters.json — the master ("summary") articles for create-master-articles.sh (grouped by the model code
CS508DC: colour suffixes and S_/S2_/S3_ battery-set prefixes are its variants; the plain article joins them):
  { value (= shop SKU), name, grossPrice, netPrice, upc (= real EAN from `ean_code`), sku, weight, length,
    width, height, group1 (brand VONROC), group2 (category path), classification (top category),
    description, documentNote (long description + "Lieferumfang"), imageUrl, images (gallery), stock }

The feed exposes no weights/dimensions (`weight` is null on every product): estimates per top category
and price. Out-of-stock articles are kept with stock 0, the rest get a deterministic opening stock of
2…60 (bulky tools, fewer pieces than fragrances).

  python3 build-catalog.py [catalog-raw.json] catalog.json      # fetches what the raw file lacks (resumable)
"""
import html, json, os, re, sys, time, urllib.request, urllib.error, collections
PAGE = 10   # the shop's Varnish answers 503 "Backend fetch failed" above ~10 products with all fields

BASE = "https://vonroc.de"
HDR = {"User-Agent": "Mozilla/5.0", "Content-Type": "application/json", "Store": "de"}
args = sys.argv[1:]
out_path = args[-1] if args else "catalog.json"
raw_path = args[0] if len(args) > 1 else os.path.join(os.path.dirname(out_path) or ".", "catalog-raw.json")

FIELDS = """sku name url_key stock_status only_x_left_in_stock ean_code
  description{html} short_description{html} image{url} media_gallery{url}
  categories{name level url_path} price_range{minimum_price{final_price{value} regular_price{value}}}"""

def gql(query, tries=5):
    req = urllib.request.Request(f"{BASE}/graphql", data=json.dumps({"query": query}).encode(), headers=HDR)
    for attempt in range(tries):
        try:
            with urllib.request.urlopen(req, timeout=120) as r:
                d = json.load(r)
            break
        except urllib.error.HTTPError as e:          # 503 "Backend fetch failed" on heavy pages → back off
            if attempt == tries - 1 or e.code < 500: raise
            time.sleep(5 * (attempt + 1))
    if d.get("errors") and not d.get("data"):
        raise RuntimeError(d["errors"][0]["message"])
    return d["data"]

# The shop's Varnish fails any GraphQL response above ~40 KB ("Backend fetch failed"), so the catalog is
# fetched in two passes: everything but the long description (10 per page, ~15 KB) and then the descriptions
# alone (4 per page); a page that still fails is fetched product by product (filter sku eq).
LIGHT = re.sub(r"(?<![_a-z])description\{html\} ", "", FIELDS)   # keep short_description
def fetch_page(fields, size, page):
    try:
        return gql(f'{{ products(search:"", pageSize:{size}, currentPage:{page}) {{ total_count items {{ {fields} }} }} }}', tries=2)["products"]
    except Exception as e:
        print(f"  page {page}/{size}: {str(e)[:50]} -> per SKU")
        d = gql(f'{{ products(search:"", pageSize:{size}, currentPage:{page}) {{ total_count items {{ sku }} }} }}')["products"]
        items = []
        for i in d["items"]:
            try:
                items += gql(f'{{ products(filter:{{sku:{{eq:"{i["sku"]}"}}}}, pageSize:1) {{ items {{ {fields} }} }} }}', tries=2)["products"]["items"]
            except Exception as e2:
                print(f"    {i['sku']}: {str(e2)[:50]} -> skipped"); items.append({"sku": i["sku"]})
        return {"total_count": d["total_count"], "items": items}

docs = json.load(open(raw_path, encoding="utf-8")) if os.path.exists(raw_path) else []
by_sku = {x["sku"]: x for x in docs}
if not docs or any("name" not in x for x in docs):                       # pass 1: everything but the description
    page, fresh = 1, []
    while True:
        d = fetch_page(LIGHT, PAGE, page); fresh += d["items"]
        print(f"page {page}: {len(fresh)}/{d['total_count']}")
        if page * PAGE >= d["total_count"] or not d["items"]: break
        page += 1; time.sleep(0.3)
    for x in fresh:
        if x["sku"] in by_sku: by_sku[x["sku"]].update(x)
        else: by_sku[x["sku"]] = x; docs.append(x)
    json.dump(docs, open(raw_path, "w", encoding="utf-8"), ensure_ascii=False)
if any("description" not in x for x in docs):                            # pass 2: descriptions, 4 per page
    page, got = 1, 0
    while True:
        d = fetch_page("sku description{html}", 4, page)
        for x in d["items"]:
            if x.get("description") and x["sku"] in by_sku: by_sku[x["sku"]]["description"] = x["description"]; got += 1
        if page % 25 == 0: print(f"descriptions page {page}: {got}")
        if page * 4 >= d["total_count"] or not d["items"]: break
        page += 1; time.sleep(0.3)
    print(f"descriptions: {got}/{len(docs)}")
    json.dump(docs, open(raw_path, "w", encoding="utf-8"), ensure_ascii=False)
raw = json.load(open(raw_path, encoding="utf-8"))

# url_path segment → category name (from every category seen on any product)
seg_name = {}
for p in raw:
    for c in p.get("categories") or []:
        seg_name[c["url_path"]] = c["name"]
PROMO = ("angebote", "sale", "outlet", "neu", "neuheiten", "marken", "black-friday", "bestseller", "geschenk", "top")
def cat_path(p):
    best = []
    for c in p.get("categories") or []:
        segs = c["url_path"].split("/")
        if segs[0].split("-")[0] in PROMO: continue
        path = [seg_name.get("/".join(segs[:i + 1]), segs[i].replace("-", " ").title()) for i in range(len(segs))]
        if len(path) > len(best): best = path
    return best

# (kg, L, W, H in cm) by top category, scaled by price
SIZE = {"Werkzeug": (2.2, 40, 30, 12), "Garten": (3.5, 60, 40, 20), "Haus": (2.5, 45, 35, 25), "Wasser": (4.0, 45, 35, 25),
        "Zubehör": (0.4, 25, 18, 6), "Ersatzteile": (0.3, 20, 15, 5)}
def size_for(top, name, price):
    kg, l, w, h = SIZE.get(top, (1.5, 35, 25, 12))
    n = name.lower()
    if any(k in n for k in ("zubehör", "ersatz", "filter", "sägeblatt", "bohrer", "bit", "schleif", "akku ", "ladegerät", "schlauch", "düse", "messer")):
        kg, l, w, h = (0.45, 25, 18, 6)
        if price >= 60: kg, l, w, h = (1.2, 30, 22, 10)
        return kg, l, w, h
    if price >= 300: return round(kg * 5, 1), int(l * 1.8), int(w * 1.4), int(h * 2)
    if price >= 150: return round(kg * 3, 1), int(l * 1.5), int(w * 1.3), int(h * 1.6)
    if price >= 70: return round(kg * 1.8, 1), int(l * 1.2), int(w * 1.1), int(h * 1.3)
    if price < 25: return round(kg * 0.35, 2), int(l * 0.6), int(w * 0.6), int(h * 0.6)
    return kg, l, w, h

def strip_html(s):
    s = re.sub(r"<[^>]+>", " ", s or "")
    return html.unescape(re.sub(r"\s+", " ", s)).strip()

def stock_for(sku, in_stock):
    if not in_stock: return 0
    return 2 + (sum(ord(c) for c in sku) * 7919) % 59          # deterministic 2…60

items, seen = [], set()
for p in raw:
    sku = (p.get("sku") or "").strip()
    if not sku or sku in seen: continue
    seen.add(sku)
    pr = ((p.get("price_range") or {}).get("minimum_price") or {})
    gross = float(((pr.get("final_price") or {}).get("value")) or 0)
    regular = float(((pr.get("regular_price") or {}).get("value")) or gross)
    path = cat_path(p)
    top = path[0] if path else "Sonstiges"
    name = re.sub(r"\s*\|\s*VONROC\s*$", "", (p.get("name") or "").strip())
    kg, l, w, h = size_for(top, name, gross)
    in_stock = p.get("stock_status") == "IN_STOCK"
    short = strip_html((p.get("short_description") or {}).get("html"))
    long_ = strip_html((p.get("description") or {}).get("html"))
    attrs = {a["code"]: a.get("value") for a in ((p.get("custom_attributesV2") or {}).get("items") or []) if a.get("value") is not None}  # (not fetched: custom_attributesV2 crashes the backend on some products)
    box = ""
    try:
        parts = json.loads(attrs.get("in_the_box") or "[]")
        box = ", ".join(f"{x.get('amount', '')} {x.get('name', '')}".strip() for x in parts if x.get("name"))
    except Exception: pass
    note = long_
    if box: note = (note + " · Lieferumfang: " + box) if note else "Lieferumfang: " + box
    gallery = [g["url"] for g in p.get("media_gallery") or [] if g.get("url")]
    img = (p.get("image") or {}).get("url") or (gallery[0] if gallery else None)
    items.append({
        "value": sku, "name": name[:120],
        "grossPrice": round(gross, 2), "netPrice": round(gross / 1.19, 2), "regularPrice": round(regular, 2), "vat": 19,
        "upc": (p.get("ean_code") or "").strip(), "sku": sku,
        "weight": kg, "length": l, "width": w, "height": h,
        "group1": "VONROC", "group2": " > ".join(path)[:255], "classification": top[:255],
        "description": (short or (strip_html(attrs.get("pros_description")) or f"VONROC · {' > '.join(path)}"))[:255],
        "documentNote": note[:1900],
        "imageUrl": img, "images": gallery[:6],
        "stock": stock_for(sku, in_stock), "salable": in_stock,
        "shopUrl": f"{BASE}/{p.get('url_key')}",
    })

# master ("summary") articles: SKUs sharing the model code (two letters + three digits + two letters, e.g.
# CS508DC) — colour suffixes (BH502AC_WHITE) and set prefixes (S_/S2_/S3_ = with 1/2/… batteries) are variants.
CODE = re.compile(r"[A-Z]{2}\d{3}[A-Z]{2}")
by_code = collections.defaultdict(list)
for it in items:
    m = CODE.search(it["value"])
    if m: by_code[m.group(0)].append(it)
values = {it["value"] for it in items}
SEP = re.compile(r"\s*(?:\||–|-|,|/|mit|inkl\.?)\s*[^|–\-,/]*$", re.I)
masters = []
for code, vs in sorted(by_code.items()):
    if len(vs) < 2: continue
    key = code if code not in values else f"{code}-VAR"
    base = next((v for v in vs if v["value"] == code), None) or next((v for v in vs if v["salable"]), vs[0])
    common = os.path.commonprefix([v["name"] for v in vs]).strip()
    if len(common) < len(base["name"]): common = SEP.sub("", common).strip(" -–|,/")   # cut the broken last token
    name = common if len(common) >= 12 else base["name"]
    masters.append({
        "value": key, "name": name[:120], "variants": [v["value"] for v in vs],
        "group1": "VONROC", "group2": base["group2"], "classification": base["classification"],
        "description": (f"Stammartikel · {len(vs)} Varianten: " + ", ".join(v["value"] for v in vs))[:255],
        "documentNote": base["documentNote"], "imageUrl": base["imageUrl"], "shopUrl": base["shopUrl"],
    })
    for v in vs: v["parentValue"] = key
json.dump(masters, open(os.path.join(os.path.dirname(out_path) or ".", "masters.json"), "w", encoding="utf-8"), ensure_ascii=False, indent=0)
print(f"{len(masters)} master articles for {sum(len(m['variants']) for m in masters)} variants -> masters.json")
json.dump(items, open(out_path, "w", encoding="utf-8"), ensure_ascii=False, indent=0)
print(f"{len(items)} articles -> {out_path}")
print("in stock", sum(1 for i in items if i["salable"]), "| no image", sum(1 for i in items if not i["imageUrl"]),
      "| no EAN", sum(1 for i in items if not i["upc"]), "| price 0", sum(1 for i in items if not i["grossPrice"]),
      "| no category", sum(1 for i in items if not i["group2"]))
print("top categories", collections.Counter(i["classification"] for i in items).most_common())