#!/usr/bin/env python3
"""Scout one candidate host: robots, the discovery page, one listing; say what a collector would find.

Sources of truth: `app/services/collectors/robots.py` (the one robots policy), `fetch.py` (content
validated, never status), `avolta.py` and `shopify.py` (the parsers a new source would run under).
The scout reuses those and adds no matcher of its own: the count it prints is the count the
collector's own parser makes of the page, so a fixture it saves is a test of the real thing.

    ../.venv-dev/bin/python scripts/scout-source.py https://london-gatwick.worlddutyfree.com --platform avolta
    ../.venv-dev/bin/python scripts/scout-source.py https://ec.attenza.net --platform shopify --fixture-dir tests/fixtures/scout
    ../.venv-dev/bin/python scripts/scout-source.py https://www.shopdutyfree.com --platform hub

Per host at most three requests, each after the host's published crawl delay (the slower of
theirs and `--delay`): `robots.txt`; the discovery page (`/en/` for Avolta, which is where
`collect()` starts; `/products.json?limit=250` for Shopify, which IS the listing); one drinks
category page for Avolta (the first the collector's own link rule finds). `hub` reads the
platform's storefront registry once (`/en/` of the hub) and prints the storefronts it names.

A `SourceBlocked` anywhere, or 401/403 at robots, prints REFUSED and the host is never asked
again in the run. A hostname that does not resolve is "no such storefront", not a refusal. A
storefront that answers with another host's pages (a dead legacy subdomain that 301s to the
hub) is "no storefront" too. Announce the run in `.logs/runs/` before the first request; every
line here is a request to a retailer.

A second pass on a host (`--robots-file <the robots.txt the first pass saved>`) reads no
robots.txt and so costs the host two requests, not three; the first pass on 19 Sep read the
tree root as its listing, which is a landing page or a carousel, and the grid is a child.

`--fixture-dir` saves what the tasks after the scout test on: the host's robots.txt, and the
listing trimmed to facts (tiles only, `<img>` and description blocks stripped; Shopify products
through `facts_only`, at most ten). `--report` appends one JSON line per host for the table.
"""

from __future__ import annotations

import argparse
import json
import pathlib
import re
import socket
import sys
import time
import urllib.error
from datetime import UTC, datetime
from urllib.parse import urlsplit

sys.path.insert(0, str(pathlib.Path(__file__).resolve().parents[1]))

from app.services.collectors import robots as robots_mod  # noqa: E402
from app.services.collectors.avolta import (  # noqa: E402
    _CONFIGURABLE_SUFFIX,
    _DECLARED_CURRENCY_RE,
    _ITEM_SPLIT,
    _LINK_RE,
    _NOT_A_CATEGORY,
    _SYMBOL_RE,
    declared_currency,
    parse_product_variants,
)
from app.services.collectors.base import facts_only  # noqa: E402
from app.services.collectors.fetch import FetchError, SourceBlocked, fetch  # noqa: E402
from app.services.collectors.shopify import FEED_PATH, PAGE_SIZE, shelf_vertical  # noqa: E402
from app.services.normalize import clean_gtin, gtin_from_sku  # noqa: E402

# Markup a fixture must not carry: imagery and copy are expression (agents.md, "collect facts,
# not expression"); the tile's facts survive without them and the parser reads them the same.
# `<img>` and `<image>` tags, and the microdata metas that point at a picture or carry copy.
_IMG_RE = re.compile(r"<(?:img|image)\b[^>]*>|<meta itemprop=\"(?:image|description)\"[^>]*>", re.I)
_DESCRIPTION_RE = re.compile(
    r'<div class="product(?:-item)?-description[^"]*"[^>]*>.*?</div>', re.S | re.I
)
_SCRIPT_RE = re.compile(r"<script\b.*?</script>", re.S | re.I)
_STYLE_RE = re.compile(r"<style\b.*?</style>", re.S | re.I)
_INPUT_RE = re.compile(r"<input\b[^>]*>", re.I)
_SPACE_RE = re.compile(r"[ \t]*\n[ \t\n]*")
# A tile ends at its list item; what follows the last tile is the page's footer, not a fact.
_TILE_END_RE = re.compile(r"</li>", re.I)
_STOREFRONT_URL_RE = re.compile(r"https?://([a-z0-9-]+)\.(shopdutyfree|worlddutyfree)\.com", re.I)
_LOCATIONS_RE = re.compile(r'"locations"\s*:\s*(\[.*?\])\s*[,}]', re.S)
MAX_SHOPIFY_FIXTURE_PRODUCTS = 10
# The grid's own currency declaration, in the spelling every Avolta category page used on
# 19 Sep 2026 (`"currency_code": "USD"` per tile); `declared_currency()` reads only the
# `priceCurrency` microdata, which the grids do not carry (running list, Stream A, P3).
_CURRENCY_CODE_RE = re.compile(r'"currency_code"\s*:\s*"([A-Z]{3})"')


def page_currency(page: str) -> tuple[str | None, str]:
    """The currency the page declares and the form it declared it in: the collector's own
    reading first, then the grid's `currency_code` when every occurrence agrees."""
    code = declared_currency(page)
    if code:
        return code, "priceCurrency"
    codes = set(_CURRENCY_CODE_RE.findall(page))
    if len(codes) == 1:
        return codes.pop(), "currency_code"
    return None, "none" if not codes else "mixed"


def _slug(base_url: str) -> str:
    return urlsplit(base_url).hostname.replace(".", "-")


def _resolves(base_url: str) -> bool:
    try:
        socket.getaddrinfo(urlsplit(base_url).hostname, 443)
        return True
    except socket.gaierror:
        return False


def _is_name_failure(exc: BaseException) -> bool:
    cause = exc.__cause__
    return isinstance(cause, urllib.error.URLError) and isinstance(cause.reason, socket.gaierror)


def _read_robots(base_url: str, result: dict, robots_file: pathlib.Path | None = None) -> robots_mod.Robots | None:
    """The host's rules, or None with the reason written into `result`. With `robots_file`, the
    robots.txt the scout saved minutes earlier is evaluated instead of read again: a second pass
    on a host (a listing the first pass mischose) costs the host no extra request for its rules."""
    if robots_file is not None:
        text = robots_file.read_text(encoding="utf-8")
        robots = robots_mod.parse(text, host=base_url.rstrip("/"))
        result["robots_status"] = "saved"
        result["robots_text"] = text
        return robots
    try:
        robots = robots_mod.read(base_url, fresh=True)
    except SourceBlocked as exc:
        result.update(outcome="REFUSED", detail=str(exc)[:200])
        return None
    except robots_mod.RobotsUnavailable as exc:
        if _is_name_failure(exc):
            result.update(outcome="no such storefront", detail="hostname does not resolve")
        else:
            result.update(outcome="robots unavailable", detail=str(exc)[:200])
        return None
    result["robots_status"] = robots.status
    result["robots_text"] = _robots_text(base_url)
    return robots


_last_robots_body: dict[str, str] = {}
_original_fetcher = robots_mod._default_fetcher


def _remembering_fetcher(url: str) -> tuple[int, bytes]:
    """The reader's own fetcher, remembering each robots.txt body so the fixture can carry the
    text the reader evaluated: one request, read once, saved as read."""
    status, body = _original_fetcher(url)
    _last_robots_body[url.rsplit("/robots.txt", 1)[0]] = body.decode("utf-8", "replace")
    return status, body


robots_mod._default_fetcher = _remembering_fetcher


def _robots_text(base_url: str) -> str:
    return _last_robots_body.get(base_url.rstrip("/"), "")


def _wait(seconds: float, what: str) -> None:
    print(f"  waiting {seconds:.0f}s before {what}", flush=True)
    time.sleep(seconds)


def _save(fixture_dir: pathlib.Path | None, name: str, text: str) -> None:
    if fixture_dir is None:
        return
    fixture_dir.mkdir(parents=True, exist_ok=True)
    (fixture_dir / name).write_text(text, encoding="utf-8")
    print(f"  saved {fixture_dir / name} ({len(text.encode())} bytes)")


def _trim_avolta_listing(page: str, *, base_url: str, url: str, when: str) -> str:
    """The listing reduced to its tiles, each stripped of imagery, copy and scripts, with the
    page's own currency declaration kept so `declared_currency(fixture)` answers as the page did."""
    code, form = page_currency(page)
    head = [
        f"<!-- Trimmed from {url} on {when} by scripts/scout-source.py: the tiles of one category",
        "     page, <img>, description blocks and scripts stripped; the currency the page declared",
        f"     ({form}) is kept as the one line below. Facts, never expression. -->",
    ]
    if code:
        head.append(f'<meta itemprop="priceCurrency" content="{code}"/>')
    tiles = []
    for chunk in _ITEM_SPLIT.split(page)[1:]:
        end = _TILE_END_RE.search(chunk)
        if end:
            chunk = chunk[: end.end()]
        for pattern in (_SCRIPT_RE, _STYLE_RE, _IMG_RE, _DESCRIPTION_RE, _INPUT_RE):
            chunk = pattern.sub("", chunk)
        tiles.append('<div class="product-item-info' + _SPACE_RE.sub("\n", chunk))
    return "\n".join(head) + "\n" + "\n".join(tiles) + "\n"


def scout_avolta(base_url: str, delay: float, fixture_dir: pathlib.Path | None,
                 robots_file: pathlib.Path | None = None, raw_dir: pathlib.Path | None = None) -> dict:
    result: dict = {"host": urlsplit(base_url).hostname, "platform": "avolta", "requests": 0}
    if not _resolves(base_url):
        result.update(outcome="no such storefront", detail="hostname does not resolve")
        return result
    robots = _read_robots(base_url, result, robots_file)
    result["requests"] += 0 if robots_file else 1
    if robots is None:
        return result
    wait = robots.delay_for(delay)
    result["delay"] = wait
    result["published_delay"] = robots.crawl_delay
    result["en_allowed"] = robots.allows("/en/")
    if not result["en_allowed"]:
        result.update(outcome="REFUSED", detail="robots.txt disallows /en/")
        return result
    _save(fixture_dir, f"{_slug(base_url)}_robots.txt", result.pop("robots_text", ""))

    _wait(wait, "the discovery page /en/")
    try:
        home = fetch(f"{base_url}/en/", accept="text/html")
    except SourceBlocked as exc:
        result.update(outcome="REFUSED", detail=str(exc)[:200])
        return result
    except FetchError as exc:
        result.update(outcome="error", detail=str(exc)[:200])
        return result
    finally:
        result["requests"] += 1
    _save(raw_dir, f"{_slug(base_url)}_en.html", home.text)
    links = sorted(u for u in set(_LINK_RE["liquor"].findall(home.text)) if not _NOT_A_CATEGORY.search(u))
    hosts_linked = {urlsplit(u).hostname for u in links}
    own = urlsplit(base_url).hostname
    if links and own not in hosts_linked:
        result.update(outcome="no storefront", detail=f"serves another host's pages ({', '.join(sorted(hosts_linked))})")
        return result
    allowed = [u for u in links if robots.allows(u)]
    shallow = [u for u in allowed if u.rstrip("/").count("/") <= 6]
    categories = [u for u in (shallow or allowed) if "view-all" not in u and "digital" not in u] or (shallow or allowed)
    result["drinks_categories"] = len(categories)
    if not categories:
        result.update(outcome="no drinks categories on /en/", detail=f"{len(links)} drinks link(s), {len(allowed)} allowed")
        return result
    # The collector walks the root first and then its children; the root is a landing page
    # (shopdutyfree.com: no tile) or a carousel (worlddutyfree.com: the same products at every
    # airport), so the page that says what a grid holds is a child, one level below the root.
    children = [u for u in categories if u.rstrip("/").count("/") == 6]
    first = (children or categories)[0]
    result["categories"] = [u.split("/", 3)[-1] for u in categories]
    result["first_category"] = first
    result["p2_allowed"] = robots.allows(f"{first}?p=2")
    result["page_one_only"] = not result["p2_allowed"]

    _wait(wait, f"the listing {first}")
    try:
        listing = fetch(first, accept="text/html")
    except SourceBlocked as exc:
        result.update(outcome="REFUSED", detail=str(exc)[:200])
        return result
    except FetchError as exc:
        result.update(outcome="error", detail=str(exc)[:200])
        return result
    finally:
        result["requests"] += 1
    _save(raw_dir, f"{_slug(base_url)}_listing.html", listing.text)
    tiles = parse_product_variants(listing.text)
    symbol = _SYMBOL_RE.search(listing.text)
    code, form = page_currency(listing.text)
    result["currency"] = code or "unconfirmed"
    result["currency_form"] = form
    result["symbol"] = symbol.group(1).strip() if symbol else None
    result["tiles"] = len(tiles)
    result["configurable_tiles"] = sum(1 for t in tiles if t["sku"].endswith(_CONFIGURABLE_SUFFIX))
    with_barcode = sum(1 for t in tiles if gtin_from_sku(t["sku"]))
    result["with_barcode"] = with_barcode
    result["barcode_share"] = round(with_barcode / len(tiles), 2) if tiles else 0.0
    result["outcome"] = "ok" if tiles else "no tiles in the HTML (client-rendered grid?)"
    when = datetime.now(UTC).strftime("%Y-%m-%d")
    _save(fixture_dir, f"{_slug(base_url)}_listing.html",
          _trim_avolta_listing(listing.text, base_url=base_url, url=first, when=when))
    return result


def scout_shopify(base_url: str, delay: float, fixture_dir: pathlib.Path | None,
                  robots_file: pathlib.Path | None = None, raw_dir: pathlib.Path | None = None) -> dict:
    result: dict = {"host": urlsplit(base_url).hostname, "platform": "shopify", "requests": 0}
    if not _resolves(base_url):
        result.update(outcome="no such storefront", detail="hostname does not resolve")
        return result
    robots = _read_robots(base_url, result, robots_file)
    result["requests"] += 0 if robots_file else 1
    if robots is None:
        return result
    wait = robots.delay_for(delay)
    result["delay"] = wait
    result["published_delay"] = robots.crawl_delay
    feed = f"{base_url}{FEED_PATH}?limit={PAGE_SIZE}"
    result["feed_allowed"] = robots.allows(FEED_PATH) and robots.allows(feed)
    if not result["feed_allowed"]:
        result.update(outcome="REFUSED", detail=f"robots.txt disallows {FEED_PATH}")
        return result
    _save(fixture_dir, f"{_slug(base_url)}_robots.txt", result.pop("robots_text", ""))

    _wait(wait, "the feed " + FEED_PATH)
    try:
        payload = fetch(feed).json()
    except SourceBlocked as exc:
        result.update(outcome="REFUSED", detail=str(exc)[:200])
        return result
    except (FetchError, ValueError) as exc:
        result.update(outcome="error", detail=str(exc)[:200])
        return result
    finally:
        result["requests"] += 1
    products = payload.get("products", []) if isinstance(payload, dict) else []
    in_scope = [p for p in products if shelf_vertical(p) is not None]
    shelves = sorted({str(p.get("product_type") or "") for p in products if p.get("product_type")})
    variants = [v for p in in_scope for v in p.get("variants") or []]
    with_barcode = sum(1 for v in variants if clean_gtin(v.get("barcode")) or gtin_from_sku(v.get("sku")))
    currencies = sorted({
        str(price.get("price", {}).get("currency_code"))
        for v in variants for price in (v.get("presentment_prices") or [])
        if isinstance(price, dict) and price.get("price", {}).get("currency_code")
    })
    result.update(
        products=len(products), in_scope=len(in_scope),
        liquor=sum(1 for p in in_scope if shelf_vertical(p) == "liquor"),
        beauty=sum(1 for p in in_scope if shelf_vertical(p) == "beauty"),
        tiles=len(variants), with_barcode=with_barcode,
        barcode_share=round(with_barcode / len(variants), 2) if variants else 0.0,
        currency=currencies[0] if len(currencies) == 1 else "unconfirmed",
        shelves=shelves[:40], page_one_only=False,
        outcome="ok" if products else "empty feed",
    )
    when = datetime.now(UTC).strftime("%Y-%m-%d")
    sample = [
        {k: v for k, v in facts_only(p).items()}
        for p in in_scope[:MAX_SHOPIFY_FIXTURE_PRODUCTS]
    ]
    _save(fixture_dir, f"{_slug(base_url)}_products.json", json.dumps({
        "_note": f"Trimmed from {feed} on {when} by scripts/scout-source.py: the first "
                 f"{len(sample)} in-scope products through facts_only (no images, no body_html).",
        "products": sample,
    }, indent=1, ensure_ascii=False) + "\n")
    return result


def scout_hub(base_url: str, delay: float, fixture_dir: pathlib.Path | None,
              robots_file: pathlib.Path | None = None, raw_dir: pathlib.Path | None = None) -> dict:
    """The platform's own storefront registry, read once: which subdomains exist, by name."""
    result: dict = {"host": urlsplit(base_url).hostname, "platform": "hub", "requests": 0}
    robots = _read_robots(base_url, result)
    result["requests"] += 1
    if robots is None:
        return result
    wait = robots.delay_for(delay)
    result["delay"] = wait
    if not robots.allows("/en/"):
        result.update(outcome="REFUSED", detail="robots.txt disallows /en/")
        return result
    _wait(wait, "the storefront registry /en/")
    try:
        home = fetch(f"{base_url}/en/", accept="text/html")
    except SourceBlocked as exc:
        result.update(outcome="REFUSED", detail=str(exc)[:200])
        return result
    except FetchError as exc:
        result.update(outcome="error", detail=str(exc)[:200])
        return result
    finally:
        result["requests"] += 1
    _save(raw_dir, f"{_slug(base_url)}_en.html", home.text)
    storefronts = sorted({f"{m.group(1).lower()}.{m.group(2).lower()}.com" for m in _STOREFRONT_URL_RE.finditer(home.text)})
    registry = None
    match = _LOCATIONS_RE.search(home.text)
    if match:
        try:
            registry = json.loads(match.group(1))
        except ValueError:
            registry = None
    result.update(outcome="ok", storefronts=storefronts, registry_entries=len(registry) if isinstance(registry, list) else 0)
    if isinstance(registry, list):
        _save(fixture_dir, f"{_slug(base_url)}_locations.json", json.dumps(facts_only(registry), indent=1, ensure_ascii=False) + "\n")
    return result


def main() -> int:
    ap = argparse.ArgumentParser()
    ap.add_argument("base_url")
    ap.add_argument("--platform", choices=("avolta", "shopify", "hub"), required=True)
    ap.add_argument("--fixture-dir", type=pathlib.Path, default=None)
    ap.add_argument("--delay", type=float, default=1.0, help="our own floor; the host's wins when slower")
    ap.add_argument("--report", type=pathlib.Path, default=None, help="append one JSON line per host here")
    ap.add_argument("--robots-file", type=pathlib.Path, default=None,
                    help="evaluate this saved robots.txt instead of reading the host's (a second pass)")
    ap.add_argument("--raw-dir", type=pathlib.Path, default=None,
                    help="keep the pages as served here, for diagnosis; never a repo path")
    args = ap.parse_args()
    base_url = args.base_url.rstrip("/")
    if not base_url.startswith("https://"):
        base_url = "https://" + base_url.split("://", 1)[-1]

    print(f"scout {base_url} ({args.platform})", flush=True)
    started = time.monotonic()
    result = {"avolta": scout_avolta, "shopify": scout_shopify, "hub": scout_hub}[args.platform](
        base_url, args.delay, args.fixture_dir, robots_file=args.robots_file, raw_dir=args.raw_dir
    )
    result.pop("robots_text", None)
    result["seconds"] = round(time.monotonic() - started)
    result["at"] = datetime.now(UTC).isoformat(timespec="minutes")
    for key, value in result.items():
        if key in ("shelves", "storefronts"):
            print(f"  {key}: {', '.join(value)}")
        else:
            print(f"  {key}: {value}")
    if args.report:
        args.report.parent.mkdir(parents=True, exist_ok=True)
        with args.report.open("a", encoding="utf-8") as fh:
            fh.write(json.dumps(result, ensure_ascii=False) + "\n")
    return 0 if result.get("outcome") == "ok" else 3 if result.get("outcome") == "REFUSED" else 2


if __name__ == "__main__":
    sys.exit(main())