"""Beauty widening (task A6): two trees per store, targeted beauty pages, page-1-only
stores re-reading what they hold, shelves in Spanish, sizes no item comes in.
Each class pins a measured failure: Panama's whole "Licores" shelf invisible to an
English-only filter (181 of 395 drinks passed), an alphabetical category cap that dropped
whisky and wine at Heathrow, four stores whose robots forbid the page-2 query string, and
two mists stored at seven litres.
"""
from pathlib import Path
import pytest
from app.services.collectors import avolta
from app.services.collectors.ari import vertical_of_url, wanted_urls
from app.services.collectors.avolta import STORES, AvoltaCollector
from app.services.collectors.base import ListingRef
from app.services.collectors.fetch import FetchError, Page
from app.services.collectors.robots import parse as parse_robots
from app.services.collectors.shopify import not_offered, shelf_vertical
from app.services.collectors.targets import BeautyTargets, load_beauty_targets
from app.services.normalize import size_is_implausible
FIXTURES = Path(__file__).parent / "fixtures"
PDP = (FIXTURES / "avolta_product_page.html").read_text()
LHR = next(s for s in STORES if s.code == "LHR")
class TestShopifyShelves:
"""The shop's own shelf names decide the family, in the shop's own language."""
@pytest.mark.parametrize("product", [
{"product_type": "Licores", "tags": ["licores", "Whisky"]}, # the Attenza drinks shelf
{"product_type": "licores", "tags": []},
{"product_type": "", "tags": ["Vinos", "Tintos"]},
{"product_type": "", "tags": ["Rones"]},
{"product_type": "", "tags": ["Champanas"]},
{"product_type": "", "tags": ["Cordiales"]},
{"product_type": "Scotch Whisky", "tags": []},
{"product_type": "Icewine", "tags": []},
{"product_type": "Eau de Vie", "tags": []},
])
def test_drinks_shelves_in_english_and_spanish(self, product):
assert shelf_vertical(product) == "liquor"
@pytest.mark.parametrize("product", [
{"product_type": "Fragancias", "tags": ["fragancias", "Fragancias Femeninas"]},
{"product_type": "Women's Perfume", "tags": ["EDP", "Perfume Section"]},
{"product_type": "Men's Cologne", "tags": []},
{"product_type": "Unisex", "tags": ["EDP"]},
{"product_type": "Rostro", "tags": ["tratamientos"]},
{"product_type": "Cuerpo", "tags": ["tratamientos"]},
{"product_type": "Capilar", "tags": []},
{"product_type": "Body Mist", "tags": []},
{"product_type": "Sunscreen", "tags": []},
])
def test_beauty_shelves(self, product):
assert shelf_vertical(product) == "beauty"
@pytest.mark.parametrize("product", [
{"product_type": "Audio", "tags": ["Electrónica"]}, # "ron" inside electrónica must not fire
{"product_type": "Reloj Análogo", "tags": []},
{"product_type": "Lentes", "tags": []},
{"product_type": "Chocolates", "tags": []},
{"product_type": "", "tags": []},
])
def test_everything_else_is_out_of_scope(self, product):
assert shelf_vertical(product) is None
def test_the_title_is_never_consulted(self):
"""A skincare product named "Immortelle" is beauty, whatever "rum" inside a word says."""
assert shelf_vertical({"title": "Immortelle Divine Cream", "product_type": "Cuerpo", "tags": []}) == "beauty"
assert shelf_vertical({"title": "Rum Cake", "product_type": "Chocolates", "tags": []}) is None
def test_a_shop_hidden_product_is_not_on_the_shelf(self):
assert not_offered({"tags": ["fragancias", "auto-oculto", "auto-oculto-precio-0"]})
assert not_offered({"tags": "Apagar, fragancias"})
assert not not_offered({"tags": ["fragancias"]})
class TestDublinTrees:
def test_prefixes_name_the_tree(self):
assert vertical_of_url("https://www.dublinandcorkdutyfree.ie/alcohol/spirits/gin/irish-gin-70cl/362013.html?lang=en_IE") == "liquor"
assert vertical_of_url("https://www.dublinandcorkdutyfree.ie/beauty/fragrance/x/y-100ml/1.html") == "beauty"
assert vertical_of_url("https://www.dublinandcorkdutyfree.ie/gifts/x/1.html") is None
def test_drinks_walk_whole_and_beauty_only_the_targeted_lines(self):
targets = BeautyTargets([{"brand": "Lancôme", "line": "vie est belle", "size_ml": 50}])
locs = [
"https://x.ie/alcohol/whiskey/some-whiskey-70cl/1.html",
"https://x.ie/beauty/fragrance/womens/la-vie-est-belle-eau-de-parfum-50ml/2.html",
"https://x.ie/beauty/fragrance/womens/some-other-scent-50ml/3.html",
"https://x.ie/beauty/fragrance/womens/la-vie-est-belle-eau-de-parfum-100ml/4.html",
]
assert [u.rsplit("/", 2)[-2] for u, _ in wanted_urls(locs, targets)] == [
"some-whiskey-70cl", "la-vie-est-belle-eau-de-parfum-50ml",
]
def test_no_targets_means_no_beauty_pages_never_all_of_them(self):
locs = ["https://x.ie/beauty/fragrance/womens/la-vie-est-belle-eau-de-parfum-50ml/2.html"]
assert list(wanted_urls(locs, BeautyTargets([]))) == []
class TestTargets:
def test_brand_and_line_words_match_a_tile_name(self):
targets = BeautyTargets([{"brand": "Jean Paul Gaultier", "line": "beau narcisse", "size_ml": 125}])
assert targets.matches("Le Beau Narcisse Eau de Parfum 125ml", "Jean Paul Gaultier")
assert targets.matches("jean-paul-gaultier-le-beau-narcisse-edp-125ml")
assert not targets.matches("Le Beau Eau de Toilette 125ml", "Jean Paul Gaultier")
def test_a_slug_without_the_brand_needs_the_size_and_a_distinctive_word(self):
targets = BeautyTargets([{"brand": "Jean Paul Gaultier", "line": "beau", "size_ml": 125},
{"brand": "Lancôme", "line": "vie est belle", "size_ml": 50}])
assert not targets.matches("le-beau-eau-de-toilette-125ml") # "beau" alone is not distinctive
assert targets.matches("la-vie-est-belle-eau-de-parfum-50ml")
assert not targets.matches("la-vie-est-belle-eau-de-parfum-100ml")
def test_the_repo_list_loads_and_is_forty(self):
assert len(load_beauty_targets()) == 40
def _home(*links: str) -> str:
return "".join(f'x' for u in links)
def _tile(sku: str, name: str, brand: str, price: str, url: str) -> str:
return (
f''
)
class FakeSite:
"""A storefront made of a dict of URL -> HTML; every fetch is recorded."""
def __init__(self, pages: dict[str, str]) -> None:
self.pages, self.fetched = pages, []
def fetch(self, url, accept=None, delay=0.0, **_):
self.fetched.append(url)
if url not in self.pages:
raise FetchError(f"{url} returned HTTP 404")
return Page(url=url, status=200, body=self.pages[url].encode())
@pytest.fixture
def lhr(monkeypatch):
base = LHR.base_url
robots = parse_robots("User-agent: *\nDisallow: /*?\nCrawl-delay: 30\n", host=base)
monkeypatch.setattr(avolta, "check_allowed", lambda *a, **k: robots)
monkeypatch.setattr(avolta, "load_beauty_targets", lambda *a, **k: BeautyTargets(
[{"brand": "Dior", "line": "sauvage", "size_ml": 100}]
))
site = FakeSite({
f"{base}/en/": _home(f"{base}/en/64/liquor/whisky", f"{base}/en/137/beauty/fragrance"),
f"{base}/en/64/liquor/whisky": _tile("1001", "Talisker 10 70cl", "Talisker", "40.00", f"{base}/en/64/talisker-10-70cl"),
f"{base}/en/137/beauty/fragrance":
_tile("2001", "Sauvage Eau de Toilette 100ml", "Dior", "90.00", f"{base}/en/137/dior-sauvage-100ml")
+ _tile("2002-P", "Sauvage Eau de Parfum", "Dior", "70.00", f"{base}/en/137/dior-sauvage-edp")
+ _tile("2003-P", "Bleu de Chanel", "Chanel", "80.00", f"{base}/en/137/bleu-de-chanel"),
f"{base}/en/137/dior-sauvage-edp": (FIXTURES / "avolta_configurable_product.html").read_text(),
f"{base}/en/64/held-bottle": PDP,
})
monkeypatch.setattr(avolta, "fetch", site.fetch)
collector = AvoltaCollector(LHR)
return collector, site, base
class TestAvoltaWalk:
def test_two_trees_each_with_their_own_slugs(self, lhr):
collector, site, base = lhr
home = site.pages[f"{base}/en/"]
robots = avolta.check_allowed()
assert collector._category_urls(home, robots, "liquor") == [f"{base}/en/64/liquor/whisky"]
assert collector._category_urls(home, robots, "beauty") == [f"{base}/en/137/beauty/fragrance"]
def test_a_cap_that_drops_a_category_says_so_naming_the_store(self, lhr, caplog):
collector, site, base = lhr
many = _home(*(f"{base}/en/64/liquor/cat-{i:02d}" for i in range(30)))
with caplog.at_level("WARNING"):
urls = collector._category_urls(many, avolta.check_allowed(), "liquor")
assert len(urls) == avolta.MAX_CATEGORIES_PER_VERTICAL
assert "categories_capped" in caplog.text and "store=LHR" in caplog.text and "cat-29" in caplog.text
def test_a_query_blocked_store_reads_page_one_and_fetches_only_targeted_beauty_pages(self, lhr):
collector, site, base = lhr
rows = list(collector.collect(delay=0.0))
assert not any("?p=" in u for u in site.fetched), "page 2 was requested at a store that forbids it"
# the simple beauty tile is free; the targeted family costs one fetch; Chanel's does not
assert f"{base}/en/137/dior-sauvage-edp" in site.fetched
assert f"{base}/en/137/bleu-de-chanel" not in site.fetched
by_sku = {r.source_sku: r for r in rows}
assert by_sku["1001"].vertical == "liquor"
assert by_sku["2001"].vertical == "beauty" and by_sku["2001"].price == 90.0
assert "2002-P::1000" in by_sku and by_sku["2002-P::1000"].vertical == "beauty"
assert "2003-P" not in by_sku and not any(k.startswith("2003-P") for k in by_sku)
def test_held_urls_the_walk_did_not_reach_are_re_read_once(self, lhr):
collector, site, base = lhr
collector.held_listings = [
ListingRef("1001", f"{base}/en/64/talisker-10-70cl", "LHR", "liquor"), # seen on page 1: not re-read
ListingRef("5074177", f"{base}/en/64/held-bottle", "LHR", "beauty"), # page-2 product: re-read
ListingRef("9999", f"{base}/en/64/gone-bottle", "LHR", "liquor"), # 404: gone, no row
ListingRef("8888::700", f"{base}/en/64/held-bottle", "LHR", "beauty"), # same URL: one fetch
]
rows = list(collector.collect(delay=0.0))
assert site.fetched.count(f"{base}/en/64/held-bottle") == 1
assert f"{base}/en/64/talisker-10-70cl" not in site.fetched
held = next(r for r in rows if r.source_sku == "5074177")
assert held.price == 19.95 and held.currency == "EUR" and held.vertical == "beauty"
def test_a_store_that_may_page_does_not_re_read_held_urls(self, lhr, monkeypatch):
collector, site, base = lhr
open_robots = parse_robots("User-agent: *\nCrawl-delay: 30\n", host=base)
monkeypatch.setattr(avolta, "check_allowed", lambda *a, **k: open_robots)
collector.held_listings = [ListingRef("5074177", f"{base}/en/64/held-bottle", "LHR", "beauty")]
list(collector.collect(delay=0.0))
assert f"{base}/en/64/held-bottle" not in site.fetched
assert f"{base}/en/64/liquor/whisky?p=2" in site.fetched
class TestImplausibleSizes:
"""The four rows the first audit listed (2026-09-05), and the large ones that are real."""
@pytest.mark.parametrize(("vertical", "name", "quantity_ml"), [
("beauty", "CHEIROSA 76", 7624),
("beauty", "DELICIA DRENCH 59", 5924),
("liquor", "15yo Sherry Cask", 7000),
("liquor", "18yo Port Cask", 7000),
])
def test_the_misreads_are_unknown_sizes(self, vertical, name, quantity_ml):
assert size_is_implausible(vertical, name, quantity_ml)
@pytest.mark.parametrize(("vertical", "name", "quantity_ml"), [
("liquor", "Moët & Chandon Impérial Brut 6L", 6000),
("liquor", "Faustino Rivero Red BiB 5L", 5000),
("liquor", "Paulaner Weissbier 24x0,5L DS", 12000),
("liquor", "Heineken Original 24x330ml", 7920),
("liquor", "Royal Export 5.8% 24x0.33LDS", 7920), # the feed's suffix glued to the unit
("beauty", "Cheirosa 39 24cl", 240),
("beauty", "Hand Soap 1.5L", 1500),
])
def test_large_sizes_the_name_vouches_for_stay(self, vertical, name, quantity_ml):
assert not size_is_implausible(vertical, name, quantity_ml)
class TestDublinLeafPaths:
"""Dublin lists 204 beauty product variants only under a flat leaf category, no /beauty/ above it."""
@pytest.mark.parametrize("url", [
"https://www.dublinandcorkdutyfree.ie/eau-de-toilette/red-door-eau-de-toilette-100ml/000389.html?lang=en_IE",
"https://www.dublinandcorkdutyfree.ie/mascara/hypnose-mascara/001200.html",
"https://www.dublinandcorkdutyfree.ie/day-cream/hydra-zen-50ml/001201.html",
])
def test_a_leaf_segment_names_the_beauty_tree(self, url):
assert vertical_of_url(url) == "beauty"
@pytest.mark.parametrize("url", [
"https://www.dublinandcorkdutyfree.ie/souvenirs/celtic/a-mug/000001.html",
"https://www.dublinandcorkdutyfree.ie/premium-chocolate/butlers-box/000002.html",
"https://www.dublinandcorkdutyfree.ie/fashion-and-accessories/bags/a-bag/000003.html",
])
def test_souvenirs_chocolate_and_fashion_are_not(self, url):
assert vertical_of_url(url) is None