"""The line above the product, the variation vocabulary, and the form: the pure rules. Sources of truth: this module, `models/catalog.py` (ProductLine, VariationAlias, Product.line_id), `cli.backfill_lines` / `backfill_variations`, `tests/test_lines.py`. Decided by rian on 12 Sep (`.logs/planning/streams/M-merging.md`): every listing is standardised into brand | line | variation | size; the line is the real product ("1 Million"), the variation is the juice (Eau de Toilette, Parfum, Elixir), the size is the bottle; a product is one line at one variation and one size; the page will be the line. Everything here is a pure function of the collected text, so `rederive` can re-run it over stored raw with no network and a test can pin every row that once went wrong. The key must be conservative in one direction only: two spellings of one line must meet, and two lines must never meet by accident. Where the rules cannot tell ("XV" against "15", "Aromatic Essence" against the plain line), they keep the words and leave the join to a person through the suggestion queue; a wrong merge costs more than a missed one. Beauty strips the concentration and variation words out of the line and hands them to the variation. Drinks strip nothing of the expression: a 12 and an 18 are different lines, and "original", "reserve", "XO", "black label" stay, because the catalogue's own stopword list once deleted exactly the words that tell an expression from its sibling and put a medal on the wrong bottle (agents.md). The audience is part of a beauty line ("Eternity for Men" and "Eternity for Women" are different bottles); its spellings are folded so shops agree. """ from __future__ import annotations import re import unicodedata from collections import Counter from collections.abc import Iterable from app.services.normalize import ( _ABV_RE, _NOISE_RE, _SIZE_PATTERNS, _strip_brand_prefix, brand_key, is_multipack, looks_like_set, ) #: The verticals whose products carry a variation at all. A drink's "variation" is its #: line (a 12 and an 18), so it has none; confectionery and tobacco likewise. VARIATION_VERTICALS = frozenset({"beauty"}) #: The concentration phrases and the variation qualifiers, longest phrase first, each to #: its label. Read before the brand and the line, because "Eau de Parfum" must never leave #: a stray "eau" or "de" in the line. _VARIATION_RULES: list[tuple[re.Pattern[str], str]] = [ (re.compile(r"\babsolu\s+de\s+parfum\b", re.I), "absolu"), (re.compile(r"\b[eé]lixir\s+de\s+parfum\b", re.I), "elixir"), (re.compile(r"\bessence\s+de\s+parfum\b", re.I), "parfum"), (re.compile(r"\bextrait\s+de\s+parfum\b", re.I), "parfum"), (re.compile(r"\beau\s+de\s+parfum\b", re.I), "edp"), (re.compile(r"\beau\s+de\s+toilette\b", re.I), "edt"), (re.compile(r"\beau\s+de\s+cologne\b", re.I), "edc"), (re.compile(r"\bedp\b", re.I), "edp"), (re.compile(r"\bedt\b", re.I), "edt"), (re.compile(r"\bedc\b", re.I), "edc"), (re.compile(r"\bcologne\b", re.I), "edc"), (re.compile(r"\bextrait\b", re.I), "parfum"), (re.compile(r"\bparfum\b", re.I), "parfum"), (re.compile(r"\bperfume\b", re.I), "parfum"), (re.compile(r"\b(?:body\s+|hair\s+)?(?:mist|brume|bruma)\b", re.I), "mist"), (re.compile(r"\b[eé]lixir\b", re.I), "elixir"), (re.compile(r"\bintense\b", re.I), "intense"), (re.compile(r"\bextr[eêè]me\b", re.I), "extreme"), (re.compile(r"\babsolu(?:e|te)?\b", re.I), "absolu"), ] _CONCENTRATIONS = ("edp", "edt", "edc", "parfum", "mist") _QUALIFIERS = ("intense", "extreme", "absolu") #: What a person reads for each canonical variation. VARIATION_DISPLAY = { "edp": "Eau de Parfum", "edt": "Eau de Toilette", "edc": "Eau de Cologne", "parfum": "Parfum", "mist": "Mist", "elixir": "Elixir", "intense": "Intense", "extreme": "Extreme", "absolu": "Absolu", } # Words that describe the format, the packaging or the edition, never the line. _FORMAT_WORDS = frozenset({ "spray", "vapo", "vaporisateur", "vaporizador", "vaporizer", "natural", "rechargeable", "recargable", "refillable", "refill", "recharge", "jumbo", "repack", "new", "travel", "exclusive", "edition", "limited", "collector", "collectors", "set", "coffret", "gift", "giftset", "duo", "trio", "kit", "pack", "twinpack", "twin", "triple", "tripack", "tri", "bundle", "estuche", "miniature", "miniatures", "mini", "deluxe", "size", "bottle", "bottles", "btl", "ml", "cl", "oz", "fl", "gr", "g", "ltr", "lt", "gp", "gb", "tube", "tin", "box", "carton", "canister", "case", "can", "pet", "trx", "ck", "vol", "abv", "proof", }) # Connectors a shop adds or omits at will. The articles among them leave the key but # stay in a name a person reads ("Le Male", "The One"); the rest go from either end. _ARTICLES = frozenset({"the", "le", "la", "les", "l", "el", "los", "las", "il", "lo", "gli", "der", "die", "das"}) _CONNECTORS = frozenset({"by", "for", "pour", "and", "et", "y", "with", "de", "du", "des", "di", "da", "of", "a", "an"}) | _ARTICLES # The audience IS part of a beauty line; its spellings are folded so shops agree. _AUDIENCE = {"homme": "men", "hommes": "men", "man": "men", "him": "men", "men": "men", "male": "men", "herren": "men", "uomo": "men", "hombre": "men", "femme": "women", "femmes": "women", "woman": "women", "her": "women", "female": "women", "women": "women", "damen": "women", "donna": "women", "mujer": "women", "ladies": "women"} # Category and region words a drinks shop appends to the expression, never the line. _DRINK_WORDS = frozenset({ "whisky", "whiskey", "whiskies", "scotch", "bourbon", "single", "malt", "malts", "blended", "blend", "grain", "gin", "vodka", "rum", "rhum", "ron", "tequila", "mezcal", "cognac", "brandy", "armagnac", "calvados", "liqueur", "liqueurs", "liquor", "liquer", "wine", "wines", "champagne", "prosecco", "cava", "sparkling", "beer", "lager", "cider", "spirit", "spirits", "dry", "london", "premium", "old", "aged", "years", "year", "yrs", "yr", "yo", "ans", "jahre", "anos", "highland", "highlands", "speyside", "islay", "lowland", "campbeltown", "scotland", "scottish", "ireland", "irish", "kentucky", "tennessee", "sweden", "swedish", "france", "french", "mexico", "mexican", "japan", "japanese", "swiss", "brut", }) #: A category word that names a cask is the expression, not the category: "Rum Cask", #: "Port Cask Finish", "Sherry Cask Selection" keep their first word. _CASK_WORDS = frozenset({"cask", "casks", "finish", "finished", "barrel", "barrels", "wood", "matured"}) _GLYPHS_RE = re.compile(r"[™®©℠*]") _DOTTED_RE = re.compile(r"\b(?:[a-z]\.){1,4}[a-z]\.?(?![a-z0-9])", re.I) # X.O, V.S.O.P. -> xo, vsop _APOSTROPHE_RE = re.compile(r"['’`]") _NUMBERED_RE = re.compile(r"\bn(?:o|°|º)?\.?\s*(\d+)\b") # N°5, No. 5, No5 -> no 5 _EXTRA_SIZE_RE = re.compile(r"\b\d+(?:[.,]\d+)?\s*(?:fl\.?\s*oz|oz|g|gr|kg)\b", re.I) _PACK_RE = re.compile(r"\b\d{1,2}\s*[x×]\s*\d+(?:[.,]\d+)?\s*(?:ml|cl|l)\b", re.I) _AGE_RE = re.compile(r"\b(\d{1,2})\s*(?:years?|yrs?|yo|y|ans|jahre|anos)\b(?:\s*old)?", re.I) _AGED_RE = re.compile(r"\baged\s+(\d{1,2})\b", re.I) _PUNCT_RE = re.compile(r"[^a-z0-9]+") _REFILL_RE = re.compile(r"\b(refill|recharge|rechargeable|refillable|recargable)\b", re.I) KEY_MAX = 150 def _fold(text: str | None) -> str: """Lowercase ASCII with the glyphs a shop decorates a name with removed.""" text = unicodedata.normalize("NFKD", text or "").encode("ascii", "ignore").decode().lower() text = _GLYPHS_RE.sub(" ", text) text = _DOTTED_RE.sub(lambda m: m.group(0).replace(".", ""), text) return _APOSTROPHE_RE.sub("", text) def _strip_measures(text: str) -> str: """Sizes, packs, strengths and the packaging noise, gone.""" text = _NOISE_RE.sub(" ", text) text = _PACK_RE.sub(" ", text) for pattern, _ in _SIZE_PATTERNS: text = pattern.sub(" ", text) text = _EXTRA_SIZE_RE.sub(" ", text) return _ABV_RE.sub(" ", text) def variation_of(name: str | None, vertical: str | None) -> tuple[str, str]: """`(raw, canonical)` of a name's variation: the vocabulary words found, in the order the shop wrote them ("elixir parfum intense"), and the one they mean ("elixir"). Both empty when the name says nothing or the vertical has no variations. The alias table may map a raw wording elsewhere; the rule is the default it starts from.""" if vertical not in VARIATION_VERTICALS or not name: return "", "" text = _strip_measures(_fold(name)) found: list[tuple[int, str]] = [] for pattern, label in _VARIATION_RULES: for match in pattern.finditer(text): found.append((match.start(), label)) # Blank the span in place so a shorter rule cannot re-read it ("parfum" inside # "eau de parfum") and the positions of later words stay where the shop put them. text = text[: match.start()] + " " * (match.end() - match.start()) + text[match.end():] if not found: return "", "" labels = [label for _, label in sorted(found)] raw = " ".join(dict.fromkeys(labels)) return raw, canonical_variation(raw) def canonical_variation(raw: str) -> str: """The rule's reading of a raw wording: Elixir dominates; else the first concentration named, then its qualifiers in a fixed order; a bare qualifier stands alone.""" labels = raw.split() if not labels: return "" if "elixir" in labels: return "elixir" base = next((label for label in labels if label in _CONCENTRATIONS), "") quals = [q for q in _QUALIFIERS if q in labels] return " ".join([*( [base] if base else []), *quals]) def display_variation(canonical: str) -> str: """"edt intense" -> "Eau de Toilette Intense".""" return " ".join(VARIATION_DISPLAY.get(label, label.title()) for label in canonical.split()) def form_of(name: str | None) -> str: """The form a name declares that keeps it off the bottle's own row: "set" (a coffret or kit, once folded a Boss Bottled gift set into the bottle), "refill" (the pod, not the jar), "pack" (a case of several); empty for a single item.""" if not name: return "" if looks_like_set(name): return "set" if _REFILL_RE.search(name): return "refill" if is_multipack(name): return "pack" return "" def resolve_alias(rows: dict, row_id: int | None, hops: int = 4): """The row an id points at once its `canonical_id` chain is followed (a brand's house, a line's canonical line); None for an unknown id. Bounded, so a cycle cannot hang.""" row = rows.get(row_id) if row_id else None for _ in range(hops): if row is None or not row.canonical_id or row.canonical_id == row.id: break row = rows.get(row.canonical_id, row) return row def _brand_sequences(brand: str | None, house: str | None) -> list[list[str]]: out: list[list[str]] = [] for text in (brand_key(brand), _fold(brand), brand_key(house), _fold(house)): words = _PUNCT_RE.sub(" ", text).split() if words and words not in out: out.append(words) return out def line_key(name: str | None, *, brand: str | None = None, house: str | None = None, vertical: str | None = None) -> str: """The line's key: the name with everything that is not the line removed, as words. The brand comes off the head first (`normalize._strip_brand_prefix`, which knows that "Nina Ricci Ricci Ricci" is a scent called Ricci Ricci), then any brand word left anywhere ("Flower by Kenzo" is Kenzo's Flower) as long as a word remains. Empty is the house's namesake line ("Jameson" itself). """ text = _strip_measures(_NUMBERED_RE.sub(r"no \1", _fold(name))) beauty = vertical in VARIATION_VERTICALS if beauty: for pattern, _ in _VARIATION_RULES: text = pattern.sub(" ", text) else: text = _AGED_RE.sub(r"\1", text) text = _AGE_RE.sub(r"\1", text) sequences = _brand_sequences(brand, house) tokens = _strip_brand_prefix(_PUNCT_RE.sub(" ", text).split(), sequences) # The house's own name may sit anywhere ("Flower by Kenzo", "Bleu de Chanel"); only a # brand's LAST word leaves the middle of a name, and only while a word remains. The # first word never does: the second "paris" in PSG's "paris" cap tells it from the # plain cap, and "Nina Ricci Nina" is a scent called Nina. last_words = {seq[-1] for seq in sequences if seq} without = [t for t in tokens if t not in last_words] if without: tokens = without drop = _FORMAT_WORDS | _CONNECTORS if beauty else _DRINK_WORDS | _FORMAT_WORDS | _CONNECTORS kept = [] for index, token in enumerate(tokens): if beauty: token = _AUDIENCE.get(token, token) elif token in _DRINK_WORDS and index + 1 < len(tokens) and tokens[index + 1] in _CASK_WORDS: kept.append(token) continue if token in drop: continue kept.append(token) key = " ".join(kept) return key[:KEY_MAX].rstrip() def brand_words_of(*names: str | None) -> frozenset[str]: """Every word of every spelling of a house, folded, for the display to leave out.""" return frozenset(w for name in names for seq in _brand_sequences(name, None) for w in seq) def _display_from(sample: str, brand_words: frozenset[str], vertical: str | None, *, prefix_only: bool) -> str: """The sample name with the brand, the measures, the format and (beauty) the variation words taken out and everything else left as the shop cased and spelt it.""" text = _GLYPHS_RE.sub(" ", sample or "") text = _strip_measures(text) if vertical in VARIATION_VERTICALS: for pattern, _ in _VARIATION_RULES: text = pattern.sub(" ", text) else: text = _AGED_RE.sub(r"\1", text) text = _AGE_RE.sub(r"\1", text) tokens = text.split() folded = [_PUNCT_RE.sub("", _fold(t)) for t in tokens] keep = [] at_head = True for index, (token, fold) in enumerate(zip(tokens, folded)): if not fold: continue if fold in brand_words and (at_head or not prefix_only): continue at_head = False if (vertical not in VARIATION_VERTICALS and fold in _DRINK_WORDS and index + 1 < len(folded) and folded[index + 1] in _CASK_WORDS): keep.append((token.strip("-.:;,()[]"), fold)) continue if fold in _FORMAT_WORDS or (vertical not in VARIATION_VERTICALS and fold in _DRINK_WORDS): continue keep.append((token.strip("-.:;,()[]"), fold)) keep = [(t, f) for t, f in keep if t] while (keep and keep[0][1] in _CONNECTORS and keep[0][1] not in _ARTICLES and not (len(keep) == 2 and keep[1][1] in _AUDIENCE)): keep.pop(0) while keep and keep[-1][1] in _CONNECTORS: keep.pop() return " ".join(t for t, _ in keep) def display_line_name(key: str, samples: Iterable[str] | Counter, house_name: str, *, brand_words: frozenset[str] = frozenset(), vertical: str | None = None) -> str: """A spelling to show for a line: the most common product name with the brand, the measures, the format and the variation words removed and the rest as the shop wrote it (articles, apostrophes and accents kept: "Le Male", "L'Interdit", "Terre d'Hermès"), accepted only when it folds back to the key; the house's own name for the namesake line; the key's words in title case when no sample fits.""" if not key: return house_name counter = samples if isinstance(samples, Counter) else Counter(samples) for sample, _ in counter.most_common(): for prefix_only in (False, True): display = _display_from(sample, brand_words, vertical, prefix_only=prefix_only) if display and line_key(display, vertical=vertical) == key: return display return " ".join(w.upper() if w in ("xo", "vsop", "vs", "xxo") else w.title() for w in key.split()) def line_slug(house_slug: str, key: str) -> str: """The address a line page will answer at: the house's slug, then the key's words.""" words = key.replace(" ", "-") return f"{house_slug}-{words}" if words else house_slug