"""Turn a flat, inconsistently-filed library into collections and tags. The imported libraries carry years of ad-hoc filing: documents uploaded one at a time with no relationship to each other, an "Other Documents" bucket holding tens of unrelated things, and people's bios filed as newsletters. This command reads the documents that are actually there and proposes structure: * **Collections** from same-day batches — the papers from one AGM, a year's insurance renewal — which is how they were genuinely delivered. * **Tags** for the cross-cutting facts a category can't express (the year, "AGM", "insurance", "bio"). * **Recategorisation** where the title plainly disagrees with the filing. Always rehearse first; nothing is written without --apply: python manage.py organise_documents # report only python manage.py organise_documents --apply """ import re from collections import defaultdict from django.core.management.base import BaseCommand from django.db import transaction from django.utils.text import slugify from documents.models import Category, Collection, Document, Tag # Title patterns -> the category a document plainly belongs in. # Ordered: the first match wins, so the specific comes before the general. # Deliberately avoids the corporation's own name ("strata plan #", "strata corp"), # which appears in most titles on these sites and classifies nothing. RULES = [ ("insurance", ["insurance", "reinsurance", "d&o", "lineslip", "ipl0", "b174", "b0600", "certificate", "liability", "policy schedule"]), ("minutes", ["minute", "meeting", "agm", "annual general"]), ("financials", ["financial", "budget", "revenue", "balance sheet", "audit", "capex", "reserve fund", "income statement", "statement", "presentation", "analysis"]), ("governance", ["bylaw", "by-law", "management agreement", "rental and management", "declaration", "unit entitlement", "proxy", "resolution", "rules and regulation"]), ("newsletters", ["newsletter", "owners' update", "owner update", "sand prints"]), ] # A person's bio/resume: says so, or the title is *only* a name. BIO_HINTS = ["bio", "resume", "curriculum vitae", " cv"] NAME_RE = re.compile(r"^[A-Z][a-zA-Z'\-]+(?:\s+[A-Z]\.?)?(?:\s+[A-Z][a-zA-Z'\-]+){1,2}$") # Words that mean a title describes a document, not a person — guards against # "Palms Management Presentation" being read as somebody's name. NOT_A_NAME = { "strata", "palms", "sands", "shore", "club", "resort", "management", "presentation", "report", "minutes", "plan", "board", "committee", "annual", "general", "meeting", "financial", "financials", "budget", "insurance", "certificate", "policy", "statement", "documents", "document", "update", "newsletter", "agm", "owners", "owner", "building", "buildings", "external", "works", "appendix", "corp", "excess", "primary", "capex", "study", "notice", } YEAR_RE = re.compile(r"(20[0-2]\d)") # Each site invented its own category names over the years. Fold the legacy ones # into the standard set so the three portals read the same and no site shows # both "Insurance" and "Insurance Documents". LEGACY_MERGES = { "insurance documents": "insurance", "meeting minutes": "minutes", "owners' newsletters": "newsletters", "owner's newsletters": "newsletters", "owner's newsletter": "newsletters", "communications": "newsletters", "strata capex": "financials", "minutes & financials": "minutes", "bylaws": "governance", "management agreements": "governance", "other documents": "other", } def looks_like_a_person(title): stripped = title.strip() lowered = stripped.lower() if any(hint in lowered for hint in BIO_HINTS): return True if not NAME_RE.match(stripped): return False # Every word must be plausibly part of a person's name. return not any(word.strip(".,'").lower() in NOT_A_NAME for word in stripped.split()) def classify(title): """Return (category_key, tags) implied by the title alone.""" lowered = title.lower() tags = set() year = YEAR_RE.search(title) if year: tags.add(year.group(1)) if "agm" in lowered or "annual general meeting" in lowered: tags.add("AGM") if looks_like_a_person(title): tags.add("bio") return "board-and-people", tags for key, needles in RULES: if any(needle in lowered for needle in needles): tags.add(key.rstrip("s") if key != "financials" else "financials") return key, tags return None, tags def collection_title(titles, date): """Name a batch after what it plainly is.""" joined = " ".join(titles).lower() year = None for title in titles: found = YEAR_RE.search(title) if found: year = found.group(1) break year = year or (str(date.year) if date else "") # People first: a batch of bios often includes one stray AGM paper, and # "AGM documents" would be the wrong name for eleven biographies. if sum(looks_like_a_person(t) for t in titles) >= max(2, len(titles) // 2): return f"Board member biographies {year}".strip() if any(w in joined for w in ("insurance", "reinsurance", "d&o", "lineslip", "ipl0")): return f"{year} insurance renewal".strip() if "agm" in joined or "annual general meeting" in joined: return f"{year} AGM documents".strip() if "budget" in joined or "financial" in joined: return f"{year} financial documents".strip() if "capex" in joined: return f"{year} CapEx study".strip() return f"Documents added {date:%B %Y}" if date else "Documents" class Command(BaseCommand): help = "Group documents into collections, tag them, and fix obvious misfiling." def add_arguments(self, parser): parser.add_argument("--apply", action="store_true", help="write changes") parser.add_argument("--min-batch", type=int, default=2, help="smallest same-day group that becomes a collection") def handle(self, *args, **options): apply_changes = options["apply"] out = self.stdout wanted = { "minutes": "Minutes", "financials": "Financials", "insurance": "Insurance", "governance": "Governance", "newsletters": "Newsletters", "board-and-people": "Board & People", "other": "Other", } categories = {} created_categories = [] for index, (key, name) in enumerate(wanted.items()): existing = Category.objects.filter(slug=key).first() if existing: categories[key] = existing continue created_categories.append(name) if apply_changes: categories[key] = Category.objects.create(name=name, slug=key, order=index) else: categories[key] = None documents = list(Document.objects.select_related("category").all()) recategorised, tagged = [], 0 by_day = defaultdict(list) for document in documents: key, tags = classify(document.title) if key: target = categories.get(key) current = document.category.name if document.category else None # Only a genuine change counts — comparing by NAME keeps the dry # run honest when the target category doesn't exist yet. if (target and document.category_id != target.id) or ( target is None and current != wanted[key] ): recategorised.append((document, document.category, target or wanted[key])) if tags: tagged += 1 if document.published_date: by_day[document.published_date].append(document) batches = {day: docs for day, docs in by_day.items() if len(docs) >= options["min_batch"]} out.write(self.style.MIGRATE_HEADING( f"\n{'APPLYING' if apply_changes else 'DRY RUN'} — {len(documents)} documents")) out.write(self.style.MIGRATE_LABEL("\nCategories")) out.write(f" standard set: {', '.join(wanted.values())}") if created_categories: out.write(f" to create : {', '.join(created_categories)}") out.write(f" refiled : {len(recategorised)}") for document, was, now in recategorised[:8]: name = now.name if hasattr(now, "name") else now out.write(f" {document.title[:44]:46} {str(was or '—')[:18]:20} -> {name}") if len(recategorised) > 8: out.write(f" … and {len(recategorised) - 8} more") out.write(self.style.MIGRATE_LABEL("\nCollections")) out.write(f" same-day batches: {len(batches)} covering " f"{sum(len(v) for v in batches.values())} documents") for day in sorted(batches, reverse=True)[:8]: docs = batches[day] out.write(f" {day} {collection_title([d.title for d in docs], day)!r} " f"({len(docs)})") out.write(self.style.MIGRATE_LABEL("\nTags")) out.write(f" documents receiving at least one tag: {tagged}") if not apply_changes: out.write(self.style.WARNING("\nNothing written. Re-run with --apply.\n")) return with transaction.atomic(): for document, _was, now in recategorised: document.category = now document.save(update_fields=["category"]) tag_cache = {} for document in documents: _key, tags = classify(document.title) for label in tags: tag = tag_cache.get(label) if tag is None: tag, _ = Tag.objects.get_or_create( slug=slugify(label)[:70], defaults={"name": label} ) tag_cache[label] = tag document.tags.add(tag) used = set(Collection.objects.values_list("slug", flat=True)) made = 0 for day, docs in batches.items(): if any(d.collection_id for d in docs): continue title = collection_title([d.title for d in docs], day) base = slugify(title)[:200] or f"batch-{day}" slug, suffix = base, 2 while slug in used: slug, suffix = f"{base}-{suffix}", suffix + 1 used.add(slug) collection = Collection.objects.create( title=title, slug=slug, occurred_on=day, ) for document in docs: document.collection = collection document.save(update_fields=["collection"]) made += 1 # Fold legacy category names into the standard set, then retire the # emptied ones so each portal shows one coherent list. merged, removed = 0, 0 for legacy in list(Category.objects.all()): target_key = LEGACY_MERGES.get(legacy.name.strip().lower()) if not target_key: continue target = categories.get(target_key) if target is None: target = Category.objects.filter(slug=target_key).first() if target is None: target = Category.objects.create( name=wanted[target_key], slug=target_key, order=90 ) categories[target_key] = target if target.id == legacy.id: continue merged += legacy.documents.count() legacy.documents.update(category=target) legacy.delete() removed += 1 # Give every category a distinct colour, in display order. from documents.models import CATEGORY_PALETTE for position, category in enumerate(Category.objects.all()): wanted_colour = CATEGORY_PALETTE[position % len(CATEGORY_PALETTE)] if category.color != wanted_colour: category.color = wanted_colour category.save(update_fields=["color"]) out.write(self.style.SUCCESS( f"\nApplied: {len(recategorised)} refiled, {made} collections, " f"{len(tag_cache)} tags, {merged} documents merged out of " f"{removed} legacy categories.\n"))