"""Re-file the library using what the documents actually say. The first pass grouped and categorised from titles alone. That was enough to build structure, but titles have been edited by hand over the years and some batches were "catching up" — several years of paperwork uploaded on one day, which the date-based grouping then treated as one event. This reads each document's contents and proposes: * **Per-document categories** — a single AGM batch legitimately contains minutes, financials and insurance, so the category belongs on the document, not on the group. * **Descriptive group names** — "Documents added March 2024" becomes what the batch actually is. * **Splitting catch-up batches** — when one group's documents are about several different years, they were never one event; they are ungrouped. * **Title corrections** where the title plainly contradicts the file. Dry run by default: python manage.py reorganise python manage.py reorganise --apply """ import re from collections import Counter _YEAR = re.compile(r"\b(20[0-3]\d)\b") def _year_in(title): """The year stated in a title, if any — the most reliable subject marker.""" found = _YEAR.findall(title or "") return int(found[0]) if found else None from django.core.management.base import BaseCommand from django.db import transaction from django.utils.text import slugify from backoffice.classify import ( classify_text, content_date, subject_year, suggest_title, ) from backoffice.content import extract_text from documents.models import Category, Collection, Document CATEGORY_NAMES = { "minutes": "Minutes", "financials": "Financials", "insurance": "Insurance", "governance": "Governance", "newsletters": "Newsletters", "board-and-people": "Board & People", } def name_for_batch(facts, when): """A descriptive name for a batch, from what its documents are. Named after what the batch mostly *contains*, not what it mentions: board biographies and insurance renewals both talk about the AGM, so an AGM-first rule labelled them all "AGM documents". """ kinds = Counter(f["category"] for f in facts if f["category"]) years = Counter(f["year"] for f in facts if f["year"]) year = str(years.most_common(1)[0][0]) if years else (str(when.year) if when else "") if not kinds: return "" dominant, count = kinds.most_common(1)[0] majority = count >= max(2, (len(facts) + 1) // 2) def says_agm(fact): blob = f"{fact['title']} {(fact['text'] or '')[:600]}".lower() return "annual general meeting" in blob or "agm" in fact["title"].lower() if majority: if dominant == "board-and-people": return f"Board member biographies {year}".strip() if dominant == "insurance": return f"{year} insurance renewal".strip() if dominant == "governance": return f"{year} governance documents".strip() if dominant == "minutes": # Only minutes decide whether this was the AGM. agm = any(says_agm(f) for f in facts if f["category"] == "minutes") return f"{year} AGM documents".strip() if agm else f"{year} meeting papers".strip() if dominant == "financials": return f"{year} financial documents".strip() # A genuinely mixed batch anchored by minutes is the paperwork from one # meeting — the case worth naming for the event rather than the contents. if kinds.get("minutes") and any(says_agm(f) for f in facts if f["category"] == "minutes"): return f"{year} AGM documents".strip() if kinds.get("minutes"): return f"{year} meeting papers".strip() return "" class Command(BaseCommand): help = "Re-file documents and groups using their contents." def add_arguments(self, parser): parser.add_argument("--apply", action="store_true", help="write the changes") parser.add_argument("--limit", type=int, default=0, help="only read N documents") def handle(self, *args, **options): apply_changes = options["apply"] out = self.stdout categories = {} for key, name in CATEGORY_NAMES.items(): existing = Category.objects.filter(slug=key).first() if existing: categories[key] = existing documents = Document.objects.select_related("category", "collection") if options["limit"]: documents = documents[: options["limit"]] out.write("Reading documents…") facts = {} unreadable = 0 for document in documents: text = extract_text(document.file) if not text: unreadable += 1 key, confidence, evidence = classify_text(text, document.title) facts[document.pk] = { "doc": document, "title": document.title, "text": text, "category": key, "confidence": confidence, "evidence": evidence, "year": subject_year(document.title, text), "title_year": _year_in(document.title), "date": content_date(text), "kind": document.file_kind, } # ---- per-document categories ----------------------------------- recategorised = [] for fact in facts.values(): key = fact["category"] if not key or key not in categories or fact["confidence"] < 0.25: continue if fact["doc"].category_id != categories[key].id: recategorised.append((fact, categories[key])) # ---- titles ---------------------------------------------------- retitled = [] for fact in facts.values(): better = suggest_title(fact["text"], fact["title"], fact["kind"]) if better and better.lower() != fact["title"].lower(): retitled.append((fact, better)) # ---- groups ---------------------------------------------------- renamed, split = [], [] for collection in Collection.objects.all(): members = [facts[d.pk] for d in collection.documents.all() if d.pk in facts] if not members: continue # A catch-up batch: several years of paperwork uploaded on one day. # Judged on the years written in the TITLES, which state a # document's subject; body years are unreliable because an # insurance renewal or a set of minutes legitimately cites earlier # years without being about them. title_years = sorted({f["title_year"] for f in members if f["title_year"]}) spread = len(title_years) > 1 and (title_years[-1] - title_years[0]) >= 1 # Minutes anchor a batch to a real event, and the paperwork from one # meeting legitimately covers more than one year (last year's # accounts, next year's budget). Without that anchor, documents from # different years arriving together were a catch-up upload. anchored = any(f["category"] == "minutes" for f in members) if spread and len(members) > 1 and not anchored: split.append((collection, title_years, len(members))) continue # Only the placeholder names are replaced. A group already called # "2023 insurance renewal" says more than any rule can infer, and # renaming it on a guess loses information. generic = collection.title.lower().startswith("documents added") if not generic: continue better = name_for_batch(members, collection.occurred_on) if better and better.lower() != collection.title.lower(): renamed.append((collection, better)) # Two batches can legitimately describe the same thing in one year; # give the later one its month so the names stay distinguishable. seen_names = {c.title.lower() for c in Collection.objects.all()} deduped = [] for collection, better in renamed: candidate = better if candidate.lower() in seen_names and collection.occurred_on: candidate = f"{better} ({collection.occurred_on:%B})" suffix = 2 while candidate.lower() in seen_names: candidate, suffix = f"{better} ({suffix})", suffix + 1 seen_names.add(candidate.lower()) deduped.append((collection, candidate)) renamed = deduped # ---- report ---------------------------------------------------- out.write(self.style.MIGRATE_HEADING( f"\n{'APPLYING' if apply_changes else 'DRY RUN'} — read {len(facts)} documents" f" ({unreadable} unreadable)")) out.write(self.style.MIGRATE_LABEL(f"\nCategories from contents ({len(recategorised)})")) for fact, category in recategorised[:12]: was = fact["doc"].category.name if fact["doc"].category else "—" out.write(f" {fact['title'][:42]:44} {was[:14]:16} -> {category.name:16}" f" ({fact['confidence']}) {','.join(fact['evidence'][:2])}") if len(recategorised) > 12: out.write(f" … and {len(recategorised) - 12} more") out.write(self.style.MIGRATE_LABEL(f"\nTitles corrected ({len(retitled)})")) for fact, better in retitled[:10]: out.write(f" {fact['title'][:44]:46} -> {better}") out.write(self.style.MIGRATE_LABEL(f"\nGroups renamed ({len(renamed)})")) for collection, better in renamed[:14]: out.write(f" {collection.title[:44]:46} -> {better}") out.write(self.style.MIGRATE_LABEL(f"\nCatch-up batches to ungroup ({len(split)})")) for collection, years, count in split[:10]: out.write(f" {collection.title[:40]:42} {count} docs spanning " f"{', '.join(str(y) for y in years)}") if not apply_changes: out.write(self.style.WARNING("\nNothing written. Re-run with --apply.\n")) return with transaction.atomic(): for fact, category in recategorised: fact["doc"].category = category fact["doc"].save(update_fields=["category"]) for fact, better in retitled: fact["doc"].title = better[:255] fact["doc"].save(update_fields=["title"]) for collection, years, _count in split: collection.documents.update(collection=None) collection.delete() used = set(Collection.objects.values_list("slug", flat=True)) for collection, better in renamed: base = slugify(better)[:200] or collection.slug slug, suffix = base, 2 while slug in used and slug != collection.slug: slug, suffix = f"{base}-{suffix}", suffix + 1 used.add(slug) collection.title, collection.slug = better[:200], slug collection.save(update_fields=["title", "slug"]) out.write(self.style.SUCCESS( f"\nApplied: {len(recategorised)} refiled, {len(retitled)} retitled, " f"{len(renamed)} groups renamed, {len(split)} batches ungrouped.\n"))