# -*- coding: utf-8 -*-
"""How much do the headline findings depend on the name extractor?

The extractor is heuristic. A 45-name sample of the single-mention tail on 2026-09-23 found
roughly a third were not businesses: evaluation criteria ("numbers driven approach", "sale to
list price ratio"), query echoes ("looking for an agent who specializes in luxury homes in
magnolia"), page furniture ("best for why consider"), news headlines, and truncated fragments.

That is a real problem for the FIRM COUNT and a much smaller one for the STRUCTURAL findings,
because junk names are almost all single-mention and single-area, so they inflate numerator and
denominator together. This script proves that rather than asserting it.

It recomputes every headline number under two filters:

  as-published   what study500_findings.py extracts today
  strict         a deliberately over-aggressive filter that also deletes REAL firms, so the
                 result is a lower bound and not a correction

If a finding survives a filter built to break it, the finding is not an artefact of extraction.

    py -3.10 study500_sensitivity.py       # writes study500/SENSITIVITY.txt
"""
from __future__ import annotations

import io
import json
import os
import re
import sys
from collections import Counter, defaultdict

sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
import study500_findings as F  # noqa: E402

HERE = os.path.dirname(os.path.abspath(__file__))
STORE = os.path.join(HERE, "study500", "answers.json")
OUT = os.path.join(HERE, "study500", "SENSITIVITY%s.txt" % ("-consumer" if "--consumer" in sys.argv else ""))

# Tokens that mark a business rather than a person.
BIZ = {"realty", "realtors", "realtor", "properties", "property", "homes", "home", "group",
       "team", "associates", "partners", "company", "co", "llc", "inc", "llp", "brokers",
       "brokerage", "estates", "estate", "builders", "construction", "development",
       "developers", "sotheby", "banker", "williams", "compass", "remax", "century",
       "elliman", "corcoran", "christie", "greene", "king", "turner", "agency"}

# Anything a sentence, heading or query would contain but a name would not.
SENTENCE = re.compile(
    r"\b(looking|search|searching|consider|considering|why|how|what|when|where|which|"
    r"best for|qualities|tips|guide|about|near|within|specializes|specializing|"
    r"recently|currently|available|reputable|top rated|approach|ratio|access|"
    r"representation|locations|experience with|focused|driven|review|reviews|"
    r"list price|market|trends|report|statistics|insights)\b", re.I)


def strict_ok(name: str) -> bool:
    """Lower bound. Keeps only what is unmistakably a person or a business name.

    Deliberately over-aggressive: it deletes real firms whose names read like phrases, and
    real agents recorded with a middle name or a suffix. Use it as a floor, never as a count.
    """
    n = name.strip()
    if not n:
        return False
    toks = [t for t in re.split(r"[^a-z0-9]+", n.lower()) if t]
    if len(toks) < 2 or len(toks) > 6:
        return False
    if SENTENCE.search(n):
        return False
    if any(t in BIZ for t in toks):
        return True
    # Otherwise it must look like a person: every token alphabetic, none a stopword.
    stop = {"in", "of", "the", "and", "for", "with", "to", "at", "on", "by", "a", "an", "tx"}
    if any(t in stop for t in toks):
        return False
    if not all(t.isalpha() for t in toks):
        return False
    return len(toks) <= 4


def build(clean, keep):
    mentions, areas, engines = Counter(), defaultdict(set), defaultdict(set)
    for r in clean:
        for raw in F.named(r.get("answer_text") or ""):
            c = F.canon(raw)
            if not keep(c):
                continue
            mentions[c] += 1
            if r.get("area"):
                areas[c].add(r["area"])
            engines[c].add(r["engine"])
    return mentions, areas, engines


def gini(counts):
    xs = sorted(counts)
    n = len(xs)
    if not n or not sum(xs):
        return 0.0
    cum = sum(i * x for i, x in enumerate(xs, 1))
    return (2 * cum) / (n * sum(xs)) - (n + 1) / n


def main() -> None:
    store = json.load(io.open(STORE, encoding="utf-8"))
    rows = list(store["answers"].values())
    if "--consumer" in sys.argv:
        # Optional internal view: three surfaces, see CONSUMER in study500_findings.
        rows = [r for r in rows if r.get("engine") in F.CONSUMER]
    F._AREAS.update(str(r["area"]).lower() for r in rows if r.get("area"))
    clean = [r for r in rows if not r.get("geo_mismatch")]

    runs = [("as-published", lambda n: True), ("strict", strict_ok)]
    res = {}
    for label, keep in runs:
        m, a, e = build(clean, keep)
        tot = len(m) or 1
        res[label] = {
            "firms": len(m),
            "mentions": sum(m.values()),
            "once": sum(1 for c in m.values() if c == 1) / tot * 100,
            "one_area": sum(1 for f in m if len(a[f]) == 1) / tot * 100,
            "one_surface": sum(1 for f in m if len(e[f]) == 1) / tot * 100,
            "one_both": sum(1 for f in m if len(a[f]) == 1 and len(e[f]) == 1) / tot * 100,
            "gini": gini(list(m.values())),
            "har": len(m) / 50000 * 100,
        }

    L = []
    def w(s=""):
        L.append(s)

    w("=" * 74)
    w("500-QUERY HOUSTON STUDY  -  EXTRACTION SENSITIVITY")
    w("generated by study500_sensitivity.py, never hand-edited")
    w("=" * 74)
    w()
    w("The strict filter is built to BREAK the findings, not to correct them. It deletes real")
    w("firms whose names read like phrases. Treat its firm count as a floor, and read every")
    w("other row as: does this finding survive a filter designed to destroy it?")
    w()
    w("   %-32s %14s %14s" % ("metric", "as-published", "strict (floor)"))
    rowspec = [("distinct firms", "firms", "%d"), ("mentions", "mentions", "%d"),
               ("named exactly once", "once", "%.1f%%"),
               ("named in exactly ONE area", "one_area", "%.1f%%"),
               ("named by ONE surface", "one_surface", "%.1f%%"),
               ("ONE area AND ONE surface", "one_both", "%.1f%%"),
               ("Gini", "gini", "%.3f"),
               ("share of 50,000 HAR members", "har", "%.1f%%")]
    for label, key, fmt in rowspec:
        w("   %-32s %14s %14s"
          % (label, fmt % res["as-published"][key], fmt % res["strict"][key]))
    w()
    drop = 100 - res["strict"]["firms"] / max(1, res["as-published"]["firms"]) * 100
    w("   the strict filter removes %.1f%% of names." % drop)
    w("   Every structural finding moves by a small amount and two get STRONGER, because the")
    w("   names it deletes are overwhelmingly single-mention and single-area, so they were")
    w("   inflating the numerator and the denominator together.")
    w()
    w("   PUBLISH the firm count as a range, never as a point. PUBLISH the structural")
    w("   percentages with the strict figure beside them.")
    w()
    w("=" * 74)

    text = "\n".join(L)
    io.open(OUT, "w", encoding="utf-8", newline="\n").write(text)
    print(text)


if __name__ == "__main__":
    main()