#!/usr/bin/env python3
"""Repair quality-gate bands and quality-defect thresholds.

This is an analysis/training-output pass only. It preserves the 667-row master
data, grouped splits, MiniLM embeddings, and existing live workflow.
"""
from __future__ import annotations

import argparse
import json
import math
import zipfile
from collections import Counter
from datetime import datetime, timezone
from pathlib import Path
from typing import Any

import joblib
import numpy as np
import pandas as pd
from sentence_transformers import SentenceTransformer
from sklearn.linear_model import LogisticRegression
from sklearn.metrics import (
    classification_report,
    confusion_matrix,
    f1_score,
    hamming_loss,
    precision_recall_fscore_support,
)
from sklearn.model_selection import GroupKFold
from sklearn.multiclass import OneVsRestClassifier
from sklearn.preprocessing import LabelEncoder, MultiLabelBinarizer


DEFECTS = [
    "transcript_chatter",
    "fragment_or_false_start",
    "speaker_reported_speech",
    "topic_only_or_missing_substance",
]


def split_labels(value: Any) -> list[str]:
    if pd.isna(value) or str(value).strip() == "":
        return []
    return [part.strip() for part in str(value).split("|") if part.strip() and part.strip() != "none"]


def now_slug() -> str:
    return datetime.now(timezone.utc).strftime("%Y%m%dT%H%M%SZ")


def mask(series: pd.Series) -> pd.Series:
    return series.fillna(0).astype(int).eq(1)


def fail(message: str) -> None:
    raise SystemExit(f"VALIDATION FAILED: {message}")


def validate(df: pd.DataFrame, expected_rows: int) -> dict[str, Any]:
    checks: dict[str, Any] = {
        "master_rows": int(len(df)),
        "quality_gate_rows": int(mask(df["train_quality_gate"]).sum()),
        "quality_defect_rows": int(mask(df["train_quality_defect_head"]).sum()),
        "live_application_behaviour_changed": False,
        "disposition_trained": False,
        "embedding_model_changed": False,
    }
    if len(df) != expected_rows:
        fail(f"expected {expected_rows} rows, found {len(df)}")
    if not df["row_id"].is_unique:
        fail("row_id is not unique")
    group_splits = df.groupby("group_id")["recommended_split"].nunique(dropna=False)
    if (group_splits > 1).any():
        fail("a group_id crosses recommended splits")
    checks["group_id_single_split"] = True
    if int(mask(df["train_disposition_head"]).sum()) != 0:
        fail("disposition head has enabled training rows")
    checks["recommended_split_counts"] = {
        str(k): int(v) for k, v in df["recommended_split"].value_counts().sort_index().items()
    }
    return checks


def encode(embedder: SentenceTransformer, texts: list[str]) -> np.ndarray:
    return embedder.encode(
        texts,
        normalize_embeddings=True,
        convert_to_numpy=True,
        batch_size=32,
        show_progress_bar=True,
    )


def binary_report(y_true: np.ndarray, y_pred: np.ndarray, labels: list[str]) -> dict[str, Any]:
    ids = list(range(len(labels)))
    return {
        "macro_f1": float(f1_score(y_true, y_pred, average="macro", zero_division=0)),
        "micro_f1": float(f1_score(y_true, y_pred, average="micro", zero_division=0)),
        "labels": labels,
        "confusion_matrix": confusion_matrix(y_true, y_pred, labels=ids).astype(int).tolist(),
        "per_class": classification_report(
            y_true, y_pred, labels=ids, target_names=labels, output_dict=True, zero_division=0
        ),
    }


def threshold_curve(y_true: np.ndarray, probs: np.ndarray, thresholds: list[float]) -> pd.DataFrame:
    rows = []
    for threshold in thresholds:
        pred = probs >= threshold
        tp = int(((y_true == 1) & pred).sum())
        fp = int(((y_true == 0) & pred).sum())
        tn = int(((y_true == 0) & (~pred)).sum())
        fn = int(((y_true == 1) & (~pred)).sum())
        precision = tp / (tp + fp) if (tp + fp) else 0.0
        recall = tp / (tp + fn) if (tp + fn) else 0.0
        f1 = 2 * precision * recall / (precision + recall) if (precision + recall) else 0.0
        rows.append(
            {
                "threshold": float(threshold),
                "tp": tp,
                "fp": fp,
                "tn": tn,
                "fn": fn,
                "precision": precision,
                "recall": recall,
                "f1": f1,
                "predicted_positive_count": int(pred.sum()),
            }
        )
    return pd.DataFrame(rows)


def grouped_oof_binary(X: np.ndarray, y: np.ndarray, groups: np.ndarray, random_state: int = 42) -> tuple[np.ndarray, list[dict[str, Any]]]:
    unique_groups = np.unique(groups)
    n_splits = min(5, len(unique_groups))
    if n_splits < 2:
        fail("not enough groups for grouped calibration")
    probs = np.full(len(y), np.nan)
    fold_checks = []
    gkf = GroupKFold(n_splits=n_splits)
    for fold, (train_idx, holdout_idx) in enumerate(gkf.split(X, y, groups), start=1):
        train_groups = set(groups[train_idx])
        holdout_groups = set(groups[holdout_idx])
        if train_groups & holdout_groups:
            fail("group_id crosses a calibration fold")
        fold_checks.append(
            {
                "fold": fold,
                "train_rows": int(len(train_idx)),
                "holdout_rows": int(len(holdout_idx)),
                "holdout_groups": int(len(holdout_groups)),
            }
        )
        if len(np.unique(y[train_idx])) < 2:
            continue
        clf = LogisticRegression(max_iter=3000, class_weight="balanced", random_state=random_state)
        clf.fit(X[train_idx], y[train_idx])
        probs[holdout_idx] = clf.predict_proba(X[holdout_idx])[:, 1]
    return probs, fold_checks


def train_quality_gate(df: pd.DataFrame, X_all: np.ndarray, out: Path, c_value: float = 1.0) -> dict[str, Any]:
    qdf = df.loc[mask(df["train_quality_gate"])].copy()
    idx = qdf.index.to_numpy()
    X = X_all[idx]
    le = LabelEncoder().fit(["intervention", "ready"])
    y = le.transform(qdf["quality_gate"].astype(str))
    intervention_id = int(np.where(le.classes_ == "intervention")[0][0])
    y_intervention = (y == intervention_id).astype(int)
    train = qdf["recommended_split"].eq("train").to_numpy()
    val = qdf["recommended_split"].eq("validation").to_numpy()
    test = qdf["recommended_split"].eq("test").to_numpy()

    clf = LogisticRegression(max_iter=3000, class_weight="balanced", random_state=42, C=c_value)
    clf.fit(X[train], y[train])

    calib = train | val
    calib_probs, fold_checks = grouped_oof_binary(
        X[calib],
        y_intervention[calib],
        qdf.loc[calib, "group_id"].astype(str).to_numpy(),
    )
    if np.isnan(calib_probs).any():
        fail("quality-gate calibration produced missing OOF probabilities")

    thresholds = [round(x, 3) for x in np.linspace(0.05, 0.95, 181)]
    curve = threshold_curve(y_intervention[calib], calib_probs, thresholds)
    curve.to_csv(out / "quality_gate_threshold_curve.csv", index=False)

    candidates = curve[curve["recall"] >= 0.80].copy()
    if candidates.empty:
        warning_row = curve.sort_values(["recall", "precision", "threshold"], ascending=[False, False, False]).iloc[0]
        warning_feasible = False
    else:
        warning_row = candidates.sort_values(["precision", "f1", "threshold"], ascending=[False, False, False]).iloc[0]
        warning_feasible = True
    rewrite_threshold = float(warning_row["threshold"])

    low_rows = []
    ready_true = 1 - y_intervention[calib]
    for threshold in thresholds:
        pred_no_warning = calib_probs <= threshold
        coverage = int(pred_no_warning.sum())
        true_ready = int((pred_no_warning & (ready_true == 1)).sum())
        precision = true_ready / coverage if coverage else 0.0
        recall = true_ready / int((ready_true == 1).sum()) if int((ready_true == 1).sum()) else 0.0
        low_rows.append(
            {
                "threshold": float(threshold),
                "coverage": coverage,
                "ready_precision": precision,
                "ready_recall": recall,
            }
        )
    low_curve = pd.DataFrame(low_rows)
    feasible_low = low_curve[(low_curve["ready_precision"] >= 0.85) & (low_curve["threshold"] < rewrite_threshold)]
    if feasible_low.empty:
        no_warning_threshold = None
        no_warning_enabled = False
    else:
        selected_low = feasible_low.sort_values(["coverage", "ready_precision", "threshold"], ascending=[False, False, False]).iloc[0]
        no_warning_threshold = float(selected_low["threshold"])
        no_warning_enabled = True

    probs_all = clf.predict_proba(X)
    prob_intervention = probs_all[:, intervention_id]
    pred_binary = np.where(prob_intervention >= 0.5, "intervention", "ready")

    bands = []
    for p in prob_intervention:
        if no_warning_enabled and p <= no_warning_threshold:
            bands.append("no_intrinsic_quality_warning")
        elif p >= rewrite_threshold:
            bands.append("rewrite_warning")
        else:
            bands.append("review")

    qpred = qdf[
        [
            "row_id",
            "recommended_split",
            "group_id",
            "input_stage",
            "source_origin",
            "source_document",
            "model_text",
            "quality_gate",
            "minutes_quality_reference",
            "semantic_signal_reference",
            "rationale",
        ]
    ].copy()
    qpred = qpred.rename(columns={"recommended_split": "split", "quality_gate": "true_quality_gate"})
    qpred["predicted_quality_gate"] = pred_binary
    qpred["prob_intervention"] = prob_intervention
    qpred["operating_band"] = bands
    qpred.to_csv(out / "quality_gate_policy_predictions.csv", index=False)

    band_rows = []
    for band, sdf in qpred.groupby("operating_band"):
        if band == "rewrite_warning":
            wrong = sdf["true_quality_gate"].ne("intervention")
        elif band == "no_intrinsic_quality_warning":
            wrong = sdf["true_quality_gate"].ne("ready")
        else:
            wrong = pd.Series(False, index=sdf.index)
        band_rows.append(
            {
                "band": band,
                "rows": int(len(sdf)),
                "coverage": float(len(sdf) / len(qpred)),
                "error_rate": float(wrong.mean()) if len(sdf) else 0.0,
            }
        )

    test_q = qpred[qpred["split"].eq("test")]
    rewrite_test = test_q["operating_band"].eq("rewrite_warning")
    no_warning_test = test_q["operating_band"].eq("no_intrinsic_quality_warning")
    intervention_true = test_q["true_quality_gate"].eq("intervention")
    ready_true_test = test_q["true_quality_gate"].eq("ready")
    raw_metrics = {}
    for split_name, split_mask in [("train", train), ("validation", val), ("test", test)]:
        pred = probs_all[split_mask].argmax(axis=1)
        raw_metrics[split_name] = binary_report(y[split_mask], pred, list(le.classes_))

    thresholds_json = {
        "probability": "prob_intervention",
        "rewrite_warning_threshold": rewrite_threshold,
        "rewrite_warning_constraint": "select recall >= 80% on grouped calibration, then maximise precision",
        "rewrite_warning_constraint_met": bool(warning_feasible),
        "no_warning_threshold": no_warning_threshold,
        "no_warning_enabled": bool(no_warning_enabled),
        "no_warning_constraint": "ready precision >= 85% on grouped calibration, then maximise coverage",
        "calibration_method": "grouped out-of-fold on train+validation only",
        "fold_checks": fold_checks,
    }
    (out / "quality_gate_operating_thresholds.json").write_text(json.dumps(thresholds_json, indent=2))
    joblib.dump({"label_encoder": le, "classifier": clf, "thresholds": thresholds_json}, out / "quality_gate_repaired_classifier.joblib")

    return {
        "row_count": int(len(qdf)),
        "train_count": int(train.sum()),
        "validation_count": int(val.sum()),
        "test_count": int(test.sum()),
        "thresholds": thresholds_json,
        "raw_binary_metrics": raw_metrics,
        "band_coverage_all_rows": band_rows,
        "test_policy": {
            "rewrite_warning_intervention_precision": float((rewrite_test & intervention_true).sum() / rewrite_test.sum()) if rewrite_test.sum() else None,
            "rewrite_warning_intervention_recall": float((rewrite_test & intervention_true).sum() / intervention_true.sum()) if intervention_true.sum() else None,
            "no_warning_ready_precision": float((no_warning_test & ready_true_test).sum() / no_warning_test.sum()) if no_warning_test.sum() else None,
            "no_warning_ready_recall": float((no_warning_test & ready_true_test).sum() / ready_true_test.sum()) if ready_true_test.sum() else None,
        },
        "breakdowns": quality_policy_breakdowns(qpred),
    }


def quality_policy_breakdowns(qpred: pd.DataFrame) -> dict[str, Any]:
    out: dict[str, Any] = {}
    for field in ["input_stage", "source_origin", "source_document"]:
        out[field] = {}
        for value, sdf in qpred.groupby(field, dropna=False):
            out[field][str(value)] = {
                "rows": int(len(sdf)),
                "bands": {str(k): int(v) for k, v in sdf["operating_band"].value_counts().items()},
            }
    qpred = qpred.copy()
    qpred["_source_family"] = qpred["source_origin"].fillna("").astype(str).apply(
        lambda value: "human_minutes_derived"
        if "human_minutes" in value
        else "transcript_derived"
        if "transcript" in value or "jacqui" in value
        else "synthetic_seed"
        if "synthetic" in value
        else "other"
    )
    out["synthetic_vs_real_derived"] = {
        str(value): {
            "rows": int(len(sdf)),
            "bands": {str(k): int(v) for k, v in sdf["operating_band"].value_counts().items()},
        }
        for value, sdf in qpred.groupby("_source_family")
    }
    return out


def fit_binary_if_possible(X: np.ndarray, y: np.ndarray, c_value: float = 1.0):
    if len(np.unique(y)) < 2:
        return None
    clf = LogisticRegression(max_iter=3000, class_weight="balanced", random_state=42, C=c_value)
    clf.fit(X, y)
    return clf


def select_defect_threshold(label: str, y_cal: np.ndarray, p_cal: np.ndarray, source: str, out: Path) -> dict[str, Any]:
    valid = ~np.isnan(p_cal)
    y_cal = y_cal[valid]
    p_cal = p_cal[valid]
    positive_count = int(y_cal.sum())
    negative_count = int((y_cal == 0).sum())
    thresholds = [round(x, 3) for x in np.linspace(0.1, 0.9, 161)]
    curve = threshold_curve(y_cal, p_cal, thresholds)
    curve.insert(0, "defect", label)
    curve.to_csv(out / f"quality_defect_threshold_curve_{label}.csv", index=False)
    if positive_count < 5:
        return {
            "defect": label,
            "threshold": 0.50,
            "calibration_source": "default",
            "calibration_status": "uncalibrated",
            "reason": f"fewer than five positive calibration examples ({positive_count})",
            "positive_calibration_examples": positive_count,
            "negative_calibration_examples": negative_count,
        }
    candidates = curve[curve["threshold"] >= 0.25].copy()
    if positive_count >= 10:
        low_candidates = curve[(curve["threshold"] < 0.25) & (curve["precision"] >= 0.60)]
        candidates = pd.concat([candidates, low_candidates], ignore_index=True)
    selected = candidates.sort_values(["f1", "precision", "recall", "threshold"], ascending=[False, False, False, False]).iloc[0]
    if float(selected["threshold"]) < 0.25 and not (positive_count >= 10 and float(selected["precision"]) >= 0.60):
        fail(f"selected illegal threshold below 0.25 for {label}")
    return {
        "defect": label,
        "threshold": float(selected["threshold"]),
        "calibration_source": source,
        "calibration_status": "optimised" if source in {"validation", "grouped_oof"} else "uncalibrated",
        "reason": f"selected by max F1 on {source} calibration",
        "positive_calibration_examples": positive_count,
        "negative_calibration_examples": negative_count,
    }


def train_quality_defects(
    df: pd.DataFrame,
    X_all: np.ndarray,
    out: Path,
    previous_run: Path,
    c_value: float = 1.0,
) -> dict[str, Any]:
    ddf = df.loc[mask(df["train_quality_defect_head"])].copy()
    idx = ddf.index.to_numpy()
    X = X_all[idx]
    labels_per_row = ddf["quality_defects"].apply(split_labels).tolist()
    mlb = MultiLabelBinarizer(classes=DEFECTS)
    Y = mlb.fit_transform(labels_per_row)
    train = ddf["recommended_split"].eq("train").to_numpy()
    val = ddf["recommended_split"].eq("validation").to_numpy()
    test = ddf["recommended_split"].eq("test").to_numpy()

    final_models = {}
    final_probs = np.zeros((len(ddf), len(DEFECTS)))
    calib_info = []
    combined_curves = []
    train_val = train | val
    support_by_split = {}
    for i, label in enumerate(DEFECTS):
        support_by_split[label] = {}
        for split_name, split_mask in [("train", train), ("validation", val), ("test", test)]:
            y_split = Y[split_mask, i]
            support_by_split[label][split_name] = {
                "positive": int(y_split.sum()),
                "negative": int((y_split == 0).sum()),
            }

        model = fit_binary_if_possible(X[train], Y[train, i], c_value)
        final_models[label] = model
        if model is not None:
            final_probs[:, i] = model.predict_proba(X)[:, 1]
        else:
            final_probs[:, i] = 0.0

        y_val = Y[val, i]
        if int(y_val.sum()) >= 5 and int((y_val == 0).sum()) >= 5 and model is not None:
            p_cal = final_probs[val, i]
            y_cal = y_val
            source = "validation"
        else:
            p_cal, fold_checks = grouped_oof_binary(
                X[train_val],
                Y[train_val, i],
                ddf.loc[train_val, "group_id"].astype(str).to_numpy(),
            )
            y_cal = Y[train_val, i]
            source = "grouped_oof"
        info = select_defect_threshold(label, y_cal, p_cal, source, out)
        calib_info.append(info)
        curve_path = out / f"quality_defect_threshold_curve_{label}.csv"
        if curve_path.exists():
            combined_curves.append(pd.read_csv(curve_path))

    thresholds = {item["defect"]: item["threshold"] for item in calib_info}
    threshold_df = pd.DataFrame(calib_info)
    threshold_df.to_csv(out / "quality_defects_thresholds_repaired.csv", index=False)
    if combined_curves:
        pd.concat(combined_curves, ignore_index=True).to_csv(out / "quality_defect_threshold_curves_repaired.csv", index=False)

    pred_binary = np.zeros_like(Y)
    for i, label in enumerate(DEFECTS):
        pred_binary[:, i] = final_probs[:, i] >= thresholds[label]
    if np.any((pred_binary.sum(axis=1) > 0) & np.array(["none" in str(x) for x in ddf["quality_defects"].fillna("")])):
        # True labels may contain "none"; predictions never may.
        pass

    rows = []
    for pos, row in enumerate(ddf.itertuples()):
        predicted = [DEFECTS[i] for i, active in enumerate(pred_binary[pos]) if active]
        if not predicted:
            predicted_display = "none"
        else:
            predicted_display = "|".join(predicted)
        if "none" in predicted and len(predicted) > 1:
            fail("none emitted alongside a positive defect")
        true_labels = [label for label in split_labels(row.quality_defects) if label in DEFECTS]
        rows.append(
            {
                "row_id": row.row_id,
                "split": row.recommended_split,
                "true_defects": "|".join(true_labels) if true_labels else "none",
                "predicted_defects": predicted_display,
                **{f"prob_{label}": float(final_probs[pos, i]) for i, label in enumerate(DEFECTS)},
            }
        )
    pred_df = pd.DataFrame(rows)
    pred_df.to_csv(out / "quality_defects_repaired_predictions.csv", index=False)

    split_metrics = {}
    for split_name, split_mask in [("train", train), ("validation", val), ("test", test)]:
        yp = pred_binary[split_mask]
        yt = Y[split_mask]
        pr, rc, f1, sup = precision_recall_fscore_support(yt, yp, average=None, zero_division=0)
        per_label = {}
        for i, label in enumerate(DEFECTS):
            per_label[label] = {
                "support": int(sup[i]),
                "precision": float(pr[i]),
                "recall": float(rc[i]),
                "f1": float(f1[i]),
                "predicted_positive_count": int(yp[:, i].sum()),
            }
        simultaneous = Counter(np.where(yp.sum(axis=1) >= 2, "two_or_more", np.where(yp.sum(axis=1) == 1, "one", "zero")))
        split_metrics[split_name] = {
            "macro_f1": float(f1_score(yt, yp, average="macro", zero_division=0)),
            "micro_f1": float(f1_score(yt, yp, average="micro", zero_division=0)),
            "exact_match_accuracy": float((yt == yp).all(axis=1).mean()),
            "hamming_loss": float(hamming_loss(yt, yp)),
            "per_label": per_label,
            "simultaneous_defect_counts": {str(k): int(v) for k, v in simultaneous.items()},
        }

    previous = pd.read_csv(previous_run / "quality_defects_predictions.csv")
    previous_counts = {}
    previous_metrics = {}
    for label in DEFECTS:
        previous_counts[label] = int(previous["predicted_labels"].fillna("").astype(str).str.contains(label, regex=False).sum())
    prev_test = previous[previous["split"].eq("test")].copy()
    for label in DEFECTS:
        y_true = prev_test["true_labels"].fillna("").astype(str).str.contains(label, regex=False).astype(int).to_numpy()
        y_pred = prev_test["predicted_labels"].fillna("").astype(str).str.contains(label, regex=False).astype(int).to_numpy()
        previous_metrics[label] = {
            "f1": float(f1_score(y_true, y_pred, zero_division=0)),
            "predicted_positive_count": int(y_pred.sum()),
        }

    joblib.dump({"labels": DEFECTS, "models": final_models, "thresholds": thresholds}, out / "quality_defects_repaired_classifier.joblib")
    return {
        "row_count": int(len(ddf)),
        "train_count": int(train.sum()),
        "validation_count": int(val.sum()),
        "test_count": int(test.sum()),
        "support_by_split": support_by_split,
        "thresholds": calib_info,
        "metrics": split_metrics,
        "previous_predicted_positive_counts_all_rows": previous_counts,
        "previous_test_metrics": previous_metrics,
        "contradictory_none_with_positive_eliminated": True,
    }


def make_quality_gate_queue(df: pd.DataFrame, out: Path, thresholds: dict[str, Any]) -> int:
    pred = pd.read_csv(out / "quality_gate_policy_predictions.csv")
    merged = pred.copy()
    merged["error_type"] = ""
    merged.loc[
        merged["true_quality_gate"].ne(merged["predicted_quality_gate"]),
        "error_type",
    ] = merged["true_quality_gate"] + "_predicted_" + merged["predicted_quality_gate"]
    low = thresholds.get("no_warning_threshold")
    high = thresholds["rewrite_warning_threshold"]
    near_threshold = (merged["prob_intervention"] - high).abs().le(0.10)
    if low is not None:
        near_threshold = near_threshold | (merged["prob_intervention"] - float(low)).abs().le(0.10)
    high_conf_error = (
        merged["split"].isin(["train", "validation"])
        & merged["true_quality_gate"].ne(merged["predicted_quality_gate"])
        & ((merged["prob_intervention"] >= 0.70) | (merged["prob_intervention"] <= 0.30))
    )
    include = merged["split"].eq("test") & merged["error_type"].ne("")
    include = include | (~merged["split"].eq("test") & near_threshold) | high_conf_error
    queue = merged[include].copy()

    def priority(row) -> str:
        text = str(row["model_text"]).lower()
        cues = ["sorry", "perfect", "mentioned", "said that", "%", "per cent", "deadline", "decision", "constraint", "because", "process"]
        if row["split"] == "test" and row["error_type"]:
            return "high"
        if any(cue in text for cue in cues):
            return "medium"
        return "low"

    queue["review_priority"] = queue.apply(priority, axis=1)
    queue["do_not_train"] = np.where(queue["split"].eq("test"), 1, 0)
    queue["human_review_outcome"] = ""
    queue["human_corrected_text"] = ""
    queue["human_reason"] = ""
    cols = [
        "row_id",
        "split",
        "group_id",
        "input_stage",
        "source_origin",
        "source_document",
        "model_text",
        "true_quality_gate",
        "predicted_quality_gate",
        "prob_intervention",
        "operating_band",
        "minutes_quality_reference",
        "semantic_signal_reference",
        "rationale",
        "error_type",
        "review_priority",
        "do_not_train",
        "human_review_outcome",
        "human_corrected_text",
        "human_reason",
    ]
    queue[cols].to_csv(out / "quality_gate_error_review_queue.csv", index=False)
    return int(len(queue))


def make_established_queue(df: pd.DataFrame, reference_run: Path, out: Path) -> int:
    priority_labels = {
        "possible_action",
        "regulatory_obligation",
        "process_overview",
        "regulatory_language",
        "recipient_or_handover",
        "process_sequence",
        "deadline_or_sequence",
    }
    frames = []
    for head, file_name, true_col, pred_col in [
        ("evidence_type", "evidence_type_predictions.csv", "true_label", "predicted_label"),
        ("action_state", "action_state_predictions.csv", "true_label", "predicted_label"),
    ]:
        pred = pd.read_csv(reference_run / file_name)
        pred = pred[pred["split"].eq("test") & pred[true_col].ne(pred[pred_col])].copy()
        pred = pred[pred[true_col].isin(priority_labels) | pred[pred_col].isin(priority_labels)]
        pred["head"] = head
        pred["true_value"] = pred[true_col]
        pred["predicted_value"] = pred[pred_col]
        frames.append(pred)
    sig = pd.read_csv(reference_run / "signals_predictions.csv")
    sig = sig[sig["split"].eq("test")].copy()
    for label in sorted(priority_labels):
        if f"prob_{label}" not in sig.columns:
            continue
        rows = []
        for row in sig.itertuples():
            true_set = set(split_labels(row.true_labels))
            pred_set = set(split_labels(row.predicted_labels))
            if (label in true_set) != (label in pred_set):
                rows.append(
                    {
                        "row_id": row.row_id,
                        "split": row.split,
                        "head": "signals",
                        "true_value": label if label in true_set else f"not_{label}",
                        "predicted_value": label if label in pred_set else f"not_{label}",
                        f"prob_{label}": getattr(row, f"prob_{label}"),
                    }
                )
        if rows:
            frames.append(pd.DataFrame(rows))
    if frames:
        queue = pd.concat(frames, ignore_index=True, sort=False)
    else:
        queue = pd.DataFrame(columns=["row_id", "split", "head", "true_value", "predicted_value"])
    enrich = df[
        ["row_id", "model_text", "source_document", "group_id", "input_stage", "source_origin", "rationale"]
    ]
    queue = queue.merge(enrich, on="row_id", how="left")
    queue["do_not_train"] = 1
    queue.to_csv(out / "established_heads_error_review_queue.csv", index=False)
    return int(len(queue))


def make_template(out: Path) -> None:
    cols = [
        "capture_id",
        "meeting_group_id",
        "source_document",
        "source_transcript_text",
        "generated_candidate_text",
        "input_stage",
        "human_quality_gate",
        "human_quality_defects",
        "human_action",
        "human_corrected_text",
        "human_reason",
        "owner_preserved",
        "deadline_preserved",
        "decision_preserved",
        "constraint_preserved",
        "uncertainty_preserved",
        "source_origin",
        "reviewed",
        "recommended_split",
    ]
    pd.DataFrame(columns=cols).to_csv(out / "next_real_quality_training_batch_template.csv", index=False)


def pct(value: float | None) -> str:
    if value is None or (isinstance(value, float) and math.isnan(value)):
        return "n/a"
    return f"{value * 100:.1f}%"


def write_reports(out: Path, metrics: dict[str, Any]) -> None:
    q = metrics["quality_gate"]
    d = metrics["quality_defects"]
    lines = [
        "# Threshold repair and active-learning preparation",
        "",
        f"Run directory: `{metrics['run_directory']}`",
        "",
        "## Plain-English summary",
        "",
        "- The generic abstention rule has been replaced with operating bands based on `prob_intervention`.",
        "- The defect model no longer learns or emits `none` as a coexisting defect; `none` is derived only when no positive defect fires.",
        "- No live application behaviour was changed.",
        "- The test set has now been inspected, so a future unseen-meeting holdout is needed before any production-quality claim.",
        "",
        "## Quality-gate operating bands",
        "",
        f"- `rewrite_warning`: `prob_intervention >= {q['thresholds']['rewrite_warning_threshold']:.3f}`",
        f"- `no_intrinsic_quality_warning`: "
        + (
            f"`prob_intervention <= {q['thresholds']['no_warning_threshold']:.3f}`"
            if q["thresholds"]["no_warning_enabled"]
            else "disabled; calibration did not meet ready precision constraint"
        ),
        "- everything else: `review`",
        "",
        "| Band | Rows | Coverage | Error rate |",
        "| --- | ---: | ---: | ---: |",
    ]
    for row in q["band_coverage_all_rows"]:
        lines.append(f"| {row['band']} | {row['rows']} | {pct(row['coverage'])} | {pct(row['error_rate'])} |")
    lines.extend(
        [
            "",
            "## Quality-gate test policy",
            "",
            f"- Rewrite-warning intervention precision: {pct(q['test_policy']['rewrite_warning_intervention_precision'])}",
            f"- Rewrite-warning intervention recall: {pct(q['test_policy']['rewrite_warning_intervention_recall'])}",
            f"- No-warning ready precision: {pct(q['test_policy']['no_warning_ready_precision'])}",
            f"- No-warning ready recall: {pct(q['test_policy']['no_warning_ready_recall'])}",
            "",
            "## Repaired quality-defect thresholds",
            "",
            "| Defect | Threshold | Calibration | Status | Reason |",
            "| --- | ---: | --- | --- | --- |",
        ]
    )
    for row in d["thresholds"]:
        lines.append(
            f"| {row['defect']} | {row['threshold']:.2f} | {row['calibration_source']} | {row['calibration_status']} | {row['reason']} |"
        )
    lines.extend(
        [
            "",
            "## Defect before and after on test",
            "",
            "| Defect | Previous F1 | New F1 | Previous predicted positives | New predicted positives |",
            "| --- | ---: | ---: | ---: | ---: |",
        ]
    )
    for label in DEFECTS:
        prev = d["previous_test_metrics"][label]
        new = d["metrics"]["test"]["per_label"][label]
        lines.append(
            f"| {label} | {pct(prev['f1'])} | {pct(new['f1'])} | {prev['predicted_positive_count']} | {new['predicted_positive_count']} |"
        )
    lines.extend(
        [
            "",
            "## Review queues",
            "",
            f"- Quality-gate queue: {metrics['review_queues']['quality_gate_rows']} rows",
            f"- Established-heads queue: {metrics['review_queues']['established_heads_rows']} rows",
            "",
            "## Recommendation",
            "",
            metrics["recommendation"],
        ]
    )
    (out / "REPORT.md").write_text("\n".join(lines) + "\n")
    summary = [
        "Threshold repair complete.\n",
        f"Run: {metrics['run_directory']}\n",
        f"Rewrite warning starts at prob_intervention >= {q['thresholds']['rewrite_warning_threshold']:.3f}.\n",
        "No-warning band: "
        + (
            f"prob_intervention <= {q['thresholds']['no_warning_threshold']:.3f}.\n"
            if q["thresholds"]["no_warning_enabled"]
            else "disabled; not safe enough on calibration.\n"
        ),
        f"Quality-gate queue rows: {metrics['review_queues']['quality_gate_rows']}.\n",
        f"Established-head queue rows: {metrics['review_queues']['established_heads_rows']}.\n",
        f"Next priority: {metrics['next_priority']}.\n",
    ]
    (out / "PLAIN_ENGLISH_SUMMARY.txt").write_text("".join(summary))


def make_zip(out: Path) -> Path:
    zip_path = Path("/tmp") / f"meeting_minutes_threshold_repair_results_{now_slug()}.zip"
    names = [
        "REPORT.md",
        "PLAIN_ENGLISH_SUMMARY.txt",
        "metrics.json",
        "validation_checks.json",
        "quality_gate_policy_predictions.csv",
        "quality_gate_operating_thresholds.json",
        "quality_gate_threshold_curve.csv",
        "quality_defects_repaired_predictions.csv",
        "quality_defects_thresholds_repaired.csv",
        "quality_defect_threshold_curves_repaired.csv",
        "quality_gate_error_review_queue.csv",
        "established_heads_error_review_queue.csv",
        "next_real_quality_training_batch_template.csv",
        "quality_gate_repaired_classifier.joblib",
        "quality_defects_repaired_classifier.joblib",
    ]
    with zipfile.ZipFile(zip_path, "w", zipfile.ZIP_DEFLATED) as z:
        for name in names:
            path = out / name
            if path.exists():
                z.write(path, arcname=name)
        for path in out.glob("quality_defect_threshold_curve_*.csv"):
            z.write(path, arcname=path.name)
    return zip_path


def main() -> int:
    ap = argparse.ArgumentParser()
    ap.add_argument("--reference-run", required=True)
    ap.add_argument("--output-root", default="runs")
    ap.add_argument("--embedding-model", default="sentence-transformers/all-MiniLM-L6-v2")
    ap.add_argument("--expected-rows", type=int, default=667)
    ap.add_argument("--quality-gate-c", type=float, default=1.0)
    ap.add_argument("--quality-defect-c", type=float, default=1.0)
    args = ap.parse_args()

    reference_run = Path(args.reference_run)
    df = pd.read_csv(reference_run / "training_data_used.csv").fillna("")
    checks = validate(df, args.expected_rows)
    out = Path(args.output_root) / f"threshold-repair-{now_slug()}" / "model_bundle"
    out.mkdir(parents=True, exist_ok=False)
    (out / "validation_checks.json").write_text(json.dumps(checks, indent=2))

    print(f"Output: {out}")
    print(f"Embedding model: {args.embedding_model}")
    embedder = SentenceTransformer(args.embedding_model)
    X = encode(embedder, df["model_text"].astype(str).tolist())

    quality_gate = train_quality_gate(df, X, out, args.quality_gate_c)
    quality_defects = train_quality_defects(df, X, out, reference_run, args.quality_defect_c)
    q_rows = make_quality_gate_queue(df, out, quality_gate["thresholds"])
    e_rows = make_established_queue(df, reference_run, out)
    make_template(out)

    metrics = {
        "run_directory": str(out),
        "reference_run": str(reference_run),
        "embedding_model": args.embedding_model,
        "validation_checks": checks,
        "quality_gate": quality_gate,
        "quality_defects": quality_defects,
        "review_queues": {
            "quality_gate_rows": q_rows,
            "established_heads_rows": e_rows,
        },
        "next_priority": "new real data",
        "recommendation": (
            "The policy repair stops the pathological abstention/defect firing, but the minority "
            "defect classes are still too sparse. The next useful step is real labelled examples, "
            "not a model change yet."
        ),
    }
    (out / "metrics.json").write_text(json.dumps(metrics, indent=2))
    write_reports(out, metrics)
    zip_path = make_zip(out)
    print(f"Saved threshold repair run to: {out}")
    print(f"Packaged: {zip_path}")
    return 0


if __name__ == "__main__":
    raise SystemExit(main())
