"""Accuracy-style metrics on the cached 5-fold test predictions (run after run_experiments.py).

Besides the paper's error metrics, these ask "how often is the prediction right?":
  exact star       rounded prediction equals the true star rating
  within 1 star    |prediction - truth| <= 1
  good-hotel acc   predicted >= 4 agrees with true >= 4 (77 % of ratings are >= 4, so also balanced accuracy)
  ROC-AUC          how well predictions rank good (>= 4) above not-good stays
  R^2              share of rating variance explained

Also scores "DMCCF+ blend": the equal-weight average of DMCCF+ and SVD. The 50/50 weight is a fixed
default checked on the inner split (dev_select.py's split), not tuned on the test folds.
"""
from pathlib import Path

import numpy as np
import pandas as pd
from scipy.stats import ttest_rel
from sklearn.metrics import balanced_accuracy_score, r2_score, roc_auc_score

import deepmccf as D

HERE = Path(__file__).resolve().parent
PREDS = HERE / "results" / "preds"
METHODS = {"SVD": "SVD", "Paper model": "Proposed_DMCCF", "DMCCF+": "DMCCFp_improved"}


def scores(y, p):
    good, pg = y >= 4, p >= 4
    return {"MAE": np.abs(p - y).mean(), "exact star": (np.rint(np.clip(p, 1, 5)) == y).mean(),
            "within 1 star": (np.abs(p - y) <= 1).mean(), "good-hotel acc": (pg == good).mean(),
            "balanced acc": balanced_accuracy_score(good, pg), "ROC-AUC": roc_auc_score(good, p), "R2": r2_score(y, p)}


def main():
    df = D.load_ratings(HERE / "data" / "tripadvisor_mc.csv")
    rows = []
    for f in D.make_folds(len(df)):
        y = df.overall.values[f.test].astype(float)
        P = {m: np.load(PREDS / f"{k}__f{f.k}.npz")["pred"].astype(float) for m, k in METHODS.items()}
        P["DMCCF+ blend (with SVD)"] = 0.5 * P["DMCCF+"] + 0.5 * P["SVD"]
        for m, p in P.items():
            rows.append({"method": m, "fold": f.k, **scores(y, p)})
    d = pd.DataFrame(rows)
    d.to_csv(HERE / "results" / "accuracy_per_fold.csv", index=False)
    mean = d.groupby("method", sort=False).mean().drop(columns="fold")
    mean.to_csv(HERE / "results" / "accuracy_metrics.csv")
    tests = []
    for m in ["DMCCF+", "DMCCF+ blend (with SVD)"]:
        for c in mean.columns:
            a, b = d[d.method == m][c].values, d[d.method == "Paper model"][c].values
            better = (a < b) if c == "MAE" else (a > b)
            tests.append({"method": m, "metric": c, "vs paper model": f"{b.mean():.4f} -> {a.mean():.4f}",
                          "better folds": int(better.sum()), "p": ttest_rel(a, b).pvalue})
    t = pd.DataFrame(tests)
    t.to_csv(HERE / "results" / "accuracy_tests.csv", index=False)
    print(mean.round(4).to_string(), "\nalways predicting 'good' scores", round((df.overall >= 4).mean(), 4), "\n")
    print(t.round(4).to_string(index=False))


if __name__ == "__main__":
    main()