"""Run every experiment in Section 4 of the paper and write the result tables to results/. Table 5 : MAE of each criterion predicted by the Criteria Ratings DNN Table 6 : the proposed model against SVD, SVD++, SlopeOne, Baseline estimates, KNN with baseline and a single DNN (MAE, F1, F2, FCP, MAP, MRR) Table 3 : optimizer comparison (criteria DNN optimizer; whole-model MAE) Table 4 : hidden-layer configurations (criteria DNN layers; whole-model MAE) Fig. 4 : embedding vector size (criteria DNN embedding; whole-model MAE) Test-set predictions are cached in results/preds/, so a rerun only computes what is missing. Delete that folder to start over. Usage: python run_experiments.py [--skip-ablations] [--folds 5] """ from __future__ import annotations import argparse import json import time import warnings from dataclasses import replace from pathlib import Path import numpy as np import pandas as pd import torch import deepmccf as D import improved as I PLUS = "DMCCF+ (improved)" HERE = Path(__file__).resolve().parent RESULTS = HERE / "results" PREDS = RESULTS / "preds" METRICS = ["MAE", "RMSE", "F1", "F2", "FCP", "MAP", "MRR"] warnings.filterwarnings("ignore", category=UserWarning) def cache_key(method: str) -> str: return method.replace(" ", "_").replace("+", "p").replace("(", "").replace(")", "") class Runner: def __init__(self, n_folds: int = 5, with_plus: bool = True): self.with_plus = with_plus torch.set_num_threads(max(1, (torch.get_num_threads() or 2))) self.df = D.load_ratings(HERE / "data" / "tripadvisor_mc.csv") self.folds = D.make_folds(len(self.df), n_splits=5)[:n_folds] self._overall = {} PREDS.mkdir(parents=True, exist_ok=True) def overall_dnn(self, fold): if fold.k not in self._overall: # the aggregation DNN does not depend on the criteria DNN self._overall[fold.k] = D.train_overall_dnn(self.df, fold, D.DMCCFConfig()) return self._overall[fold.k] def get(self, key: str, fold, compute) -> dict: """Cached test predictions for one method on one fold.""" path = PREDS / f"{key}__f{fold.k}.npz" if path.exists(): z = np.load(path, allow_pickle=False) return {k: z[k] for k in z.files} | json.loads(str(z["meta"])) t0 = time.time() out = compute() meta = {"seconds": round(time.time() - t0, 1), **out.get("meta", {})} arrays = {k: np.asarray(v) for k, v in out.items() if k != "meta"} np.savez(path, meta=json.dumps(meta), **arrays) print(f" {key:28s} fold {fold.k} {meta['seconds']:6.1f}s MAE {self.mae(fold, arrays['pred']):.4f}" + (f" restarts {meta.get('restarts')}" if meta.get("restarts") else ""), flush=True) return arrays | meta def mae(self, fold, pred): return float(np.abs(self.df.overall.values[fold.test] - pred).mean()) # --- methods ----------------------------------------------------------------------------------- def dmccf(self, fold, cfg: D.DMCCFConfig = D.DMCCFConfig()): def compute(): r = D.run_dmccf(self.df, fold, cfg, overall=self.overall_dnn(fold)) ci = r["criteria_info"] return {"pred": r["pred"], "criteria_mae": r["criteria_mae"], "meta": {"restarts": ci["restarts"], "best_epoch": ci["best_epoch"], "overall_best_epoch": r["overall_info"]["best_epoch"]}} return compute def single_dnn(self, fold): def compute(): r = D.run_single_dnn(self.df, fold) return {"pred": r["pred"], "meta": {"restarts": r["info"]["restarts"], "best_epoch": r["info"]["best_epoch"]}} return compute def surprise(self, fold, name): return lambda: {"pred": D.run_surprise(self.df, fold, name)["pred"]} # --- experiments ------------------------------------------------------------------------------- def main_table(self): methods = [(n, lambda f, n=n: self.surprise(f, n)) for n in D.surprise_algorithms()] methods += [("Single DNN", self.single_dnn), ("Proposed (DMCCF)", self.dmccf)] if self.with_plus: methods += [(PLUS, self.plus)] rows, crit, crit_plus = [], [], [] for fold in self.folds: u, y = self.df.u.values[fold.test], self.df.overall.values[fold.test].astype(float) for name, make in methods: r = self.get(cache_key(name), fold, make(fold)) rows.append({"method": name, "fold": fold.k, **D.evaluate(u, y, r["pred"].astype(float)), "seconds": r["seconds"], "restarts": r.get("restarts", 0)}) if name == "Proposed (DMCCF)": crit.append(r["criteria_mae"]) elif name == PLUS: crit_plus.append(r["criteria_mae"]) per_fold = pd.DataFrame(rows) per_fold.to_csv(RESULTS / "table6_per_fold.csv", index=False) means = per_fold.groupby("method", sort=False)[METRICS].mean() stds = per_fold.groupby("method", sort=False)[METRICS].std(ddof=0) t6 = pd.concat({"mean": means, "std": stds}, axis=1) t6.to_csv(RESULTS / "table6.csv") crit = np.array(crit) c_mean, c_std = crit.mean(0), crit.std(0) fold_means = crit.mean(1) t5 = pd.DataFrame({ "ours_mean": [*c_mean, fold_means.mean()], "ours_std": [*c_std, fold_means.std()], "paper": D.PAPER_TABLE5.values, }, index=D.CRITERIA_LABELS + ["Mean"]) if crit_plus: cp = np.array(crit_plus) t5["improved_mean"] = [*cp.mean(0), cp.mean(1).mean()] t5.to_csv(RESULTS / "table5.csv") return per_fold, t5 def plus(self, fold): def compute(): r = I.run_plus(self.df, fold, I.PlusConfig()) return {"pred": r["pred"], "criteria_mae": r["criteria_mae"]} return compute def improvement(self, per_fold: pd.DataFrame) -> pd.DataFrame: """DMCCF+ against the paper's model and the best baseline: MAE overall, for warm and cold users, and a paired t-test over the 5 folds.""" from scipy.stats import ttest_rel rows = [] for fold in self.folds: te = fold.test y = self.df.overall.values[te] warm = np.isin(self.df.u.values[te], self.df.u.values[fold.train_full]) for name in ["SVD", "Proposed (DMCCF)", PLUS]: p = np.load(PREDS / f"{cache_key(name)}__f{fold.k}.npz")["pred"] e = np.abs(p - y) rows.append({"method": name, "fold": fold.k, "MAE": e.mean(), "MAE warm users": e[warm].mean(), "MAE cold users": e[~warm].mean(), "within 1 star": (e <= 1).mean(), "exact after rounding": (np.rint(np.clip(p, 1, 5)) == y).mean()}) d = pd.DataFrame(rows) out = d.groupby("method", sort=False).mean().drop(columns="fold") plus = d[d.method == PLUS].MAE.values for ref in ["SVD", "Proposed (DMCCF)"]: other = d[d.method == ref].MAE.values out.loc[ref, "PLUS better on folds"] = int((plus < other).sum()) out.loc[ref, "paired t-test p"] = ttest_rel(plus, other).pvalue out.loc[ref, "relative MAE reduction"] = 1 - plus.mean() / other.mean() out.to_csv(RESULTS / "improvement.csv") return out def ablation(self, name, variants: dict[str, D.DMCCFConfig], paper: dict | None = None): rows = [] for fold in self.folds: for label, cfg in variants.items(): key = f"abl_{name}_{label}".replace(",", "-").replace("[", "").replace("]", "") r = self.get(key, fold, self.dmccf(fold, cfg)) rows.append({"setting": label, "fold": fold.k, "MAE": self.mae(fold, r["pred"]), "restarts": r.get("restarts", 0)}) df = pd.DataFrame(rows) out = df.groupby("setting", sort=False).agg(ours_mean=("MAE", "mean"), ours_std=("MAE", lambda s: s.std(ddof=0)), restarts=("restarts", "sum")) if paper is not None: out["paper"] = pd.Series(paper) out.to_csv(RESULTS / f"ablation_{name}.csv") return out def ablations(self): base = D.DMCCFConfig() opt = {o: replace(base, train=replace(base.train, optimizer=o, lr=None)) for o in ["adadelta", "adagrad", "adam", "adamax", "rmsprop", "sgd"]} layers = {} for n in range(1, 7): h = tuple(8 * 2 ** j for j in range(n))[::-1] # n=3 -> (32, 16, 8), written [8,16,32] in the paper layers["[" + ",".join(str(x) for x in h[::-1]) + "]"] = replace(base, criteria_hidden=h) emb = {str(e): replace(base, emb=e) for e in [8, 16, 32, 64, 128]} print("Table 3 (optimizers)"); t3 = self.ablation("optimizer", opt, D.PAPER_TABLE3.to_dict()) print("Table 4 (hidden layers)"); t4 = self.ablation("layers", layers, D.PAPER_TABLE4.to_dict()) print("Fig. 4 (embedding size)"); f4 = self.ablation("embedding", emb) return t3, t4, f4 def main(): ap = argparse.ArgumentParser() ap.add_argument("--skip-ablations", action="store_true") ap.add_argument("--folds", type=int, default=5) a = ap.parse_args() R = Runner(a.folds) print(f"{len(R.df):,} ratings, {R.df.u.nunique():,} users, {R.df.i.nunique():,} hotels; {len(R.folds)} folds") print("Tables 5 and 6"); per_fold, t5 = R.main_table() summary = D.summarise(per_fold, ["MAE", "F1", "F2", "FCP", "MAP", "MRR"]) print(summary.to_string(), "\n", t5.round(4).to_string(), sep="") print(R.improvement(per_fold).round(4).to_string()) if not a.skip_ablations: for t in R.ablations(): print(t.round(4).to_string()) if __name__ == "__main__": main()