140 lines (139 with data), 4.1 kB

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "from glob import glob\n",
    "import pickle\n",
    "\n",
    "import numpy as np\n",
    "import scipy\n",
    "import scipy.stats"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# TODO: uncomment only two experiments\n",
    "EXPERIMENTS = (\n",
    "#     '0_baseline',\n",
    "#     '1_mixup',\n",
    "#     '2_mixup_nowd',\n",
    "#     '3_uda1',\n",
    "#     '4_uda2',\n",
    "#     '5_uda1_mixup_nowd'\n",
    ")\n",
    "\n",
    "# TODO: uncomment the section related to the selected evaluation dataset\n",
    "# if True:\n",
    "#     DATASET = 'oai_imo'\n",
    "#     CLASSES = list(range(1, 5))\n",
    "#     ATLAS = 'segm'\n",
    "# if True:\n",
    "#     DATASET = 'okoa'\n",
    "#     CLASSES = (1, 2)\n",
    "#     ATLAS = 'okoa'\n",
    "\n",
    "# TODO: specify path to your `results` directory\n",
    "path_results_root = ''\n",
    "\n",
    "path_base = os.path.join(path_results_root,\n",
    "                         (f'{EXPERIMENTS[0]}/logs_{DATASET}_test'\n",
    "                          f'/cache_{DATASET}_test_{ATLAS}_volumew_paired.pkl'))\n",
    "path_eval = os.path.join(path_results_root,\n",
    "                         (f'{EXPERIMENTS[1]}/logs_{DATASET}_test'\n",
    "                          f'/cache_{DATASET}_test_{ATLAS}_volumew_paired.pkl'))\n",
    "\n",
    "with open(path_base, 'rb') as f:\n",
    "    dict_base = pickle.load(f)\n",
    "\n",
    "with open(path_eval, 'rb') as f:\n",
    "    dict_eval = pickle.load(f)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "kl_ranges = {\n",
    "    '0': (0, ),\n",
    "    '1': (1, ),\n",
    "    '2': (2, ),\n",
    "    '3': (3, ),\n",
    "    '4': (4, ),\n",
    "    'all': (0, 1, 2, 3, 4),\n",
    "}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "dsc_base = np.asarray(dict_base['dice_score'])\n",
    "dsc_eval = np.asarray(dict_eval['dice_score'])\n",
    "\n",
    "for kl_name, kl_values in kl_ranges.items():\n",
    "    sel_base = np.isin(np.asarray(dict_base['KL']), kl_values)\n",
    "    sel_eval = np.isin(np.asarray(dict_eval['KL']), kl_values)\n",
    "    assert np.all(np.equal(sel_base, sel_eval))\n",
    "    sel = sel_base\n",
    "    if not np.sum(sel):\n",
    "        continue\n",
    "    \n",
    "    print(f'------ KL: {kl_name} ------')\n",
    "\n",
    "    print('--- Wilcoxon signed-rank, two-sided ---')\n",
    "    res_2side = [scipy.stats.wilcoxon(dsc_base[sel, c],\n",
    "                                      dsc_eval[sel, c])\n",
    "                 for c in CLASSES]\n",
    "    print(*res_2side, sep='\\n')\n",
    "\n",
    "    print('--- Wilcoxon signed-rank, one-sided, less ---')\n",
    "    res_1side = [scipy.stats.wilcoxon(dsc_base[sel, c],\n",
    "                                      dsc_eval[sel, c],\n",
    "                                      alternative='less')\n",
    "                 for c in CLASSES]\n",
    "    print(*res_1side, sep='\\n')\n",
    "\n",
    "    print('--- Wilcoxon signed-rank, one-sided, greater ---')\n",
    "    res_1side = [scipy.stats.wilcoxon(dsc_base[sel, c],\n",
    "                                      dsc_eval[sel, c],\n",
    "                                      alternative='greater')\n",
    "                 for c in CLASSES]\n",
    "    print(*res_1side, sep='\\n')"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}