--- a
+++ b/notebooks/Statistical_tests.ipynb
@@ -0,0 +1,139 @@
+{
+ "cells": [
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "import os\n",
+    "from glob import glob\n",
+    "import pickle\n",
+    "\n",
+    "import numpy as np\n",
+    "import scipy\n",
+    "import scipy.stats"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# TODO: uncomment only two experiments\n",
+    "EXPERIMENTS = (\n",
+    "#     '0_baseline',\n",
+    "#     '1_mixup',\n",
+    "#     '2_mixup_nowd',\n",
+    "#     '3_uda1',\n",
+    "#     '4_uda2',\n",
+    "#     '5_uda1_mixup_nowd'\n",
+    ")\n",
+    "\n",
+    "# TODO: uncomment the section related to the selected evaluation dataset\n",
+    "# if True:\n",
+    "#     DATASET = 'oai_imo'\n",
+    "#     CLASSES = list(range(1, 5))\n",
+    "#     ATLAS = 'segm'\n",
+    "# if True:\n",
+    "#     DATASET = 'okoa'\n",
+    "#     CLASSES = (1, 2)\n",
+    "#     ATLAS = 'okoa'\n",
+    "\n",
+    "# TODO: specify path to your `results` directory\n",
+    "path_results_root = ''\n",
+    "\n",
+    "path_base = os.path.join(path_results_root,\n",
+    "                         (f'{EXPERIMENTS[0]}/logs_{DATASET}_test'\n",
+    "                          f'/cache_{DATASET}_test_{ATLAS}_volumew_paired.pkl'))\n",
+    "path_eval = os.path.join(path_results_root,\n",
+    "                         (f'{EXPERIMENTS[1]}/logs_{DATASET}_test'\n",
+    "                          f'/cache_{DATASET}_test_{ATLAS}_volumew_paired.pkl'))\n",
+    "\n",
+    "with open(path_base, 'rb') as f:\n",
+    "    dict_base = pickle.load(f)\n",
+    "\n",
+    "with open(path_eval, 'rb') as f:\n",
+    "    dict_eval = pickle.load(f)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "kl_ranges = {\n",
+    "    '0': (0, ),\n",
+    "    '1': (1, ),\n",
+    "    '2': (2, ),\n",
+    "    '3': (3, ),\n",
+    "    '4': (4, ),\n",
+    "    'all': (0, 1, 2, 3, 4),\n",
+    "}"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "dsc_base = np.asarray(dict_base['dice_score'])\n",
+    "dsc_eval = np.asarray(dict_eval['dice_score'])\n",
+    "\n",
+    "for kl_name, kl_values in kl_ranges.items():\n",
+    "    sel_base = np.isin(np.asarray(dict_base['KL']), kl_values)\n",
+    "    sel_eval = np.isin(np.asarray(dict_eval['KL']), kl_values)\n",
+    "    assert np.all(np.equal(sel_base, sel_eval))\n",
+    "    sel = sel_base\n",
+    "    if not np.sum(sel):\n",
+    "        continue\n",
+    "    \n",
+    "    print(f'------ KL: {kl_name} ------')\n",
+    "\n",
+    "    print('--- Wilcoxon signed-rank, two-sided ---')\n",
+    "    res_2side = [scipy.stats.wilcoxon(dsc_base[sel, c],\n",
+    "                                      dsc_eval[sel, c])\n",
+    "                 for c in CLASSES]\n",
+    "    print(*res_2side, sep='\\n')\n",
+    "\n",
+    "    print('--- Wilcoxon signed-rank, one-sided, less ---')\n",
+    "    res_1side = [scipy.stats.wilcoxon(dsc_base[sel, c],\n",
+    "                                      dsc_eval[sel, c],\n",
+    "                                      alternative='less')\n",
+    "                 for c in CLASSES]\n",
+    "    print(*res_1side, sep='\\n')\n",
+    "\n",
+    "    print('--- Wilcoxon signed-rank, one-sided, greater ---')\n",
+    "    res_1side = [scipy.stats.wilcoxon(dsc_base[sel, c],\n",
+    "                                      dsc_eval[sel, c],\n",
+    "                                      alternative='greater')\n",
+    "                 for c in CLASSES]\n",
+    "    print(*res_1side, sep='\\n')"
+   ]
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "Python 3",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.7.3"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 2
+}