Compare_performance.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pickle\n",
    "a_auc_map = pickle.load(open('a_auc_map', 'rb'))\n",
    "h_auc_map = pickle.load(open('h_auc_map', 'rb'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "dict_keys(['defect_camel_1_2', 'defect_eclipse_2_0', 'defect_eclipse_3_0', 'defect_prop_2', 'defect_xalan_2_6'])\n"
     ]
    }
   ],
   "source": [
    "print(a_auc_map.keys())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "dict_keys(['H_eclipse_2_0_xgboost', 'H_eclipse_2_0_randomforest', 'H_eclipse_2_0_lr', 'H_camel_1_2_xgboost', 'H_camel_1_2_randomforest', 'H_camel_1_2_lr', 'H_eclipse_3_0_xgboost', 'H_eclipse_3_0_randomforest', 'H_eclipse_3_0_lr', 'H_prop_2_xgboost', 'H_prop_2_randomforest', 'H_prop_2_lr', 'H_xalan_2_6_xgboost', 'H_xalan_2_6_randomforest', 'H_xalan_2_6_lr'])\n"
     ]
    }
   ],
   "source": [
    "print(h_auc_map.keys())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "import scipy.stats"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "camel_1_2\n",
      "H_camel_1_2_xgboost\n",
      "WilcoxonResult(statistic=4.0, pvalue=6.436988241970729e-18)\n",
      "H_camel_1_2_randomforest\n",
      "WilcoxonResult(statistic=1165.0, pvalue=4.824465542235559e-06)\n",
      "H_camel_1_2_lr\n",
      "WilcoxonResult(statistic=1648.0, pvalue=0.0038959032430327762)\n",
      "eclipse_2_0\n",
      "H_eclipse_2_0_xgboost\n",
      "WilcoxonResult(statistic=2.0, pvalue=6.055964751718375e-18)\n",
      "H_eclipse_2_0_randomforest\n",
      "WilcoxonResult(statistic=1.0, pvalue=5.873890051096796e-18)\n",
      "H_eclipse_2_0_lr\n",
      "WilcoxonResult(statistic=0.0, pvalue=5.697220948724935e-18)\n",
      "eclipse_3_0\n",
      "H_eclipse_3_0_xgboost\n",
      "WilcoxonResult(statistic=0.0, pvalue=5.697220948724935e-18)\n",
      "H_eclipse_3_0_randomforest\n",
      "WilcoxonResult(statistic=15.0, pvalue=8.996130287159537e-18)\n",
      "H_eclipse_3_0_lr\n",
      "WilcoxonResult(statistic=1.0, pvalue=5.873890051096796e-18)\n",
      "prop_2\n",
      "H_prop_2_xgboost\n",
      "WilcoxonResult(statistic=1613.0, pvalue=0.0026242868364230896)\n",
      "H_prop_2_randomforest\n",
      "WilcoxonResult(statistic=0.0, pvalue=5.697220948724935e-18)\n",
      "H_prop_2_lr\n",
      "WilcoxonResult(statistic=0.0, pvalue=5.697220948724935e-18)\n",
      "xalan_2_6\n",
      "H_xalan_2_6_xgboost\n",
      "WilcoxonResult(statistic=1647.0, pvalue=0.00866714288192058)\n",
      "H_xalan_2_6_randomforest\n",
      "WilcoxonResult(statistic=729.0, pvalue=1.1008884217359586e-09)\n",
      "H_xalan_2_6_lr\n",
      "WilcoxonResult(statistic=29.0, pvalue=1.3745537655624765e-17)\n"
     ]
    }
   ],
   "source": [
    "wil_pval=[]\n",
    "for key in a_auc_map.keys():\n",
    "    kk = key.split(\"defect_\")[1]\n",
    "    print(kk)\n",
    "    for hkey in h_auc_map.keys():\n",
    "        if kk in hkey:\n",
    "            print(hkey)\n",
    "            aa = a_auc_map[key]\n",
    "            hh = h_auc_map[hkey]\n",
    "            wil_test = scipy.stats.wilcoxon(aa,hh)\n",
    "            print(wil_test)\n",
    "            wil_pval.append(wil_test.pvalue)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([ True,  True,  True,  True,  True,  True,  True,  True,  True,\n",
       "        True,  True,  True,  True,  True,  True])"
      ]
     },
     "execution_count": 48,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from statsmodels.stats import multitest \n",
    "multitest.multipletests(pvals=wil_pval,method='holm')[1]<0.05"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "def cliffsDelta(lst1, lst2, **dull):\n",
    "\n",
    "    \"\"\"Returns delta and true if there are more than 'dull' differences\"\"\"\n",
    "    if not dull:\n",
    "        dull = {'small': 0.147, 'medium': 0.33, 'large': 0.474} # effect sizes from (Hess and Kromrey, 2004)\n",
    "    m, n = len(lst1), len(lst2)\n",
    "    lst2 = sorted(lst2)\n",
    "    j = more = less = 0\n",
    "    for repeats, x in runs(sorted(lst1)):\n",
    "        while j <= (n - 1) and lst2[j] < x:\n",
    "            j += 1\n",
    "        more += j*repeats\n",
    "        while j <= (n - 1) and lst2[j] == x:\n",
    "            j += 1\n",
    "        less += (n - j)*repeats\n",
    "    d = (more - less) / (m*n)\n",
    "    size = lookup_size(d, dull)\n",
    "    return d, size\n",
    "\n",
    "\n",
    "def lookup_size(delta: float, dull: dict) -> str:\n",
    "    \"\"\"\n",
    "    :type delta: float\n",
    "    :type dull: dict, a dictionary of small, medium, large thresholds.\n",
    "    \"\"\"\n",
    "    delta = abs(delta)\n",
    "    if delta < dull['small']:\n",
    "        return 'negligible'\n",
    "    if dull['small'] <= delta < dull['medium']:\n",
    "        return 'small'\n",
    "    if dull['medium'] <= delta < dull['large']:\n",
    "        return 'medium'\n",
    "    if delta >= dull['large']:\n",
    "        return 'large'\n",
    "\n",
    "\n",
    "def runs(lst):\n",
    "    \"\"\"Iterator, chunks repeated values\"\"\"\n",
    "    for j, two in enumerate(lst):\n",
    "        if j == 0:\n",
    "            one, i = two, 0\n",
    "        if one != two:\n",
    "            yield j - i, one\n",
    "            i = j\n",
    "        one = two\n",
    "    yield j - i + 1, two\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'large'"
      ]
     },
     "execution_count": 42,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cliffsDelta(aa,hh)[1]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "camel_1_2\n",
      "H_camel_1_2_xgboost\n",
      "H_camel_1_2_randomforest\n",
      "H_camel_1_2_lr\n",
      "eclipse_2_0\n",
      "H_eclipse_2_0_xgboost\n",
      "H_eclipse_2_0_randomforest\n",
      "H_eclipse_2_0_lr\n",
      "eclipse_3_0\n",
      "H_eclipse_3_0_xgboost\n",
      "H_eclipse_3_0_randomforest\n",
      "H_eclipse_3_0_lr\n",
      "prop_2\n",
      "H_prop_2_xgboost\n",
      "H_prop_2_randomforest\n",
      "H_prop_2_lr\n",
      "xalan_2_6\n",
      "H_xalan_2_6_xgboost\n",
      "H_xalan_2_6_randomforest\n",
      "H_xalan_2_6_lr\n"
     ]
    }
   ],
   "source": [
    "cliff_result={}\n",
    "for key in a_auc_map.keys():\n",
    "    kk = key.split(\"defect_\")[1]\n",
    "    print(kk)\n",
    "    for hkey in h_auc_map.keys():\n",
    "        if kk in hkey:\n",
    "            print(hkey)\n",
    "            aa = a_auc_map[key]\n",
    "            hh = h_auc_map[hkey]\n",
    "\n",
    "            cliff_result[hkey] =             cliffsDelta(aa,hh)[1]\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'H_camel_1_2_xgboost': 'large',\n",
       " 'H_camel_1_2_randomforest': 'small',\n",
       " 'H_camel_1_2_lr': 'small',\n",
       " 'H_eclipse_2_0_xgboost': 'medium',\n",
       " 'H_eclipse_2_0_randomforest': 'large',\n",
       " 'H_eclipse_2_0_lr': 'large',\n",
       " 'H_eclipse_3_0_xgboost': 'medium',\n",
       " 'H_eclipse_3_0_randomforest': 'large',\n",
       " 'H_eclipse_3_0_lr': 'large',\n",
       " 'H_prop_2_xgboost': 'negligible',\n",
       " 'H_prop_2_randomforest': 'large',\n",
       " 'H_prop_2_lr': 'large',\n",
       " 'H_xalan_2_6_xgboost': 'negligible',\n",
       " 'H_xalan_2_6_randomforest': 'small',\n",
       " 'H_xalan_2_6_lr': 'large'}"
      ]
     },
     "execution_count": 44,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cliff_result"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}