{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Couldn't import dot_parser, loading of dot files will not be possible.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Using Theano backend.\n"
     ]
    }
   ],
   "source": [
    "import mhcflurry\n",
    "from mhcflurry.dataset import Dataset\n",
    "import os\n",
    "from collections import defaultdict"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Dataset(n=137654, alleles=['ELA-A1', 'Gogo-B0101', 'H-2-DB', 'H-2-DD', 'H-2-KB', 'H-2-KD', 'H-2-KK', 'H-2-LD', 'HLA-A0101', 'HLA-A0201', 'HLA-A0202', 'HLA-A0203', 'HLA-A0205', 'HLA-A0206', 'HLA-A0207', 'HLA-A0210', 'HLA-A0211', 'HLA-A0212', 'HLA-A0216', 'HLA-A0219', 'HLA-A0250', 'HLA-A0301', 'HLA-A0302', 'HLA-A11', 'HLA-A1101', 'HLA-A2', 'HLA-A2301', 'HLA-A2402', 'HLA-A2403', 'HLA-A2501', 'HLA-A26', 'HLA-A2601', 'HLA-A2602', 'HLA-A2603', 'HLA-A2902', 'HLA-A3001', 'HLA-A3002', 'HLA-A3101', 'HLA-A3201', 'HLA-A3301', 'HLA-A6601', 'HLA-A6801', 'HLA-A6802', 'HLA-A6901', 'HLA-A8001', 'HLA-B0702', 'HLA-B0801', 'HLA-B0802', 'HLA-B0803', 'HLA-B1402', 'HLA-B1501', 'HLA-B1502', 'HLA-B1503', 'HLA-B1509', 'HLA-B1517', 'HLA-B1801', 'HLA-B2701', 'HLA-B2702', 'HLA-B2703', 'HLA-B2705', 'HLA-B3501', 'HLA-B3503', 'HLA-B3508', 'HLA-B3801', 'HLA-B3901', 'HLA-B4001', 'HLA-B4002', 'HLA-B4201', 'HLA-B44', 'HLA-B4402', 'HLA-B4403', 'HLA-B4501', 'HLA-B4601', 'HLA-B4801', 'HLA-B5101', 'HLA-B5301', 'HLA-B5401', 'HLA-B5701', 'HLA-B5801', 'HLA-B5802', 'HLA-B7', 'HLA-B7301', 'HLA-E0101', 'Mamu-A01', 'Mamu-A02', 'Mamu-A07', 'Mamu-A11', 'Mamu-A2201', 'Mamu-A2601', 'Mamu-B01', 'Mamu-B03', 'Mamu-B04', 'Mamu-B08', 'Mamu-B17', 'Mamu-B52', 'Patr-A0101', 'Patr-A0301', 'Patr-A0401', 'Patr-A0602', 'Patr-A0701', 'Patr-A0901', 'Patr-B0101', 'Patr-B0901', 'Patr-B1301', 'Patr-B1701', 'Patr-B2401'])"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "bd2009 = Dataset.from_csv(\"../data/bdata.2009.mhci.public.1.txt\")\n",
    "bd2009"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "bd2009_dict = bd2009.to_nested_dictionary()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "dict_keys(['HLA-B5101', 'H-2-KD', 'HLA-A3002', 'HLA-A0210', 'Patr-B0101', 'HLA-A0206', 'Mamu-B17', 'HLA-A0203', 'HLA-B2702', 'HLA-A8001', 'HLA-B4002', 'H-2-DD', 'HLA-A1101', 'HLA-A0201', 'H-2-LD', 'HLA-B4501', 'HLA-A0205', 'HLA-A0202', 'HLA-A0212', 'HLA-B5301', 'HLA-A6901', 'HLA-B0802', 'HLA-B5802', 'HLA-A11', 'HLA-B1402', 'HLA-A2', 'HLA-A0207', 'H-2-KB', 'HLA-B3508', 'HLA-A6801', 'HLA-A0302', 'Patr-A0602', 'HLA-A0301', 'HLA-A3101', 'HLA-A0216', 'Mamu-A02', 'HLA-B4001', 'Patr-B0901', 'HLA-B1509', 'HLA-B0801', 'HLA-A2601', 'HLA-B4402', 'HLA-A26', 'Patr-B2401', 'HLA-B3501', 'HLA-A0219', 'HLA-A2301', 'HLA-B2705', 'HLA-A0211', 'HLA-B2703', 'Mamu-A2201', 'HLA-B1503', 'HLA-B4601', 'ELA-A1', 'HLA-B4801', 'HLA-A2403', 'HLA-B4403', 'Patr-B1301', 'Mamu-B08', 'HLA-B1501', 'HLA-B3503', 'HLA-B0702', 'Mamu-A2601', 'HLA-A2902', 'HLA-A6802', 'Mamu-A07', 'HLA-A0101', 'HLA-B44', 'HLA-A2602', 'Patr-A0901', 'HLA-B3801', 'HLA-E0101', 'Mamu-B03', 'H-2-DB', 'HLA-B4201', 'HLA-A3301', 'HLA-B5701', 'HLA-A2603', 'Mamu-B52', 'Patr-A0101', 'HLA-B1502', 'HLA-B7', 'Gogo-B0101', 'HLA-A3001', 'H-2-KK', 'HLA-B1517', 'Patr-B1701', 'HLA-B3901', 'Mamu-A11', 'HLA-B7301', 'Mamu-B04', 'HLA-A6601', 'HLA-A2501', 'Patr-A0401', 'Mamu-B01', 'HLA-A3201', 'HLA-B0803', 'Patr-A0701', 'HLA-B5801', 'Patr-A0301', 'Mamu-A01', 'HLA-A2402', 'HLA-B1801', 'HLA-A0250', 'HLA-B5401', 'HLA-B2701'])"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "bd2009_dict.keys()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "from mhcflurry.imputation_helpers import imputer_from_name\n",
    "from fancyimpute import MICE\n",
    "\n",
    "imputation_algos = {\n",
    "    name: imputer_from_name(name, min_value=0)\n",
    "    for name in [\"knn\", \"softimpute\", \"svd\"]\n",
    "}\n",
    "\n",
    "imputation_algos[\"mice-burnin5-iters25\"] = MICE(\n",
    "    n_burn_in=5,\n",
    "    n_imputations=25)\n",
    "\n",
    "imputation_algos[\"mice-burnin10-iters25\"] = MICE(\n",
    "    n_burn_in=10,\n",
    "    n_imputations=25)\n",
    "\n",
    "\n",
    "imputation_algos[\"mice-burnin10-iters50\"] = MICE(\n",
    "    n_burn_in=10,\n",
    "    n_imputations=50)\n",
    "\n",
    "imputation_algos[\"mice-sparse5-burnin5-iters25\"] = MICE(\n",
    "    n_burn_in=5,\n",
    "    n_imputations=25,\n",
    "    n_nearest_columns=5)\n",
    "\n",
    "imputation_algos[\"mice-sparse25-burnin5-iters25\"] = MICE(\n",
    "    n_burn_in=5,\n",
    "    n_imputations=25,\n",
    "    n_nearest_columns=25)\n",
    "\n",
    "imputation_algos[\"mice-sparse5-burnin5-iters50\"] = MICE(\n",
    "    n_burn_in=5,\n",
    "    n_imputations=50,\n",
    "    n_nearest_columns=5)\n",
    "\n",
    "\n",
    "imputation_algos[\"mice-sparse25-burnin5-iters50\"] = MICE(\n",
    "    n_burn_in=5,\n",
    "    n_imputations=50,\n",
    "    n_nearest_columns=25)\n",
    "\n",
    "\n",
    "imputation_algos[\"mice-sparse25-burnin5-iters50\"] = MICE(\n",
    "    n_burn_in=5,\n",
    "    n_imputations=50,\n",
    "    n_nearest_columns=25)\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "KendalltauResult(correlation=-0.33333333333333326, pvalue=0.60150814411369402)"
      ]
     },
     "execution_count": 48,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from sklearn.metrics import f1_score, roc_auc_score \n",
    "from scipy.stats import kendalltau\n",
    "\n",
    "kendalltau([0.908, 0.2, -3], [0.38, -1, 0.4])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "\n",
    "def evaluate_imputed_ic50s(imputed_dataset, test_dataset, binding_cutoff=500.0, filename=None):\n",
    "    \"\"\"\n",
    "    Keep only pMHCs contained in both datasets, returns DataFrame with the following columns:\n",
    "        - allele\n",
    "        - peptide\n",
    "        - test_ic50\n",
    "        - imputed_ic50\n",
    "    \"\"\"\n",
    "    imputed_dict = imputed_dataset.to_nested_dictionary()\n",
    "    test_ic50s = []\n",
    "    imputed_ic50s = []\n",
    "    for (allele, peptide, ic50) in zip(test_dataset.alleles, test_dataset.peptides, test_dataset.affinities):\n",
    "        allele_dict = imputed_dict.get(allele, {})\n",
    "        if peptide in allele_dict:\n",
    "            imputed_ic50 = allele_dict[peptide]\n",
    "        else:\n",
    "            continue\n",
    "        test_ic50s.append(ic50)\n",
    "        imputed_ic50s.append(imputed_ic50)\n",
    "    test_ic50s = np.asarray(test_ic50s)\n",
    "    imputed_ic50s = np.asarray(imputed_ic50s)\n",
    "    print(\"Keeping %d of %d imputed values, %d test values\" % (\n",
    "        len(test_ic50s),\n",
    "        len(imputed_dataset),\n",
    "        len(test_dataset)))\n",
    "    return dict(\n",
    "        f1=f1_score(y_true=test_ic50s<=binding_cutoff, y_pred=imputed_ic50s<=binding_cutoff),\n",
    "        auc=roc_auc_score(y_true=test_ic50s<=binding_cutoff, y_score=-imputed_ic50s),\n",
    "        tau=kendalltau(test_ic50s, imputed_ic50s).correlation\n",
    "    )\n",
    "\n",
    "def evaluate_imputation_method(imputer, train, test):\n",
    "    \"\"\"\n",
    "    Returns dictionary with \"auc\", \"f1\", \"tau\" keys.\n",
    "    \"\"\"\n",
    "    train_imputed = train.impute_missing_values(\n",
    "        imputation_method=imputer,\n",
    "        min_observations_per_peptide=2,\n",
    "        min_observations_per_allele=2)\n",
    "    return evaluate_imputed_ic50s(imputed_dataset=train_imputed, test_dataset=test)\n",
    "\n",
    "def scores_generator(all_data, imputation_algos, n_folds=3):\n",
    "    \"\"\"\n",
    "    Generator yields (imputer_name, cv_fold_idx, scores_dict)\n",
    "    \"\"\"\n",
    "    for i, (train, test) in enumerate(all_data.cross_validation_iterator(n_folds=n_folds)):\n",
    "        print(\"=== CV fold %d/%d\" % (i + 1, n_folds))\n",
    "        for name, imputer in imputation_algos.items():\n",
    "            print(\"--> Running imputation algorithm %s\" % name)\n",
    "            scores = evaluate_imputation_method(imputer, train, test)\n",
    "            print(scores)\n",
    "            yield (name, i, scores)\n",
    "\n",
    "def evaluate_imputation_methods(filename, all_data, imputation_algos, n_folds=3):\n",
    "    \"\"\"\n",
    "    Writes out results to filename and also returns dictionary from algorithms to score collections.\n",
    "    \"\"\"\n",
    "    f = open(filename, \"w\")\n",
    "    f.write(\"imputation_algorithm,cv_fold,f1,tau,auc\\n\")\n",
    "    results = defaultdict(list)\n",
    "    for name, cv_fold_idx, scores in scores_generator(all_data, imputation_algos, n_folds):\n",
    "        line = \"%s,%d,%f,%f,%f\\n\" % (name, cv_fold_idx, scores[\"f1\"], scores[\"tau\"], scores[\"auc\"])\n",
    "        print(line)\n",
    "        f.write(line)\n",
    "        f.flush()\n",
    "        results[name].append(scores)\n",
    "    f.close()\n",
    "    return results"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "=== CV fold 1/3\n",
      "--> Running imputation algorithm mice-burnin5-iters25\n",
      "Dropping 11113 peptides with <2 observations\n",
      "Dropping 9 alleles with <2 observations: ['ELA-A1', 'HLA-B2701', 'HLA-B3508', 'HLA-B4201', 'HLA-B44', 'Mamu-A07', 'Mamu-B04', 'Patr-A0602', 'Patr-B1701']\n",
      "[MICE] Completing matrix with shape (15655, 95)\n",
      "[MICE] Starting imputation round 1/30, elapsed time 0.047\n",
      "[MICE] Starting imputation round 2/30, elapsed time 3.708\n",
      "[MICE] Starting imputation round 3/30, elapsed time 7.330\n",
      "[MICE] Starting imputation round 4/30, elapsed time 11.206\n",
      "[MICE] Starting imputation round 5/30, elapsed time 14.837\n",
      "[MICE] Starting imputation round 6/30, elapsed time 18.625\n",
      "[MICE] Starting imputation round 7/30, elapsed time 22.519\n",
      "[MICE] Starting imputation round 8/30, elapsed time 26.480\n",
      "[MICE] Starting imputation round 9/30, elapsed time 30.414\n",
      "[MICE] Starting imputation round 10/30, elapsed time 34.524\n",
      "[MICE] Starting imputation round 11/30, elapsed time 38.387\n",
      "[MICE] Starting imputation round 12/30, elapsed time 42.512\n",
      "[MICE] Starting imputation round 13/30, elapsed time 46.885\n",
      "[MICE] Starting imputation round 14/30, elapsed time 50.729\n",
      "[MICE] Starting imputation round 15/30, elapsed time 54.445\n",
      "[MICE] Starting imputation round 16/30, elapsed time 58.134\n",
      "[MICE] Starting imputation round 17/30, elapsed time 61.835\n",
      "[MICE] Starting imputation round 18/30, elapsed time 65.711\n",
      "[MICE] Starting imputation round 19/30, elapsed time 69.797\n",
      "[MICE] Starting imputation round 20/30, elapsed time 73.877\n",
      "[MICE] Starting imputation round 21/30, elapsed time 77.772\n",
      "[MICE] Starting imputation round 22/30, elapsed time 81.798\n",
      "[MICE] Starting imputation round 23/30, elapsed time 85.576\n",
      "[MICE] Starting imputation round 24/30, elapsed time 89.441\n",
      "[MICE] Starting imputation round 25/30, elapsed time 93.543\n",
      "[MICE] Starting imputation round 26/30, elapsed time 97.355\n",
      "[MICE] Starting imputation round 27/30, elapsed time 101.234\n",
      "[MICE] Starting imputation round 28/30, elapsed time 105.177\n",
      "[MICE] Starting imputation round 29/30, elapsed time 108.856\n",
      "[MICE] Starting imputation round 30/30, elapsed time 112.930\n",
      "Keeping 36024 of 1487225 imputed values, 45885 test values\n",
      "{'f1': 0.56205353638433153, 'tau': 0.44226875402066285, 'auc': 0.84626025276143801}\n",
      "mice-burnin5-iters25,0,0.562054,0.442269,0.846260\n",
      "\n",
      "--> Running imputation algorithm mice-burnin10-iters50\n",
      "Dropping 11113 peptides with <2 observations\n",
      "Dropping 9 alleles with <2 observations: ['ELA-A1', 'HLA-B2701', 'HLA-B3508', 'HLA-B4201', 'HLA-B44', 'Mamu-A07', 'Mamu-B04', 'Patr-A0602', 'Patr-B1701']\n",
      "[MICE] Completing matrix with shape (15655, 95)\n",
      "[MICE] Starting imputation round 1/60, elapsed time 0.044\n",
      "[MICE] Starting imputation round 2/60, elapsed time 3.730\n",
      "[MICE] Starting imputation round 3/60, elapsed time 7.216\n",
      "[MICE] Starting imputation round 4/60, elapsed time 10.694\n",
      "[MICE] Starting imputation round 5/60, elapsed time 14.096\n",
      "[MICE] Starting imputation round 6/60, elapsed time 17.493\n",
      "[MICE] Starting imputation round 7/60, elapsed time 21.151\n",
      "[MICE] Starting imputation round 8/60, elapsed time 24.788\n",
      "[MICE] Starting imputation round 9/60, elapsed time 28.398\n",
      "[MICE] Starting imputation round 10/60, elapsed time 32.107\n",
      "[MICE] Starting imputation round 11/60, elapsed time 35.742\n",
      "[MICE] Starting imputation round 12/60, elapsed time 39.312\n",
      "[MICE] Starting imputation round 13/60, elapsed time 42.845\n",
      "[MICE] Starting imputation round 14/60, elapsed time 46.404\n",
      "[MICE] Starting imputation round 15/60, elapsed time 49.977\n",
      "[MICE] Starting imputation round 16/60, elapsed time 53.447\n",
      "[MICE] Starting imputation round 17/60, elapsed time 56.968\n",
      "[MICE] Starting imputation round 18/60, elapsed time 60.485\n",
      "[MICE] Starting imputation round 19/60, elapsed time 64.091\n",
      "[MICE] Starting imputation round 20/60, elapsed time 67.628\n",
      "[MICE] Starting imputation round 21/60, elapsed time 71.218\n",
      "[MICE] Starting imputation round 22/60, elapsed time 74.819\n",
      "[MICE] Starting imputation round 23/60, elapsed time 78.386\n",
      "[MICE] Starting imputation round 24/60, elapsed time 81.962\n",
      "[MICE] Starting imputation round 25/60, elapsed time 85.476\n",
      "[MICE] Starting imputation round 26/60, elapsed time 89.116\n",
      "[MICE] Starting imputation round 27/60, elapsed time 92.688\n",
      "[MICE] Starting imputation round 28/60, elapsed time 96.305\n",
      "[MICE] Starting imputation round 29/60, elapsed time 99.781\n",
      "[MICE] Starting imputation round 30/60, elapsed time 103.253\n",
      "[MICE] Starting imputation round 31/60, elapsed time 106.823\n",
      "[MICE] Starting imputation round 32/60, elapsed time 110.399\n",
      "[MICE] Starting imputation round 33/60, elapsed time 114.005\n",
      "[MICE] Starting imputation round 34/60, elapsed time 117.619\n",
      "[MICE] Starting imputation round 35/60, elapsed time 121.385\n",
      "[MICE] Starting imputation round 36/60, elapsed time 125.214\n",
      "[MICE] Starting imputation round 37/60, elapsed time 128.768\n",
      "[MICE] Starting imputation round 38/60, elapsed time 132.395\n",
      "[MICE] Starting imputation round 39/60, elapsed time 136.011\n",
      "[MICE] Starting imputation round 40/60, elapsed time 139.568\n",
      "[MICE] Starting imputation round 41/60, elapsed time 143.108\n",
      "[MICE] Starting imputation round 42/60, elapsed time 146.709\n",
      "[MICE] Starting imputation round 43/60, elapsed time 150.172\n",
      "[MICE] Starting imputation round 44/60, elapsed time 153.683\n",
      "[MICE] Starting imputation round 45/60, elapsed time 157.520\n",
      "[MICE] Starting imputation round 46/60, elapsed time 161.167\n",
      "[MICE] Starting imputation round 47/60, elapsed time 164.761\n",
      "[MICE] Starting imputation round 48/60, elapsed time 168.379\n",
      "[MICE] Starting imputation round 49/60, elapsed time 172.013\n",
      "[MICE] Starting imputation round 50/60, elapsed time 175.736\n",
      "[MICE] Starting imputation round 51/60, elapsed time 179.661\n",
      "[MICE] Starting imputation round 52/60, elapsed time 183.259\n",
      "[MICE] Starting imputation round 53/60, elapsed time 186.929\n",
      "[MICE] Starting imputation round 54/60, elapsed time 190.380\n",
      "[MICE] Starting imputation round 55/60, elapsed time 193.961\n",
      "[MICE] Starting imputation round 56/60, elapsed time 197.409\n",
      "[MICE] Starting imputation round 57/60, elapsed time 200.799\n",
      "[MICE] Starting imputation round 58/60, elapsed time 204.311\n",
      "[MICE] Starting imputation round 59/60, elapsed time 208.169\n",
      "[MICE] Starting imputation round 60/60, elapsed time 211.733\n",
      "Keeping 36024 of 1487225 imputed values, 45885 test values\n",
      "{'f1': 0.57174445234146731, 'tau': 0.45326663598809636, 'auc': 0.85177290155761964}\n",
      "mice-burnin10-iters50,0,0.571744,0.453267,0.851773\n",
      "\n",
      "--> Running imputation algorithm softimpute\n",
      "Dropping 11113 peptides with <2 observations\n",
      "Dropping 9 alleles with <2 observations: ['ELA-A1', 'HLA-B2701', 'HLA-B3508', 'HLA-B4201', 'HLA-B44', 'Mamu-A07', 'Mamu-B04', 'Patr-A0602', 'Patr-B1701']\n",
      "[SoftImpute] Max Singular Value of X_init = 1210.696785\n",
      "[SoftImpute] Iter 1: observed MAE=0.571483 rank=85\n",
      "[SoftImpute] Iter 2: observed MAE=0.571819 rank=85\n",
      "[SoftImpute] Iter 3: observed MAE=0.572093 rank=85\n",
      "[SoftImpute] Iter 4: observed MAE=0.572260 rank=84\n",
      "[SoftImpute] Iter 5: observed MAE=0.572363 rank=84\n",
      "[SoftImpute] Iter 6: observed MAE=0.572468 rank=84\n",
      "[SoftImpute] Iter 7: observed MAE=0.572555 rank=83\n",
      "[SoftImpute] Iter 8: observed MAE=0.572541 rank=82\n",
      "[SoftImpute] Iter 9: observed MAE=0.572500 rank=82\n",
      "[SoftImpute] Iter 10: observed MAE=0.572408 rank=81\n",
      "[SoftImpute] Iter 11: observed MAE=0.572301 rank=81\n",
      "[SoftImpute] Iter 12: observed MAE=0.572170 rank=81\n",
      "[SoftImpute] Iter 13: observed MAE=0.571987 rank=81\n",
      "[SoftImpute] Iter 14: observed MAE=0.571689 rank=80\n",
      "[SoftImpute] Iter 15: observed MAE=0.571064 rank=79\n",
      "[SoftImpute] Iter 16: observed MAE=0.570320 rank=78\n",
      "[SoftImpute] Iter 17: observed MAE=0.569673 rank=77\n",
      "[SoftImpute] Iter 18: observed MAE=0.569055 rank=76\n",
      "[SoftImpute] Iter 19: observed MAE=0.568479 rank=76\n",
      "[SoftImpute] Iter 20: observed MAE=0.567838 rank=75\n",
      "[SoftImpute] Iter 21: observed MAE=0.567170 rank=75\n",
      "[SoftImpute] Iter 22: observed MAE=0.566533 rank=74\n",
      "[SoftImpute] Iter 23: observed MAE=0.565831 rank=73\n",
      "[SoftImpute] Iter 24: observed MAE=0.565057 rank=73\n",
      "[SoftImpute] Iter 25: observed MAE=0.564481 rank=73\n",
      "[SoftImpute] Iter 26: observed MAE=0.563796 rank=72\n",
      "[SoftImpute] Iter 27: observed MAE=0.563036 rank=71\n",
      "[SoftImpute] Iter 28: observed MAE=0.562192 rank=70\n",
      "[SoftImpute] Iter 29: observed MAE=0.561340 rank=69\n",
      "[SoftImpute] Iter 30: observed MAE=0.560464 rank=69\n",
      "[SoftImpute] Iter 31: observed MAE=0.559746 rank=68\n",
      "[SoftImpute] Iter 32: observed MAE=0.559133 rank=68\n",
      "[SoftImpute] Iter 33: observed MAE=0.558487 rank=67\n",
      "[SoftImpute] Iter 34: observed MAE=0.557854 rank=67\n",
      "[SoftImpute] Iter 35: observed MAE=0.557301 rank=66\n",
      "[SoftImpute] Iter 36: observed MAE=0.556707 rank=65\n",
      "[SoftImpute] Iter 37: observed MAE=0.556104 rank=65\n",
      "[SoftImpute] Iter 38: observed MAE=0.555602 rank=65\n",
      "[SoftImpute] Iter 39: observed MAE=0.554953 rank=64\n",
      "[SoftImpute] Iter 40: observed MAE=0.554449 rank=64\n",
      "[SoftImpute] Iter 41: observed MAE=0.553932 rank=63\n",
      "[SoftImpute] Iter 42: observed MAE=0.553428 rank=63\n",
      "[SoftImpute] Iter 43: observed MAE=0.552932 rank=62\n",
      "[SoftImpute] Iter 44: observed MAE=0.552424 rank=62\n",
      "[SoftImpute] Iter 45: observed MAE=0.551982 rank=62\n",
      "[SoftImpute] Iter 46: observed MAE=0.551480 rank=61\n",
      "[SoftImpute] Iter 47: observed MAE=0.550998 rank=61\n",
      "[SoftImpute] Iter 48: observed MAE=0.550561 rank=60\n",
      "[SoftImpute] Iter 49: observed MAE=0.550107 rank=60\n",
      "[SoftImpute] Iter 50: observed MAE=0.549591 rank=59\n",
      "[SoftImpute] Iter 51: observed MAE=0.549129 rank=58\n",
      "[SoftImpute] Iter 52: observed MAE=0.548692 rank=58\n",
      "[SoftImpute] Iter 53: observed MAE=0.548320 rank=58\n",
      "[SoftImpute] Iter 54: observed MAE=0.547894 rank=57\n",
      "[SoftImpute] Iter 55: observed MAE=0.547517 rank=57\n",
      "[SoftImpute] Iter 56: observed MAE=0.547189 rank=57\n",
      "[SoftImpute] Iter 57: observed MAE=0.546883 rank=56\n",
      "[SoftImpute] Iter 58: observed MAE=0.546527 rank=56\n",
      "[SoftImpute] Iter 59: observed MAE=0.546223 rank=56\n",
      "[SoftImpute] Iter 60: observed MAE=0.545946 rank=56\n",
      "[SoftImpute] Iter 61: observed MAE=0.545686 rank=56\n",
      "[SoftImpute] Iter 62: observed MAE=0.545438 rank=56\n",
      "[SoftImpute] Iter 63: observed MAE=0.545201 rank=56\n",
      "[SoftImpute] Iter 64: observed MAE=0.544972 rank=56\n",
      "[SoftImpute] Iter 65: observed MAE=0.544749 rank=56\n",
      "[SoftImpute] Iter 66: observed MAE=0.544533 rank=56\n",
      "[SoftImpute] Iter 67: observed MAE=0.544286 rank=55\n",
      "[SoftImpute] Iter 68: observed MAE=0.544047 rank=55\n",
      "[SoftImpute] Iter 69: observed MAE=0.543833 rank=55\n",
      "[SoftImpute] Iter 70: observed MAE=0.543598 rank=54\n",
      "[SoftImpute] Iter 71: observed MAE=0.543354 rank=54\n",
      "[SoftImpute] Iter 72: observed MAE=0.543149 rank=54\n",
      "[SoftImpute] Iter 73: observed MAE=0.542959 rank=54\n",
      "[SoftImpute] Iter 74: observed MAE=0.542778 rank=54\n",
      "[SoftImpute] Iter 75: observed MAE=0.542603 rank=54\n",
      "[SoftImpute] Iter 76: observed MAE=0.542433 rank=54\n",
      "[SoftImpute] Iter 77: observed MAE=0.542267 rank=54\n",
      "[SoftImpute] Iter 78: observed MAE=0.542104 rank=54\n",
      "[SoftImpute] Iter 79: observed MAE=0.541945 rank=54\n",
      "[SoftImpute] Iter 80: observed MAE=0.541789 rank=53\n",
      "[SoftImpute] Iter 81: observed MAE=0.541590 rank=53\n",
      "[SoftImpute] Iter 82: observed MAE=0.541425 rank=53\n",
      "[SoftImpute] Iter 83: observed MAE=0.541274 rank=53\n",
      "[SoftImpute] Iter 84: observed MAE=0.541130 rank=53\n",
      "[SoftImpute] Iter 85: observed MAE=0.540992 rank=53\n",
      "[SoftImpute] Iter 86: observed MAE=0.540859 rank=53\n",
      "[SoftImpute] Iter 87: observed MAE=0.540714 rank=52\n",
      "[SoftImpute] Iter 88: observed MAE=0.540567 rank=52\n",
      "[SoftImpute] Iter 89: observed MAE=0.540434 rank=52\n",
      "[SoftImpute] Iter 90: observed MAE=0.540307 rank=52\n",
      "[SoftImpute] Iter 91: observed MAE=0.540176 rank=51\n",
      "[SoftImpute] Iter 92: observed MAE=0.540037 rank=51\n",
      "[SoftImpute] Iter 93: observed MAE=0.539911 rank=51\n",
      "[SoftImpute] Iter 94: observed MAE=0.539792 rank=51\n",
      "[SoftImpute] Iter 95: observed MAE=0.539678 rank=51\n",
      "[SoftImpute] Iter 96: observed MAE=0.539566 rank=51\n",
      "[SoftImpute] Iter 97: observed MAE=0.539457 rank=51\n",
      "[SoftImpute] Iter 98: observed MAE=0.539349 rank=51\n",
      "[SoftImpute] Iter 99: observed MAE=0.539243 rank=51\n",
      "[SoftImpute] Iter 100: observed MAE=0.539138 rank=51\n",
      "[SoftImpute] Stopped after iteration 100 for lambda=24.213936\n",
      "Keeping 36024 of 1487225 imputed values, 45885 test values\n",
      "{'f1': 0.60126207848550584, 'tau': 0.4313736122656695, 'auc': 0.84451550307432588}\n",
      "softimpute,0,0.601262,0.431374,0.844516\n",
      "\n",
      "--> Running imputation algorithm mice-burnin10-iters25\n",
      "Dropping 11113 peptides with <2 observations\n",
      "Dropping 9 alleles with <2 observations: ['ELA-A1', 'HLA-B2701', 'HLA-B3508', 'HLA-B4201', 'HLA-B44', 'Mamu-A07', 'Mamu-B04', 'Patr-A0602', 'Patr-B1701']\n",
      "[MICE] Completing matrix with shape (15655, 95)\n",
      "[MICE] Starting imputation round 1/35, elapsed time 0.042\n",
      "[MICE] Starting imputation round 2/35, elapsed time 3.827\n",
      "[MICE] Starting imputation round 3/35, elapsed time 7.621\n",
      "[MICE] Starting imputation round 4/35, elapsed time 11.678\n",
      "[MICE] Starting imputation round 5/35, elapsed time 15.461\n",
      "[MICE] Starting imputation round 6/35, elapsed time 19.626\n",
      "[MICE] Starting imputation round 7/35, elapsed time 23.840\n",
      "[MICE] Starting imputation round 8/35, elapsed time 27.878\n",
      "[MICE] Starting imputation round 9/35, elapsed time 32.126\n",
      "[MICE] Starting imputation round 10/35, elapsed time 35.846\n",
      "[MICE] Starting imputation round 11/35, elapsed time 39.692\n",
      "[MICE] Starting imputation round 12/35, elapsed time 43.658\n",
      "[MICE] Starting imputation round 13/35, elapsed time 48.196\n",
      "[MICE] Starting imputation round 14/35, elapsed time 52.106\n",
      "[MICE] Starting imputation round 15/35, elapsed time 55.765\n",
      "[MICE] Starting imputation round 16/35, elapsed time 59.325\n",
      "[MICE] Starting imputation round 17/35, elapsed time 62.911\n",
      "[MICE] Starting imputation round 18/35, elapsed time 66.489\n",
      "[MICE] Starting imputation round 19/35, elapsed time 70.038\n",
      "[MICE] Starting imputation round 20/35, elapsed time 73.832\n",
      "[MICE] Starting imputation round 21/35, elapsed time 77.703\n",
      "[MICE] Starting imputation round 22/35, elapsed time 81.717\n",
      "[MICE] Starting imputation round 23/35, elapsed time 85.702\n",
      "[MICE] Starting imputation round 24/35, elapsed time 89.386\n",
      "[MICE] Starting imputation round 25/35, elapsed time 93.447\n",
      "[MICE] Starting imputation round 26/35, elapsed time 97.761\n",
      "[MICE] Starting imputation round 27/35, elapsed time 101.673\n",
      "[MICE] Starting imputation round 28/35, elapsed time 105.365\n",
      "[MICE] Starting imputation round 29/35, elapsed time 109.062\n",
      "[MICE] Starting imputation round 30/35, elapsed time 112.825\n",
      "[MICE] Starting imputation round 31/35, elapsed time 116.620\n",
      "[MICE] Starting imputation round 32/35, elapsed time 120.492\n",
      "[MICE] Starting imputation round 33/35, elapsed time 124.256\n",
      "[MICE] Starting imputation round 34/35, elapsed time 127.916\n",
      "[MICE] Starting imputation round 35/35, elapsed time 131.539\n",
      "Keeping 36024 of 1487225 imputed values, 45885 test values\n",
      "{'f1': 0.56414653156664063, 'tau': 0.43801891232263229, 'auc': 0.84493387224859762}\n",
      "mice-burnin10-iters25,0,0.564147,0.438019,0.844934\n",
      "\n",
      "--> Running imputation algorithm mice-sparse5-burnin5-iters50\n",
      "Dropping 11113 peptides with <2 observations\n",
      "Dropping 9 alleles with <2 observations: ['ELA-A1', 'HLA-B2701', 'HLA-B3508', 'HLA-B4201', 'HLA-B44', 'Mamu-A07', 'Mamu-B04', 'Patr-A0602', 'Patr-B1701']\n",
      "[MICE] Completing matrix with shape (15655, 95)\n",
      "[MICE] Starting imputation round 1/55, elapsed time 0.039\n",
      "[MICE] Starting imputation round 2/55, elapsed time 0.422\n",
      "[MICE] Starting imputation round 3/55, elapsed time 0.856\n",
      "[MICE] Starting imputation round 4/55, elapsed time 1.240\n",
      "[MICE] Starting imputation round 5/55, elapsed time 1.679\n",
      "[MICE] Starting imputation round 6/55, elapsed time 2.097\n",
      "[MICE] Starting imputation round 7/55, elapsed time 2.552\n",
      "[MICE] Starting imputation round 8/55, elapsed time 2.956\n",
      "[MICE] Starting imputation round 9/55, elapsed time 3.412\n",
      "[MICE] Starting imputation round 10/55, elapsed time 3.972\n",
      "[MICE] Starting imputation round 11/55, elapsed time 4.523\n",
      "[MICE] Starting imputation round 12/55, elapsed time 4.962\n",
      "[MICE] Starting imputation round 13/55, elapsed time 5.408\n",
      "[MICE] Starting imputation round 14/55, elapsed time 5.899\n",
      "[MICE] Starting imputation round 15/55, elapsed time 6.344\n",
      "[MICE] Starting imputation round 16/55, elapsed time 6.822\n",
      "[MICE] Starting imputation round 17/55, elapsed time 7.302\n",
      "[MICE] Starting imputation round 18/55, elapsed time 7.836\n",
      "[MICE] Starting imputation round 19/55, elapsed time 8.298\n",
      "[MICE] Starting imputation round 20/55, elapsed time 8.761\n",
      "[MICE] Starting imputation round 21/55, elapsed time 9.224\n",
      "[MICE] Starting imputation round 22/55, elapsed time 9.670\n",
      "[MICE] Starting imputation round 23/55, elapsed time 10.150\n",
      "[MICE] Starting imputation round 24/55, elapsed time 10.631\n",
      "[MICE] Starting imputation round 25/55, elapsed time 11.107\n",
      "[MICE] Starting imputation round 26/55, elapsed time 11.567\n",
      "[MICE] Starting imputation round 27/55, elapsed time 11.994\n",
      "[MICE] Starting imputation round 28/55, elapsed time 12.469\n",
      "[MICE] Starting imputation round 29/55, elapsed time 12.999\n",
      "[MICE] Starting imputation round 30/55, elapsed time 13.474\n",
      "[MICE] Starting imputation round 31/55, elapsed time 13.926\n",
      "[MICE] Starting imputation round 32/55, elapsed time 14.387\n",
      "[MICE] Starting imputation round 33/55, elapsed time 14.828\n",
      "[MICE] Starting imputation round 34/55, elapsed time 15.247\n",
      "[MICE] Starting imputation round 35/55, elapsed time 15.671\n",
      "[MICE] Starting imputation round 36/55, elapsed time 16.118\n",
      "[MICE] Starting imputation round 37/55, elapsed time 16.577\n",
      "[MICE] Starting imputation round 38/55, elapsed time 17.066\n",
      "[MICE] Starting imputation round 39/55, elapsed time 17.546\n",
      "[MICE] Starting imputation round 40/55, elapsed time 18.056\n",
      "[MICE] Starting imputation round 41/55, elapsed time 18.566\n",
      "[MICE] Starting imputation round 42/55, elapsed time 19.050\n",
      "[MICE] Starting imputation round 43/55, elapsed time 19.522\n",
      "[MICE] Starting imputation round 44/55, elapsed time 19.984\n",
      "[MICE] Starting imputation round 45/55, elapsed time 20.458\n",
      "[MICE] Starting imputation round 46/55, elapsed time 20.961\n",
      "[MICE] Starting imputation round 47/55, elapsed time 21.435\n",
      "[MICE] Starting imputation round 48/55, elapsed time 21.928\n",
      "[MICE] Starting imputation round 49/55, elapsed time 22.416\n",
      "[MICE] Starting imputation round 50/55, elapsed time 22.907\n",
      "[MICE] Starting imputation round 51/55, elapsed time 23.487\n",
      "[MICE] Starting imputation round 52/55, elapsed time 24.023\n",
      "[MICE] Starting imputation round 53/55, elapsed time 24.643\n",
      "[MICE] Starting imputation round 54/55, elapsed time 25.295\n",
      "[MICE] Starting imputation round 55/55, elapsed time 25.830\n",
      "Keeping 36024 of 1487225 imputed values, 45885 test values\n",
      "{'f1': 0.50628521889900302, 'tau': 0.41122034869977581, 'auc': 0.83167035189573701}\n",
      "mice-sparse5-burnin5-iters50,0,0.506285,0.411220,0.831670\n",
      "\n",
      "--> Running imputation algorithm mice-sparse5-burnin5-iters25\n",
      "Dropping 11113 peptides with <2 observations\n",
      "Dropping 9 alleles with <2 observations: ['ELA-A1', 'HLA-B2701', 'HLA-B3508', 'HLA-B4201', 'HLA-B44', 'Mamu-A07', 'Mamu-B04', 'Patr-A0602', 'Patr-B1701']\n",
      "[MICE] Completing matrix with shape (15655, 95)\n",
      "[MICE] Starting imputation round 1/30, elapsed time 0.039\n",
      "[MICE] Starting imputation round 2/30, elapsed time 0.440\n",
      "[MICE] Starting imputation round 3/30, elapsed time 0.872\n",
      "[MICE] Starting imputation round 4/30, elapsed time 1.305\n",
      "[MICE] Starting imputation round 5/30, elapsed time 1.733\n",
      "[MICE] Starting imputation round 6/30, elapsed time 2.159\n",
      "[MICE] Starting imputation round 7/30, elapsed time 2.602\n",
      "[MICE] Starting imputation round 8/30, elapsed time 3.046\n",
      "[MICE] Starting imputation round 9/30, elapsed time 3.496\n",
      "[MICE] Starting imputation round 10/30, elapsed time 3.938\n",
      "[MICE] Starting imputation round 11/30, elapsed time 4.361\n",
      "[MICE] Starting imputation round 12/30, elapsed time 4.821\n",
      "[MICE] Starting imputation round 13/30, elapsed time 5.277\n",
      "[MICE] Starting imputation round 14/30, elapsed time 5.738\n",
      "[MICE] Starting imputation round 15/30, elapsed time 6.249\n",
      "[MICE] Starting imputation round 16/30, elapsed time 6.710\n",
      "[MICE] Starting imputation round 17/30, elapsed time 7.168\n",
      "[MICE] Starting imputation round 18/30, elapsed time 7.673\n",
      "[MICE] Starting imputation round 19/30, elapsed time 8.142\n",
      "[MICE] Starting imputation round 20/30, elapsed time 8.649\n",
      "[MICE] Starting imputation round 21/30, elapsed time 9.159\n",
      "[MICE] Starting imputation round 22/30, elapsed time 9.690\n",
      "[MICE] Starting imputation round 23/30, elapsed time 10.170\n",
      "[MICE] Starting imputation round 24/30, elapsed time 10.632\n",
      "[MICE] Starting imputation round 25/30, elapsed time 11.118\n",
      "[MICE] Starting imputation round 26/30, elapsed time 11.570\n",
      "[MICE] Starting imputation round 27/30, elapsed time 12.034\n",
      "[MICE] Starting imputation round 28/30, elapsed time 12.490\n",
      "[MICE] Starting imputation round 29/30, elapsed time 12.922\n",
      "[MICE] Starting imputation round 30/30, elapsed time 13.356\n",
      "Keeping 36024 of 1487225 imputed values, 45885 test values\n",
      "{'f1': 0.49732896777528862, 'tau': 0.38496820078532085, 'auc': 0.81659458779198979}\n",
      "mice-sparse5-burnin5-iters25,0,0.497329,0.384968,0.816595\n",
      "\n",
      "--> Running imputation algorithm mice-sparse25-burnin5-iters25\n",
      "Dropping 11113 peptides with <2 observations\n",
      "Dropping 9 alleles with <2 observations: ['ELA-A1', 'HLA-B2701', 'HLA-B3508', 'HLA-B4201', 'HLA-B44', 'Mamu-A07', 'Mamu-B04', 'Patr-A0602', 'Patr-B1701']\n",
      "[MICE] Completing matrix with shape (15655, 95)\n",
      "[MICE] Starting imputation round 1/30, elapsed time 0.038\n",
      "[MICE] Starting imputation round 2/30, elapsed time 0.844\n",
      "[MICE] Starting imputation round 3/30, elapsed time 1.815\n",
      "[MICE] Starting imputation round 4/30, elapsed time 2.754\n",
      "[MICE] Starting imputation round 5/30, elapsed time 3.709\n",
      "[MICE] Starting imputation round 6/30, elapsed time 4.647\n",
      "[MICE] Starting imputation round 7/30, elapsed time 5.602\n",
      "[MICE] Starting imputation round 8/30, elapsed time 6.537\n",
      "[MICE] Starting imputation round 9/30, elapsed time 7.489\n",
      "[MICE] Starting imputation round 10/30, elapsed time 8.432\n",
      "[MICE] Starting imputation round 11/30, elapsed time 9.385\n",
      "[MICE] Starting imputation round 12/30, elapsed time 10.329\n",
      "[MICE] Starting imputation round 13/30, elapsed time 11.300\n",
      "[MICE] Starting imputation round 14/30, elapsed time 12.266\n",
      "[MICE] Starting imputation round 15/30, elapsed time 13.213\n",
      "[MICE] Starting imputation round 16/30, elapsed time 14.170\n",
      "[MICE] Starting imputation round 17/30, elapsed time 15.122\n",
      "[MICE] Starting imputation round 18/30, elapsed time 16.081\n",
      "[MICE] Starting imputation round 19/30, elapsed time 17.040\n",
      "[MICE] Starting imputation round 20/30, elapsed time 17.986\n",
      "[MICE] Starting imputation round 21/30, elapsed time 18.968\n",
      "[MICE] Starting imputation round 22/30, elapsed time 20.244\n",
      "[MICE] Starting imputation round 23/30, elapsed time 21.360\n",
      "[MICE] Starting imputation round 24/30, elapsed time 22.558\n",
      "[MICE] Starting imputation round 25/30, elapsed time 23.576\n",
      "[MICE] Starting imputation round 26/30, elapsed time 24.773\n",
      "[MICE] Starting imputation round 27/30, elapsed time 25.823\n",
      "[MICE] Starting imputation round 28/30, elapsed time 26.807\n",
      "[MICE] Starting imputation round 29/30, elapsed time 27.793\n",
      "[MICE] Starting imputation round 30/30, elapsed time 28.837\n",
      "Keeping 36024 of 1487225 imputed values, 45885 test values\n",
      "{'f1': 0.57251848968470209, 'tau': 0.45060830650205214, 'auc': 0.85165766690820011}\n",
      "mice-sparse25-burnin5-iters25,0,0.572518,0.450608,0.851658\n",
      "\n",
      "--> Running imputation algorithm svd\n",
      "Dropping 11113 peptides with <2 observations\n",
      "Dropping 9 alleles with <2 observations: ['ELA-A1', 'HLA-B2701', 'HLA-B3508', 'HLA-B4201', 'HLA-B44', 'Mamu-A07', 'Mamu-B04', 'Patr-A0602', 'Patr-B1701']\n",
      "[IterativeSVD] Iter 1: observed MAE=6.312350\n",
      "[IterativeSVD] Iter 2: observed MAE=4.804277\n",
      "[IterativeSVD] Iter 3: observed MAE=3.621238\n",
      "[IterativeSVD] Iter 4: observed MAE=2.503448\n",
      "[IterativeSVD] Iter 5: observed MAE=1.945234\n",
      "[IterativeSVD] Iter 6: observed MAE=1.693694\n",
      "[IterativeSVD] Iter 7: observed MAE=1.544717\n",
      "[IterativeSVD] Iter 8: observed MAE=1.444813\n",
      "[IterativeSVD] Iter 9: observed MAE=1.371306\n",
      "[IterativeSVD] Iter 10: observed MAE=1.313013\n",
      "[IterativeSVD] Iter 11: observed MAE=1.263992\n",
      "[IterativeSVD] Iter 12: observed MAE=1.221109\n",
      "[IterativeSVD] Iter 13: observed MAE=1.182588\n",
      "[IterativeSVD] Iter 14: observed MAE=1.147352\n",
      "[IterativeSVD] Iter 15: observed MAE=1.114834\n",
      "[IterativeSVD] Iter 16: observed MAE=1.084818\n",
      "[IterativeSVD] Iter 17: observed MAE=1.057043\n",
      "[IterativeSVD] Iter 18: observed MAE=1.031316\n",
      "[IterativeSVD] Iter 19: observed MAE=1.007606\n",
      "[IterativeSVD] Iter 20: observed MAE=0.985624\n",
      "[IterativeSVD] Iter 21: observed MAE=0.965119\n",
      "[IterativeSVD] Iter 22: observed MAE=0.945989\n",
      "[IterativeSVD] Iter 23: observed MAE=0.928138\n",
      "[IterativeSVD] Iter 24: observed MAE=0.911408\n",
      "[IterativeSVD] Iter 25: observed MAE=0.895686\n",
      "[IterativeSVD] Iter 26: observed MAE=0.880906\n",
      "[IterativeSVD] Iter 27: observed MAE=0.866998\n",
      "[IterativeSVD] Iter 28: observed MAE=0.853913\n",
      "[IterativeSVD] Iter 29: observed MAE=0.841567\n",
      "[IterativeSVD] Iter 30: observed MAE=0.829902\n",
      "[IterativeSVD] Iter 31: observed MAE=0.818847\n",
      "[IterativeSVD] Iter 32: observed MAE=0.808353\n",
      "[IterativeSVD] Iter 33: observed MAE=0.798343\n",
      "[IterativeSVD] Iter 34: observed MAE=0.788792\n",
      "[IterativeSVD] Iter 35: observed MAE=0.779646\n",
      "[IterativeSVD] Iter 36: observed MAE=0.770877\n",
      "[IterativeSVD] Iter 37: observed MAE=0.762450\n",
      "[IterativeSVD] Iter 38: observed MAE=0.754331\n",
      "[IterativeSVD] Iter 39: observed MAE=0.746494\n",
      "[IterativeSVD] Iter 40: observed MAE=0.738905\n",
      "[IterativeSVD] Iter 41: observed MAE=0.731551\n",
      "[IterativeSVD] Iter 42: observed MAE=0.724411\n",
      "[IterativeSVD] Iter 43: observed MAE=0.717458\n",
      "[IterativeSVD] Iter 44: observed MAE=0.710675\n",
      "[IterativeSVD] Iter 45: observed MAE=0.704056\n",
      "[IterativeSVD] Iter 46: observed MAE=0.697573\n",
      "[IterativeSVD] Iter 47: observed MAE=0.691212\n",
      "[IterativeSVD] Iter 48: observed MAE=0.684973\n",
      "[IterativeSVD] Iter 49: observed MAE=0.678837\n",
      "[IterativeSVD] Iter 50: observed MAE=0.672793\n",
      "[IterativeSVD] Iter 51: observed MAE=0.666833\n",
      "[IterativeSVD] Iter 52: observed MAE=0.660942\n",
      "[IterativeSVD] Iter 53: observed MAE=0.655110\n",
      "[IterativeSVD] Iter 54: observed MAE=0.649341\n",
      "[IterativeSVD] Iter 55: observed MAE=0.643627\n",
      "[IterativeSVD] Iter 56: observed MAE=0.637966\n",
      "[IterativeSVD] Iter 57: observed MAE=0.632358\n",
      "[IterativeSVD] Iter 58: observed MAE=0.626798\n",
      "[IterativeSVD] Iter 59: observed MAE=0.621288\n",
      "[IterativeSVD] Iter 60: observed MAE=0.615847\n",
      "[IterativeSVD] Iter 61: observed MAE=0.610476\n",
      "[IterativeSVD] Iter 62: observed MAE=0.605181\n",
      "[IterativeSVD] Iter 63: observed MAE=0.599960\n",
      "[IterativeSVD] Iter 64: observed MAE=0.594823\n",
      "[IterativeSVD] Iter 65: observed MAE=0.589774\n",
      "[IterativeSVD] Iter 66: observed MAE=0.584820\n",
      "[IterativeSVD] Iter 67: observed MAE=0.579967\n",
      "[IterativeSVD] Iter 68: observed MAE=0.575217\n",
      "[IterativeSVD] Iter 69: observed MAE=0.570567\n",
      "[IterativeSVD] Iter 70: observed MAE=0.566021\n",
      "[IterativeSVD] Iter 71: observed MAE=0.561585\n",
      "[IterativeSVD] Iter 72: observed MAE=0.557253\n",
      "[IterativeSVD] Iter 73: observed MAE=0.553018\n",
      "[IterativeSVD] Iter 74: observed MAE=0.548882\n",
      "[IterativeSVD] Iter 75: observed MAE=0.544843\n",
      "[IterativeSVD] Iter 76: observed MAE=0.540898\n",
      "[IterativeSVD] Iter 77: observed MAE=0.537043\n",
      "[IterativeSVD] Iter 78: observed MAE=0.533277\n",
      "[IterativeSVD] Iter 79: observed MAE=0.529599\n",
      "[IterativeSVD] Iter 80: observed MAE=0.526007\n",
      "[IterativeSVD] Iter 81: observed MAE=0.522495\n",
      "[IterativeSVD] Iter 82: observed MAE=0.519062\n",
      "[IterativeSVD] Iter 83: observed MAE=0.515711\n",
      "[IterativeSVD] Iter 84: observed MAE=0.512430\n",
      "[IterativeSVD] Iter 85: observed MAE=0.509218\n",
      "[IterativeSVD] Iter 86: observed MAE=0.506072\n",
      "[IterativeSVD] Iter 87: observed MAE=0.502992\n",
      "[IterativeSVD] Iter 88: observed MAE=0.499975\n",
      "[IterativeSVD] Iter 89: observed MAE=0.497018\n",
      "[IterativeSVD] Iter 90: observed MAE=0.494117\n",
      "[IterativeSVD] Iter 91: observed MAE=0.491277\n",
      "[IterativeSVD] Iter 92: observed MAE=0.488493\n",
      "[IterativeSVD] Iter 93: observed MAE=0.485761\n",
      "[IterativeSVD] Iter 94: observed MAE=0.483080\n",
      "[IterativeSVD] Iter 95: observed MAE=0.480449\n",
      "[IterativeSVD] Iter 96: observed MAE=0.477865\n",
      "[IterativeSVD] Iter 97: observed MAE=0.475328\n",
      "[IterativeSVD] Iter 98: observed MAE=0.472835\n",
      "[IterativeSVD] Iter 99: observed MAE=0.470385\n",
      "[IterativeSVD] Iter 100: observed MAE=0.467978\n",
      "[IterativeSVD] Iter 101: observed MAE=0.465614\n",
      "[IterativeSVD] Iter 102: observed MAE=0.463291\n",
      "[IterativeSVD] Iter 103: observed MAE=0.461004\n",
      "[IterativeSVD] Iter 104: observed MAE=0.458755\n",
      "[IterativeSVD] Iter 105: observed MAE=0.456541\n",
      "Keeping 36024 of 1487225 imputed values, 45885 test values\n",
      "{'f1': 0.59432367853604517, 'tau': 0.407370451433619, 'auc': 0.82957656005816982}\n",
      "svd,0,0.594324,0.407370,0.829577\n",
      "\n",
      "--> Running imputation algorithm knn\n",
      "Dropping 11113 peptides with <2 observations\n",
      "Dropping 9 alleles with <2 observations: ['ELA-A1', 'HLA-B2701', 'HLA-B3508', 'HLA-B4201', 'HLA-B44', 'Mamu-A07', 'Mamu-B04', 'Patr-A0602', 'Patr-B1701']\n",
      "Computing pairwise distances between 95 samples\n",
      "Computing distances for sample #1/95, elapsed time: 0.148\n",
      "Imputing row 1/95 with 15650 missing columns, elapsed time: 1.026\n",
      "Imputing row 11/95 with 11973 missing columns, elapsed time: 2.528\n",
      "Imputing row 21/95 with 11760 missing columns, elapsed time: 4.021\n",
      "Imputing row 31/95 with 13791 missing columns, elapsed time: 5.380\n",
      "Imputing row 41/95 with 13535 missing columns, elapsed time: 6.852\n",
      "Imputing row 51/95 with 15577 missing columns, elapsed time: 8.340\n",
      "Imputing row 61/95 with 15559 missing columns, elapsed time: 9.865\n",
      "Imputing row 71/95 with 14996 missing columns, elapsed time: 11.555\n",
      "Imputing row 81/95 with 15542 missing columns, elapsed time: 13.139\n",
      "Imputing row 91/95 with 15388 missing columns, elapsed time: 14.789\n",
      "[KNN] Warning: 168420/1487225 still missing after imputation, replacing with 0\n",
      "Keeping 36024 of 1487225 imputed values, 45885 test values\n",
      "{'f1': 0.56693229759592845, 'tau': 0.42421450117403825, 'auc': 0.8141242639235905}\n",
      "knn,0,0.566932,0.424215,0.814124\n",
      "\n",
      "--> Running imputation algorithm mice-sparse25-burnin5-iters50\n",
      "Dropping 11113 peptides with <2 observations\n",
      "Dropping 9 alleles with <2 observations: ['ELA-A1', 'HLA-B2701', 'HLA-B3508', 'HLA-B4201', 'HLA-B44', 'Mamu-A07', 'Mamu-B04', 'Patr-A0602', 'Patr-B1701']\n",
      "[MICE] Completing matrix with shape (15655, 95)\n",
      "[MICE] Starting imputation round 1/55, elapsed time 0.035\n",
      "[MICE] Starting imputation round 2/55, elapsed time 0.850\n",
      "[MICE] Starting imputation round 3/55, elapsed time 1.805\n",
      "[MICE] Starting imputation round 4/55, elapsed time 2.772\n",
      "[MICE] Starting imputation round 5/55, elapsed time 3.709\n",
      "[MICE] Starting imputation round 6/55, elapsed time 4.643\n",
      "[MICE] Starting imputation round 7/55, elapsed time 5.622\n",
      "[MICE] Starting imputation round 8/55, elapsed time 6.570\n",
      "[MICE] Starting imputation round 9/55, elapsed time 7.550\n",
      "[MICE] Starting imputation round 10/55, elapsed time 8.504\n",
      "[MICE] Starting imputation round 11/55, elapsed time 9.470\n",
      "[MICE] Starting imputation round 12/55, elapsed time 10.434\n",
      "[MICE] Starting imputation round 13/55, elapsed time 11.403\n",
      "[MICE] Starting imputation round 14/55, elapsed time 12.369\n",
      "[MICE] Starting imputation round 15/55, elapsed time 13.329\n",
      "[MICE] Starting imputation round 16/55, elapsed time 14.273\n",
      "[MICE] Starting imputation round 17/55, elapsed time 15.229\n",
      "[MICE] Starting imputation round 18/55, elapsed time 16.189\n",
      "[MICE] Starting imputation round 19/55, elapsed time 17.163\n",
      "[MICE] Starting imputation round 20/55, elapsed time 18.138\n",
      "[MICE] Starting imputation round 21/55, elapsed time 19.088\n",
      "[MICE] Starting imputation round 22/55, elapsed time 20.056\n",
      "[MICE] Starting imputation round 23/55, elapsed time 21.020\n",
      "[MICE] Starting imputation round 24/55, elapsed time 21.989\n",
      "[MICE] Starting imputation round 25/55, elapsed time 22.936\n",
      "[MICE] Starting imputation round 26/55, elapsed time 23.912\n",
      "[MICE] Starting imputation round 27/55, elapsed time 24.858\n",
      "[MICE] Starting imputation round 28/55, elapsed time 25.807\n",
      "[MICE] Starting imputation round 29/55, elapsed time 26.804\n",
      "[MICE] Starting imputation round 30/55, elapsed time 27.782\n",
      "[MICE] Starting imputation round 31/55, elapsed time 28.737\n",
      "[MICE] Starting imputation round 32/55, elapsed time 29.723\n",
      "[MICE] Starting imputation round 33/55, elapsed time 30.697\n",
      "[MICE] Starting imputation round 34/55, elapsed time 31.662\n",
      "[MICE] Starting imputation round 35/55, elapsed time 32.631\n",
      "[MICE] Starting imputation round 36/55, elapsed time 33.594\n",
      "[MICE] Starting imputation round 37/55, elapsed time 34.562\n",
      "[MICE] Starting imputation round 38/55, elapsed time 35.516\n",
      "[MICE] Starting imputation round 39/55, elapsed time 36.488\n",
      "[MICE] Starting imputation round 40/55, elapsed time 37.461\n",
      "[MICE] Starting imputation round 41/55, elapsed time 38.429\n",
      "[MICE] Starting imputation round 42/55, elapsed time 39.385\n",
      "[MICE] Starting imputation round 43/55, elapsed time 40.331\n",
      "[MICE] Starting imputation round 44/55, elapsed time 41.308\n",
      "[MICE] Starting imputation round 45/55, elapsed time 42.259\n",
      "[MICE] Starting imputation round 46/55, elapsed time 43.227\n",
      "[MICE] Starting imputation round 47/55, elapsed time 44.179\n",
      "[MICE] Starting imputation round 48/55, elapsed time 45.130\n",
      "[MICE] Starting imputation round 49/55, elapsed time 46.104\n",
      "[MICE] Starting imputation round 50/55, elapsed time 47.078\n",
      "[MICE] Starting imputation round 51/55, elapsed time 48.047\n",
      "[MICE] Starting imputation round 52/55, elapsed time 49.081\n",
      "[MICE] Starting imputation round 53/55, elapsed time 50.096\n",
      "[MICE] Starting imputation round 54/55, elapsed time 51.114\n",
      "[MICE] Starting imputation round 55/55, elapsed time 52.124\n",
      "Keeping 36024 of 1487225 imputed values, 45885 test values\n",
      "{'f1': 0.57644189157290115, 'tau': 0.46210587171103662, 'auc': 0.85635393654702785}\n",
      "mice-sparse25-burnin5-iters50,0,0.576442,0.462106,0.856354\n",
      "\n",
      "=== CV fold 2/3\n",
      "--> Running imputation algorithm mice-burnin5-iters25\n",
      "Dropping 11218 peptides with <2 observations\n",
      "Dropping 11 alleles with <2 observations: ['ELA-A1', 'HLA-A26', 'HLA-A6601', 'HLA-B2701', 'HLA-B3508', 'HLA-B44', 'HLA-E0101', 'Mamu-A07', 'Mamu-B04', 'Patr-B0901', 'Patr-B1701']\n",
      "[MICE] Completing matrix with shape (15638, 93)\n",
      "[MICE] Starting imputation round 1/30, elapsed time 0.034\n",
      "[MICE] Starting imputation round 2/30, elapsed time 3.168\n",
      "[MICE] Starting imputation round 3/30, elapsed time 6.345\n",
      "[MICE] Starting imputation round 4/30, elapsed time 9.511\n",
      "[MICE] Starting imputation round 5/30, elapsed time 12.629\n",
      "[MICE] Starting imputation round 6/30, elapsed time 15.801\n",
      "[MICE] Starting imputation round 7/30, elapsed time 18.948\n",
      "[MICE] Starting imputation round 8/30, elapsed time 22.108\n",
      "[MICE] Starting imputation round 9/30, elapsed time 25.307\n",
      "[MICE] Starting imputation round 10/30, elapsed time 28.426\n",
      "[MICE] Starting imputation round 11/30, elapsed time 31.571\n",
      "[MICE] Starting imputation round 12/30, elapsed time 34.764\n",
      "[MICE] Starting imputation round 13/30, elapsed time 37.896\n",
      "[MICE] Starting imputation round 14/30, elapsed time 41.020\n",
      "[MICE] Starting imputation round 15/30, elapsed time 44.163\n",
      "[MICE] Starting imputation round 16/30, elapsed time 47.362\n",
      "[MICE] Starting imputation round 17/30, elapsed time 50.485\n",
      "[MICE] Starting imputation round 18/30, elapsed time 53.627\n",
      "[MICE] Starting imputation round 19/30, elapsed time 56.779\n",
      "[MICE] Starting imputation round 20/30, elapsed time 60.027\n",
      "[MICE] Starting imputation round 21/30, elapsed time 63.274\n",
      "[MICE] Starting imputation round 22/30, elapsed time 66.572\n",
      "[MICE] Starting imputation round 23/30, elapsed time 69.849\n",
      "[MICE] Starting imputation round 24/30, elapsed time 73.363\n",
      "[MICE] Starting imputation round 25/30, elapsed time 76.857\n",
      "[MICE] Starting imputation round 26/30, elapsed time 80.032\n",
      "[MICE] Starting imputation round 27/30, elapsed time 83.192\n",
      "[MICE] Starting imputation round 28/30, elapsed time 86.360\n",
      "[MICE] Starting imputation round 29/30, elapsed time 89.492\n",
      "[MICE] Starting imputation round 30/30, elapsed time 92.686\n",
      "Keeping 35916 of 1454334 imputed values, 45885 test values\n",
      "{'f1': 0.56271943823811044, 'tau': 0.43846472238199519, 'auc': 0.84570899028181235}\n",
      "mice-burnin5-iters25,1,0.562719,0.438465,0.845709\n",
      "\n",
      "--> Running imputation algorithm mice-burnin10-iters50\n",
      "Dropping 11218 peptides with <2 observations\n",
      "Dropping 11 alleles with <2 observations: ['ELA-A1', 'HLA-A26', 'HLA-A6601', 'HLA-B2701', 'HLA-B3508', 'HLA-B44', 'HLA-E0101', 'Mamu-A07', 'Mamu-B04', 'Patr-B0901', 'Patr-B1701']\n",
      "[MICE] Completing matrix with shape (15638, 93)\n",
      "[MICE] Starting imputation round 1/60, elapsed time 0.036\n",
      "[MICE] Starting imputation round 2/60, elapsed time 3.487\n",
      "[MICE] Starting imputation round 3/60, elapsed time 6.685\n",
      "[MICE] Starting imputation round 4/60, elapsed time 9.972\n",
      "[MICE] Starting imputation round 5/60, elapsed time 13.371\n",
      "[MICE] Starting imputation round 6/60, elapsed time 16.656\n",
      "[MICE] Starting imputation round 7/60, elapsed time 19.850\n",
      "[MICE] Starting imputation round 8/60, elapsed time 22.964\n",
      "[MICE] Starting imputation round 9/60, elapsed time 26.160\n",
      "[MICE] Starting imputation round 10/60, elapsed time 29.354\n",
      "[MICE] Starting imputation round 11/60, elapsed time 32.509\n",
      "[MICE] Starting imputation round 12/60, elapsed time 35.675\n",
      "[MICE] Starting imputation round 13/60, elapsed time 38.858\n",
      "[MICE] Starting imputation round 14/60, elapsed time 42.008\n",
      "[MICE] Starting imputation round 15/60, elapsed time 45.145\n",
      "[MICE] Starting imputation round 16/60, elapsed time 48.289\n",
      "[MICE] Starting imputation round 17/60, elapsed time 51.451\n",
      "[MICE] Starting imputation round 18/60, elapsed time 54.618\n",
      "[MICE] Starting imputation round 19/60, elapsed time 57.787\n",
      "[MICE] Starting imputation round 20/60, elapsed time 60.930\n",
      "[MICE] Starting imputation round 21/60, elapsed time 64.079\n",
      "[MICE] Starting imputation round 22/60, elapsed time 67.205\n",
      "[MICE] Starting imputation round 23/60, elapsed time 70.360\n",
      "[MICE] Starting imputation round 24/60, elapsed time 73.523\n",
      "[MICE] Starting imputation round 25/60, elapsed time 76.668\n",
      "[MICE] Starting imputation round 26/60, elapsed time 79.882\n",
      "[MICE] Starting imputation round 27/60, elapsed time 83.013\n",
      "[MICE] Starting imputation round 28/60, elapsed time 86.344\n",
      "[MICE] Starting imputation round 29/60, elapsed time 89.601\n",
      "[MICE] Starting imputation round 30/60, elapsed time 92.908\n",
      "[MICE] Starting imputation round 31/60, elapsed time 96.410\n",
      "[MICE] Starting imputation round 32/60, elapsed time 99.952\n",
      "[MICE] Starting imputation round 33/60, elapsed time 103.385\n",
      "[MICE] Starting imputation round 34/60, elapsed time 106.867\n",
      "[MICE] Starting imputation round 35/60, elapsed time 110.197\n",
      "[MICE] Starting imputation round 36/60, elapsed time 113.582\n",
      "[MICE] Starting imputation round 37/60, elapsed time 117.192\n",
      "[MICE] Starting imputation round 38/60, elapsed time 120.402\n",
      "[MICE] Starting imputation round 39/60, elapsed time 123.895\n",
      "[MICE] Starting imputation round 40/60, elapsed time 127.702\n",
      "[MICE] Starting imputation round 41/60, elapsed time 131.116\n",
      "[MICE] Starting imputation round 42/60, elapsed time 134.228\n",
      "[MICE] Starting imputation round 43/60, elapsed time 137.357\n",
      "[MICE] Starting imputation round 44/60, elapsed time 140.573\n",
      "[MICE] Starting imputation round 45/60, elapsed time 143.743\n",
      "[MICE] Starting imputation round 46/60, elapsed time 147.075\n",
      "[MICE] Starting imputation round 47/60, elapsed time 150.315\n",
      "[MICE] Starting imputation round 48/60, elapsed time 153.558\n",
      "[MICE] Starting imputation round 49/60, elapsed time 156.833\n",
      "[MICE] Starting imputation round 50/60, elapsed time 160.016\n",
      "[MICE] Starting imputation round 51/60, elapsed time 163.205\n",
      "[MICE] Starting imputation round 52/60, elapsed time 166.380\n",
      "[MICE] Starting imputation round 53/60, elapsed time 169.611\n",
      "[MICE] Starting imputation round 54/60, elapsed time 172.779\n",
      "[MICE] Starting imputation round 55/60, elapsed time 176.038\n",
      "[MICE] Starting imputation round 56/60, elapsed time 179.218\n",
      "[MICE] Starting imputation round 57/60, elapsed time 182.458\n",
      "[MICE] Starting imputation round 58/60, elapsed time 185.653\n",
      "[MICE] Starting imputation round 59/60, elapsed time 188.833\n",
      "[MICE] Starting imputation round 60/60, elapsed time 191.988\n",
      "Keeping 35916 of 1454334 imputed values, 45885 test values\n",
      "{'f1': 0.57047086991221074, 'tau': 0.45622216692192352, 'auc': 0.8512872689091352}\n",
      "mice-burnin10-iters50,1,0.570471,0.456222,0.851287\n",
      "\n",
      "--> Running imputation algorithm softimpute\n",
      "Dropping 11218 peptides with <2 observations\n",
      "Dropping 11 alleles with <2 observations: ['ELA-A1', 'HLA-A26', 'HLA-A6601', 'HLA-B2701', 'HLA-B3508', 'HLA-B44', 'HLA-E0101', 'Mamu-A07', 'Mamu-B04', 'Patr-B0901', 'Patr-B1701']\n",
      "[SoftImpute] Max Singular Value of X_init = 1209.010447\n",
      "[SoftImpute] Iter 1: observed MAE=0.570236 rank=83\n",
      "[SoftImpute] Iter 2: observed MAE=0.570512 rank=83\n",
      "[SoftImpute] Iter 3: observed MAE=0.570685 rank=82\n",
      "[SoftImpute] Iter 4: observed MAE=0.570723 rank=82\n",
      "[SoftImpute] Iter 5: observed MAE=0.570759 rank=82\n",
      "[SoftImpute] Iter 6: observed MAE=0.570769 rank=82\n",
      "[SoftImpute] Iter 7: observed MAE=0.570768 rank=82\n",
      "[SoftImpute] Iter 8: observed MAE=0.570739 rank=82\n",
      "[SoftImpute] Iter 9: observed MAE=0.570656 rank=81\n",
      "[SoftImpute] Iter 10: observed MAE=0.570463 rank=80\n",
      "[SoftImpute] Iter 11: observed MAE=0.570169 rank=80\n",
      "[SoftImpute] Iter 12: observed MAE=0.569886 rank=80\n",
      "[SoftImpute] Iter 13: observed MAE=0.569392 rank=78\n",
      "[SoftImpute] Iter 14: observed MAE=0.568621 rank=78\n",
      "[SoftImpute] Iter 15: observed MAE=0.568149 rank=78\n",
      "[SoftImpute] Iter 16: observed MAE=0.567786 rank=78\n",
      "[SoftImpute] Iter 17: observed MAE=0.567259 rank=76\n",
      "[SoftImpute] Iter 18: observed MAE=0.566602 rank=76\n",
      "[SoftImpute] Iter 19: observed MAE=0.566116 rank=76\n",
      "[SoftImpute] Iter 20: observed MAE=0.565572 rank=75\n",
      "[SoftImpute] Iter 21: observed MAE=0.564820 rank=73\n",
      "[SoftImpute] Iter 22: observed MAE=0.563879 rank=73\n",
      "[SoftImpute] Iter 23: observed MAE=0.562980 rank=72\n",
      "[SoftImpute] Iter 24: observed MAE=0.562294 rank=71\n",
      "[SoftImpute] Iter 25: observed MAE=0.561654 rank=71\n",
      "[SoftImpute] Iter 26: observed MAE=0.560928 rank=70\n",
      "[SoftImpute] Iter 27: observed MAE=0.560073 rank=69\n",
      "[SoftImpute] Iter 28: observed MAE=0.559350 rank=69\n",
      "[SoftImpute] Iter 29: observed MAE=0.558668 rank=68\n",
      "[SoftImpute] Iter 30: observed MAE=0.557986 rank=68\n",
      "[SoftImpute] Iter 31: observed MAE=0.557383 rank=67\n",
      "[SoftImpute] Iter 32: observed MAE=0.556820 rank=67\n",
      "[SoftImpute] Iter 33: observed MAE=0.556254 rank=66\n",
      "[SoftImpute] Iter 34: observed MAE=0.555735 rank=66\n",
      "[SoftImpute] Iter 35: observed MAE=0.555069 rank=65\n",
      "[SoftImpute] Iter 36: observed MAE=0.554531 rank=65\n",
      "[SoftImpute] Iter 37: observed MAE=0.554053 rank=65\n",
      "[SoftImpute] Iter 38: observed MAE=0.553587 rank=64\n",
      "[SoftImpute] Iter 39: observed MAE=0.552926 rank=63\n",
      "[SoftImpute] Iter 40: observed MAE=0.552347 rank=63\n",
      "[SoftImpute] Iter 41: observed MAE=0.551879 rank=63\n",
      "[SoftImpute] Iter 42: observed MAE=0.551423 rank=62\n",
      "[SoftImpute] Iter 43: observed MAE=0.550969 rank=62\n",
      "[SoftImpute] Iter 44: observed MAE=0.550556 rank=62\n",
      "[SoftImpute] Iter 45: observed MAE=0.550146 rank=61\n",
      "[SoftImpute] Iter 46: observed MAE=0.549653 rank=61\n",
      "[SoftImpute] Iter 47: observed MAE=0.549187 rank=60\n",
      "[SoftImpute] Iter 48: observed MAE=0.548712 rank=60\n",
      "[SoftImpute] Iter 49: observed MAE=0.548283 rank=59\n",
      "[SoftImpute] Iter 50: observed MAE=0.547800 rank=59\n",
      "[SoftImpute] Iter 51: observed MAE=0.547285 rank=57\n",
      "[SoftImpute] Iter 52: observed MAE=0.546846 rank=57\n",
      "[SoftImpute] Iter 53: observed MAE=0.546482 rank=57\n",
      "[SoftImpute] Iter 54: observed MAE=0.546153 rank=57\n",
      "[SoftImpute] Iter 55: observed MAE=0.545828 rank=56\n",
      "[SoftImpute] Iter 56: observed MAE=0.545471 rank=56\n",
      "[SoftImpute] Iter 57: observed MAE=0.545159 rank=56\n",
      "[SoftImpute] Iter 58: observed MAE=0.544873 rank=56\n",
      "[SoftImpute] Iter 59: observed MAE=0.544606 rank=56\n",
      "[SoftImpute] Iter 60: observed MAE=0.544350 rank=56\n",
      "[SoftImpute] Iter 61: observed MAE=0.544106 rank=56\n",
      "[SoftImpute] Iter 62: observed MAE=0.543869 rank=56\n",
      "[SoftImpute] Iter 63: observed MAE=0.543642 rank=56\n",
      "[SoftImpute] Iter 64: observed MAE=0.543422 rank=56\n",
      "[SoftImpute] Iter 65: observed MAE=0.543163 rank=55\n",
      "[SoftImpute] Iter 66: observed MAE=0.542925 rank=55\n",
      "[SoftImpute] Iter 67: observed MAE=0.542709 rank=55\n",
      "[SoftImpute] Iter 68: observed MAE=0.542505 rank=55\n",
      "[SoftImpute] Iter 69: observed MAE=0.542308 rank=55\n",
      "[SoftImpute] Iter 70: observed MAE=0.542075 rank=54\n",
      "[SoftImpute] Iter 71: observed MAE=0.541857 rank=54\n",
      "[SoftImpute] Iter 72: observed MAE=0.541587 rank=53\n",
      "[SoftImpute] Iter 73: observed MAE=0.541370 rank=53\n",
      "[SoftImpute] Iter 74: observed MAE=0.541178 rank=53\n",
      "[SoftImpute] Iter 75: observed MAE=0.540997 rank=53\n",
      "[SoftImpute] Iter 76: observed MAE=0.540826 rank=53\n",
      "[SoftImpute] Iter 77: observed MAE=0.540662 rank=53\n",
      "[SoftImpute] Iter 78: observed MAE=0.540503 rank=53\n",
      "[SoftImpute] Iter 79: observed MAE=0.540318 rank=52\n",
      "[SoftImpute] Iter 80: observed MAE=0.540137 rank=52\n",
      "[SoftImpute] Iter 81: observed MAE=0.539977 rank=52\n",
      "[SoftImpute] Iter 82: observed MAE=0.539826 rank=52\n",
      "[SoftImpute] Iter 83: observed MAE=0.539681 rank=52\n",
      "[SoftImpute] Iter 84: observed MAE=0.539541 rank=52\n",
      "[SoftImpute] Iter 85: observed MAE=0.539404 rank=52\n",
      "[SoftImpute] Iter 86: observed MAE=0.539218 rank=51\n",
      "[SoftImpute] Iter 87: observed MAE=0.539065 rank=51\n",
      "[SoftImpute] Iter 88: observed MAE=0.538928 rank=51\n",
      "[SoftImpute] Iter 89: observed MAE=0.538799 rank=51\n",
      "[SoftImpute] Iter 90: observed MAE=0.538676 rank=51\n",
      "[SoftImpute] Iter 91: observed MAE=0.538556 rank=51\n",
      "[SoftImpute] Iter 92: observed MAE=0.538440 rank=51\n",
      "[SoftImpute] Iter 93: observed MAE=0.538327 rank=51\n",
      "[SoftImpute] Iter 94: observed MAE=0.538217 rank=51\n",
      "[SoftImpute] Iter 95: observed MAE=0.538109 rank=51\n",
      "[SoftImpute] Iter 96: observed MAE=0.538004 rank=51\n",
      "[SoftImpute] Iter 97: observed MAE=0.537893 rank=50\n",
      "[SoftImpute] Iter 98: observed MAE=0.537755 rank=50\n",
      "[SoftImpute] Iter 99: observed MAE=0.537640 rank=50\n",
      "[SoftImpute] Iter 100: observed MAE=0.537534 rank=50\n",
      "[SoftImpute] Stopped after iteration 100 for lambda=24.180209\n",
      "Keeping 35916 of 1454334 imputed values, 45885 test values\n",
      "{'f1': 0.60052091011843334, 'tau': 0.42918596108342905, 'auc': 0.84827216612190215}\n",
      "softimpute,1,0.600521,0.429186,0.848272\n",
      "\n",
      "--> Running imputation algorithm mice-burnin10-iters25\n",
      "Dropping 11218 peptides with <2 observations\n",
      "Dropping 11 alleles with <2 observations: ['ELA-A1', 'HLA-A26', 'HLA-A6601', 'HLA-B2701', 'HLA-B3508', 'HLA-B44', 'HLA-E0101', 'Mamu-A07', 'Mamu-B04', 'Patr-B0901', 'Patr-B1701']\n",
      "[MICE] Completing matrix with shape (15638, 93)\n",
      "[MICE] Starting imputation round 1/35, elapsed time 0.040\n",
      "[MICE] Starting imputation round 2/35, elapsed time 3.289\n",
      "[MICE] Starting imputation round 3/35, elapsed time 6.506\n",
      "[MICE] Starting imputation round 4/35, elapsed time 9.691\n",
      "[MICE] Starting imputation round 5/35, elapsed time 12.842\n",
      "[MICE] Starting imputation round 6/35, elapsed time 15.932\n",
      "[MICE] Starting imputation round 7/35, elapsed time 19.061\n",
      "[MICE] Starting imputation round 8/35, elapsed time 22.320\n",
      "[MICE] Starting imputation round 9/35, elapsed time 25.469\n",
      "[MICE] Starting imputation round 10/35, elapsed time 28.627\n",
      "[MICE] Starting imputation round 11/35, elapsed time 31.831\n",
      "[MICE] Starting imputation round 12/35, elapsed time 34.935\n",
      "[MICE] Starting imputation round 13/35, elapsed time 38.048\n",
      "[MICE] Starting imputation round 14/35, elapsed time 41.190\n",
      "[MICE] Starting imputation round 15/35, elapsed time 44.503\n",
      "[MICE] Starting imputation round 16/35, elapsed time 47.842\n",
      "[MICE] Starting imputation round 17/35, elapsed time 51.184\n",
      "[MICE] Starting imputation round 18/35, elapsed time 54.533\n",
      "[MICE] Starting imputation round 19/35, elapsed time 58.023\n",
      "[MICE] Starting imputation round 20/35, elapsed time 61.360\n",
      "[MICE] Starting imputation round 21/35, elapsed time 64.573\n",
      "[MICE] Starting imputation round 22/35, elapsed time 67.991\n",
      "[MICE] Starting imputation round 23/35, elapsed time 71.330\n",
      "[MICE] Starting imputation round 24/35, elapsed time 74.887\n",
      "[MICE] Starting imputation round 25/35, elapsed time 78.513\n",
      "[MICE] Starting imputation round 26/35, elapsed time 81.896\n",
      "[MICE] Starting imputation round 27/35, elapsed time 85.184\n",
      "[MICE] Starting imputation round 28/35, elapsed time 88.380\n",
      "[MICE] Starting imputation round 29/35, elapsed time 91.874\n",
      "[MICE] Starting imputation round 30/35, elapsed time 95.359\n",
      "[MICE] Starting imputation round 31/35, elapsed time 98.761\n",
      "[MICE] Starting imputation round 32/35, elapsed time 102.255\n",
      "[MICE] Starting imputation round 33/35, elapsed time 105.757\n",
      "[MICE] Starting imputation round 34/35, elapsed time 109.111\n",
      "[MICE] Starting imputation round 35/35, elapsed time 112.569\n",
      "Keeping 35916 of 1454334 imputed values, 45885 test values\n",
      "{'f1': 0.56753134116533943, 'tau': 0.43878017897369653, 'auc': 0.84387444593296346}\n",
      "mice-burnin10-iters25,1,0.567531,0.438780,0.843874\n",
      "\n",
      "--> Running imputation algorithm mice-sparse5-burnin5-iters50\n",
      "Dropping 11218 peptides with <2 observations\n",
      "Dropping 11 alleles with <2 observations: ['ELA-A1', 'HLA-A26', 'HLA-A6601', 'HLA-B2701', 'HLA-B3508', 'HLA-B44', 'HLA-E0101', 'Mamu-A07', 'Mamu-B04', 'Patr-B0901', 'Patr-B1701']\n",
      "[MICE] Completing matrix with shape (15638, 93)\n",
      "[MICE] Starting imputation round 1/55, elapsed time 0.039\n",
      "[MICE] Starting imputation round 2/55, elapsed time 0.459\n",
      "[MICE] Starting imputation round 3/55, elapsed time 0.858\n",
      "[MICE] Starting imputation round 4/55, elapsed time 1.277\n",
      "[MICE] Starting imputation round 5/55, elapsed time 1.690\n",
      "[MICE] Starting imputation round 6/55, elapsed time 2.061\n",
      "[MICE] Starting imputation round 7/55, elapsed time 2.512\n",
      "[MICE] Starting imputation round 8/55, elapsed time 2.964\n",
      "[MICE] Starting imputation round 9/55, elapsed time 3.417\n",
      "[MICE] Starting imputation round 10/55, elapsed time 3.867\n",
      "[MICE] Starting imputation round 11/55, elapsed time 4.291\n",
      "[MICE] Starting imputation round 12/55, elapsed time 4.744\n",
      "[MICE] Starting imputation round 13/55, elapsed time 5.170\n",
      "[MICE] Starting imputation round 14/55, elapsed time 5.608\n",
      "[MICE] Starting imputation round 15/55, elapsed time 6.045\n",
      "[MICE] Starting imputation round 16/55, elapsed time 6.466\n",
      "[MICE] Starting imputation round 17/55, elapsed time 6.968\n",
      "[MICE] Starting imputation round 18/55, elapsed time 7.458\n",
      "[MICE] Starting imputation round 19/55, elapsed time 7.916\n",
      "[MICE] Starting imputation round 20/55, elapsed time 8.343\n",
      "[MICE] Starting imputation round 21/55, elapsed time 8.804\n",
      "[MICE] Starting imputation round 22/55, elapsed time 9.282\n",
      "[MICE] Starting imputation round 23/55, elapsed time 9.779\n",
      "[MICE] Starting imputation round 24/55, elapsed time 10.247\n",
      "[MICE] Starting imputation round 25/55, elapsed time 10.699\n",
      "[MICE] Starting imputation round 26/55, elapsed time 11.143\n",
      "[MICE] Starting imputation round 27/55, elapsed time 11.599\n",
      "[MICE] Starting imputation round 28/55, elapsed time 12.050\n",
      "[MICE] Starting imputation round 29/55, elapsed time 12.535\n",
      "[MICE] Starting imputation round 30/55, elapsed time 13.063\n",
      "[MICE] Starting imputation round 31/55, elapsed time 13.579\n",
      "[MICE] Starting imputation round 32/55, elapsed time 14.087\n",
      "[MICE] Starting imputation round 33/55, elapsed time 14.559\n",
      "[MICE] Starting imputation round 34/55, elapsed time 15.085\n",
      "[MICE] Starting imputation round 35/55, elapsed time 15.638\n",
      "[MICE] Starting imputation round 36/55, elapsed time 16.184\n",
      "[MICE] Starting imputation round 37/55, elapsed time 16.733\n",
      "[MICE] Starting imputation round 38/55, elapsed time 17.204\n",
      "[MICE] Starting imputation round 39/55, elapsed time 17.703\n",
      "[MICE] Starting imputation round 40/55, elapsed time 18.155\n",
      "[MICE] Starting imputation round 41/55, elapsed time 18.636\n",
      "[MICE] Starting imputation round 42/55, elapsed time 19.079\n",
      "[MICE] Starting imputation round 43/55, elapsed time 19.532\n",
      "[MICE] Starting imputation round 44/55, elapsed time 20.061\n",
      "[MICE] Starting imputation round 45/55, elapsed time 20.592\n",
      "[MICE] Starting imputation round 46/55, elapsed time 21.074\n",
      "[MICE] Starting imputation round 47/55, elapsed time 21.612\n",
      "[MICE] Starting imputation round 48/55, elapsed time 22.125\n",
      "[MICE] Starting imputation round 49/55, elapsed time 22.574\n",
      "[MICE] Starting imputation round 50/55, elapsed time 23.009\n",
      "[MICE] Starting imputation round 51/55, elapsed time 23.444\n",
      "[MICE] Starting imputation round 52/55, elapsed time 23.899\n",
      "[MICE] Starting imputation round 53/55, elapsed time 24.389\n",
      "[MICE] Starting imputation round 54/55, elapsed time 24.791\n",
      "[MICE] Starting imputation round 55/55, elapsed time 25.179\n",
      "Keeping 35916 of 1454334 imputed values, 45885 test values\n",
      "{'f1': 0.49559412550066756, 'tau': 0.41537947357800742, 'auc': 0.83267441972876155}\n",
      "mice-sparse5-burnin5-iters50,1,0.495594,0.415379,0.832674\n",
      "\n",
      "--> Running imputation algorithm mice-sparse5-burnin5-iters25\n",
      "Dropping 11218 peptides with <2 observations\n",
      "Dropping 11 alleles with <2 observations: ['ELA-A1', 'HLA-A26', 'HLA-A6601', 'HLA-B2701', 'HLA-B3508', 'HLA-B44', 'HLA-E0101', 'Mamu-A07', 'Mamu-B04', 'Patr-B0901', 'Patr-B1701']\n",
      "[MICE] Completing matrix with shape (15638, 93)\n",
      "[MICE] Starting imputation round 1/30, elapsed time 0.047\n",
      "[MICE] Starting imputation round 2/30, elapsed time 0.502\n",
      "[MICE] Starting imputation round 3/30, elapsed time 0.917\n",
      "[MICE] Starting imputation round 4/30, elapsed time 1.322\n",
      "[MICE] Starting imputation round 5/30, elapsed time 1.745\n",
      "[MICE] Starting imputation round 6/30, elapsed time 2.203\n",
      "[MICE] Starting imputation round 7/30, elapsed time 2.587\n",
      "[MICE] Starting imputation round 8/30, elapsed time 3.023\n",
      "[MICE] Starting imputation round 9/30, elapsed time 3.465\n",
      "[MICE] Starting imputation round 10/30, elapsed time 3.917\n",
      "[MICE] Starting imputation round 11/30, elapsed time 4.353\n",
      "[MICE] Starting imputation round 12/30, elapsed time 4.797\n",
      "[MICE] Starting imputation round 13/30, elapsed time 5.248\n",
      "[MICE] Starting imputation round 14/30, elapsed time 5.708\n",
      "[MICE] Starting imputation round 15/30, elapsed time 6.162\n",
      "[MICE] Starting imputation round 16/30, elapsed time 6.608\n",
      "[MICE] Starting imputation round 17/30, elapsed time 7.049\n",
      "[MICE] Starting imputation round 18/30, elapsed time 7.494\n",
      "[MICE] Starting imputation round 19/30, elapsed time 7.943\n",
      "[MICE] Starting imputation round 20/30, elapsed time 8.417\n",
      "[MICE] Starting imputation round 21/30, elapsed time 8.872\n",
      "[MICE] Starting imputation round 22/30, elapsed time 9.317\n",
      "[MICE] Starting imputation round 23/30, elapsed time 9.765\n",
      "[MICE] Starting imputation round 24/30, elapsed time 10.255\n",
      "[MICE] Starting imputation round 25/30, elapsed time 10.715\n",
      "[MICE] Starting imputation round 26/30, elapsed time 11.099\n",
      "[MICE] Starting imputation round 27/30, elapsed time 11.563\n",
      "[MICE] Starting imputation round 28/30, elapsed time 12.012\n",
      "[MICE] Starting imputation round 29/30, elapsed time 12.441\n",
      "[MICE] Starting imputation round 30/30, elapsed time 12.885\n",
      "Keeping 35916 of 1454334 imputed values, 45885 test values\n",
      "{'f1': 0.4977832949104451, 'tau': 0.39219582382581281, 'auc': 0.8244997596242154}\n",
      "mice-sparse5-burnin5-iters25,1,0.497783,0.392196,0.824500\n",
      "\n",
      "--> Running imputation algorithm mice-sparse25-burnin5-iters25\n",
      "Dropping 11218 peptides with <2 observations\n",
      "Dropping 11 alleles with <2 observations: ['ELA-A1', 'HLA-A26', 'HLA-A6601', 'HLA-B2701', 'HLA-B3508', 'HLA-B44', 'HLA-E0101', 'Mamu-A07', 'Mamu-B04', 'Patr-B0901', 'Patr-B1701']\n",
      "[MICE] Completing matrix with shape (15638, 93)\n",
      "[MICE] Starting imputation round 1/30, elapsed time 0.048\n",
      "[MICE] Starting imputation round 2/30, elapsed time 1.274\n",
      "[MICE] Starting imputation round 3/30, elapsed time 2.245\n",
      "[MICE] Starting imputation round 4/30, elapsed time 3.223\n",
      "[MICE] Starting imputation round 5/30, elapsed time 4.224\n",
      "[MICE] Starting imputation round 6/30, elapsed time 5.332\n",
      "[MICE] Starting imputation round 7/30, elapsed time 6.895\n",
      "[MICE] Starting imputation round 8/30, elapsed time 8.420\n",
      "[MICE] Starting imputation round 9/30, elapsed time 9.916\n",
      "[MICE] Starting imputation round 10/30, elapsed time 10.932\n",
      "[MICE] Starting imputation round 11/30, elapsed time 11.837\n",
      "[MICE] Starting imputation round 12/30, elapsed time 12.749\n",
      "[MICE] Starting imputation round 13/30, elapsed time 13.672\n",
      "[MICE] Starting imputation round 14/30, elapsed time 14.616\n",
      "[MICE] Starting imputation round 15/30, elapsed time 15.969\n",
      "[MICE] Starting imputation round 16/30, elapsed time 17.136\n",
      "[MICE] Starting imputation round 17/30, elapsed time 18.022\n",
      "[MICE] Starting imputation round 18/30, elapsed time 18.922\n",
      "[MICE] Starting imputation round 19/30, elapsed time 19.830\n",
      "[MICE] Starting imputation round 20/30, elapsed time 20.723\n",
      "[MICE] Starting imputation round 21/30, elapsed time 21.617\n",
      "[MICE] Starting imputation round 22/30, elapsed time 22.504\n",
      "[MICE] Starting imputation round 23/30, elapsed time 23.392\n",
      "[MICE] Starting imputation round 24/30, elapsed time 24.291\n",
      "[MICE] Starting imputation round 25/30, elapsed time 25.177\n",
      "[MICE] Starting imputation round 26/30, elapsed time 26.087\n",
      "[MICE] Starting imputation round 27/30, elapsed time 27.006\n",
      "[MICE] Starting imputation round 28/30, elapsed time 27.911\n",
      "[MICE] Starting imputation round 29/30, elapsed time 29.483\n",
      "[MICE] Starting imputation round 30/30, elapsed time 30.758\n",
      "Keeping 35916 of 1454334 imputed values, 45885 test values\n",
      "{'f1': 0.57940828402366862, 'tau': 0.44798053818230804, 'auc': 0.8518500496389928}\n",
      "mice-sparse25-burnin5-iters25,1,0.579408,0.447981,0.851850\n",
      "\n",
      "--> Running imputation algorithm svd\n",
      "Dropping 11218 peptides with <2 observations\n",
      "Dropping 11 alleles with <2 observations: ['ELA-A1', 'HLA-A26', 'HLA-A6601', 'HLA-B2701', 'HLA-B3508', 'HLA-B44', 'HLA-E0101', 'Mamu-A07', 'Mamu-B04', 'Patr-B0901', 'Patr-B1701']\n",
      "[IterativeSVD] Iter 1: observed MAE=6.313576\n",
      "[IterativeSVD] Iter 2: observed MAE=4.811756\n",
      "[IterativeSVD] Iter 3: observed MAE=3.615716\n",
      "[IterativeSVD] Iter 4: observed MAE=2.509971\n",
      "[IterativeSVD] Iter 5: observed MAE=1.949668\n",
      "[IterativeSVD] Iter 6: observed MAE=1.695925\n",
      "[IterativeSVD] Iter 7: observed MAE=1.546240\n",
      "[IterativeSVD] Iter 8: observed MAE=1.446675\n",
      "[IterativeSVD] Iter 9: observed MAE=1.373712\n",
      "[IterativeSVD] Iter 10: observed MAE=1.315973\n",
      "[IterativeSVD] Iter 11: observed MAE=1.267692\n",
      "[IterativeSVD] Iter 12: observed MAE=1.225604\n",
      "[IterativeSVD] Iter 13: observed MAE=1.187816\n",
      "[IterativeSVD] Iter 14: observed MAE=1.153097\n",
      "[IterativeSVD] Iter 15: observed MAE=1.120765\n",
      "[IterativeSVD] Iter 16: observed MAE=1.090495\n",
      "[IterativeSVD] Iter 17: observed MAE=1.062255\n",
      "[IterativeSVD] Iter 18: observed MAE=1.036151\n",
      "[IterativeSVD] Iter 19: observed MAE=1.012147\n",
      "[IterativeSVD] Iter 20: observed MAE=0.990105\n",
      "[IterativeSVD] Iter 21: observed MAE=0.969852\n",
      "[IterativeSVD] Iter 22: observed MAE=0.951179\n",
      "[IterativeSVD] Iter 23: observed MAE=0.933883\n",
      "[IterativeSVD] Iter 24: observed MAE=0.917790\n",
      "[IterativeSVD] Iter 25: observed MAE=0.902790\n",
      "[IterativeSVD] Iter 26: observed MAE=0.888728\n",
      "[IterativeSVD] Iter 27: observed MAE=0.875542\n",
      "[IterativeSVD] Iter 28: observed MAE=0.863104\n",
      "[IterativeSVD] Iter 29: observed MAE=0.851340\n",
      "[IterativeSVD] Iter 30: observed MAE=0.840175\n",
      "[IterativeSVD] Iter 31: observed MAE=0.829517\n",
      "[IterativeSVD] Iter 32: observed MAE=0.819338\n",
      "[IterativeSVD] Iter 33: observed MAE=0.809607\n",
      "[IterativeSVD] Iter 34: observed MAE=0.800263\n",
      "[IterativeSVD] Iter 35: observed MAE=0.791275\n",
      "[IterativeSVD] Iter 36: observed MAE=0.782587\n",
      "[IterativeSVD] Iter 37: observed MAE=0.774159\n",
      "[IterativeSVD] Iter 38: observed MAE=0.765970\n",
      "[IterativeSVD] Iter 39: observed MAE=0.758015\n",
      "[IterativeSVD] Iter 40: observed MAE=0.750274\n",
      "[IterativeSVD] Iter 41: observed MAE=0.742729\n",
      "[IterativeSVD] Iter 42: observed MAE=0.735358\n",
      "[IterativeSVD] Iter 43: observed MAE=0.728132\n",
      "[IterativeSVD] Iter 44: observed MAE=0.721032\n",
      "[IterativeSVD] Iter 45: observed MAE=0.714046\n",
      "[IterativeSVD] Iter 46: observed MAE=0.707163\n",
      "[IterativeSVD] Iter 47: observed MAE=0.700367\n",
      "[IterativeSVD] Iter 48: observed MAE=0.693650\n",
      "[IterativeSVD] Iter 49: observed MAE=0.686993\n",
      "[IterativeSVD] Iter 50: observed MAE=0.680396\n",
      "[IterativeSVD] Iter 51: observed MAE=0.673860\n",
      "[IterativeSVD] Iter 52: observed MAE=0.667383\n",
      "[IterativeSVD] Iter 53: observed MAE=0.660978\n",
      "[IterativeSVD] Iter 54: observed MAE=0.654650\n",
      "[IterativeSVD] Iter 55: observed MAE=0.648405\n",
      "[IterativeSVD] Iter 56: observed MAE=0.642249\n",
      "[IterativeSVD] Iter 57: observed MAE=0.636187\n",
      "[IterativeSVD] Iter 58: observed MAE=0.630237\n",
      "[IterativeSVD] Iter 59: observed MAE=0.624408\n",
      "[IterativeSVD] Iter 60: observed MAE=0.618698\n",
      "[IterativeSVD] Iter 61: observed MAE=0.613113\n",
      "[IterativeSVD] Iter 62: observed MAE=0.607659\n",
      "[IterativeSVD] Iter 63: observed MAE=0.602345\n",
      "[IterativeSVD] Iter 64: observed MAE=0.597170\n",
      "[IterativeSVD] Iter 65: observed MAE=0.592131\n",
      "[IterativeSVD] Iter 66: observed MAE=0.587231\n",
      "[IterativeSVD] Iter 67: observed MAE=0.582473\n",
      "[IterativeSVD] Iter 68: observed MAE=0.577847\n",
      "[IterativeSVD] Iter 69: observed MAE=0.573353\n",
      "[IterativeSVD] Iter 70: observed MAE=0.568981\n",
      "[IterativeSVD] Iter 71: observed MAE=0.564731\n",
      "[IterativeSVD] Iter 72: observed MAE=0.560597\n",
      "[IterativeSVD] Iter 73: observed MAE=0.556576\n",
      "[IterativeSVD] Iter 74: observed MAE=0.552667\n",
      "[IterativeSVD] Iter 75: observed MAE=0.548865\n",
      "[IterativeSVD] Iter 76: observed MAE=0.545163\n",
      "[IterativeSVD] Iter 77: observed MAE=0.541557\n",
      "[IterativeSVD] Iter 78: observed MAE=0.538045\n",
      "[IterativeSVD] Iter 79: observed MAE=0.534617\n",
      "[IterativeSVD] Iter 80: observed MAE=0.531268\n",
      "[IterativeSVD] Iter 81: observed MAE=0.527993\n",
      "[IterativeSVD] Iter 82: observed MAE=0.524789\n",
      "[IterativeSVD] Iter 83: observed MAE=0.521651\n",
      "[IterativeSVD] Iter 84: observed MAE=0.518581\n",
      "[IterativeSVD] Iter 85: observed MAE=0.515574\n",
      "[IterativeSVD] Iter 86: observed MAE=0.512627\n",
      "[IterativeSVD] Iter 87: observed MAE=0.509737\n",
      "[IterativeSVD] Iter 88: observed MAE=0.506908\n",
      "[IterativeSVD] Iter 89: observed MAE=0.504133\n",
      "[IterativeSVD] Iter 90: observed MAE=0.501412\n",
      "[IterativeSVD] Iter 91: observed MAE=0.498739\n",
      "[IterativeSVD] Iter 92: observed MAE=0.496116\n",
      "[IterativeSVD] Iter 93: observed MAE=0.493541\n",
      "[IterativeSVD] Iter 94: observed MAE=0.491012\n",
      "[IterativeSVD] Iter 95: observed MAE=0.488528\n",
      "[IterativeSVD] Iter 96: observed MAE=0.486088\n",
      "[IterativeSVD] Iter 97: observed MAE=0.483690\n",
      "[IterativeSVD] Iter 98: observed MAE=0.481332\n",
      "[IterativeSVD] Iter 99: observed MAE=0.479014\n",
      "[IterativeSVD] Iter 100: observed MAE=0.476733\n",
      "[IterativeSVD] Iter 101: observed MAE=0.474488\n",
      "[IterativeSVD] Iter 102: observed MAE=0.472278\n",
      "[IterativeSVD] Iter 103: observed MAE=0.470101\n",
      "Keeping 35916 of 1454334 imputed values, 45885 test values\n",
      "{'f1': 0.60450387051372267, 'tau': 0.40533112974874363, 'auc': 0.83428691986097669}\n",
      "svd,1,0.604504,0.405331,0.834287\n",
      "\n",
      "--> Running imputation algorithm knn\n",
      "Dropping 11218 peptides with <2 observations\n",
      "Dropping 11 alleles with <2 observations: ['ELA-A1', 'HLA-A26', 'HLA-A6601', 'HLA-B2701', 'HLA-B3508', 'HLA-B44', 'HLA-E0101', 'Mamu-A07', 'Mamu-B04', 'Patr-B0901', 'Patr-B1701']\n",
      "Computing pairwise distances between 93 samples\n",
      "Computing distances for sample #1/93, elapsed time: 0.132\n",
      "Imputing row 1/93 with 15630 missing columns, elapsed time: 0.966\n",
      "Imputing row 11/93 with 11917 missing columns, elapsed time: 2.447\n",
      "Imputing row 21/93 with 11772 missing columns, elapsed time: 3.966\n",
      "Imputing row 31/93 with 15514 missing columns, elapsed time: 5.510\n",
      "Imputing row 41/93 with 14208 missing columns, elapsed time: 7.034\n",
      "Imputing row 51/93 with 15405 missing columns, elapsed time: 8.787\n",
      "Imputing row 61/93 with 13889 missing columns, elapsed time: 10.362\n",
      "Imputing row 71/93 with 14975 missing columns, elapsed time: 11.820\n",
      "Imputing row 81/93 with 15613 missing columns, elapsed time: 13.280\n",
      "Imputing row 91/93 with 15496 missing columns, elapsed time: 14.774\n",
      "[KNN] Warning: 139651/1454334 still missing after imputation, replacing with 0\n",
      "Keeping 35916 of 1454334 imputed values, 45885 test values\n",
      "{'f1': 0.56865111289890058, 'tau': 0.41751083790369364, 'auc': 0.80831089920788002}\n",
      "knn,1,0.568651,0.417511,0.808311\n",
      "\n",
      "--> Running imputation algorithm mice-sparse25-burnin5-iters50\n",
      "Dropping 11218 peptides with <2 observations\n",
      "Dropping 11 alleles with <2 observations: ['ELA-A1', 'HLA-A26', 'HLA-A6601', 'HLA-B2701', 'HLA-B3508', 'HLA-B44', 'HLA-E0101', 'Mamu-A07', 'Mamu-B04', 'Patr-B0901', 'Patr-B1701']\n",
      "[MICE] Completing matrix with shape (15638, 93)\n",
      "[MICE] Starting imputation round 1/55, elapsed time 0.035\n",
      "[MICE] Starting imputation round 2/55, elapsed time 0.744\n",
      "[MICE] Starting imputation round 3/55, elapsed time 1.514\n",
      "[MICE] Starting imputation round 4/55, elapsed time 2.464\n",
      "[MICE] Starting imputation round 5/55, elapsed time 3.414\n",
      "[MICE] Starting imputation round 6/55, elapsed time 4.332\n",
      "[MICE] Starting imputation round 7/55, elapsed time 5.360\n",
      "[MICE] Starting imputation round 8/55, elapsed time 6.330\n",
      "[MICE] Starting imputation round 9/55, elapsed time 7.314\n",
      "[MICE] Starting imputation round 10/55, elapsed time 8.292\n",
      "[MICE] Starting imputation round 11/55, elapsed time 9.249\n",
      "[MICE] Starting imputation round 12/55, elapsed time 10.244\n",
      "[MICE] Starting imputation round 13/55, elapsed time 11.196\n",
      "[MICE] Starting imputation round 14/55, elapsed time 12.217\n",
      "[MICE] Starting imputation round 15/55, elapsed time 13.232\n",
      "[MICE] Starting imputation round 16/55, elapsed time 14.202\n",
      "[MICE] Starting imputation round 17/55, elapsed time 15.141\n",
      "[MICE] Starting imputation round 18/55, elapsed time 16.088\n",
      "[MICE] Starting imputation round 19/55, elapsed time 17.066\n",
      "[MICE] Starting imputation round 20/55, elapsed time 18.036\n",
      "[MICE] Starting imputation round 21/55, elapsed time 18.990\n",
      "[MICE] Starting imputation round 22/55, elapsed time 19.944\n",
      "[MICE] Starting imputation round 23/55, elapsed time 20.907\n",
      "[MICE] Starting imputation round 24/55, elapsed time 21.895\n",
      "[MICE] Starting imputation round 25/55, elapsed time 22.853\n",
      "[MICE] Starting imputation round 26/55, elapsed time 23.840\n",
      "[MICE] Starting imputation round 27/55, elapsed time 24.849\n",
      "[MICE] Starting imputation round 28/55, elapsed time 25.871\n",
      "[MICE] Starting imputation round 29/55, elapsed time 26.847\n",
      "[MICE] Starting imputation round 30/55, elapsed time 27.838\n",
      "[MICE] Starting imputation round 31/55, elapsed time 28.787\n",
      "[MICE] Starting imputation round 32/55, elapsed time 29.792\n",
      "[MICE] Starting imputation round 33/55, elapsed time 30.876\n",
      "[MICE] Starting imputation round 34/55, elapsed time 31.961\n",
      "[MICE] Starting imputation round 35/55, elapsed time 33.027\n",
      "[MICE] Starting imputation round 36/55, elapsed time 33.983\n",
      "[MICE] Starting imputation round 37/55, elapsed time 35.001\n",
      "[MICE] Starting imputation round 38/55, elapsed time 36.003\n",
      "[MICE] Starting imputation round 39/55, elapsed time 36.948\n",
      "[MICE] Starting imputation round 40/55, elapsed time 37.891\n",
      "[MICE] Starting imputation round 41/55, elapsed time 38.904\n",
      "[MICE] Starting imputation round 42/55, elapsed time 39.897\n",
      "[MICE] Starting imputation round 43/55, elapsed time 40.922\n",
      "[MICE] Starting imputation round 44/55, elapsed time 41.938\n",
      "[MICE] Starting imputation round 45/55, elapsed time 42.894\n",
      "[MICE] Starting imputation round 46/55, elapsed time 43.861\n",
      "[MICE] Starting imputation round 47/55, elapsed time 44.811\n",
      "[MICE] Starting imputation round 48/55, elapsed time 45.858\n",
      "[MICE] Starting imputation round 49/55, elapsed time 46.860\n",
      "[MICE] Starting imputation round 50/55, elapsed time 47.819\n",
      "[MICE] Starting imputation round 51/55, elapsed time 48.768\n",
      "[MICE] Starting imputation round 52/55, elapsed time 49.729\n",
      "[MICE] Starting imputation round 53/55, elapsed time 50.697\n",
      "[MICE] Starting imputation round 54/55, elapsed time 51.673\n",
      "[MICE] Starting imputation round 55/55, elapsed time 52.634\n",
      "Keeping 35916 of 1454334 imputed values, 45885 test values\n",
      "{'f1': 0.57172724348665172, 'tau': 0.45974621933804261, 'auc': 0.85810703529367083}\n",
      "mice-sparse25-burnin5-iters50,1,0.571727,0.459746,0.858107\n",
      "\n",
      "=== CV fold 3/3\n",
      "--> Running imputation algorithm mice-burnin5-iters25\n",
      "Dropping 11120 peptides with <2 observations\n",
      "Dropping 9 alleles with <2 observations: ['ELA-A1', 'HLA-A26', 'HLA-B2701', 'HLA-E0101', 'Mamu-A07', 'Mamu-B04', 'Patr-A0602', 'Patr-B0901', 'Patr-B1701']\n",
      "[MICE] Completing matrix with shape (15623, 95)\n",
      "[MICE] Starting imputation round 1/30, elapsed time 0.036\n",
      "[MICE] Starting imputation round 2/30, elapsed time 3.804\n",
      "[MICE] Starting imputation round 3/30, elapsed time 7.485\n",
      "[MICE] Starting imputation round 4/30, elapsed time 11.286\n",
      "[MICE] Starting imputation round 5/30, elapsed time 15.006\n",
      "[MICE] Starting imputation round 6/30, elapsed time 18.677\n",
      "[MICE] Starting imputation round 7/30, elapsed time 22.368\n",
      "[MICE] Starting imputation round 8/30, elapsed time 25.884\n",
      "[MICE] Starting imputation round 9/30, elapsed time 29.608\n",
      "[MICE] Starting imputation round 10/30, elapsed time 33.135\n",
      "[MICE] Starting imputation round 11/30, elapsed time 36.573\n",
      "[MICE] Starting imputation round 12/30, elapsed time 40.126\n",
      "[MICE] Starting imputation round 13/30, elapsed time 43.533\n",
      "[MICE] Starting imputation round 14/30, elapsed time 47.118\n",
      "[MICE] Starting imputation round 15/30, elapsed time 50.627\n",
      "[MICE] Starting imputation round 16/30, elapsed time 54.240\n",
      "[MICE] Starting imputation round 17/30, elapsed time 57.729\n",
      "[MICE] Starting imputation round 18/30, elapsed time 61.390\n",
      "[MICE] Starting imputation round 19/30, elapsed time 65.046\n",
      "[MICE] Starting imputation round 20/30, elapsed time 68.634\n",
      "[MICE] Starting imputation round 21/30, elapsed time 72.116\n",
      "[MICE] Starting imputation round 22/30, elapsed time 75.575\n",
      "[MICE] Starting imputation round 23/30, elapsed time 79.017\n",
      "[MICE] Starting imputation round 24/30, elapsed time 82.543\n",
      "[MICE] Starting imputation round 25/30, elapsed time 86.022\n",
      "[MICE] Starting imputation round 26/30, elapsed time 89.595\n",
      "[MICE] Starting imputation round 27/30, elapsed time 93.115\n",
      "[MICE] Starting imputation round 28/30, elapsed time 96.634\n",
      "[MICE] Starting imputation round 29/30, elapsed time 100.418\n",
      "[MICE] Starting imputation round 30/30, elapsed time 103.904\n",
      "Keeping 35864 of 1484185 imputed values, 45884 test values\n",
      "{'f1': 0.56035837786859477, 'tau': 0.43728242463957318, 'auc': 0.8449851675890433}\n",
      "mice-burnin5-iters25,2,0.560358,0.437282,0.844985\n",
      "\n",
      "--> Running imputation algorithm mice-burnin10-iters50\n",
      "Dropping 11120 peptides with <2 observations\n",
      "Dropping 9 alleles with <2 observations: ['ELA-A1', 'HLA-A26', 'HLA-B2701', 'HLA-E0101', 'Mamu-A07', 'Mamu-B04', 'Patr-A0602', 'Patr-B0901', 'Patr-B1701']\n",
      "[MICE] Completing matrix with shape (15623, 95)\n",
      "[MICE] Starting imputation round 1/60, elapsed time 0.038\n",
      "[MICE] Starting imputation round 2/60, elapsed time 3.633\n",
      "[MICE] Starting imputation round 3/60, elapsed time 7.126\n",
      "[MICE] Starting imputation round 4/60, elapsed time 10.768\n",
      "[MICE] Starting imputation round 5/60, elapsed time 14.631\n",
      "[MICE] Starting imputation round 6/60, elapsed time 18.263\n",
      "[MICE] Starting imputation round 7/60, elapsed time 22.233\n",
      "[MICE] Starting imputation round 8/60, elapsed time 25.712\n",
      "[MICE] Starting imputation round 9/60, elapsed time 29.410\n",
      "[MICE] Starting imputation round 10/60, elapsed time 33.070\n",
      "[MICE] Starting imputation round 11/60, elapsed time 36.673\n",
      "[MICE] Starting imputation round 12/60, elapsed time 40.198\n",
      "[MICE] Starting imputation round 13/60, elapsed time 43.751\n",
      "[MICE] Starting imputation round 14/60, elapsed time 47.457\n",
      "[MICE] Starting imputation round 15/60, elapsed time 51.178\n",
      "[MICE] Starting imputation round 16/60, elapsed time 54.542\n",
      "[MICE] Starting imputation round 17/60, elapsed time 57.961\n",
      "[MICE] Starting imputation round 18/60, elapsed time 61.455\n",
      "[MICE] Starting imputation round 19/60, elapsed time 65.012\n",
      "[MICE] Starting imputation round 20/60, elapsed time 68.601\n",
      "[MICE] Starting imputation round 21/60, elapsed time 72.158\n",
      "[MICE] Starting imputation round 22/60, elapsed time 75.936\n",
      "[MICE] Starting imputation round 23/60, elapsed time 79.795\n",
      "[MICE] Starting imputation round 24/60, elapsed time 83.438\n",
      "[MICE] Starting imputation round 25/60, elapsed time 87.114\n",
      "[MICE] Starting imputation round 26/60, elapsed time 90.710\n",
      "[MICE] Starting imputation round 27/60, elapsed time 94.296\n",
      "[MICE] Starting imputation round 28/60, elapsed time 97.884\n",
      "[MICE] Starting imputation round 29/60, elapsed time 101.491\n",
      "[MICE] Starting imputation round 30/60, elapsed time 105.026\n",
      "[MICE] Starting imputation round 31/60, elapsed time 108.544\n",
      "[MICE] Starting imputation round 32/60, elapsed time 112.255\n",
      "[MICE] Starting imputation round 33/60, elapsed time 115.843\n",
      "[MICE] Starting imputation round 34/60, elapsed time 119.446\n",
      "[MICE] Starting imputation round 35/60, elapsed time 122.791\n",
      "[MICE] Starting imputation round 36/60, elapsed time 126.150\n",
      "[MICE] Starting imputation round 37/60, elapsed time 129.466\n",
      "[MICE] Starting imputation round 38/60, elapsed time 132.850\n",
      "[MICE] Starting imputation round 39/60, elapsed time 136.352\n",
      "[MICE] Starting imputation round 40/60, elapsed time 139.707\n",
      "[MICE] Starting imputation round 41/60, elapsed time 143.017\n",
      "[MICE] Starting imputation round 42/60, elapsed time 146.401\n",
      "[MICE] Starting imputation round 43/60, elapsed time 149.720\n",
      "[MICE] Starting imputation round 44/60, elapsed time 153.085\n",
      "[MICE] Starting imputation round 45/60, elapsed time 156.623\n",
      "[MICE] Starting imputation round 46/60, elapsed time 160.113\n",
      "[MICE] Starting imputation round 47/60, elapsed time 163.734\n",
      "[MICE] Starting imputation round 48/60, elapsed time 167.575\n",
      "[MICE] Starting imputation round 49/60, elapsed time 171.246\n",
      "[MICE] Starting imputation round 50/60, elapsed time 174.876\n",
      "[MICE] Starting imputation round 51/60, elapsed time 178.219\n",
      "[MICE] Starting imputation round 52/60, elapsed time 181.538\n",
      "[MICE] Starting imputation round 53/60, elapsed time 184.928\n",
      "[MICE] Starting imputation round 54/60, elapsed time 188.274\n",
      "[MICE] Starting imputation round 55/60, elapsed time 191.567\n",
      "[MICE] Starting imputation round 56/60, elapsed time 194.930\n",
      "[MICE] Starting imputation round 57/60, elapsed time 198.255\n",
      "[MICE] Starting imputation round 58/60, elapsed time 201.593\n",
      "[MICE] Starting imputation round 59/60, elapsed time 205.018\n",
      "[MICE] Starting imputation round 60/60, elapsed time 208.314\n",
      "Keeping 35864 of 1484185 imputed values, 45884 test values\n",
      "{'f1': 0.56092353917925197, 'tau': 0.45273359606118646, 'auc': 0.85028941803600322}\n",
      "mice-burnin10-iters50,2,0.560924,0.452734,0.850289\n",
      "\n",
      "--> Running imputation algorithm softimpute\n",
      "Dropping 11120 peptides with <2 observations\n",
      "Dropping 9 alleles with <2 observations: ['ELA-A1', 'HLA-A26', 'HLA-B2701', 'HLA-E0101', 'Mamu-A07', 'Mamu-B04', 'Patr-A0602', 'Patr-B0901', 'Patr-B1701']\n",
      "[SoftImpute] Max Singular Value of X_init = 1215.347134\n",
      "[SoftImpute] Iter 1: observed MAE=0.573800 rank=84\n",
      "[SoftImpute] Iter 2: observed MAE=0.574090 rank=84\n",
      "[SoftImpute] Iter 3: observed MAE=0.574214 rank=83\n",
      "[SoftImpute] Iter 4: observed MAE=0.574231 rank=83\n",
      "[SoftImpute] Iter 5: observed MAE=0.574272 rank=83\n",
      "[SoftImpute] Iter 6: observed MAE=0.574307 rank=83\n",
      "[SoftImpute] Iter 7: observed MAE=0.574329 rank=83\n",
      "[SoftImpute] Iter 8: observed MAE=0.574331 rank=83\n",
      "[SoftImpute] Iter 9: observed MAE=0.574300 rank=83\n",
      "[SoftImpute] Iter 10: observed MAE=0.574193 rank=82\n",
      "[SoftImpute] Iter 11: observed MAE=0.574049 rank=82\n",
      "[SoftImpute] Iter 12: observed MAE=0.573726 rank=80\n",
      "[SoftImpute] Iter 13: observed MAE=0.573294 rank=80\n",
      "[SoftImpute] Iter 14: observed MAE=0.572724 rank=79\n",
      "[SoftImpute] Iter 15: observed MAE=0.572034 rank=79\n",
      "[SoftImpute] Iter 16: observed MAE=0.571381 rank=77\n",
      "[SoftImpute] Iter 17: observed MAE=0.570660 rank=76\n",
      "[SoftImpute] Iter 18: observed MAE=0.570028 rank=76\n",
      "[SoftImpute] Iter 19: observed MAE=0.569525 rank=76\n",
      "[SoftImpute] Iter 20: observed MAE=0.569034 rank=75\n",
      "[SoftImpute] Iter 21: observed MAE=0.568344 rank=74\n",
      "[SoftImpute] Iter 22: observed MAE=0.567395 rank=73\n",
      "[SoftImpute] Iter 23: observed MAE=0.566668 rank=73\n",
      "[SoftImpute] Iter 24: observed MAE=0.565793 rank=71\n",
      "[SoftImpute] Iter 25: observed MAE=0.564867 rank=70\n",
      "[SoftImpute] Iter 26: observed MAE=0.563948 rank=70\n",
      "[SoftImpute] Iter 27: observed MAE=0.563245 rank=70\n",
      "[SoftImpute] Iter 28: observed MAE=0.562580 rank=69\n",
      "[SoftImpute] Iter 29: observed MAE=0.561837 rank=68\n",
      "[SoftImpute] Iter 30: observed MAE=0.561184 rank=68\n",
      "[SoftImpute] Iter 31: observed MAE=0.560533 rank=67\n",
      "[SoftImpute] Iter 32: observed MAE=0.559943 rank=67\n",
      "[SoftImpute] Iter 33: observed MAE=0.559385 rank=66\n",
      "[SoftImpute] Iter 34: observed MAE=0.558769 rank=65\n",
      "[SoftImpute] Iter 35: observed MAE=0.558158 rank=65\n",
      "[SoftImpute] Iter 36: observed MAE=0.557655 rank=65\n",
      "[SoftImpute] Iter 37: observed MAE=0.557201 rank=65\n",
      "[SoftImpute] Iter 38: observed MAE=0.556615 rank=63\n",
      "[SoftImpute] Iter 39: observed MAE=0.555955 rank=63\n",
      "[SoftImpute] Iter 40: observed MAE=0.555386 rank=62\n",
      "[SoftImpute] Iter 41: observed MAE=0.554816 rank=62\n",
      "[SoftImpute] Iter 42: observed MAE=0.554307 rank=62\n",
      "[SoftImpute] Iter 43: observed MAE=0.553721 rank=61\n",
      "[SoftImpute] Iter 44: observed MAE=0.553214 rank=61\n",
      "[SoftImpute] Iter 45: observed MAE=0.552792 rank=61\n",
      "[SoftImpute] Iter 46: observed MAE=0.552348 rank=60\n",
      "[SoftImpute] Iter 47: observed MAE=0.551834 rank=59\n",
      "[SoftImpute] Iter 48: observed MAE=0.551327 rank=59\n",
      "[SoftImpute] Iter 49: observed MAE=0.550828 rank=58\n",
      "[SoftImpute] Iter 50: observed MAE=0.550370 rank=58\n",
      "[SoftImpute] Iter 51: observed MAE=0.549978 rank=58\n",
      "[SoftImpute] Iter 52: observed MAE=0.549618 rank=58\n",
      "[SoftImpute] Iter 53: observed MAE=0.549280 rank=58\n",
      "[SoftImpute] Iter 54: observed MAE=0.548882 rank=57\n",
      "[SoftImpute] Iter 55: observed MAE=0.548543 rank=57\n",
      "[SoftImpute] Iter 56: observed MAE=0.548200 rank=56\n",
      "[SoftImpute] Iter 57: observed MAE=0.547843 rank=56\n",
      "[SoftImpute] Iter 58: observed MAE=0.547538 rank=56\n",
      "[SoftImpute] Iter 59: observed MAE=0.547209 rank=55\n",
      "[SoftImpute] Iter 60: observed MAE=0.546879 rank=55\n",
      "[SoftImpute] Iter 61: observed MAE=0.546602 rank=55\n",
      "[SoftImpute] Iter 62: observed MAE=0.546326 rank=54\n",
      "[SoftImpute] Iter 63: observed MAE=0.546022 rank=54\n",
      "[SoftImpute] Iter 64: observed MAE=0.545764 rank=54\n",
      "[SoftImpute] Iter 65: observed MAE=0.545527 rank=54\n",
      "[SoftImpute] Iter 66: observed MAE=0.545262 rank=53\n",
      "[SoftImpute] Iter 67: observed MAE=0.544985 rank=53\n",
      "[SoftImpute] Iter 68: observed MAE=0.544752 rank=53\n",
      "[SoftImpute] Iter 69: observed MAE=0.544538 rank=53\n",
      "[SoftImpute] Iter 70: observed MAE=0.544280 rank=52\n",
      "[SoftImpute] Iter 71: observed MAE=0.544061 rank=52\n",
      "[SoftImpute] Iter 72: observed MAE=0.543865 rank=52\n",
      "[SoftImpute] Iter 73: observed MAE=0.543681 rank=52\n",
      "[SoftImpute] Iter 74: observed MAE=0.543506 rank=52\n",
      "[SoftImpute] Iter 75: observed MAE=0.543338 rank=52\n",
      "[SoftImpute] Iter 76: observed MAE=0.543161 rank=51\n",
      "[SoftImpute] Iter 77: observed MAE=0.542953 rank=51\n",
      "[SoftImpute] Iter 78: observed MAE=0.542777 rank=51\n",
      "[SoftImpute] Iter 79: observed MAE=0.542600 rank=50\n",
      "[SoftImpute] Iter 80: observed MAE=0.542395 rank=50\n",
      "[SoftImpute] Iter 81: observed MAE=0.542225 rank=50\n",
      "[SoftImpute] Iter 82: observed MAE=0.542070 rank=50\n",
      "[SoftImpute] Iter 83: observed MAE=0.541923 rank=50\n",
      "[SoftImpute] Iter 84: observed MAE=0.541783 rank=50\n",
      "[SoftImpute] Iter 85: observed MAE=0.541647 rank=50\n",
      "[SoftImpute] Iter 86: observed MAE=0.541515 rank=50\n",
      "[SoftImpute] Iter 87: observed MAE=0.541386 rank=50\n",
      "[SoftImpute] Iter 88: observed MAE=0.541261 rank=50\n",
      "[SoftImpute] Iter 89: observed MAE=0.541139 rank=50\n",
      "[SoftImpute] Iter 90: observed MAE=0.541019 rank=50\n",
      "[SoftImpute] Iter 91: observed MAE=0.540902 rank=50\n",
      "[SoftImpute] Iter 92: observed MAE=0.540788 rank=50\n",
      "[SoftImpute] Iter 93: observed MAE=0.540676 rank=50\n",
      "[SoftImpute] Iter 94: observed MAE=0.540566 rank=50\n",
      "[SoftImpute] Iter 95: observed MAE=0.540454 rank=49\n",
      "[SoftImpute] Iter 96: observed MAE=0.540305 rank=49\n",
      "[SoftImpute] Iter 97: observed MAE=0.540183 rank=49\n",
      "[SoftImpute] Iter 98: observed MAE=0.540073 rank=49\n",
      "[SoftImpute] Iter 99: observed MAE=0.539969 rank=49\n",
      "[SoftImpute] Iter 100: observed MAE=0.539869 rank=49\n",
      "[SoftImpute] Stopped after iteration 100 for lambda=24.306943\n",
      "Keeping 35864 of 1484185 imputed values, 45884 test values\n",
      "{'f1': 0.60461485926142733, 'tau': 0.43219759301046101, 'auc': 0.8444586574239934}\n",
      "softimpute,2,0.604615,0.432198,0.844459\n",
      "\n",
      "--> Running imputation algorithm mice-burnin10-iters25\n",
      "Dropping 11120 peptides with <2 observations\n",
      "Dropping 9 alleles with <2 observations: ['ELA-A1', 'HLA-A26', 'HLA-B2701', 'HLA-E0101', 'Mamu-A07', 'Mamu-B04', 'Patr-A0602', 'Patr-B0901', 'Patr-B1701']\n",
      "[MICE] Completing matrix with shape (15623, 95)\n",
      "[MICE] Starting imputation round 1/35, elapsed time 0.034\n",
      "[MICE] Starting imputation round 2/35, elapsed time 3.460\n",
      "[MICE] Starting imputation round 3/35, elapsed time 6.794\n",
      "[MICE] Starting imputation round 4/35, elapsed time 10.103\n",
      "[MICE] Starting imputation round 5/35, elapsed time 13.447\n",
      "[MICE] Starting imputation round 6/35, elapsed time 16.772\n",
      "[MICE] Starting imputation round 7/35, elapsed time 20.080\n",
      "[MICE] Starting imputation round 8/35, elapsed time 23.410\n",
      "[MICE] Starting imputation round 9/35, elapsed time 26.773\n",
      "[MICE] Starting imputation round 10/35, elapsed time 30.089\n",
      "[MICE] Starting imputation round 11/35, elapsed time 33.472\n",
      "[MICE] Starting imputation round 12/35, elapsed time 36.823\n",
      "[MICE] Starting imputation round 13/35, elapsed time 40.155\n",
      "[MICE] Starting imputation round 14/35, elapsed time 43.590\n",
      "[MICE] Starting imputation round 15/35, elapsed time 46.909\n",
      "[MICE] Starting imputation round 16/35, elapsed time 50.319\n",
      "[MICE] Starting imputation round 17/35, elapsed time 53.710\n",
      "[MICE] Starting imputation round 18/35, elapsed time 57.150\n",
      "[MICE] Starting imputation round 19/35, elapsed time 60.863\n",
      "[MICE] Starting imputation round 20/35, elapsed time 64.682\n",
      "[MICE] Starting imputation round 21/35, elapsed time 68.290\n",
      "[MICE] Starting imputation round 22/35, elapsed time 71.977\n",
      "[MICE] Starting imputation round 23/35, elapsed time 75.520\n",
      "[MICE] Starting imputation round 24/35, elapsed time 78.992\n",
      "[MICE] Starting imputation round 25/35, elapsed time 82.743\n",
      "[MICE] Starting imputation round 26/35, elapsed time 86.300\n",
      "[MICE] Starting imputation round 27/35, elapsed time 89.772\n",
      "[MICE] Starting imputation round 28/35, elapsed time 93.161\n",
      "[MICE] Starting imputation round 29/35, elapsed time 96.652\n",
      "[MICE] Starting imputation round 30/35, elapsed time 100.166\n",
      "[MICE] Starting imputation round 31/35, elapsed time 103.745\n",
      "[MICE] Starting imputation round 32/35, elapsed time 107.461\n",
      "[MICE] Starting imputation round 33/35, elapsed time 111.089\n",
      "[MICE] Starting imputation round 34/35, elapsed time 114.677\n",
      "[MICE] Starting imputation round 35/35, elapsed time 118.037\n",
      "Keeping 35864 of 1484185 imputed values, 45884 test values\n",
      "{'f1': 0.56161710762506833, 'tau': 0.43553145978000313, 'auc': 0.84383500325995797}\n",
      "mice-burnin10-iters25,2,0.561617,0.435531,0.843835\n",
      "\n",
      "--> Running imputation algorithm mice-sparse5-burnin5-iters50\n",
      "Dropping 11120 peptides with <2 observations\n",
      "Dropping 9 alleles with <2 observations: ['ELA-A1', 'HLA-A26', 'HLA-B2701', 'HLA-E0101', 'Mamu-A07', 'Mamu-B04', 'Patr-A0602', 'Patr-B0901', 'Patr-B1701']\n",
      "[MICE] Completing matrix with shape (15623, 95)\n",
      "[MICE] Starting imputation round 1/55, elapsed time 0.034\n",
      "[MICE] Starting imputation round 2/55, elapsed time 0.386\n",
      "[MICE] Starting imputation round 3/55, elapsed time 0.776\n",
      "[MICE] Starting imputation round 4/55, elapsed time 1.185\n",
      "[MICE] Starting imputation round 5/55, elapsed time 1.589\n",
      "[MICE] Starting imputation round 6/55, elapsed time 1.986\n",
      "[MICE] Starting imputation round 7/55, elapsed time 2.400\n",
      "[MICE] Starting imputation round 8/55, elapsed time 2.811\n",
      "[MICE] Starting imputation round 9/55, elapsed time 3.218\n",
      "[MICE] Starting imputation round 10/55, elapsed time 3.632\n",
      "[MICE] Starting imputation round 11/55, elapsed time 4.039\n",
      "[MICE] Starting imputation round 12/55, elapsed time 4.474\n",
      "[MICE] Starting imputation round 13/55, elapsed time 4.904\n",
      "[MICE] Starting imputation round 14/55, elapsed time 5.330\n",
      "[MICE] Starting imputation round 15/55, elapsed time 5.754\n",
      "[MICE] Starting imputation round 16/55, elapsed time 6.169\n",
      "[MICE] Starting imputation round 17/55, elapsed time 6.582\n",
      "[MICE] Starting imputation round 18/55, elapsed time 6.989\n",
      "[MICE] Starting imputation round 19/55, elapsed time 7.394\n",
      "[MICE] Starting imputation round 20/55, elapsed time 7.810\n",
      "[MICE] Starting imputation round 21/55, elapsed time 8.214\n",
      "[MICE] Starting imputation round 22/55, elapsed time 8.637\n",
      "[MICE] Starting imputation round 23/55, elapsed time 9.062\n",
      "[MICE] Starting imputation round 24/55, elapsed time 9.503\n",
      "[MICE] Starting imputation round 25/55, elapsed time 9.939\n",
      "[MICE] Starting imputation round 26/55, elapsed time 10.367\n",
      "[MICE] Starting imputation round 27/55, elapsed time 10.839\n",
      "[MICE] Starting imputation round 28/55, elapsed time 11.277\n",
      "[MICE] Starting imputation round 29/55, elapsed time 11.680\n",
      "[MICE] Starting imputation round 30/55, elapsed time 12.145\n",
      "[MICE] Starting imputation round 31/55, elapsed time 12.578\n",
      "[MICE] Starting imputation round 32/55, elapsed time 13.013\n",
      "[MICE] Starting imputation round 33/55, elapsed time 13.448\n",
      "[MICE] Starting imputation round 34/55, elapsed time 13.890\n",
      "[MICE] Starting imputation round 35/55, elapsed time 14.312\n",
      "[MICE] Starting imputation round 36/55, elapsed time 14.735\n",
      "[MICE] Starting imputation round 37/55, elapsed time 15.158\n",
      "[MICE] Starting imputation round 38/55, elapsed time 15.572\n",
      "[MICE] Starting imputation round 39/55, elapsed time 15.984\n",
      "[MICE] Starting imputation round 40/55, elapsed time 16.398\n",
      "[MICE] Starting imputation round 41/55, elapsed time 16.809\n",
      "[MICE] Starting imputation round 42/55, elapsed time 17.218\n",
      "[MICE] Starting imputation round 43/55, elapsed time 17.626\n",
      "[MICE] Starting imputation round 44/55, elapsed time 18.038\n",
      "[MICE] Starting imputation round 45/55, elapsed time 18.449\n",
      "[MICE] Starting imputation round 46/55, elapsed time 18.862\n",
      "[MICE] Starting imputation round 47/55, elapsed time 19.276\n",
      "[MICE] Starting imputation round 48/55, elapsed time 19.688\n",
      "[MICE] Starting imputation round 49/55, elapsed time 20.130\n",
      "[MICE] Starting imputation round 50/55, elapsed time 20.563\n",
      "[MICE] Starting imputation round 51/55, elapsed time 20.983\n",
      "[MICE] Starting imputation round 52/55, elapsed time 21.403\n",
      "[MICE] Starting imputation round 53/55, elapsed time 21.822\n",
      "[MICE] Starting imputation round 54/55, elapsed time 22.187\n",
      "[MICE] Starting imputation round 55/55, elapsed time 22.560\n",
      "Keeping 35864 of 1484185 imputed values, 45884 test values\n",
      "{'f1': 0.4952834347174469, 'tau': 0.41240645045479923, 'auc': 0.83404258159155398}\n",
      "mice-sparse5-burnin5-iters50,2,0.495283,0.412406,0.834043\n",
      "\n",
      "--> Running imputation algorithm mice-sparse5-burnin5-iters25\n",
      "Dropping 11120 peptides with <2 observations\n",
      "Dropping 9 alleles with <2 observations: ['ELA-A1', 'HLA-A26', 'HLA-B2701', 'HLA-E0101', 'Mamu-A07', 'Mamu-B04', 'Patr-A0602', 'Patr-B0901', 'Patr-B1701']\n",
      "[MICE] Completing matrix with shape (15623, 95)\n",
      "[MICE] Starting imputation round 1/30, elapsed time 0.035\n",
      "[MICE] Starting imputation round 2/30, elapsed time 0.389\n",
      "[MICE] Starting imputation round 3/30, elapsed time 0.802\n",
      "[MICE] Starting imputation round 4/30, elapsed time 1.211\n",
      "[MICE] Starting imputation round 5/30, elapsed time 1.615\n",
      "[MICE] Starting imputation round 6/30, elapsed time 2.016\n",
      "[MICE] Starting imputation round 7/30, elapsed time 2.428\n",
      "[MICE] Starting imputation round 8/30, elapsed time 2.845\n",
      "[MICE] Starting imputation round 9/30, elapsed time 3.258\n",
      "[MICE] Starting imputation round 10/30, elapsed time 3.665\n",
      "[MICE] Starting imputation round 11/30, elapsed time 4.090\n",
      "[MICE] Starting imputation round 12/30, elapsed time 4.517\n",
      "[MICE] Starting imputation round 13/30, elapsed time 4.939\n",
      "[MICE] Starting imputation round 14/30, elapsed time 5.362\n",
      "[MICE] Starting imputation round 15/30, elapsed time 5.769\n",
      "[MICE] Starting imputation round 16/30, elapsed time 6.191\n",
      "[MICE] Starting imputation round 17/30, elapsed time 6.603\n",
      "[MICE] Starting imputation round 18/30, elapsed time 7.012\n",
      "[MICE] Starting imputation round 19/30, elapsed time 7.424\n",
      "[MICE] Starting imputation round 20/30, elapsed time 7.834\n",
      "[MICE] Starting imputation round 21/30, elapsed time 8.241\n",
      "[MICE] Starting imputation round 22/30, elapsed time 8.653\n",
      "[MICE] Starting imputation round 23/30, elapsed time 9.065\n",
      "[MICE] Starting imputation round 24/30, elapsed time 9.478\n",
      "[MICE] Starting imputation round 25/30, elapsed time 9.888\n",
      "[MICE] Starting imputation round 26/30, elapsed time 10.310\n",
      "[MICE] Starting imputation round 27/30, elapsed time 10.731\n",
      "[MICE] Starting imputation round 28/30, elapsed time 11.151\n",
      "[MICE] Starting imputation round 29/30, elapsed time 11.570\n",
      "[MICE] Starting imputation round 30/30, elapsed time 11.961\n",
      "Keeping 35864 of 1484185 imputed values, 45884 test values\n",
      "{'f1': 0.48014790034333998, 'tau': 0.38059552730361168, 'auc': 0.81595483381014855}\n",
      "mice-sparse5-burnin5-iters25,2,0.480148,0.380596,0.815955\n",
      "\n",
      "--> Running imputation algorithm mice-sparse25-burnin5-iters25\n",
      "Dropping 11120 peptides with <2 observations\n",
      "Dropping 9 alleles with <2 observations: ['ELA-A1', 'HLA-A26', 'HLA-B2701', 'HLA-E0101', 'Mamu-A07', 'Mamu-B04', 'Patr-A0602', 'Patr-B0901', 'Patr-B1701']\n",
      "[MICE] Completing matrix with shape (15623, 95)\n",
      "[MICE] Starting imputation round 1/30, elapsed time 0.035\n",
      "[MICE] Starting imputation round 2/30, elapsed time 0.721\n",
      "[MICE] Starting imputation round 3/30, elapsed time 1.472\n",
      "[MICE] Starting imputation round 4/30, elapsed time 2.354\n",
      "[MICE] Starting imputation round 5/30, elapsed time 3.300\n",
      "[MICE] Starting imputation round 6/30, elapsed time 4.196\n",
      "[MICE] Starting imputation round 7/30, elapsed time 5.107\n",
      "[MICE] Starting imputation round 8/30, elapsed time 6.023\n",
      "[MICE] Starting imputation round 9/30, elapsed time 6.953\n",
      "[MICE] Starting imputation round 10/30, elapsed time 7.880\n",
      "[MICE] Starting imputation round 11/30, elapsed time 8.789\n",
      "[MICE] Starting imputation round 12/30, elapsed time 9.687\n",
      "[MICE] Starting imputation round 13/30, elapsed time 10.579\n",
      "[MICE] Starting imputation round 14/30, elapsed time 11.488\n",
      "[MICE] Starting imputation round 15/30, elapsed time 12.422\n",
      "[MICE] Starting imputation round 16/30, elapsed time 13.361\n",
      "[MICE] Starting imputation round 17/30, elapsed time 14.382\n",
      "[MICE] Starting imputation round 18/30, elapsed time 15.435\n",
      "[MICE] Starting imputation round 19/30, elapsed time 16.463\n",
      "[MICE] Starting imputation round 20/30, elapsed time 17.494\n",
      "[MICE] Starting imputation round 21/30, elapsed time 18.533\n",
      "[MICE] Starting imputation round 22/30, elapsed time 19.531\n",
      "[MICE] Starting imputation round 23/30, elapsed time 20.544\n",
      "[MICE] Starting imputation round 24/30, elapsed time 21.600\n",
      "[MICE] Starting imputation round 25/30, elapsed time 22.699\n",
      "[MICE] Starting imputation round 26/30, elapsed time 23.688\n",
      "[MICE] Starting imputation round 27/30, elapsed time 24.649\n",
      "[MICE] Starting imputation round 28/30, elapsed time 25.605\n",
      "[MICE] Starting imputation round 29/30, elapsed time 26.569\n",
      "[MICE] Starting imputation round 30/30, elapsed time 27.507\n",
      "Keeping 35864 of 1484185 imputed values, 45884 test values\n",
      "{'f1': 0.57223308564724285, 'tau': 0.45054842994244582, 'auc': 0.85265124620684407}\n",
      "mice-sparse25-burnin5-iters25,2,0.572233,0.450548,0.852651\n",
      "\n",
      "--> Running imputation algorithm svd\n",
      "Dropping 11120 peptides with <2 observations\n",
      "Dropping 9 alleles with <2 observations: ['ELA-A1', 'HLA-A26', 'HLA-B2701', 'HLA-E0101', 'Mamu-A07', 'Mamu-B04', 'Patr-A0602', 'Patr-B0901', 'Patr-B1701']\n",
      "[IterativeSVD] Iter 1: observed MAE=6.308484\n",
      "[IterativeSVD] Iter 2: observed MAE=4.802776\n",
      "[IterativeSVD] Iter 3: observed MAE=3.616220\n",
      "[IterativeSVD] Iter 4: observed MAE=2.503088\n",
      "[IterativeSVD] Iter 5: observed MAE=1.950313\n",
      "[IterativeSVD] Iter 6: observed MAE=1.698869\n",
      "[IterativeSVD] Iter 7: observed MAE=1.550162\n",
      "[IterativeSVD] Iter 8: observed MAE=1.450446\n",
      "[IterativeSVD] Iter 9: observed MAE=1.377134\n",
      "[IterativeSVD] Iter 10: observed MAE=1.319046\n",
      "[IterativeSVD] Iter 11: observed MAE=1.270393\n",
      "[IterativeSVD] Iter 12: observed MAE=1.227888\n",
      "[IterativeSVD] Iter 13: observed MAE=1.189724\n",
      "[IterativeSVD] Iter 14: observed MAE=1.154709\n",
      "[IterativeSVD] Iter 15: observed MAE=1.122192\n",
      "[IterativeSVD] Iter 16: observed MAE=1.092000\n",
      "[IterativeSVD] Iter 17: observed MAE=1.063968\n",
      "[IterativeSVD] Iter 18: observed MAE=1.037961\n",
      "[IterativeSVD] Iter 19: observed MAE=1.013910\n",
      "[IterativeSVD] Iter 20: observed MAE=0.991710\n",
      "[IterativeSVD] Iter 21: observed MAE=0.971261\n",
      "[IterativeSVD] Iter 22: observed MAE=0.952386\n",
      "[IterativeSVD] Iter 23: observed MAE=0.934865\n",
      "[IterativeSVD] Iter 24: observed MAE=0.918518\n",
      "[IterativeSVD] Iter 25: observed MAE=0.903224\n",
      "[IterativeSVD] Iter 26: observed MAE=0.888860\n",
      "[IterativeSVD] Iter 27: observed MAE=0.875318\n",
      "[IterativeSVD] Iter 28: observed MAE=0.862508\n",
      "[IterativeSVD] Iter 29: observed MAE=0.850365\n",
      "[IterativeSVD] Iter 30: observed MAE=0.838808\n",
      "[IterativeSVD] Iter 31: observed MAE=0.827767\n",
      "[IterativeSVD] Iter 32: observed MAE=0.817204\n",
      "[IterativeSVD] Iter 33: observed MAE=0.807056\n",
      "[IterativeSVD] Iter 34: observed MAE=0.797304\n",
      "[IterativeSVD] Iter 35: observed MAE=0.787895\n",
      "[IterativeSVD] Iter 36: observed MAE=0.778808\n",
      "[IterativeSVD] Iter 37: observed MAE=0.770009\n",
      "[IterativeSVD] Iter 38: observed MAE=0.761468\n",
      "[IterativeSVD] Iter 39: observed MAE=0.753177\n",
      "[IterativeSVD] Iter 40: observed MAE=0.745102\n",
      "[IterativeSVD] Iter 41: observed MAE=0.737227\n",
      "[IterativeSVD] Iter 42: observed MAE=0.729531\n",
      "[IterativeSVD] Iter 43: observed MAE=0.721991\n",
      "[IterativeSVD] Iter 44: observed MAE=0.714600\n",
      "[IterativeSVD] Iter 45: observed MAE=0.707348\n",
      "[IterativeSVD] Iter 46: observed MAE=0.700215\n",
      "[IterativeSVD] Iter 47: observed MAE=0.693193\n",
      "[IterativeSVD] Iter 48: observed MAE=0.686275\n",
      "[IterativeSVD] Iter 49: observed MAE=0.679471\n",
      "[IterativeSVD] Iter 50: observed MAE=0.672770\n",
      "[IterativeSVD] Iter 51: observed MAE=0.666168\n",
      "[IterativeSVD] Iter 52: observed MAE=0.659655\n",
      "[IterativeSVD] Iter 53: observed MAE=0.653237\n",
      "[IterativeSVD] Iter 54: observed MAE=0.646918\n",
      "[IterativeSVD] Iter 55: observed MAE=0.640699\n",
      "[IterativeSVD] Iter 56: observed MAE=0.634581\n",
      "[IterativeSVD] Iter 57: observed MAE=0.628579\n",
      "[IterativeSVD] Iter 58: observed MAE=0.622696\n",
      "[IterativeSVD] Iter 59: observed MAE=0.616933\n",
      "[IterativeSVD] Iter 60: observed MAE=0.611293\n",
      "[IterativeSVD] Iter 61: observed MAE=0.605776\n",
      "[IterativeSVD] Iter 62: observed MAE=0.600388\n",
      "[IterativeSVD] Iter 63: observed MAE=0.595127\n",
      "[IterativeSVD] Iter 64: observed MAE=0.589998\n",
      "[IterativeSVD] Iter 65: observed MAE=0.585004\n",
      "[IterativeSVD] Iter 66: observed MAE=0.580145\n",
      "[IterativeSVD] Iter 67: observed MAE=0.575414\n",
      "[IterativeSVD] Iter 68: observed MAE=0.570810\n",
      "[IterativeSVD] Iter 69: observed MAE=0.566329\n",
      "[IterativeSVD] Iter 70: observed MAE=0.561968\n",
      "[IterativeSVD] Iter 71: observed MAE=0.557716\n",
      "[IterativeSVD] Iter 72: observed MAE=0.553573\n",
      "[IterativeSVD] Iter 73: observed MAE=0.549536\n",
      "[IterativeSVD] Iter 74: observed MAE=0.545606\n",
      "[IterativeSVD] Iter 75: observed MAE=0.541773\n",
      "[IterativeSVD] Iter 76: observed MAE=0.538034\n",
      "[IterativeSVD] Iter 77: observed MAE=0.534384\n",
      "[IterativeSVD] Iter 78: observed MAE=0.530823\n",
      "[IterativeSVD] Iter 79: observed MAE=0.527343\n",
      "[IterativeSVD] Iter 80: observed MAE=0.523943\n",
      "[IterativeSVD] Iter 81: observed MAE=0.520619\n",
      "[IterativeSVD] Iter 82: observed MAE=0.517371\n",
      "[IterativeSVD] Iter 83: observed MAE=0.514194\n",
      "[IterativeSVD] Iter 84: observed MAE=0.511083\n",
      "[IterativeSVD] Iter 85: observed MAE=0.508035\n",
      "[IterativeSVD] Iter 86: observed MAE=0.505048\n",
      "[IterativeSVD] Iter 87: observed MAE=0.502121\n",
      "[IterativeSVD] Iter 88: observed MAE=0.499252\n",
      "[IterativeSVD] Iter 89: observed MAE=0.496439\n",
      "[IterativeSVD] Iter 90: observed MAE=0.493680\n",
      "[IterativeSVD] Iter 91: observed MAE=0.490974\n",
      "[IterativeSVD] Iter 92: observed MAE=0.488320\n",
      "[IterativeSVD] Iter 93: observed MAE=0.485718\n",
      "[IterativeSVD] Iter 94: observed MAE=0.483163\n",
      "[IterativeSVD] Iter 95: observed MAE=0.480654\n",
      "[IterativeSVD] Iter 96: observed MAE=0.478190\n",
      "[IterativeSVD] Iter 97: observed MAE=0.475769\n",
      "[IterativeSVD] Iter 98: observed MAE=0.473389\n",
      "[IterativeSVD] Iter 99: observed MAE=0.471048\n",
      "[IterativeSVD] Iter 100: observed MAE=0.468747\n",
      "[IterativeSVD] Iter 101: observed MAE=0.466485\n",
      "[IterativeSVD] Iter 102: observed MAE=0.464259\n",
      "[IterativeSVD] Iter 103: observed MAE=0.462065\n",
      "Keeping 35864 of 1484185 imputed values, 45884 test values\n",
      "{'f1': 0.5912661737523105, 'tau': 0.40936007274256475, 'auc': 0.83055828444602131}\n",
      "svd,2,0.591266,0.409360,0.830558\n",
      "\n",
      "--> Running imputation algorithm knn\n",
      "Dropping 11120 peptides with <2 observations\n",
      "Dropping 9 alleles with <2 observations: ['ELA-A1', 'HLA-A26', 'HLA-B2701', 'HLA-E0101', 'Mamu-A07', 'Mamu-B04', 'Patr-A0602', 'Patr-B0901', 'Patr-B1701']\n",
      "Computing pairwise distances between 95 samples\n",
      "Computing distances for sample #1/95, elapsed time: 0.146\n",
      "Imputing row 1/95 with 15617 missing columns, elapsed time: 0.997\n",
      "Imputing row 11/95 with 12002 missing columns, elapsed time: 2.447\n",
      "Imputing row 21/95 with 11623 missing columns, elapsed time: 4.263\n",
      "Imputing row 31/95 with 15483 missing columns, elapsed time: 5.959\n",
      "Imputing row 41/95 with 12506 missing columns, elapsed time: 7.698\n",
      "Imputing row 51/95 with 15477 missing columns, elapsed time: 9.225\n",
      "Imputing row 61/95 with 15053 missing columns, elapsed time: 10.841\n",
      "Imputing row 71/95 with 14926 missing columns, elapsed time: 12.421\n",
      "Imputing row 81/95 with 15505 missing columns, elapsed time: 13.994\n",
      "Imputing row 91/95 with 15362 missing columns, elapsed time: 15.593\n",
      "[KNN] Warning: 147870/1484185 still missing after imputation, replacing with 0\n",
      "Keeping 35864 of 1484185 imputed values, 45884 test values\n",
      "{'f1': 0.56735046139547241, 'tau': 0.41434210029094604, 'auc': 0.80876881629160025}\n",
      "knn,2,0.567350,0.414342,0.808769\n",
      "\n",
      "--> Running imputation algorithm mice-sparse25-burnin5-iters50\n",
      "Dropping 11120 peptides with <2 observations\n",
      "Dropping 9 alleles with <2 observations: ['ELA-A1', 'HLA-A26', 'HLA-B2701', 'HLA-E0101', 'Mamu-A07', 'Mamu-B04', 'Patr-A0602', 'Patr-B0901', 'Patr-B1701']\n",
      "[MICE] Completing matrix with shape (15623, 95)\n",
      "[MICE] Starting imputation round 1/55, elapsed time 0.037\n",
      "[MICE] Starting imputation round 2/55, elapsed time 0.790\n",
      "[MICE] Starting imputation round 3/55, elapsed time 1.631\n",
      "[MICE] Starting imputation round 4/55, elapsed time 2.714\n",
      "[MICE] Starting imputation round 5/55, elapsed time 3.709\n",
      "[MICE] Starting imputation round 6/55, elapsed time 4.719\n",
      "[MICE] Starting imputation round 7/55, elapsed time 5.868\n",
      "[MICE] Starting imputation round 8/55, elapsed time 6.841\n",
      "[MICE] Starting imputation round 9/55, elapsed time 7.773\n",
      "[MICE] Starting imputation round 10/55, elapsed time 8.731\n",
      "[MICE] Starting imputation round 11/55, elapsed time 9.666\n",
      "[MICE] Starting imputation round 12/55, elapsed time 10.880\n",
      "[MICE] Starting imputation round 13/55, elapsed time 12.006\n",
      "[MICE] Starting imputation round 14/55, elapsed time 13.079\n",
      "[MICE] Starting imputation round 15/55, elapsed time 14.114\n",
      "[MICE] Starting imputation round 16/55, elapsed time 15.099\n",
      "[MICE] Starting imputation round 17/55, elapsed time 16.085\n",
      "[MICE] Starting imputation round 18/55, elapsed time 17.036\n",
      "[MICE] Starting imputation round 19/55, elapsed time 17.996\n",
      "[MICE] Starting imputation round 20/55, elapsed time 19.053\n",
      "[MICE] Starting imputation round 21/55, elapsed time 20.197\n",
      "[MICE] Starting imputation round 22/55, elapsed time 21.333\n",
      "[MICE] Starting imputation round 23/55, elapsed time 22.520\n",
      "[MICE] Starting imputation round 24/55, elapsed time 23.688\n",
      "[MICE] Starting imputation round 25/55, elapsed time 24.733\n",
      "[MICE] Starting imputation round 26/55, elapsed time 25.712\n",
      "[MICE] Starting imputation round 27/55, elapsed time 26.709\n",
      "[MICE] Starting imputation round 28/55, elapsed time 27.626\n",
      "[MICE] Starting imputation round 29/55, elapsed time 28.569\n",
      "[MICE] Starting imputation round 30/55, elapsed time 29.514\n",
      "[MICE] Starting imputation round 31/55, elapsed time 30.437\n",
      "[MICE] Starting imputation round 32/55, elapsed time 31.388\n",
      "[MICE] Starting imputation round 33/55, elapsed time 32.375\n",
      "[MICE] Starting imputation round 34/55, elapsed time 33.310\n",
      "[MICE] Starting imputation round 35/55, elapsed time 34.338\n",
      "[MICE] Starting imputation round 36/55, elapsed time 35.339\n",
      "[MICE] Starting imputation round 37/55, elapsed time 36.308\n",
      "[MICE] Starting imputation round 38/55, elapsed time 37.265\n",
      "[MICE] Starting imputation round 39/55, elapsed time 38.231\n",
      "[MICE] Starting imputation round 40/55, elapsed time 39.178\n",
      "[MICE] Starting imputation round 41/55, elapsed time 40.123\n",
      "[MICE] Starting imputation round 42/55, elapsed time 41.128\n",
      "[MICE] Starting imputation round 43/55, elapsed time 42.128\n",
      "[MICE] Starting imputation round 44/55, elapsed time 43.171\n",
      "[MICE] Starting imputation round 45/55, elapsed time 44.167\n",
      "[MICE] Starting imputation round 46/55, elapsed time 45.184\n",
      "[MICE] Starting imputation round 47/55, elapsed time 46.173\n",
      "[MICE] Starting imputation round 48/55, elapsed time 47.165\n",
      "[MICE] Starting imputation round 49/55, elapsed time 48.100\n",
      "[MICE] Starting imputation round 50/55, elapsed time 49.026\n",
      "[MICE] Starting imputation round 51/55, elapsed time 49.951\n",
      "[MICE] Starting imputation round 52/55, elapsed time 50.909\n",
      "[MICE] Starting imputation round 53/55, elapsed time 51.853\n",
      "[MICE] Starting imputation round 54/55, elapsed time 52.793\n",
      "[MICE] Starting imputation round 55/55, elapsed time 53.771\n",
      "Keeping 35864 of 1484185 imputed values, 45884 test values\n",
      "{'f1': 0.56551067375605102, 'tau': 0.45627413469754474, 'auc': 0.8566217697766445}\n",
      "mice-sparse25-burnin5-iters50,2,0.565511,0.456274,0.856622\n",
      "\n"
     ]
    }
   ],
   "source": [
    "results = evaluate_imputation_methods(\"Compare-Imputation.csv\", bd2009, imputation_algos)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "score_names = [\"f1\", \"tau\", \"auc\"]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "df = pd.read_csv(\"Compare-Imputation.csv\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 59,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "df[\"combined-score\"] = df[\"f1\"] + df[\"tau\"] + 2 * (df[\"auc\"] - 0.5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 66,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>cv_fold</th>\n",
       "      <th>f1</th>\n",
       "      <th>tau</th>\n",
       "      <th>auc</th>\n",
       "      <th>combined-score</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>imputation_algorithm</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>mice-sparse5-burnin5-iters25</th>\n",
       "      <td>1</td>\n",
       "      <td>0.491753</td>\n",
       "      <td>0.385920</td>\n",
       "      <td>0.819017</td>\n",
       "      <td>1.515707</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>mice-sparse5-burnin5-iters50</th>\n",
       "      <td>1</td>\n",
       "      <td>0.499054</td>\n",
       "      <td>0.413002</td>\n",
       "      <td>0.832796</td>\n",
       "      <td>1.577647</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>knn</th>\n",
       "      <td>1</td>\n",
       "      <td>0.567644</td>\n",
       "      <td>0.418689</td>\n",
       "      <td>0.810401</td>\n",
       "      <td>1.607136</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>svd</th>\n",
       "      <td>1</td>\n",
       "      <td>0.596698</td>\n",
       "      <td>0.407354</td>\n",
       "      <td>0.831474</td>\n",
       "      <td>1.667000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>mice-burnin10-iters25</th>\n",
       "      <td>1</td>\n",
       "      <td>0.564432</td>\n",
       "      <td>0.437443</td>\n",
       "      <td>0.844214</td>\n",
       "      <td>1.690304</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>mice-burnin5-iters25</th>\n",
       "      <td>1</td>\n",
       "      <td>0.561710</td>\n",
       "      <td>0.439339</td>\n",
       "      <td>0.845651</td>\n",
       "      <td>1.692352</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>mice-burnin10-iters50</th>\n",
       "      <td>1</td>\n",
       "      <td>0.567713</td>\n",
       "      <td>0.454074</td>\n",
       "      <td>0.851116</td>\n",
       "      <td>1.724020</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>softimpute</th>\n",
       "      <td>1</td>\n",
       "      <td>0.602133</td>\n",
       "      <td>0.430919</td>\n",
       "      <td>0.845749</td>\n",
       "      <td>1.724550</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>mice-sparse25-burnin5-iters25</th>\n",
       "      <td>1</td>\n",
       "      <td>0.574720</td>\n",
       "      <td>0.449712</td>\n",
       "      <td>0.852053</td>\n",
       "      <td>1.728538</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>mice-sparse25-burnin5-iters50</th>\n",
       "      <td>1</td>\n",
       "      <td>0.571227</td>\n",
       "      <td>0.459375</td>\n",
       "      <td>0.857028</td>\n",
       "      <td>1.744657</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                               cv_fold        f1       tau       auc  \\\n",
       "imputation_algorithm                                                   \n",
       "mice-sparse5-burnin5-iters25         1  0.491753  0.385920  0.819017   \n",
       "mice-sparse5-burnin5-iters50         1  0.499054  0.413002  0.832796   \n",
       "knn                                  1  0.567644  0.418689  0.810401   \n",
       "svd                                  1  0.596698  0.407354  0.831474   \n",
       "mice-burnin10-iters25                1  0.564432  0.437443  0.844214   \n",
       "mice-burnin5-iters25                 1  0.561710  0.439339  0.845651   \n",
       "mice-burnin10-iters50                1  0.567713  0.454074  0.851116   \n",
       "softimpute                           1  0.602133  0.430919  0.845749   \n",
       "mice-sparse25-burnin5-iters25        1  0.574720  0.449712  0.852053   \n",
       "mice-sparse25-burnin5-iters50        1  0.571227  0.459375  0.857028   \n",
       "\n",
       "                               combined-score  \n",
       "imputation_algorithm                           \n",
       "mice-sparse5-burnin5-iters25         1.515707  \n",
       "mice-sparse5-burnin5-iters50         1.577647  \n",
       "knn                                  1.607136  \n",
       "svd                                  1.667000  \n",
       "mice-burnin10-iters25                1.690304  \n",
       "mice-burnin5-iters25                 1.692352  \n",
       "mice-burnin10-iters50                1.724020  \n",
       "softimpute                           1.724550  \n",
       "mice-sparse25-burnin5-iters25        1.728538  \n",
       "mice-sparse25-burnin5-iters50        1.744657  "
      ]
     },
     "execution_count": 66,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.groupby(\"imputation_algorithm\").mean().sort_values(by=\"combined-score\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.4.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}
