graphkit-learn/notebooks/.ipynb_checkpoints/run_marginalizedkernel_acyclic-checkpoint.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "- This script take as input a kernel matrix\n",
      "and returns the classification or regression performance\n",
      "- The kernel matrix can be calculated using any of the graph kernels approaches\n",
      "- The criteria used for prediction are SVM for classification and kernel Ridge regression for regression\n",
      "- For predition we divide the data in training, validation and test. For each split, we first train on the train data, \n",
      "then evaluate the performance on the validation. We choose the optimal parameters for the validation set and finally\n",
      "provide the corresponding performance on the test set. If more than one split is performed, the final results \n",
      "correspond to the average of the performances on the test sets. \n",
      "\n",
      "@references\n",
      "    https://github.com/eghisu/GraphKernels/blob/master/GraphKernelsCollection/python_scripts/compute_perf_gk.py\n",
      "\n",
      "\n",
      " Loading dataset from file...\n",
      "[ -23.7   14.    37.3  109.7   10.8   39.    42.    66.6  135.   148.5\n",
      "   40.    34.6   32.    63.    53.5   67.    64.4   84.7   95.5   92.\n",
      "   84.4  154.   156.   166.   183.    70.3   63.6   52.5   59.    59.5\n",
      "   55.2   88.    83.   104.5  102.    92.   107.4  123.2  112.5  118.5\n",
      "  101.5  173.7  165.5  181.    99.5   92.3   90.1   80.2   82.    91.2\n",
      "   91.5   81.2   93.    69.    86.3   82.   103.   103.5   96.   112.   104.\n",
      "  132.5  123.5  120.3  145.   144.2  142.8  132.   134.2  137.   139.\n",
      "  133.6  120.4  120.   137.   195.8  177.2  181.   185.9  175.7  186.   211.\n",
      "  125.   118.   117.1  107.   102.5  112.    97.4   91.5   87.6  106.5\n",
      "  101.    99.3   90.   137.   114.   126.   124.   140.5  157.5  146.   145.\n",
      "  141.   171.   166.   155.   145.   159.   138.   142.   159.   163.5\n",
      "  229.5  142.   125.   132.   130.5  125.   122.   121.   122.2  112.   106.\n",
      "  114.5  151.   128.5  109.5  126.   147.   158.   147.   165.   188.9\n",
      "  170.   178.   148.5  165.   177.   167.   195.   226.   215.   201.   205.\n",
      "  151.5  165.5  157.   139.   163.   153.5  139.   162.   173.   159.5\n",
      "  159.5  155.5  141.   126.   164.   163.   166.5  146.   165.   159.   195.\n",
      "  218.   250.   235.   186.5  156.5  162.   162.   170.2  173.2  186.8\n",
      "  173.   187.   174.   188.5  199.   228.   215.   216.   240. ]\n",
      "\n",
      " --- This is a regression problem ---\n",
      "\n",
      " --- calculating kernel matrix when termimation probability = 0.1 ---\n",
      "\n",
      " Loading the kernel matrix from file...\n",
      "[[ 0.0287062   0.0124634   0.00444444 ...,  0.00606061  0.00606061\n",
      "   0.00606061]\n",
      " [ 0.0124634   0.01108958  0.00333333 ...,  0.00454545  0.00454545\n",
      "   0.00454545]\n",
      " [ 0.00444444  0.00333333  0.0287062  ...,  0.00819912  0.00819912\n",
      "   0.00975875]\n",
      " ..., \n",
      " [ 0.00606061  0.00454545  0.00819912 ...,  0.02846735  0.02836907\n",
      "   0.02896354]\n",
      " [ 0.00606061  0.00454545  0.00819912 ...,  0.02836907  0.02831424\n",
      "   0.0288712 ]\n",
      " [ 0.00606061  0.00454545  0.00975875 ...,  0.02896354  0.0288712\n",
      "   0.02987915]]\n",
      "\n",
      " Mean performance on val set: 51.192412\n",
      "With standard deviation: 58.804642\n",
      "\n",
      " Mean performance on test set: 18.518782\n",
      "With standard deviation: 7.749004\n",
      "\n",
      " --- calculating kernel matrix when termimation probability = 0.2 ---\n",
      "\n",
      " Loading the kernel matrix from file...\n",
      "[[ 0.06171557  0.03856471  0.01777778 ...,  0.02424242  0.02424242\n",
      "   0.02424242]\n",
      " [ 0.03856471  0.03579176  0.01333333 ...,  0.01818182  0.01818182\n",
      "   0.01818182]\n",
      " [ 0.01777778  0.01333333  0.06171557 ...,  0.02994207  0.02994207\n",
      "   0.03262072]\n",
      " ..., \n",
      " [ 0.02424242  0.01818182  0.02994207 ...,  0.07442109  0.07434207\n",
      "   0.07383563]\n",
      " [ 0.02424242  0.01818182  0.02994207 ...,  0.07434207  0.07430377\n",
      "   0.07376068]\n",
      " [ 0.02424242  0.01818182  0.03262072 ...,  0.07383563  0.07376068\n",
      "   0.07366354]]\n",
      "\n",
      " Mean performance on val set: 56.692288\n",
      "With standard deviation: 58.162153\n",
      "\n",
      " Mean performance on test set: 17.899091\n",
      "With standard deviation: 6.591042\n",
      "\n",
      " --- calculating kernel matrix when termimation probability = 0.3 ---\n",
      "\n",
      " Loading the kernel matrix from file...\n",
      "[[ 0.09803909  0.07202114  0.04       ...,  0.05454545  0.05454545\n",
      "   0.05454545]\n",
      " [ 0.07202114  0.06853421  0.03       ...,  0.04090909  0.04090909\n",
      "   0.04090909]\n",
      " [ 0.04        0.03        0.09803909 ...,  0.06368916  0.06368916\n",
      "   0.06678704]\n",
      " ..., \n",
      " [ 0.05454545  0.04090909  0.06368916 ...,  0.12892852  0.12891455\n",
      "   0.12734365]\n",
      " [ 0.05454545  0.04090909  0.06368916 ...,  0.12891455  0.12892664\n",
      "   0.12733207]\n",
      " [ 0.05454545  0.04090909  0.06678704 ...,  0.12734365  0.12733207\n",
      "   0.1261675 ]]\n",
      "\n",
      " Mean performance on val set: 54.360795\n",
      "With standard deviation: 61.733054\n",
      "\n",
      " Mean performance on test set: 18.392352\n",
      "With standard deviation: 7.101611\n",
      "\n",
      " --- calculating kernel matrix when termimation probability = 0.4 ---\n",
      "\n",
      " Loading the kernel matrix from file...\n",
      "[[ 0.13888889  0.11120616  0.07111111 ...,  0.0969697   0.0969697\n",
      "   0.0969697 ]\n",
      " [ 0.11120616  0.10756609  0.05333333 ...,  0.07272727  0.07272727\n",
      "   0.07272727]\n",
      " [ 0.07111111  0.05333333  0.13888889 ...,  0.10909713  0.10909713\n",
      "   0.11216176]\n",
      " ..., \n",
      " [ 0.0969697   0.07272727  0.10909713 ...,  0.19178929  0.19182091\n",
      "   0.18963212]\n",
      " [ 0.0969697   0.07272727  0.10909713 ...,  0.19182091  0.19186661\n",
      "   0.18966477]\n",
      " [ 0.0969697   0.07272727  0.11216176 ...,  0.18963212  0.18966477\n",
      "   0.18786824]]\n",
      "\n",
      " Mean performance on val set: 44.518253\n",
      "With standard deviation: 44.478206\n",
      "\n",
      " Mean performance on test set: 19.623259\n",
      "With standard deviation: 6.248069\n",
      "\n",
      " --- calculating kernel matrix when termimation probability = 0.5 ---\n",
      "\n",
      " Loading the kernel matrix from file...\n",
      "[[ 0.18518519  0.15591398  0.11111111 ...,  0.15151515  0.15151515\n",
      "   0.15151515]\n",
      " [ 0.15591398  0.15254237  0.08333333 ...,  0.11363636  0.11363636\n",
      "   0.11363636]\n",
      " [ 0.11111111  0.08333333  0.18518519 ...,  0.16617791  0.16617791\n",
      "   0.16890214]\n",
      " ..., \n",
      " [ 0.15151515  0.11363636  0.16617791 ...,  0.26386999  0.26391515\n",
      "   0.26158184]\n",
      " [ 0.15151515  0.11363636  0.16617791 ...,  0.26391515  0.26396688\n",
      "   0.26162729]\n",
      " [ 0.15151515  0.11363636  0.16890214 ...,  0.26158184  0.26162729\n",
      "   0.25964592]]\n",
      "\n",
      " Mean performance on val set: 42.848719\n",
      "With standard deviation: 39.189276\n",
      "\n",
      " Mean performance on test set: 19.993624\n",
      "With standard deviation: 6.299511\n",
      "\n",
      " --- calculating kernel matrix when termimation probability = 0.6 ---\n",
      "\n",
      " Loading the kernel matrix from file...\n",
      "[[ 0.23809524  0.20664506  0.16       ...,  0.21818182  0.21818182\n",
      "   0.21818182]\n",
      " [ 0.20664506  0.20385906  0.12       ...,  0.16363636  0.16363636\n",
      "   0.16363636]\n",
      " [ 0.16        0.12        0.23809524 ...,  0.2351024   0.2351024\n",
      "   0.23727718]\n",
      " ..., \n",
      " [ 0.21818182  0.16363636  0.2351024  ...,  0.34658956  0.34662512\n",
      "   0.34454945]\n",
      " [ 0.21818182  0.16363636  0.2351024  ...,  0.34662512  0.34666325\n",
      "   0.34458505]\n",
      " [ 0.21818182  0.16363636  0.23727718 ...,  0.34454945  0.34458505\n",
      "   0.34279503]]\n",
      "\n",
      " Mean performance on val set: 39.983104\n",
      "With standard deviation: 32.270969\n",
      "\n",
      " Mean performance on test set: 20.546624\n",
      "With standard deviation: 6.261735\n",
      "\n",
      " --- calculating kernel matrix when termimation probability = 0.7 ---\n",
      "\n",
      " Loading the kernel matrix from file...\n",
      "[[ 0.2991453   0.26444601  0.21777778 ...,  0.2969697   0.2969697\n",
      "   0.2969697 ]\n",
      " [ 0.26444601  0.26246188  0.16333333 ...,  0.22272727  0.22272727\n",
      "   0.22272727]\n",
      " [ 0.21777778  0.16333333  0.2991453  ...,  0.31614548  0.31614548\n",
      "   0.31765009]\n",
      " ..., \n",
      " [ 0.2969697   0.22272727  0.31614548 ...,  0.44189997  0.44191814\n",
      "   0.44038348]\n",
      " [ 0.2969697   0.22272727  0.31614548 ...,  0.44191814  0.44193708\n",
      "   0.44040164]\n",
      " [ 0.2969697   0.22272727  0.31765009 ...,  0.44038348  0.44040164\n",
      "   0.43906772]]\n",
      "\n",
      " Mean performance on val set: 37.530308\n",
      "With standard deviation: 29.730795\n",
      "\n",
      " Mean performance on test set: 21.701779\n",
      "With standard deviation: 6.335305\n",
      "\n",
      " --- calculating kernel matrix when termimation probability = 0.8 ---\n",
      "\n",
      " Loading the kernel matrix from file...\n",
      "[[ 0.37037037  0.33093141  0.28444444 ...,  0.38787879  0.38787879\n",
      "   0.38787879]\n",
      " [ 0.33093141  0.32983023  0.21333333 ...,  0.29090909  0.29090909\n",
      "   0.29090909]\n",
      " [ 0.28444444  0.21333333  0.37037037 ...,  0.4096795   0.4096795\n",
      "   0.41049599]\n",
      " ..., \n",
      " [ 0.38787879  0.29090909  0.4096795  ...,  0.55242487  0.55243009\n",
      "   0.5515636 ]\n",
      " [ 0.38787879  0.29090909  0.4096795  ...,  0.55243009  0.55243545\n",
      "   0.55156881]\n",
      " [ 0.38787879  0.29090909  0.41049599 ...,  0.5515636   0.55156881\n",
      "   0.55081257]]\n",
      "\n",
      " Mean performance on val set: 37.110483\n",
      "With standard deviation: 21.287120\n",
      "\n",
      " Mean performance on test set: 23.148949\n",
      "With standard deviation: 6.102457\n",
      "\n",
      " --- calculating kernel matrix when termimation probability = 0.9 ---\n",
      "\n",
      " Loading the kernel matrix from file...\n",
      "[[ 0.45454545  0.40839542  0.36       ...,  0.49090909  0.49090909\n",
      "   0.49090909]\n",
      " [ 0.40839542  0.40805534  0.27       ...,  0.36818182  0.36818182\n",
      "   0.36818182]\n",
      " [ 0.36        0.27        0.45454545 ...,  0.51619708  0.51619708\n",
      "   0.51644564]\n",
      " ..., \n",
      " [ 0.49090909  0.36818182  0.51619708 ...,  0.68172189  0.68172233\n",
      "   0.68145294]\n",
      " [ 0.49090909  0.36818182  0.51619708 ...,  0.68172233  0.68172277\n",
      "   0.68145338]\n",
      " [ 0.49090909  0.36818182  0.51644564 ...,  0.68145294  0.68145338\n",
      "   0.68121781]]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      " Mean performance on val set: 30.572040\n",
      "With standard deviation: 11.057046\n",
      "\n",
      " Mean performance on test set: 24.715650\n",
      "With standard deviation: 4.891587\n",
      "\n",
      "\n",
      "    std     RMSE    p_quit\n",
      "-------  -------  --------\n",
      "7.749    18.5188       0.1\n",
      "6.59104  17.8991       0.2\n",
      "7.10161  18.3924       0.3\n",
      "6.24807  19.6233       0.4\n",
      "6.29951  19.9936       0.5\n",
      "6.26173  20.5466       0.6\n",
      "6.33531  21.7018       0.7\n",
      "6.10246  23.1489       0.8\n",
      "4.89159  24.7157       0.9\n"
     ]
    }
   ],
   "source": [
    "# Author: Elisabetta Ghisu\n",
    "\n",
    "\"\"\"\n",
    "- This script take as input a kernel matrix\n",
    "and returns the classification or regression performance\n",
    "- The kernel matrix can be calculated using any of the graph kernels approaches\n",
    "- The criteria used for prediction are SVM for classification and kernel Ridge regression for regression\n",
    "- For predition we divide the data in training, validation and test. For each split, we first train on the train data, \n",
    "then evaluate the performance on the validation. We choose the optimal parameters for the validation set and finally\n",
    "provide the corresponding performance on the test set. If more than one split is performed, the final results \n",
    "correspond to the average of the performances on the test sets. \n",
    "\n",
    "@references\n",
    "    https://github.com/eghisu/GraphKernels/blob/master/GraphKernelsCollection/python_scripts/compute_perf_gk.py\n",
    "\"\"\"\n",
    "\n",
    "print(__doc__)\n",
    "\n",
    "import sys\n",
    "import os\n",
    "import pathlib\n",
    "sys.path.insert(0, \"../\")\n",
    "from tabulate import tabulate\n",
    "\n",
    "import random\n",
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "\n",
    "from sklearn.kernel_ridge import KernelRidge # 0.17\n",
    "from sklearn.metrics import accuracy_score, mean_squared_error\n",
    "from sklearn import svm\n",
    "\n",
    "from pygraph.kernels.marginalizedKernel import marginalizedkernel\n",
    "from pygraph.utils.graphfiles import loadDataset\n",
    "\n",
    "print('\\n Loading dataset from file...')\n",
    "dataset, y = loadDataset(\"../../../../datasets/acyclic/Acyclic/dataset_bps.ds\")\n",
    "y = np.array(y)\n",
    "print(y)\n",
    "\n",
    "# setup the parameters\n",
    "model_type = 'regression' # Regression or classification problem\n",
    "print('\\n --- This is a %s problem ---' % model_type)\n",
    "\n",
    "datasize = len(dataset)\n",
    "trials = 100 # Trials for hyperparameters random search\n",
    "splits = 10 # Number of splits of the data\n",
    "alpha_grid = np.logspace(-10, 10, num = trials, base = 10) # corresponds to (2*C)^-1 in other linear models such as LogisticRegression\n",
    "C_grid = np.logspace(-10, 10, num = trials, base = 10)\n",
    "random.seed(20) # Set the seed for uniform parameter distribution\n",
    "\n",
    "# set the output path\n",
    "kernel_file_path = 'kernelmatrices_marginalized_acyclic/'\n",
    "if not os.path.exists(kernel_file_path):\n",
    "    os.makedirs(kernel_file_path)\n",
    "\n",
    "\n",
    "\"\"\"\n",
    "-  Here starts the main program\n",
    "-  First we permute the data, then for each split we evaluate corresponding performances\n",
    "-  In the end, the performances are averaged over the test sets\n",
    "\"\"\"\n",
    "val_means_pquit = []\n",
    "val_stds_pquit = []\n",
    "test_means_pquit = []\n",
    "test_stds_pquit = []\n",
    "\n",
    "\n",
    "for p_quit in np.linspace(0.1, 0.9, 9):\n",
    "    print('\\n --- calculating kernel matrix when termimation probability = %.1f ---' % p_quit)\n",
    "\n",
    "    # save kernel matrices to files / read kernel matrices from files\n",
    "    kernel_file = kernel_file_path + 'p_quit-' + str(p_quit)\n",
    "    path = pathlib.Path(kernel_file)\n",
    "    # get train set kernel matrix\n",
    "    if path.is_file():\n",
    "        print('\\n Loading the kernel matrix from file...')\n",
    "        Kmatrix = np.loadtxt(kernel_file)\n",
    "        print(Kmatrix)\n",
    "    else:\n",
    "        print('\\n Calculating kernel matrix, this could take a while...')\n",
    "        Kmatrix, run_time = marginalizedkernel(dataset, p_quit, 20, node_label = 'atom', edge_label = 'bond_type')\n",
    "        print(Kmatrix)\n",
    "        print('\\n Saving kernel matrix to file...')\n",
    "        np.savetxt(kernel_file, Kmatrix)\n",
    "\n",
    "    # Initialize the performance of the best parameter trial on validation with the corresponding performance on test\n",
    "    val_split = []\n",
    "    test_split = []\n",
    "\n",
    "    # For each split of the data\n",
    "    for j in range(10, 10 + splits):\n",
    "#         print('\\n Starting split %d...' % j)\n",
    "\n",
    "        # Set the random set for data permutation\n",
    "        random_state = int(j)\n",
    "        np.random.seed(random_state)\n",
    "        idx_perm = np.random.permutation(datasize)\n",
    "    #     print(idx_perm)\n",
    "\n",
    "        # Permute the data\n",
    "        y_perm = y[idx_perm] # targets permutation\n",
    "    #     print(y_perm)\n",
    "        Kmatrix_perm = Kmatrix[:, idx_perm] # inputs permutation\n",
    "    #     print(Kmatrix_perm)\n",
    "        Kmatrix_perm = Kmatrix_perm[idx_perm, :] # inputs permutation\n",
    "\n",
    "        # Set the training, validation and test\n",
    "        # Note: the percentage can be set up by the user\n",
    "        num_train_val = int((datasize * 90) / 100)         # 90% (of entire dataset) for training and validation\n",
    "        num_test = datasize - num_train_val              # 10% (of entire dataset) for test\n",
    "        num_train = int((num_train_val * 90) / 100) # 90% (of train + val) for training\n",
    "        num_val = num_train_val - num_train # 10% (of train + val) for validation\n",
    "\n",
    "        # Split the kernel matrix\n",
    "        Kmatrix_train = Kmatrix_perm[0:num_train, 0:num_train]\n",
    "        Kmatrix_val = Kmatrix_perm[num_train:(num_train + num_val), 0:num_train]\n",
    "        Kmatrix_test = Kmatrix_perm[(num_train + num_val):datasize, 0:num_train]\n",
    "\n",
    "        # Split the targets\n",
    "        y_train = y_perm[0:num_train]\n",
    "\n",
    "        # Normalization step (for real valued targets only)\n",
    "        if model_type == 'regression':\n",
    "#             print('\\n Normalizing output y...')\n",
    "            y_train_mean = np.mean(y_train)\n",
    "            y_train_std = np.std(y_train)\n",
    "            y_train = (y_train - y_train_mean) / float(y_train_std)\n",
    "    #         print(y)\n",
    "\n",
    "        y_val = y_perm[num_train:(num_train + num_val)]\n",
    "        y_test = y_perm[(num_train + num_val):datasize]\n",
    "\n",
    "        # Record the performance for each parameter trial respectively on validation and test set\n",
    "        perf_all_val = []\n",
    "        perf_all_test = []\n",
    "\n",
    "        # For each parameter trial\n",
    "        for i in range(trials):\n",
    "            # For regression use the Kernel Ridge method\n",
    "            if model_type == 'regression':\n",
    "    #             print('\\n Starting experiment for trial %d and parameter alpha = %3f\\n ' % (i, alpha_grid[i]))\n",
    "\n",
    "                # Fit the kernel ridge model\n",
    "                KR = KernelRidge(kernel = 'precomputed', alpha = alpha_grid[i])\n",
    "#                 KR = svm.SVR(kernel = 'precomputed', C = C_grid[i])\n",
    "                KR.fit(Kmatrix_train, y_train)\n",
    "\n",
    "                # predict on the validation and test set\n",
    "                y_pred = KR.predict(Kmatrix_val)\n",
    "                y_pred_test = KR.predict(Kmatrix_test)\n",
    "    #             print(y_pred)\n",
    "\n",
    "                # adjust prediction: needed because the training targets have been normalizaed\n",
    "                y_pred = y_pred * float(y_train_std) + y_train_mean\n",
    "    #             print(y_pred)\n",
    "                y_pred_test = y_pred_test * float(y_train_std) + y_train_mean\n",
    "    #             print(y_pred_test)\n",
    "\n",
    "                # root mean squared error on validation\n",
    "                rmse = np.sqrt(mean_squared_error(y_val, y_pred))\n",
    "                perf_all_val.append(rmse)\n",
    "\n",
    "                # root mean squared error in test \n",
    "                rmse_test = np.sqrt(mean_squared_error(y_test, y_pred_test))\n",
    "                perf_all_test.append(rmse_test)\n",
    "\n",
    "    #             print('The performance on the validation set is: %3f' % rmse)\n",
    "    #             print('The performance on the test set is: %3f' % rmse_test)\n",
    "\n",
    "        # --- FIND THE OPTIMAL PARAMETERS --- #\n",
    "        # For regression: minimise the mean squared error\n",
    "        if model_type == 'regression':\n",
    "\n",
    "            # get optimal parameter on validation (argmin mean squared error)\n",
    "            min_idx = np.argmin(perf_all_test)\n",
    "            alpha_opt = alpha_grid[min_idx]\n",
    "\n",
    "            # performance corresponding to optimal parameter on val\n",
    "            perf_val_opt = perf_all_val[min_idx]\n",
    "\n",
    "            # corresponding performance on test for the same parameter\n",
    "            perf_test_opt = perf_all_test[min_idx]\n",
    "\n",
    "#             print('The best performance is for trial %d with parameter alpha = %3f' % (min_idx, alpha_opt))\n",
    "#             print('The best performance on the validation set is: %3f' % perf_val_opt)\n",
    "#             print('The corresponding performance on test set is: %3f' % perf_test_opt)\n",
    "        \n",
    "        # append the best performance on validation\n",
    "        # at the current split\n",
    "        val_split.append(perf_val_opt)\n",
    "\n",
    "        # append the correponding performance on the test set\n",
    "        test_split.append(perf_test_opt)\n",
    "        \n",
    "    # average the results\n",
    "    # mean of the validation performances over the splits\n",
    "    val_mean = np.mean(np.asarray(val_split))\n",
    "    # std deviation of validation over the splits\n",
    "    val_std = np.std(np.asarray(val_split))\n",
    "\n",
    "    # mean of the test performances over the splits\n",
    "    test_mean = np.mean(np.asarray(test_split))\n",
    "    # std deviation of the test oer the splits\n",
    "    test_std = np.std(np.asarray(test_split))\n",
    "    \n",
    "    print('\\n Mean performance on val set: %3f' % val_mean)\n",
    "    print('With standard deviation: %3f' % val_std)\n",
    "    print('\\n Mean performance on test set: %3f' % test_mean)\n",
    "    print('With standard deviation: %3f' % test_std)\n",
    "    \n",
    "    val_means_pquit.append(val_mean)\n",
    "    val_stds_pquit.append(val_std)\n",
    "    test_means_pquit.append(test_mean)\n",
    "    test_stds_pquit.append(test_std)\n",
    "\n",
    "print('\\n') \n",
    "print(tabulate({'p_quit': np.linspace(0.1, 0.9, 9), 'RMSE': test_means_pquit, 'std': test_stds_pquit}, headers='keys'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}