{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Acyclic\n",
      "\n",
      "--- This is a regression problem ---\n",
      "\n",
      "\n",
      "1. Loading dataset from file...\n",
      "\n",
      "2. Calculating gram matrices. This could take a while...\n",
      "\n",
      " None edge weight specified. Set all weight to 1.\n",
      "\n",
      "getting sp graphs: 183it [00:00, 1871.37it/s]\n",
      "calculating kernels: 16836it [00:16, 1014.42it/s]\n",
      "\n",
      " --- shortest path kernel matrix of size 183 built in 16.947543382644653 seconds ---\n",
      "\n",
      "the gram matrix with parameters {'node_kernels': {'symb': <function deltakernel at 0x7f3a99093950>, 'nsymb': <function gaussiankernel at 0x7f3a990931e0>, 'mix': functools.partial(<function kernelproduct at 0x7f3a99088ae8>, <function deltakernel at 0x7f3a99093950>, <function gaussiankernel at 0x7f3a990931e0>)}, 'n_jobs': 8} is: \n",
      "\n",
      "\n",
      "\n",
      "1 gram matrices are calculated, 0 of which are ignored.\n",
      "\n",
      "3. Fitting and predicting using nested cross validation. This could really take a while...\n",
      "cross validation: 30it [00:12,  2.03it/s]\n",
      "\n",
      "4. Getting final performance...\n",
      "best_params_out:  [{'node_kernels': {'symb': <function deltakernel at 0x7f3a99093950>, 'nsymb': <function gaussiankernel at 0x7f3a990931e0>, 'mix': functools.partial(<function kernelproduct at 0x7f3a99088ae8>, <function deltakernel at 0x7f3a99093950>, <function gaussiankernel at 0x7f3a990931e0>)}, 'n_jobs': 8}]\n",
      "best_params_in:  [{'alpha': 1e-06}]\n",
      "\n",
      "best_val_perf:  9.55244065682399\n",
      "best_val_std:  0.5574811966683159\n",
      "final_performance:  [9.724426192585643]\n",
      "final_confidence:  [2.999822095078807]\n",
      "train_performance: [6.141755071354953]\n",
      "train_std:  [0.2732168016478284]\n",
      "\n",
      "time to calculate gram matrix with different hyper-params: 16.95±nans\n",
      "time to calculate best gram matrix: 16.95±nans\n",
      "total training time with all hyper-param choices: 32.74s\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/usr/local/lib/python3.6/dist-packages/numpy/core/_methods.py:140: RuntimeWarning: Degrees of freedom <= 0 for slice\n",
      "  keepdims=keepdims)\n",
      "/usr/local/lib/python3.6/dist-packages/numpy/core/_methods.py:132: RuntimeWarning: invalid value encountered in double_scalars\n",
      "  ret = ret.dtype.type(ret / rcount)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Filename: ../../gklearn/utils/model_selection_precomputed.py\n",
      "\n",
      "Line #    Mem usage    Increment   Line Contents\n",
      "================================================\n",
      "    24    115.2 MiB    115.2 MiB   @profile\n",
      "    25                             def model_selection_for_precomputed_kernel(datafile,\n",
      "    26                                                                        estimator,\n",
      "    27                                                                        param_grid_precomputed,\n",
      "    28                                                                        param_grid,\n",
      "    29                                                                        model_type,\n",
      "    30                                                                        NUM_TRIALS=30,\n",
      "    31                                                                        datafile_y=None,\n",
      "    32                                                                        extra_params=None,\n",
      "    33                                                                        ds_name='ds-unknown',\n",
      "    34                                                                        n_jobs=1,\n",
      "    35                                                                        read_gm_from_file=False):\n",
      "    36                                 \"\"\"Perform model selection, fitting and testing for precomputed kernels using nested cv. Print out neccessary data during the process then finally the results.\n",
      "    37                             \n",
      "    38                                 Parameters\n",
      "    39                                 ----------\n",
      "    40                                 datafile : string\n",
      "    41                                     Path of dataset file.\n",
      "    42                                 estimator : function\n",
      "    43                                     kernel function used to estimate. This function needs to return a gram matrix.\n",
      "    44                                 param_grid_precomputed : dictionary\n",
      "    45                                     Dictionary with names (string) of parameters used to calculate gram matrices as keys and lists of parameter settings to try as values. This enables searching over any sequence of parameter settings. Params with length 1 will be omitted.\n",
      "    46                                 param_grid : dictionary\n",
      "    47                                     Dictionary with names (string) of parameters used as penelties as keys and lists of parameter settings to try as values. This enables searching over any sequence of parameter settings. Params with length 1 will be omitted.\n",
      "    48                                 model_type : string\n",
      "    49                                     Typr of the problem, can be regression or classification.\n",
      "    50                                 NUM_TRIALS : integer\n",
      "    51                                     Number of random trials of outer cv loop. The default is 30.\n",
      "    52                                 datafile_y : string\n",
      "    53                                     Path of file storing y data. This parameter is optional depending on the given dataset file.\n",
      "    54                                 read_gm_from_file : boolean\n",
      "    55                                     Whether gram matrices are loaded from file.\n",
      "    56                             \n",
      "    57                                 Examples\n",
      "    58                                 --------\n",
      "    59                                 >>> import numpy as np\n",
      "    60                                 >>> import sys\n",
      "    61                                 >>> sys.path.insert(0, \"../\")\n",
      "    62                                 >>> from gklearn.utils.model_selection_precomputed import model_selection_for_precomputed_kernel\n",
      "    63                                 >>> from gklearn.kernels.weisfeilerLehmanKernel import weisfeilerlehmankernel\n",
      "    64                                 >>>\n",
      "    65                                 >>> datafile = '../../../../datasets/acyclic/Acyclic/dataset_bps.ds'\n",
      "    66                                 >>> estimator = weisfeilerlehmankernel\n",
      "    67                                 >>> param_grid_precomputed = {'height': [0,1,2,3,4,5,6,7,8,9,10], 'base_kernel': ['subtree']}\n",
      "    68                                 >>> param_grid = {\"alpha\": np.logspace(-2, 2, num = 10, base = 10)}\n",
      "    69                                 >>>\n",
      "    70                                 >>> model_selection_for_precomputed_kernel(datafile, estimator, param_grid_precomputed, param_grid, 'regression')\n",
      "    71                                 \"\"\"\n",
      "    72    115.2 MiB      0.0 MiB       tqdm.monitor_interval = 0\n",
      "    73                             \n",
      "    74    115.2 MiB      0.0 MiB       results_dir = '../notebooks/results/' + estimator.__name__\n",
      "    75    115.2 MiB      0.0 MiB       if not os.path.exists(results_dir):\n",
      "    76                                     os.makedirs(results_dir)\n",
      "    77                                 # a string to save all the results.\n",
      "    78    115.2 MiB      0.0 MiB       str_fw = '###################### log time: ' + datetime.datetime.now().strftime(\"%Y-%m-%d %H:%M:%S\") + '. ######################\\n\\n'\n",
      "    79    115.2 MiB      0.0 MiB       str_fw += '# This file contains results of ' + estimator.__name__ + ' on dataset ' + ds_name + ',\\n# including gram matrices, serial numbers for gram matrix figures and performance.\\n\\n'\n",
      "    80                             \n",
      "    81                                 # setup the model type\n",
      "    82    115.2 MiB      0.0 MiB       model_type = model_type.lower()\n",
      "    83    115.2 MiB      0.0 MiB       if model_type != 'regression' and model_type != 'classification':\n",
      "    84                                     raise Exception(\n",
      "    85                                         'The model type is incorrect! Please choose from regression or classification.'\n",
      "    86                                     )\n",
      "    87    115.2 MiB      0.0 MiB       print()\n",
      "    88    115.2 MiB      0.0 MiB       print('--- This is a %s problem ---' % model_type)\n",
      "    89    115.2 MiB      0.0 MiB       str_fw += 'This is a %s problem.\\n' % model_type\n",
      "    90                                 \n",
      "    91                                 # calculate gram matrices rather than read them from file.\n",
      "    92    115.2 MiB      0.0 MiB       if read_gm_from_file == False:\n",
      "    93                                     # Load the dataset\n",
      "    94    115.2 MiB      0.0 MiB           print()\n",
      "    95    115.2 MiB      0.0 MiB           print('\\n1. Loading dataset from file...')\n",
      "    96    115.2 MiB      0.0 MiB           if isinstance(datafile, str):\n",
      "    97    115.2 MiB      0.0 MiB               dataset, y_all = loadDataset(\n",
      "    98    116.3 MiB      1.1 MiB                       datafile, filename_y=datafile_y, extra_params=extra_params)\n",
      "    99                                     else: # load data directly from variable.\n",
      "   100                                         dataset = datafile\n",
      "   101                                         y_all = datafile_y                \n",
      "   102                             \n",
      "   103                                     #     import matplotlib.pyplot as plt\n",
      "   104                                     #     import networkx as nx\n",
      "   105                                     #     nx.draw_networkx(dataset[30])\n",
      "   106                                     #     plt.show()\n",
      "   107                                 \n",
      "   108                                     # Grid of parameters with a discrete number of values for each.\n",
      "   109    116.3 MiB      0.0 MiB           param_list_precomputed = list(ParameterGrid(param_grid_precomputed))\n",
      "   110    116.3 MiB      0.0 MiB           param_list = list(ParameterGrid(param_grid))\n",
      "   111                                 \n",
      "   112    116.3 MiB      0.0 MiB           gram_matrices = [\n",
      "   113                                     ]  # a list to store gram matrices for all param_grid_precomputed\n",
      "   114    116.3 MiB      0.0 MiB           gram_matrix_time = [\n",
      "   115                                     ]  # a list to store time to calculate gram matrices\n",
      "   116    116.3 MiB      0.0 MiB           param_list_pre_revised = [\n",
      "   117                                     ]  # list to store param grids precomputed ignoring the useless ones\n",
      "   118                                 \n",
      "   119                                     # calculate all gram matrices\n",
      "   120    116.3 MiB      0.0 MiB           print()\n",
      "   121    116.3 MiB      0.0 MiB           print('2. Calculating gram matrices. This could take a while...')\n",
      "   122    116.3 MiB      0.0 MiB           str_fw += '\\nII. Gram matrices.\\n\\n'\n",
      "   123    116.3 MiB      0.0 MiB           tts = time.time()  # start training time\n",
      "   124    116.3 MiB      0.0 MiB           nb_gm_ignore = 0  # the number of gram matrices those should not be considered, as they may contain elements that are not numbers (NaN)\n",
      "   125    145.3 MiB      0.0 MiB           for idx, params_out in enumerate(param_list_precomputed):\n",
      "   126    116.3 MiB      0.0 MiB               y = y_all[:]\n",
      "   127    116.3 MiB      0.0 MiB               params_out['n_jobs'] = n_jobs\n",
      "   128                             #            print(dataset)\n",
      "   129                             #            import networkx as nx\n",
      "   130                             #            nx.draw_networkx(dataset[1])\n",
      "   131                             #            plt.show()\n",
      "   132    119.5 MiB      3.1 MiB               rtn_data = estimator(dataset[:], **params_out)\n",
      "   133    119.5 MiB      0.0 MiB               Kmatrix = rtn_data[0]\n",
      "   134    119.5 MiB      0.0 MiB               current_run_time = rtn_data[1]\n",
      "   135                                         # for some kernels, some graphs in datasets may not meet the \n",
      "   136                                         # kernels' requirements for graph structure. These graphs are trimmed. \n",
      "   137    119.5 MiB      0.0 MiB               if len(rtn_data) == 3:\n",
      "   138    119.5 MiB      0.0 MiB                   idx_trim = rtn_data[2]  # the index of trimmed graph list\n",
      "   139    119.5 MiB      0.0 MiB                   y = [y[idxt] for idxt in idx_trim] # trim y accordingly\n",
      "   140                             #            Kmatrix = np.random.rand(2250, 2250)\n",
      "   141                             #            current_run_time = 0.1\n",
      "   142                                         \n",
      "   143                                         # remove graphs whose kernels with themselves are zeros\n",
      "   144    119.5 MiB      0.0 MiB               Kmatrix_diag = Kmatrix.diagonal().copy()\n",
      "   145    119.5 MiB      0.0 MiB               nb_g_ignore = 0\n",
      "   146    119.5 MiB      0.0 MiB               for idxk, diag in enumerate(Kmatrix_diag):\n",
      "   147    119.5 MiB      0.0 MiB                   if diag == 0:\n",
      "   148                                                 Kmatrix = np.delete(Kmatrix, (idxk - nb_g_ignore), axis=0)\n",
      "   149                                                 Kmatrix = np.delete(Kmatrix, (idxk - nb_g_ignore), axis=1)\n",
      "   150                                                 nb_g_ignore += 1\n",
      "   151                                         # normalization\n",
      "   152    119.5 MiB      0.0 MiB               Kmatrix_diag = Kmatrix.diagonal().copy()\n",
      "   153    119.5 MiB      0.0 MiB               for i in range(len(Kmatrix)):\n",
      "   154    119.5 MiB      0.0 MiB                   for j in range(i, len(Kmatrix)):\n",
      "   155    119.5 MiB      0.0 MiB                       Kmatrix[i][j] /= np.sqrt(Kmatrix_diag[i] * Kmatrix_diag[j])\n",
      "   156    119.5 MiB      0.0 MiB                       Kmatrix[j][i] = Kmatrix[i][j]\n",
      "   157                                 \n",
      "   158    119.5 MiB      0.0 MiB               print()\n",
      "   159    119.5 MiB      0.0 MiB               if params_out == {}:\n",
      "   160                                             print('the gram matrix is: ')\n",
      "   161                                             str_fw += 'the gram matrix is:\\n\\n'\n",
      "   162                                         else:\n",
      "   163    119.5 MiB      0.0 MiB                   print('the gram matrix with parameters', params_out, 'is: \\n\\n')\n",
      "   164    119.5 MiB      0.0 MiB                   str_fw += 'the gram matrix with parameters %s is:\\n\\n' % params_out\n",
      "   165    119.5 MiB      0.0 MiB               if len(Kmatrix) < 2:\n",
      "   166                                             nb_gm_ignore += 1\n",
      "   167                                             print('ignored, as at most only one of all its diagonal value is non-zero.')\n",
      "   168                                             str_fw += 'ignored, as at most only one of all its diagonal value is non-zero.\\n\\n'\n",
      "   169                                         else:                \n",
      "   170    119.5 MiB      0.0 MiB                   if np.isnan(Kmatrix).any(\n",
      "   171                                             ):  # if the matrix contains elements that are not numbers\n",
      "   172                                                 nb_gm_ignore += 1\n",
      "   173                                                 print('ignored, as it contains elements that are not numbers.')\n",
      "   174                                                 str_fw += 'ignored, as it contains elements that are not numbers.\\n\\n'\n",
      "   175                                             else:\n",
      "   176                             #                    print(Kmatrix)\n",
      "   177    119.5 MiB      0.0 MiB                       str_fw += np.array2string(\n",
      "   178    119.5 MiB      0.0 MiB                               Kmatrix,\n",
      "   179    119.5 MiB      0.0 MiB                               separator=',') + '\\n\\n'\n",
      "   180                             #                            separator=',',\n",
      "   181                             #                            threshold=np.inf,\n",
      "   182                             #                            floatmode='unique') + '\\n\\n'\n",
      "   183                             \n",
      "   184    119.5 MiB      0.0 MiB                       fig_file_name = results_dir + '/GM[ds]' + ds_name\n",
      "   185    119.5 MiB      0.0 MiB                       if params_out != {}:\n",
      "   186    119.5 MiB      0.0 MiB                           fig_file_name += '[params]' + str(idx)\n",
      "   187    120.3 MiB      0.7 MiB                       plt.imshow(Kmatrix)\n",
      "   188    120.4 MiB      0.1 MiB                       plt.colorbar()\n",
      "   189    145.3 MiB     24.9 MiB                       plt.savefig(fig_file_name + '.eps', format='eps', dpi=300)\n",
      "   190                             #                    plt.show()\n",
      "   191    145.3 MiB      0.0 MiB                       plt.clf()\n",
      "   192    145.3 MiB      0.0 MiB                       gram_matrices.append(Kmatrix)\n",
      "   193    145.3 MiB      0.0 MiB                       gram_matrix_time.append(current_run_time)\n",
      "   194    145.3 MiB      0.0 MiB                       param_list_pre_revised.append(params_out)\n",
      "   195    145.3 MiB      0.0 MiB                       if nb_g_ignore > 0:\n",
      "   196                                                     print(', where %d graphs are ignored as their graph kernels with themselves are zeros.' % nb_g_ignore)\n",
      "   197                                                     str_fw += ', where %d graphs are ignored as their graph kernels with themselves are zeros.' % nb_g_ignore\n",
      "   198    145.3 MiB      0.0 MiB           print()\n",
      "   199    145.3 MiB      0.0 MiB           print(\n",
      "   200    145.3 MiB      0.0 MiB               '{} gram matrices are calculated, {} of which are ignored.'.format(\n",
      "   201    145.3 MiB      0.0 MiB                   len(param_list_precomputed), nb_gm_ignore))\n",
      "   202    145.3 MiB      0.0 MiB           str_fw += '{} gram matrices are calculated, {} of which are ignored.\\n\\n'.format(len(param_list_precomputed), nb_gm_ignore)\n",
      "   203    145.3 MiB      0.0 MiB           str_fw += 'serial numbers of gram matrix figures and their corresponding parameters settings:\\n\\n'\n",
      "   204    145.3 MiB      0.0 MiB           str_fw += ''.join([\n",
      "   205    145.3 MiB      0.0 MiB               '{}: {}\\n'.format(idx, params_out)\n",
      "   206    145.3 MiB      0.0 MiB               for idx, params_out in enumerate(param_list_precomputed)\n",
      "   207                                     ])\n",
      "   208                                 \n",
      "   209    145.3 MiB      0.0 MiB           print()\n",
      "   210    145.3 MiB      0.0 MiB           if len(gram_matrices) == 0:\n",
      "   211                                         print('all gram matrices are ignored, no results obtained.')\n",
      "   212                                         str_fw += '\\nall gram matrices are ignored, no results obtained.\\n\\n'\n",
      "   213                                     else:\n",
      "   214                                         # save gram matrices to file.\n",
      "   215    145.4 MiB      0.1 MiB               np.savez(results_dir + '/' + ds_name + '.gm', \n",
      "   216    145.4 MiB      0.0 MiB                        gms=gram_matrices, params=param_list_pre_revised, y=y, \n",
      "   217    145.4 MiB      0.0 MiB                        gmtime=gram_matrix_time)\n",
      "   218                                         \n",
      "   219    145.4 MiB      0.0 MiB               print(\n",
      "   220    145.4 MiB      0.0 MiB                   '3. Fitting and predicting using nested cross validation. This could really take a while...'\n",
      "   221                                         )\n",
      "   222                                         \n",
      "   223                                         # ---- use pool.imap_unordered to parallel and track progress. ----\n",
      "   224                             #            train_pref = []\n",
      "   225                             #            val_pref = []\n",
      "   226                             #            test_pref = []\n",
      "   227                             #            def func_assign(result, var_to_assign):\n",
      "   228                             #                for idx, itm in enumerate(var_to_assign):\n",
      "   229                             #                    itm.append(result[idx])                \n",
      "   230                             #            trial_do_partial = partial(trial_do, param_list_pre_revised, param_list, y, model_type)\n",
      "   231                             #                      \n",
      "   232                             #            parallel_me(trial_do_partial, range(NUM_TRIALS), func_assign, \n",
      "   233                             #                        [train_pref, val_pref, test_pref], glbv=gram_matrices,\n",
      "   234                             #                        method='imap_unordered', n_jobs=n_jobs, chunksize=1,\n",
      "   235                             #                        itr_desc='cross validation')\n",
      "   236                                         \n",
      "   237    145.4 MiB      0.0 MiB               def init_worker(gms_toshare):\n",
      "   238                                             global G_gms\n",
      "   239                                             G_gms = gms_toshare\n",
      "   240                                         \n",
      "   241                             #            gram_matrices = np.array(gram_matrices)\n",
      "   242                             #            gms_shape = gram_matrices.shape\n",
      "   243                             #            gms_array = Array('d', np.reshape(gram_matrices.copy(), -1, order='C'))\n",
      "   244                             #            pool = Pool(processes=n_jobs, initializer=init_worker, initargs=(gms_array, gms_shape))\n",
      "   245    145.4 MiB      0.0 MiB               pool = Pool(processes=n_jobs, initializer=init_worker, initargs=(gram_matrices,))\n",
      "   246    145.4 MiB      0.0 MiB               trial_do_partial = partial(parallel_trial_do, param_list_pre_revised, param_list, y, model_type)\n",
      "   247    145.4 MiB      0.0 MiB               train_pref = []\n",
      "   248    145.4 MiB      0.0 MiB               val_pref = []\n",
      "   249    145.4 MiB      0.0 MiB               test_pref = []\n",
      "   250                             #            if NUM_TRIALS < 1000 * n_jobs:\n",
      "   251                             #                chunksize = int(NUM_TRIALS / n_jobs) + 1\n",
      "   252                             #            else:\n",
      "   253                             #                chunksize = 1000\n",
      "   254    145.4 MiB      0.0 MiB               chunksize = 1\n",
      "   255    145.4 MiB      0.0 MiB               for o1, o2, o3 in tqdm(pool.imap_unordered(trial_do_partial, range(NUM_TRIALS), chunksize), desc='cross validation', file=sys.stdout):\n",
      "   256    145.4 MiB      0.0 MiB                   train_pref.append(o1)\n",
      "   257    145.4 MiB      0.0 MiB                   val_pref.append(o2)\n",
      "   258    145.4 MiB      0.0 MiB                   test_pref.append(o3)\n",
      "   259    145.4 MiB      0.0 MiB               pool.close()\n",
      "   260    145.4 MiB      0.0 MiB               pool.join()\n",
      "   261                                 \n",
      "   262                             #            # ---- use pool.map to parallel. ----\n",
      "   263                             #            pool =  Pool(n_jobs)\n",
      "   264                             #            trial_do_partial = partial(trial_do, param_list_pre_revised, param_list, gram_matrices, y[0:250], model_type)\n",
      "   265                             #            result_perf = pool.map(trial_do_partial, range(NUM_TRIALS))\n",
      "   266                             #            train_pref = [item[0] for item in result_perf]\n",
      "   267                             #            val_pref = [item[1] for item in result_perf]\n",
      "   268                             #            test_pref = [item[2] for item in result_perf]\n",
      "   269                                 \n",
      "   270                             #            # ---- direct running, normally use a single CPU core. ----\n",
      "   271                             #            train_pref = []\n",
      "   272                             #            val_pref = []\n",
      "   273                             #            test_pref = []\n",
      "   274                             #            for i in tqdm(range(NUM_TRIALS), desc='cross validation', file=sys.stdout):\n",
      "   275                             #                o1, o2, o3 = trial_do(param_list_pre_revised, param_list, gram_matrices, y, model_type, i)\n",
      "   276                             #                train_pref.append(o1)\n",
      "   277                             #                val_pref.append(o2)\n",
      "   278                             #                test_pref.append(o3)\n",
      "   279                             #            print()\n",
      "   280                                 \n",
      "   281    145.4 MiB      0.0 MiB               print()\n",
      "   282    145.4 MiB      0.0 MiB               print('4. Getting final performance...')\n",
      "   283    145.4 MiB      0.0 MiB               str_fw += '\\nIII. Performance.\\n\\n'\n",
      "   284                                         # averages and confidences of performances on outer trials for each combination of parameters\n",
      "   285    145.4 MiB      0.0 MiB               average_train_scores = np.mean(train_pref, axis=0)\n",
      "   286                             #            print('val_pref: ', val_pref[0][0])\n",
      "   287    145.4 MiB      0.0 MiB               average_val_scores = np.mean(val_pref, axis=0)\n",
      "   288                             #            print('test_pref: ', test_pref[0][0])\n",
      "   289    145.4 MiB      0.0 MiB               average_perf_scores = np.mean(test_pref, axis=0)\n",
      "   290                                         # sample std is used here\n",
      "   291    145.4 MiB      0.0 MiB               std_train_scores = np.std(train_pref, axis=0, ddof=1)\n",
      "   292    145.4 MiB      0.0 MiB               std_val_scores = np.std(val_pref, axis=0, ddof=1)\n",
      "   293    145.4 MiB      0.0 MiB               std_perf_scores = np.std(test_pref, axis=0, ddof=1)\n",
      "   294                                 \n",
      "   295    145.4 MiB      0.0 MiB               if model_type == 'regression':\n",
      "   296    145.4 MiB      0.0 MiB                   best_val_perf = np.amin(average_val_scores)\n",
      "   297                                         else:\n",
      "   298                                             best_val_perf = np.amax(average_val_scores)\n",
      "   299                             #            print('average_val_scores: ', average_val_scores)\n",
      "   300                             #            print('best_val_perf: ', best_val_perf)\n",
      "   301                             #            print()\n",
      "   302    145.4 MiB      0.0 MiB               best_params_index = np.where(average_val_scores == best_val_perf)\n",
      "   303                                         # find smallest val std with best val perf.\n",
      "   304                                         best_val_stds = [\n",
      "   305    145.4 MiB      0.0 MiB                   std_val_scores[value][best_params_index[1][idx]]\n",
      "   306    145.4 MiB      0.0 MiB                   for idx, value in enumerate(best_params_index[0])\n",
      "   307                                         ]\n",
      "   308    145.4 MiB      0.0 MiB               min_val_std = np.amin(best_val_stds)\n",
      "   309    145.4 MiB      0.0 MiB               best_params_index = np.where(std_val_scores == min_val_std)\n",
      "   310                                         best_params_out = [\n",
      "   311    145.4 MiB      0.0 MiB                   param_list_pre_revised[i] for i in best_params_index[0]\n",
      "   312                                         ]\n",
      "   313    145.4 MiB      0.0 MiB               best_params_in = [param_list[i] for i in best_params_index[1]]\n",
      "   314    145.4 MiB      0.0 MiB               print('best_params_out: ', best_params_out)\n",
      "   315    145.4 MiB      0.0 MiB               print('best_params_in: ', best_params_in)\n",
      "   316    145.4 MiB      0.0 MiB               print()\n",
      "   317    145.4 MiB      0.0 MiB               print('best_val_perf: ', best_val_perf)\n",
      "   318    145.4 MiB      0.0 MiB               print('best_val_std: ', min_val_std)\n",
      "   319    145.4 MiB      0.0 MiB               str_fw += 'best settings of hyper-params to build gram matrix: %s\\n' % best_params_out\n",
      "   320    145.4 MiB      0.0 MiB               str_fw += 'best settings of other hyper-params: %s\\n\\n' % best_params_in\n",
      "   321    145.4 MiB      0.0 MiB               str_fw += 'best_val_perf: %s\\n' % best_val_perf\n",
      "   322    145.4 MiB      0.0 MiB               str_fw += 'best_val_std: %s\\n' % min_val_std\n",
      "   323                                 \n",
      "   324                             #            print(best_params_index)\n",
      "   325                             #            print(best_params_index[0])\n",
      "   326                             #            print(average_perf_scores)\n",
      "   327                                         final_performance = [\n",
      "   328    145.4 MiB      0.0 MiB                   average_perf_scores[value][best_params_index[1][idx]]\n",
      "   329    145.4 MiB      0.0 MiB                   for idx, value in enumerate(best_params_index[0])\n",
      "   330                                         ]\n",
      "   331                                         final_confidence = [\n",
      "   332    145.4 MiB      0.0 MiB                   std_perf_scores[value][best_params_index[1][idx]]\n",
      "   333    145.4 MiB      0.0 MiB                   for idx, value in enumerate(best_params_index[0])\n",
      "   334                                         ]\n",
      "   335    145.4 MiB      0.0 MiB               print('final_performance: ', final_performance)\n",
      "   336    145.4 MiB      0.0 MiB               print('final_confidence: ', final_confidence)\n",
      "   337    145.4 MiB      0.0 MiB               str_fw += 'final_performance: %s\\n' % final_performance\n",
      "   338    145.4 MiB      0.0 MiB               str_fw += 'final_confidence: %s\\n' % final_confidence\n",
      "   339                                         train_performance = [\n",
      "   340    145.4 MiB      0.0 MiB                   average_train_scores[value][best_params_index[1][idx]]\n",
      "   341    145.4 MiB      0.0 MiB                   for idx, value in enumerate(best_params_index[0])\n",
      "   342                                         ]\n",
      "   343                                         train_std = [\n",
      "   344    145.4 MiB      0.0 MiB                   std_train_scores[value][best_params_index[1][idx]]\n",
      "   345    145.4 MiB      0.0 MiB                   for idx, value in enumerate(best_params_index[0])\n",
      "   346                                         ]\n",
      "   347    145.4 MiB      0.0 MiB               print('train_performance: %s' % train_performance)\n",
      "   348    145.4 MiB      0.0 MiB               print('train_std: ', train_std)\n",
      "   349    145.4 MiB      0.0 MiB               str_fw += 'train_performance: %s\\n' % train_performance\n",
      "   350    145.4 MiB      0.0 MiB               str_fw += 'train_std: %s\\n\\n' % train_std\n",
      "   351                                 \n",
      "   352    145.4 MiB      0.0 MiB               print()\n",
      "   353    145.4 MiB      0.0 MiB               tt_total = time.time() - tts  # training time for all hyper-parameters\n",
      "   354    145.4 MiB      0.0 MiB               average_gram_matrix_time = np.mean(gram_matrix_time)\n",
      "   355    145.4 MiB      0.0 MiB               std_gram_matrix_time = np.std(gram_matrix_time, ddof=1)\n",
      "   356                                         best_gram_matrix_time = [\n",
      "   357    145.4 MiB      0.0 MiB                   gram_matrix_time[i] for i in best_params_index[0]\n",
      "   358                                         ]\n",
      "   359    145.4 MiB      0.0 MiB               ave_bgmt = np.mean(best_gram_matrix_time)\n",
      "   360    145.4 MiB      0.0 MiB               std_bgmt = np.std(best_gram_matrix_time, ddof=1)\n",
      "   361    145.4 MiB      0.0 MiB               print(\n",
      "   362    145.4 MiB      0.0 MiB                   'time to calculate gram matrix with different hyper-params: {:.2f}±{:.2f}s'\n",
      "   363    145.4 MiB      0.0 MiB                   .format(average_gram_matrix_time, std_gram_matrix_time))\n",
      "   364    145.4 MiB      0.0 MiB               print('time to calculate best gram matrix: {:.2f}±{:.2f}s'.format(\n",
      "   365    145.4 MiB      0.0 MiB                   ave_bgmt, std_bgmt))\n",
      "   366    145.4 MiB      0.0 MiB               print(\n",
      "   367    145.4 MiB      0.0 MiB                   'total training time with all hyper-param choices: {:.2f}s'.format(\n",
      "   368    145.4 MiB      0.0 MiB                       tt_total))\n",
      "   369    145.4 MiB      0.0 MiB               str_fw += 'time to calculate gram matrix with different hyper-params: {:.2f}±{:.2f}s\\n'.format(average_gram_matrix_time, std_gram_matrix_time)\n",
      "   370    145.4 MiB      0.0 MiB               str_fw += 'time to calculate best gram matrix: {:.2f}±{:.2f}s\\n'.format(ave_bgmt, std_bgmt)\n",
      "   371    145.4 MiB      0.0 MiB               str_fw += 'total training time with all hyper-param choices: {:.2f}s\\n\\n'.format(tt_total)\n",
      "   372                                 \n",
      "   373                                         # # save results to file\n",
      "   374                                         # np.savetxt(results_name_pre + 'average_train_scores.dt',\n",
      "   375                                         #            average_train_scores)\n",
      "   376                                         # np.savetxt(results_name_pre + 'average_val_scores', average_val_scores)\n",
      "   377                                         # np.savetxt(results_name_pre + 'average_perf_scores.dt',\n",
      "   378                                         #            average_perf_scores)\n",
      "   379                                         # np.savetxt(results_name_pre + 'std_train_scores.dt', std_train_scores)\n",
      "   380                                         # np.savetxt(results_name_pre + 'std_val_scores.dt', std_val_scores)\n",
      "   381                                         # np.savetxt(results_name_pre + 'std_perf_scores.dt', std_perf_scores)\n",
      "   382                                 \n",
      "   383                                         # np.save(results_name_pre + 'best_params_index', best_params_index)\n",
      "   384                                         # np.save(results_name_pre + 'best_params_pre.dt', best_params_out)\n",
      "   385                                         # np.save(results_name_pre + 'best_params_in.dt', best_params_in)\n",
      "   386                                         # np.save(results_name_pre + 'best_val_perf.dt', best_val_perf)\n",
      "   387                                         # np.save(results_name_pre + 'best_val_std.dt', best_val_std)\n",
      "   388                                         # np.save(results_name_pre + 'final_performance.dt', final_performance)\n",
      "   389                                         # np.save(results_name_pre + 'final_confidence.dt', final_confidence)\n",
      "   390                                         # np.save(results_name_pre + 'train_performance.dt', train_performance)\n",
      "   391                                         # np.save(results_name_pre + 'train_std.dt', train_std)\n",
      "   392                                 \n",
      "   393                                         # np.save(results_name_pre + 'gram_matrix_time.dt', gram_matrix_time)\n",
      "   394                                         # np.save(results_name_pre + 'average_gram_matrix_time.dt',\n",
      "   395                                         #         average_gram_matrix_time)\n",
      "   396                                         # np.save(results_name_pre + 'std_gram_matrix_time.dt',\n",
      "   397                                         #         std_gram_matrix_time)\n",
      "   398                                         # np.save(results_name_pre + 'best_gram_matrix_time.dt',\n",
      "   399                                         #         best_gram_matrix_time)\n",
      "   400                                 \n",
      "   401                                         # print out as table.\n",
      "   402    145.4 MiB      0.0 MiB               from collections import OrderedDict\n",
      "   403    145.4 MiB      0.0 MiB               from tabulate import tabulate\n",
      "   404    145.4 MiB      0.0 MiB               table_dict = {}\n",
      "   405    145.4 MiB      0.0 MiB               if model_type == 'regression':\n",
      "   406    145.6 MiB      0.0 MiB                   for param_in in param_list:\n",
      "   407    145.6 MiB      0.2 MiB                       param_in['alpha'] = '{:.2e}'.format(param_in['alpha'])\n",
      "   408                                         else:\n",
      "   409                                             for param_in in param_list:\n",
      "   410                                                 param_in['C'] = '{:.2e}'.format(param_in['C'])\n",
      "   411    145.6 MiB      0.0 MiB               table_dict['params'] = [{**param_out, **param_in}\n",
      "   412    145.6 MiB      0.0 MiB                                       for param_in in param_list for param_out in param_list_pre_revised]\n",
      "   413                                         table_dict['gram_matrix_time'] = [\n",
      "   414    145.6 MiB      0.0 MiB                   '{:.2f}'.format(gram_matrix_time[index_out])\n",
      "   415    145.6 MiB      0.0 MiB                   for param_in in param_list\n",
      "   416    145.6 MiB      0.0 MiB                   for index_out, _ in enumerate(param_list_pre_revised)\n",
      "   417                                         ]\n",
      "   418                                         table_dict['valid_perf'] = [\n",
      "   419    145.6 MiB      0.0 MiB                   '{:.2f}±{:.2f}'.format(average_val_scores[index_out][index_in],\n",
      "   420                                                                    std_val_scores[index_out][index_in])\n",
      "   421    145.6 MiB      0.0 MiB                   for index_in, _ in enumerate(param_list)\n",
      "   422    145.6 MiB      0.0 MiB                   for index_out, _ in enumerate(param_list_pre_revised)\n",
      "   423                                         ]\n",
      "   424                                         table_dict['test_perf'] = [\n",
      "   425    145.6 MiB      0.0 MiB                   '{:.2f}±{:.2f}'.format(average_perf_scores[index_out][index_in],\n",
      "   426                                                                    std_perf_scores[index_out][index_in])\n",
      "   427    145.6 MiB      0.0 MiB                   for index_in, _ in enumerate(param_list)\n",
      "   428    145.6 MiB      0.0 MiB                   for index_out, _ in enumerate(param_list_pre_revised)\n",
      "   429                                         ]\n",
      "   430                                         table_dict['train_perf'] = [\n",
      "   431    145.6 MiB      0.0 MiB                   '{:.2f}±{:.2f}'.format(average_train_scores[index_out][index_in],\n",
      "   432                                                                    std_train_scores[index_out][index_in])\n",
      "   433    145.6 MiB      0.0 MiB                   for index_in, _ in enumerate(param_list)\n",
      "   434    145.6 MiB      0.0 MiB                   for index_out, _ in enumerate(param_list_pre_revised)\n",
      "   435                                         ]\n",
      "   436                                         keyorder = [\n",
      "   437    145.6 MiB      0.0 MiB                   'params', 'train_perf', 'valid_perf', 'test_perf',\n",
      "   438    145.6 MiB      0.0 MiB                   'gram_matrix_time'\n",
      "   439                                         ]\n",
      "   440    145.6 MiB      0.0 MiB               print()\n",
      "   441    145.6 MiB      0.0 MiB               tb_print = tabulate(\n",
      "   442    145.6 MiB      0.0 MiB                   OrderedDict(\n",
      "   443    145.6 MiB      0.0 MiB                       sorted(table_dict.items(),\n",
      "   444    145.6 MiB      0.0 MiB                              key=lambda i: keyorder.index(i[0]))),\n",
      "   445    145.6 MiB      0.0 MiB                   headers='keys')\n",
      "   446                             #            print(tb_print)\n",
      "   447    145.6 MiB      0.0 MiB               str_fw += 'table of performance v.s. hyper-params:\\n\\n%s\\n\\n' % tb_print\n",
      "   448                                 \n",
      "   449                                 # read gram matrices from file.\n",
      "   450                                 else:    \n",
      "   451                                     # Grid of parameters with a discrete number of values for each.\n",
      "   452                             #        param_list_precomputed = list(ParameterGrid(param_grid_precomputed))\n",
      "   453                                     param_list = list(ParameterGrid(param_grid))\n",
      "   454                                 \n",
      "   455                                     # read gram matrices from file.\n",
      "   456                                     print()\n",
      "   457                                     print('2. Reading gram matrices from file...')\n",
      "   458                                     str_fw += '\\nII. Gram matrices.\\n\\nGram matrices are read from file, see last log for detail.\\n'\n",
      "   459                                     gmfile = np.load(results_dir + '/' + ds_name + '.gm.npz')\n",
      "   460                                     gram_matrices = gmfile['gms'] # a list to store gram matrices for all param_grid_precomputed\n",
      "   461                                     gram_matrix_time = gmfile['gmtime'] # time used to compute the gram matrices\n",
      "   462                                     param_list_pre_revised = gmfile['params'] # list to store param grids precomputed ignoring the useless ones\n",
      "   463                                     y = gmfile['y'].tolist()\n",
      "   464                                     \n",
      "   465                                     tts = time.time()  # start training time\n",
      "   466                             #        nb_gm_ignore = 0  # the number of gram matrices those should not be considered, as they may contain elements that are not numbers (NaN)            \n",
      "   467                                     print(\n",
      "   468                                         '3. Fitting and predicting using nested cross validation. This could really take a while...'\n",
      "   469                                     )\n",
      "   470                              \n",
      "   471                                     # ---- use pool.imap_unordered to parallel and track progress. ----\n",
      "   472                                     def init_worker(gms_toshare):\n",
      "   473                                         global G_gms\n",
      "   474                                         G_gms = gms_toshare\n",
      "   475                             \n",
      "   476                                     pool = Pool(processes=n_jobs, initializer=init_worker, initargs=(gram_matrices,))\n",
      "   477                                     trial_do_partial = partial(parallel_trial_do, param_list_pre_revised, param_list, y, model_type)\n",
      "   478                                     train_pref = []\n",
      "   479                                     val_pref = []\n",
      "   480                                     test_pref = []\n",
      "   481                                     chunksize = 1\n",
      "   482                                     for o1, o2, o3 in tqdm(pool.imap_unordered(trial_do_partial, range(NUM_TRIALS), chunksize), desc='cross validation', file=sys.stdout):\n",
      "   483                                         train_pref.append(o1)\n",
      "   484                                         val_pref.append(o2)\n",
      "   485                                         test_pref.append(o3)\n",
      "   486                                     pool.close()\n",
      "   487                                     pool.join()\n",
      "   488                                     \n",
      "   489                                     # # ---- use pool.map to parallel. ----\n",
      "   490                                     # result_perf = pool.map(trial_do_partial, range(NUM_TRIALS))\n",
      "   491                                     # train_pref = [item[0] for item in result_perf]\n",
      "   492                                     # val_pref = [item[1] for item in result_perf]\n",
      "   493                                     # test_pref = [item[2] for item in result_perf]\n",
      "   494                             \n",
      "   495                                     # # ---- use joblib.Parallel to parallel and track progress. ----\n",
      "   496                                     # trial_do_partial = partial(trial_do, param_list_pre_revised, param_list, gram_matrices, y, model_type)\n",
      "   497                                     # result_perf = Parallel(n_jobs=n_jobs, verbose=10)(delayed(trial_do_partial)(trial) for trial in range(NUM_TRIALS))\n",
      "   498                                     # train_pref = [item[0] for item in result_perf]\n",
      "   499                                     # val_pref = [item[1] for item in result_perf]\n",
      "   500                                     # test_pref = [item[2] for item in result_perf]\n",
      "   501                             \n",
      "   502                             #        # ---- direct running, normally use a single CPU core. ----\n",
      "   503                             #        train_pref = []\n",
      "   504                             #        val_pref = []\n",
      "   505                             #        test_pref = []\n",
      "   506                             #        for i in tqdm(range(NUM_TRIALS), desc='cross validation', file=sys.stdout):\n",
      "   507                             #            o1, o2, o3 = trial_do(param_list_pre_revised, param_list, gram_matrices, y, model_type, i)\n",
      "   508                             #            train_pref.append(o1)\n",
      "   509                             #            val_pref.append(o2)\n",
      "   510                             #            test_pref.append(o3)\n",
      "   511                             \n",
      "   512                                     print()\n",
      "   513                                     print('4. Getting final performance...')\n",
      "   514                                     str_fw += '\\nIII. Performance.\\n\\n'\n",
      "   515                                     # averages and confidences of performances on outer trials for each combination of parameters\n",
      "   516                                     average_train_scores = np.mean(train_pref, axis=0)\n",
      "   517                                     average_val_scores = np.mean(val_pref, axis=0)\n",
      "   518                                     average_perf_scores = np.mean(test_pref, axis=0)\n",
      "   519                                     # sample std is used here\n",
      "   520                                     std_train_scores = np.std(train_pref, axis=0, ddof=1)\n",
      "   521                                     std_val_scores = np.std(val_pref, axis=0, ddof=1)\n",
      "   522                                     std_perf_scores = np.std(test_pref, axis=0, ddof=1)\n",
      "   523                             \n",
      "   524                                     if model_type == 'regression':\n",
      "   525                                         best_val_perf = np.amin(average_val_scores)\n",
      "   526                                     else:\n",
      "   527                                         best_val_perf = np.amax(average_val_scores)\n",
      "   528                                     best_params_index = np.where(average_val_scores == best_val_perf)\n",
      "   529                                     # find smallest val std with best val perf.\n",
      "   530                                     best_val_stds = [\n",
      "   531                                         std_val_scores[value][best_params_index[1][idx]]\n",
      "   532                                         for idx, value in enumerate(best_params_index[0])\n",
      "   533                                     ]\n",
      "   534                                     min_val_std = np.amin(best_val_stds)\n",
      "   535                                     best_params_index = np.where(std_val_scores == min_val_std)\n",
      "   536                                     best_params_out = [\n",
      "   537                                         param_list_pre_revised[i] for i in best_params_index[0]\n",
      "   538                                     ]\n",
      "   539                                     best_params_in = [param_list[i] for i in best_params_index[1]]\n",
      "   540                                     print('best_params_out: ', best_params_out)\n",
      "   541                                     print('best_params_in: ', best_params_in)\n",
      "   542                                     print()\n",
      "   543                                     print('best_val_perf: ', best_val_perf)\n",
      "   544                                     print('best_val_std: ', min_val_std)\n",
      "   545                                     str_fw += 'best settings of hyper-params to build gram matrix: %s\\n' % best_params_out\n",
      "   546                                     str_fw += 'best settings of other hyper-params: %s\\n\\n' % best_params_in\n",
      "   547                                     str_fw += 'best_val_perf: %s\\n' % best_val_perf\n",
      "   548                                     str_fw += 'best_val_std: %s\\n' % min_val_std\n",
      "   549                             \n",
      "   550                                     final_performance = [\n",
      "   551                                         average_perf_scores[value][best_params_index[1][idx]]\n",
      "   552                                         for idx, value in enumerate(best_params_index[0])\n",
      "   553                                     ]\n",
      "   554                                     final_confidence = [\n",
      "   555                                         std_perf_scores[value][best_params_index[1][idx]]\n",
      "   556                                         for idx, value in enumerate(best_params_index[0])\n",
      "   557                                     ]\n",
      "   558                                     print('final_performance: ', final_performance)\n",
      "   559                                     print('final_confidence: ', final_confidence)\n",
      "   560                                     str_fw += 'final_performance: %s\\n' % final_performance\n",
      "   561                                     str_fw += 'final_confidence: %s\\n' % final_confidence\n",
      "   562                                     train_performance = [\n",
      "   563                                         average_train_scores[value][best_params_index[1][idx]]\n",
      "   564                                         for idx, value in enumerate(best_params_index[0])\n",
      "   565                                     ]\n",
      "   566                                     train_std = [\n",
      "   567                                         std_train_scores[value][best_params_index[1][idx]]\n",
      "   568                                         for idx, value in enumerate(best_params_index[0])\n",
      "   569                                     ]\n",
      "   570                                     print('train_performance: %s' % train_performance)\n",
      "   571                                     print('train_std: ', train_std)\n",
      "   572                                     str_fw += 'train_performance: %s\\n' % train_performance\n",
      "   573                                     str_fw += 'train_std: %s\\n\\n' % train_std\n",
      "   574                             \n",
      "   575                                     print()\n",
      "   576                                     average_gram_matrix_time = np.mean(gram_matrix_time)\n",
      "   577                                     std_gram_matrix_time = np.std(gram_matrix_time, ddof=1)\n",
      "   578                                     best_gram_matrix_time = [\n",
      "   579                                         gram_matrix_time[i] for i in best_params_index[0]\n",
      "   580                                     ]\n",
      "   581                                     ave_bgmt = np.mean(best_gram_matrix_time)\n",
      "   582                                     std_bgmt = np.std(best_gram_matrix_time, ddof=1)\n",
      "   583                                     print(\n",
      "   584                                         'time to calculate gram matrix with different hyper-params: {:.2f}±{:.2f}s'\n",
      "   585                                         .format(average_gram_matrix_time, std_gram_matrix_time))\n",
      "   586                                     print('time to calculate best gram matrix: {:.2f}±{:.2f}s'.format(\n",
      "   587                                         ave_bgmt, std_bgmt))\n",
      "   588                                     tt_poster = time.time() - tts  # training time with hyper-param choices who did not participate in calculation of gram matrices\n",
      "   589                                     print(\n",
      "   590                                         'training time with hyper-param choices who did not participate in calculation of gram matrices: {:.2f}s'.format(\n",
      "   591                                             tt_poster))\n",
      "   592                                     print('total training time with all hyper-param choices: {:.2f}s'.format(\n",
      "   593                                             tt_poster + np.sum(gram_matrix_time)))\n",
      "   594                             #        str_fw += 'time to calculate gram matrix with different hyper-params: {:.2f}±{:.2f}s\\n'.format(average_gram_matrix_time, std_gram_matrix_time)\n",
      "   595                             #        str_fw += 'time to calculate best gram matrix: {:.2f}±{:.2f}s\\n'.format(ave_bgmt, std_bgmt)\n",
      "   596                                     str_fw += 'training time with hyper-param choices who did not participate in calculation of gram matrices: {:.2f}s\\n\\n'.format(tt_poster)\n",
      "   597                             \n",
      "   598                                     # print out as table.\n",
      "   599                                     from collections import OrderedDict\n",
      "   600                                     from tabulate import tabulate\n",
      "   601                                     table_dict = {}\n",
      "   602                                     if model_type == 'regression':\n",
      "   603                                         for param_in in param_list:\n",
      "   604                                             param_in['alpha'] = '{:.2e}'.format(param_in['alpha'])\n",
      "   605                                     else:\n",
      "   606                                         for param_in in param_list:\n",
      "   607                                             param_in['C'] = '{:.2e}'.format(param_in['C'])\n",
      "   608                                     table_dict['params'] = [{**param_out, **param_in}\n",
      "   609                                                             for param_in in param_list for param_out in param_list_pre_revised]\n",
      "   610                             #        table_dict['gram_matrix_time'] = [\n",
      "   611                             #            '{:.2f}'.format(gram_matrix_time[index_out])\n",
      "   612                             #            for param_in in param_list\n",
      "   613                             #            for index_out, _ in enumerate(param_list_pre_revised)\n",
      "   614                             #        ]\n",
      "   615                                     table_dict['valid_perf'] = [\n",
      "   616                                         '{:.2f}±{:.2f}'.format(average_val_scores[index_out][index_in],\n",
      "   617                                                                std_val_scores[index_out][index_in])\n",
      "   618                                         for index_in, _ in enumerate(param_list)\n",
      "   619                                         for index_out, _ in enumerate(param_list_pre_revised)\n",
      "   620                                     ]\n",
      "   621                                     table_dict['test_perf'] = [\n",
      "   622                                         '{:.2f}±{:.2f}'.format(average_perf_scores[index_out][index_in],\n",
      "   623                                                                std_perf_scores[index_out][index_in])\n",
      "   624                                         for index_in, _ in enumerate(param_list)\n",
      "   625                                         for index_out, _ in enumerate(param_list_pre_revised)\n",
      "   626                                     ]\n",
      "   627                                     table_dict['train_perf'] = [\n",
      "   628                                         '{:.2f}±{:.2f}'.format(average_train_scores[index_out][index_in],\n",
      "   629                                                                std_train_scores[index_out][index_in])\n",
      "   630                                         for index_in, _ in enumerate(param_list)\n",
      "   631                                         for index_out, _ in enumerate(param_list_pre_revised)\n",
      "   632                                     ]\n",
      "   633                                     keyorder = [\n",
      "   634                                         'params', 'train_perf', 'valid_perf', 'test_perf'\n",
      "   635                                     ]\n",
      "   636                                     print()\n",
      "   637                                     tb_print = tabulate(\n",
      "   638                                         OrderedDict(\n",
      "   639                                             sorted(table_dict.items(),\n",
      "   640                                                    key=lambda i: keyorder.index(i[0]))),\n",
      "   641                                         headers='keys')\n",
      "   642                             #        print(tb_print)\n",
      "   643                                     str_fw += 'table of performance v.s. hyper-params:\\n\\n%s\\n\\n' % tb_print\n",
      "   644                             \n",
      "   645                                     # open file to save all results for this dataset.\n",
      "   646                                     if not os.path.exists(results_dir):\n",
      "   647                                         os.makedirs(results_dir)\n",
      "   648                                         \n",
      "   649                                 # open file to save all results for this dataset.\n",
      "   650    145.6 MiB      0.0 MiB       if not os.path.exists(results_dir + '/' + ds_name + '.output.txt'):\n",
      "   651                                     with open(results_dir + '/' + ds_name + '.output.txt', 'w') as f:\n",
      "   652                                         f.write(str_fw)\n",
      "   653                                 else:\n",
      "   654    145.6 MiB      0.0 MiB           with open(results_dir + '/' + ds_name + '.output.txt', 'r+') as f:\n",
      "   655    145.6 MiB      0.0 MiB               content = f.read()\n",
      "   656    145.6 MiB      0.0 MiB               f.seek(0, 0)\n",
      "   657    145.6 MiB      0.0 MiB               f.write(str_fw + '\\n\\n\\n' + content)\n",
      "\n",
      "\n",
      "\n"
     ]
    }
   ],
   "source": [
    "import functools\n",
    "import sys\n",
    "sys.path.insert(0, \"../\")\n",
    "sys.path.insert(0, \"../../\")\n",
    "from libs import *\n",
    "import multiprocessing\n",
    "\n",
    "from gklearn.kernels.spKernel import spkernel\n",
    "from gklearn.utils.kernels import deltakernel, gaussiankernel, kernelproduct\n",
    "#from gklearn.utils.model_selection_precomputed import trial_do\n",
    "\n",
    "dslist = [\n",
    "    {'name': 'Acyclic', 'dataset': '../../datasets/acyclic/dataset_bps.ds',\n",
    "        'task': 'regression'},  # node symb\n",
    "#    {'name': 'Alkane', 'dataset': '../../datasets/Alkane/dataset.ds', 'task': 'regression',\n",
    "#             'dataset_y': '../../datasets/Alkane/dataset_boiling_point_names.txt', },  \n",
    "#    # contains single node graph, node symb\n",
    "#    {'name': 'MAO', 'dataset': '../../datasets/MAO/dataset.ds', },  # node/edge symb\n",
    "#    {'name': 'PAH', 'dataset': '../../datasets/PAH/dataset.ds', },  # unlabeled\n",
    "#    {'name': 'MUTAG', 'dataset': '../../datasets/MUTAG/MUTAG.mat',\n",
    "#             'extra_params': {'am_sp_al_nl_el': [0, 0, 3, 1, 2]}},  # node/edge symb\n",
    "#    {'name': 'Letter-med', 'dataset': '../../datasets/Letter-med/Letter-med_A.txt'},\n",
    "#    # node nsymb\n",
    "#    {'name': 'ENZYMES', 'dataset': '../../datasets/ENZYMES_txt/ENZYMES_A_sparse.txt'},\n",
    "#    # node symb/nsymb\n",
    "#    {'name': 'Mutagenicity', 'dataset': '../../datasets/Mutagenicity/Mutagenicity_A.txt'},\n",
    "#    # node/edge symb\n",
    "#    {'name': 'D&D', 'dataset': '../../datasets/D&D/DD.mat',\n",
    "#     'extra_params': {'am_sp_al_nl_el': [0, 1, 2, 1, -1]}},  # node symb\n",
    "\n",
    "    #     {'name': 'COIL-DEL', 'dataset': '../../datasets/COIL-DEL/COIL-DEL_A.txt'}, # edge symb, node nsymb\n",
    "    # # #     {'name': 'BZR', 'dataset': '../../datasets/BZR_txt/BZR_A_sparse.txt'}, # node symb/nsymb\n",
    "    # # #     {'name': 'COX2', 'dataset': '../../datasets/COX2_txt/COX2_A_sparse.txt'}, # node symb/nsymb\n",
    "    #     {'name': 'Fingerprint', 'dataset': '../../datasets/Fingerprint/Fingerprint_A.txt'},\n",
    "    #\n",
    "    # #     {'name': 'DHFR', 'dataset': '../../datasets/DHFR_txt/DHFR_A_sparse.txt'}, # node symb/nsymb\n",
    "    # #     {'name': 'SYNTHETIC', 'dataset': '../../datasets/SYNTHETIC_txt/SYNTHETIC_A_sparse.txt'}, # node symb/nsymb\n",
    "    # #     {'name': 'MSRC9', 'dataset': '../../datasets/MSRC_9_txt/MSRC_9_A.txt'}, # node symb\n",
    "    # #     {'name': 'MSRC21', 'dataset': '../../datasets/MSRC_21_txt/MSRC_21_A.txt'}, # node symb\n",
    "    # #     {'name': 'FIRSTMM_DB', 'dataset': '../../datasets/FIRSTMM_DB/FIRSTMM_DB_A.txt'}, # node symb/nsymb ,edge nsymb\n",
    "\n",
    "    # #     {'name': 'PROTEINS', 'dataset': '../../datasets/PROTEINS_txt/PROTEINS_A_sparse.txt'}, # node symb/nsymb\n",
    "    # #     {'name': 'PROTEINS_full', 'dataset': '../../datasets/PROTEINS_full_txt/PROTEINS_full_A_sparse.txt'}, # node symb/nsymb\n",
    "    # #     {'name': 'AIDS', 'dataset': '../../datasets/AIDS/AIDS_A.txt'}, # node symb/nsymb, edge symb\n",
    "    #     {'name': 'NCI1', 'dataset': '../../datasets/NCI1/NCI1.mat',\n",
    "    #         'extra_params': {'am_sp_al_nl_el': [1, 1, 2, 0, -1]}}, # node symb\n",
    "    #     {'name': 'NCI109', 'dataset': '../../datasets/NCI109/NCI109.mat',\n",
    "    #         'extra_params': {'am_sp_al_nl_el': [1, 1, 2, 0, -1]}}, # node symb\n",
    "    #     {'name': 'NCI-HIV', 'dataset': '../../datasets/NCI-HIV/AIDO99SD.sdf',\n",
    "    #         'dataset_y': '../../datasets/NCI-HIV/aids_conc_may04.txt',}, # node/edge symb\n",
    "\n",
    "    #     # not working below\n",
    "    #     {'name': 'PTC_FM', 'dataset': '../../datasets/PTC/Train/FM.ds',},\n",
    "    #     {'name': 'PTC_FR', 'dataset': '../../datasets/PTC/Train/FR.ds',},\n",
    "    #     {'name': 'PTC_MM', 'dataset': '../../datasets/PTC/Train/MM.ds',},\n",
    "    #     {'name': 'PTC_MR', 'dataset': '../../datasets/PTC/Train/MR.ds',},\n",
    "]\n",
    "estimator = spkernel\n",
    "mixkernel = functools.partial(kernelproduct, deltakernel, gaussiankernel)\n",
    "param_grid_precomputed = {'node_kernels': [\n",
    "    {'symb': deltakernel, 'nsymb': gaussiankernel, 'mix': mixkernel}]}\n",
    "param_grid = [{'C': np.logspace(-10, 10, num=41, base=10)},\n",
    "              {'alpha': np.logspace(-10, 10, num=41, base=10)}]\n",
    "\n",
    "for ds in dslist:\n",
    "    print()\n",
    "    print(ds['name'])\n",
    "    model_selection_for_precomputed_kernel(\n",
    "        ds['dataset'],\n",
    "        estimator,\n",
    "        param_grid_precomputed,\n",
    "        (param_grid[1] if ('task' in ds and ds['task']\n",
    "                           == 'regression') else param_grid[0]),\n",
    "        (ds['task'] if 'task' in ds else 'classification'),\n",
    "        NUM_TRIALS=30,\n",
    "        datafile_y=(ds['dataset_y'] if 'dataset_y' in ds else None),\n",
    "        extra_params=(ds['extra_params'] if 'extra_params' in ds else None),\n",
    "        ds_name=ds['name'],\n",
    "        n_jobs=multiprocessing.cpu_count(),\n",
    "        read_gm_from_file=False)\n",
    "    print()"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.7"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
