|
123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821 |
- {
- "cells": [
- {
- "cell_type": "code",
- "execution_count": 1,
- "metadata": {
- "scrolled": false
- },
- "outputs": [
- {
- "name": "stdout",
- "output_type": "stream",
- "text": [
- "\n",
- "Acyclic\n",
- "\n",
- "--- This is a regression problem ---\n",
- "\n",
- "\n",
- "1. Loading dataset from file...\n",
- "\n",
- "2. Calculating gram matrices. This could take a while...\n",
- "\n",
- " None edge weight specified. Set all weight to 1.\n",
- "\n",
- "getting sp graphs: 183it [00:00, 1871.37it/s]\n",
- "calculating kernels: 16836it [00:16, 1014.42it/s]\n",
- "\n",
- " --- shortest path kernel matrix of size 183 built in 16.947543382644653 seconds ---\n",
- "\n",
- "the gram matrix with parameters {'node_kernels': {'symb': <function deltakernel at 0x7f3a99093950>, 'nsymb': <function gaussiankernel at 0x7f3a990931e0>, 'mix': functools.partial(<function kernelproduct at 0x7f3a99088ae8>, <function deltakernel at 0x7f3a99093950>, <function gaussiankernel at 0x7f3a990931e0>)}, 'n_jobs': 8} is: \n",
- "\n",
- "\n",
- "\n",
- "1 gram matrices are calculated, 0 of which are ignored.\n",
- "\n",
- "3. Fitting and predicting using nested cross validation. This could really take a while...\n",
- "cross validation: 30it [00:12, 2.03it/s]\n",
- "\n",
- "4. Getting final performance...\n",
- "best_params_out: [{'node_kernels': {'symb': <function deltakernel at 0x7f3a99093950>, 'nsymb': <function gaussiankernel at 0x7f3a990931e0>, 'mix': functools.partial(<function kernelproduct at 0x7f3a99088ae8>, <function deltakernel at 0x7f3a99093950>, <function gaussiankernel at 0x7f3a990931e0>)}, 'n_jobs': 8}]\n",
- "best_params_in: [{'alpha': 1e-06}]\n",
- "\n",
- "best_val_perf: 9.55244065682399\n",
- "best_val_std: 0.5574811966683159\n",
- "final_performance: [9.724426192585643]\n",
- "final_confidence: [2.999822095078807]\n",
- "train_performance: [6.141755071354953]\n",
- "train_std: [0.2732168016478284]\n",
- "\n",
- "time to calculate gram matrix with different hyper-params: 16.95±nans\n",
- "time to calculate best gram matrix: 16.95±nans\n",
- "total training time with all hyper-param choices: 32.74s\n",
- "\n"
- ]
- },
- {
- "name": "stderr",
- "output_type": "stream",
- "text": [
- "/usr/local/lib/python3.6/dist-packages/numpy/core/_methods.py:140: RuntimeWarning: Degrees of freedom <= 0 for slice\n",
- " keepdims=keepdims)\n",
- "/usr/local/lib/python3.6/dist-packages/numpy/core/_methods.py:132: RuntimeWarning: invalid value encountered in double_scalars\n",
- " ret = ret.dtype.type(ret / rcount)\n"
- ]
- },
- {
- "name": "stdout",
- "output_type": "stream",
- "text": [
- "Filename: ../../pygraph/utils/model_selection_precomputed.py\n",
- "\n",
- "Line # Mem usage Increment Line Contents\n",
- "================================================\n",
- " 24 115.2 MiB 115.2 MiB @profile\n",
- " 25 def model_selection_for_precomputed_kernel(datafile,\n",
- " 26 estimator,\n",
- " 27 param_grid_precomputed,\n",
- " 28 param_grid,\n",
- " 29 model_type,\n",
- " 30 NUM_TRIALS=30,\n",
- " 31 datafile_y=None,\n",
- " 32 extra_params=None,\n",
- " 33 ds_name='ds-unknown',\n",
- " 34 n_jobs=1,\n",
- " 35 read_gm_from_file=False):\n",
- " 36 \"\"\"Perform model selection, fitting and testing for precomputed kernels using nested cv. Print out neccessary data during the process then finally the results.\n",
- " 37 \n",
- " 38 Parameters\n",
- " 39 ----------\n",
- " 40 datafile : string\n",
- " 41 Path of dataset file.\n",
- " 42 estimator : function\n",
- " 43 kernel function used to estimate. This function needs to return a gram matrix.\n",
- " 44 param_grid_precomputed : dictionary\n",
- " 45 Dictionary with names (string) of parameters used to calculate gram matrices as keys and lists of parameter settings to try as values. This enables searching over any sequence of parameter settings. Params with length 1 will be omitted.\n",
- " 46 param_grid : dictionary\n",
- " 47 Dictionary with names (string) of parameters used as penelties as keys and lists of parameter settings to try as values. This enables searching over any sequence of parameter settings. Params with length 1 will be omitted.\n",
- " 48 model_type : string\n",
- " 49 Typr of the problem, can be regression or classification.\n",
- " 50 NUM_TRIALS : integer\n",
- " 51 Number of random trials of outer cv loop. The default is 30.\n",
- " 52 datafile_y : string\n",
- " 53 Path of file storing y data. This parameter is optional depending on the given dataset file.\n",
- " 54 read_gm_from_file : boolean\n",
- " 55 Whether gram matrices are loaded from file.\n",
- " 56 \n",
- " 57 Examples\n",
- " 58 --------\n",
- " 59 >>> import numpy as np\n",
- " 60 >>> import sys\n",
- " 61 >>> sys.path.insert(0, \"../\")\n",
- " 62 >>> from pygraph.utils.model_selection_precomputed import model_selection_for_precomputed_kernel\n",
- " 63 >>> from pygraph.kernels.weisfeilerLehmanKernel import weisfeilerlehmankernel\n",
- " 64 >>>\n",
- " 65 >>> datafile = '../../../../datasets/acyclic/Acyclic/dataset_bps.ds'\n",
- " 66 >>> estimator = weisfeilerlehmankernel\n",
- " 67 >>> param_grid_precomputed = {'height': [0,1,2,3,4,5,6,7,8,9,10], 'base_kernel': ['subtree']}\n",
- " 68 >>> param_grid = {\"alpha\": np.logspace(-2, 2, num = 10, base = 10)}\n",
- " 69 >>>\n",
- " 70 >>> model_selection_for_precomputed_kernel(datafile, estimator, param_grid_precomputed, param_grid, 'regression')\n",
- " 71 \"\"\"\n",
- " 72 115.2 MiB 0.0 MiB tqdm.monitor_interval = 0\n",
- " 73 \n",
- " 74 115.2 MiB 0.0 MiB results_dir = '../notebooks/results/' + estimator.__name__\n",
- " 75 115.2 MiB 0.0 MiB if not os.path.exists(results_dir):\n",
- " 76 os.makedirs(results_dir)\n",
- " 77 # a string to save all the results.\n",
- " 78 115.2 MiB 0.0 MiB str_fw = '###################### log time: ' + datetime.datetime.now().strftime(\"%Y-%m-%d %H:%M:%S\") + '. ######################\\n\\n'\n",
- " 79 115.2 MiB 0.0 MiB str_fw += '# This file contains results of ' + estimator.__name__ + ' on dataset ' + ds_name + ',\\n# including gram matrices, serial numbers for gram matrix figures and performance.\\n\\n'\n",
- " 80 \n",
- " 81 # setup the model type\n",
- " 82 115.2 MiB 0.0 MiB model_type = model_type.lower()\n",
- " 83 115.2 MiB 0.0 MiB if model_type != 'regression' and model_type != 'classification':\n",
- " 84 raise Exception(\n",
- " 85 'The model type is incorrect! Please choose from regression or classification.'\n",
- " 86 )\n",
- " 87 115.2 MiB 0.0 MiB print()\n",
- " 88 115.2 MiB 0.0 MiB print('--- This is a %s problem ---' % model_type)\n",
- " 89 115.2 MiB 0.0 MiB str_fw += 'This is a %s problem.\\n' % model_type\n",
- " 90 \n",
- " 91 # calculate gram matrices rather than read them from file.\n",
- " 92 115.2 MiB 0.0 MiB if read_gm_from_file == False:\n",
- " 93 # Load the dataset\n",
- " 94 115.2 MiB 0.0 MiB print()\n",
- " 95 115.2 MiB 0.0 MiB print('\\n1. Loading dataset from file...')\n",
- " 96 115.2 MiB 0.0 MiB if isinstance(datafile, str):\n",
- " 97 115.2 MiB 0.0 MiB dataset, y_all = loadDataset(\n",
- " 98 116.3 MiB 1.1 MiB datafile, filename_y=datafile_y, extra_params=extra_params)\n",
- " 99 else: # load data directly from variable.\n",
- " 100 dataset = datafile\n",
- " 101 y_all = datafile_y \n",
- " 102 \n",
- " 103 # import matplotlib.pyplot as plt\n",
- " 104 # import networkx as nx\n",
- " 105 # nx.draw_networkx(dataset[30])\n",
- " 106 # plt.show()\n",
- " 107 \n",
- " 108 # Grid of parameters with a discrete number of values for each.\n",
- " 109 116.3 MiB 0.0 MiB param_list_precomputed = list(ParameterGrid(param_grid_precomputed))\n",
- " 110 116.3 MiB 0.0 MiB param_list = list(ParameterGrid(param_grid))\n",
- " 111 \n",
- " 112 116.3 MiB 0.0 MiB gram_matrices = [\n",
- " 113 ] # a list to store gram matrices for all param_grid_precomputed\n",
- " 114 116.3 MiB 0.0 MiB gram_matrix_time = [\n",
- " 115 ] # a list to store time to calculate gram matrices\n",
- " 116 116.3 MiB 0.0 MiB param_list_pre_revised = [\n",
- " 117 ] # list to store param grids precomputed ignoring the useless ones\n",
- " 118 \n",
- " 119 # calculate all gram matrices\n",
- " 120 116.3 MiB 0.0 MiB print()\n",
- " 121 116.3 MiB 0.0 MiB print('2. Calculating gram matrices. This could take a while...')\n",
- " 122 116.3 MiB 0.0 MiB str_fw += '\\nII. Gram matrices.\\n\\n'\n",
- " 123 116.3 MiB 0.0 MiB tts = time.time() # start training time\n",
- " 124 116.3 MiB 0.0 MiB nb_gm_ignore = 0 # the number of gram matrices those should not be considered, as they may contain elements that are not numbers (NaN)\n",
- " 125 145.3 MiB 0.0 MiB for idx, params_out in enumerate(param_list_precomputed):\n",
- " 126 116.3 MiB 0.0 MiB y = y_all[:]\n",
- " 127 116.3 MiB 0.0 MiB params_out['n_jobs'] = n_jobs\n",
- " 128 # print(dataset)\n",
- " 129 # import networkx as nx\n",
- " 130 # nx.draw_networkx(dataset[1])\n",
- " 131 # plt.show()\n",
- " 132 119.5 MiB 3.1 MiB rtn_data = estimator(dataset[:], **params_out)\n",
- " 133 119.5 MiB 0.0 MiB Kmatrix = rtn_data[0]\n",
- " 134 119.5 MiB 0.0 MiB current_run_time = rtn_data[1]\n",
- " 135 # for some kernels, some graphs in datasets may not meet the \n",
- " 136 # kernels' requirements for graph structure. These graphs are trimmed. \n",
- " 137 119.5 MiB 0.0 MiB if len(rtn_data) == 3:\n",
- " 138 119.5 MiB 0.0 MiB idx_trim = rtn_data[2] # the index of trimmed graph list\n",
- " 139 119.5 MiB 0.0 MiB y = [y[idxt] for idxt in idx_trim] # trim y accordingly\n",
- " 140 # Kmatrix = np.random.rand(2250, 2250)\n",
- " 141 # current_run_time = 0.1\n",
- " 142 \n",
- " 143 # remove graphs whose kernels with themselves are zeros\n",
- " 144 119.5 MiB 0.0 MiB Kmatrix_diag = Kmatrix.diagonal().copy()\n",
- " 145 119.5 MiB 0.0 MiB nb_g_ignore = 0\n",
- " 146 119.5 MiB 0.0 MiB for idxk, diag in enumerate(Kmatrix_diag):\n",
- " 147 119.5 MiB 0.0 MiB if diag == 0:\n",
- " 148 Kmatrix = np.delete(Kmatrix, (idxk - nb_g_ignore), axis=0)\n",
- " 149 Kmatrix = np.delete(Kmatrix, (idxk - nb_g_ignore), axis=1)\n",
- " 150 nb_g_ignore += 1\n",
- " 151 # normalization\n",
- " 152 119.5 MiB 0.0 MiB Kmatrix_diag = Kmatrix.diagonal().copy()\n",
- " 153 119.5 MiB 0.0 MiB for i in range(len(Kmatrix)):\n",
- " 154 119.5 MiB 0.0 MiB for j in range(i, len(Kmatrix)):\n",
- " 155 119.5 MiB 0.0 MiB Kmatrix[i][j] /= np.sqrt(Kmatrix_diag[i] * Kmatrix_diag[j])\n",
- " 156 119.5 MiB 0.0 MiB Kmatrix[j][i] = Kmatrix[i][j]\n",
- " 157 \n",
- " 158 119.5 MiB 0.0 MiB print()\n",
- " 159 119.5 MiB 0.0 MiB if params_out == {}:\n",
- " 160 print('the gram matrix is: ')\n",
- " 161 str_fw += 'the gram matrix is:\\n\\n'\n",
- " 162 else:\n",
- " 163 119.5 MiB 0.0 MiB print('the gram matrix with parameters', params_out, 'is: \\n\\n')\n",
- " 164 119.5 MiB 0.0 MiB str_fw += 'the gram matrix with parameters %s is:\\n\\n' % params_out\n",
- " 165 119.5 MiB 0.0 MiB if len(Kmatrix) < 2:\n",
- " 166 nb_gm_ignore += 1\n",
- " 167 print('ignored, as at most only one of all its diagonal value is non-zero.')\n",
- " 168 str_fw += 'ignored, as at most only one of all its diagonal value is non-zero.\\n\\n'\n",
- " 169 else: \n",
- " 170 119.5 MiB 0.0 MiB if np.isnan(Kmatrix).any(\n",
- " 171 ): # if the matrix contains elements that are not numbers\n",
- " 172 nb_gm_ignore += 1\n",
- " 173 print('ignored, as it contains elements that are not numbers.')\n",
- " 174 str_fw += 'ignored, as it contains elements that are not numbers.\\n\\n'\n",
- " 175 else:\n",
- " 176 # print(Kmatrix)\n",
- " 177 119.5 MiB 0.0 MiB str_fw += np.array2string(\n",
- " 178 119.5 MiB 0.0 MiB Kmatrix,\n",
- " 179 119.5 MiB 0.0 MiB separator=',') + '\\n\\n'\n",
- " 180 # separator=',',\n",
- " 181 # threshold=np.inf,\n",
- " 182 # floatmode='unique') + '\\n\\n'\n",
- " 183 \n",
- " 184 119.5 MiB 0.0 MiB fig_file_name = results_dir + '/GM[ds]' + ds_name\n",
- " 185 119.5 MiB 0.0 MiB if params_out != {}:\n",
- " 186 119.5 MiB 0.0 MiB fig_file_name += '[params]' + str(idx)\n",
- " 187 120.3 MiB 0.7 MiB plt.imshow(Kmatrix)\n",
- " 188 120.4 MiB 0.1 MiB plt.colorbar()\n",
- " 189 145.3 MiB 24.9 MiB plt.savefig(fig_file_name + '.eps', format='eps', dpi=300)\n",
- " 190 # plt.show()\n",
- " 191 145.3 MiB 0.0 MiB plt.clf()\n",
- " 192 145.3 MiB 0.0 MiB gram_matrices.append(Kmatrix)\n",
- " 193 145.3 MiB 0.0 MiB gram_matrix_time.append(current_run_time)\n",
- " 194 145.3 MiB 0.0 MiB param_list_pre_revised.append(params_out)\n",
- " 195 145.3 MiB 0.0 MiB if nb_g_ignore > 0:\n",
- " 196 print(', where %d graphs are ignored as their graph kernels with themselves are zeros.' % nb_g_ignore)\n",
- " 197 str_fw += ', where %d graphs are ignored as their graph kernels with themselves are zeros.' % nb_g_ignore\n",
- " 198 145.3 MiB 0.0 MiB print()\n",
- " 199 145.3 MiB 0.0 MiB print(\n",
- " 200 145.3 MiB 0.0 MiB '{} gram matrices are calculated, {} of which are ignored.'.format(\n",
- " 201 145.3 MiB 0.0 MiB len(param_list_precomputed), nb_gm_ignore))\n",
- " 202 145.3 MiB 0.0 MiB str_fw += '{} gram matrices are calculated, {} of which are ignored.\\n\\n'.format(len(param_list_precomputed), nb_gm_ignore)\n",
- " 203 145.3 MiB 0.0 MiB str_fw += 'serial numbers of gram matrix figures and their corresponding parameters settings:\\n\\n'\n",
- " 204 145.3 MiB 0.0 MiB str_fw += ''.join([\n",
- " 205 145.3 MiB 0.0 MiB '{}: {}\\n'.format(idx, params_out)\n",
- " 206 145.3 MiB 0.0 MiB for idx, params_out in enumerate(param_list_precomputed)\n",
- " 207 ])\n",
- " 208 \n",
- " 209 145.3 MiB 0.0 MiB print()\n",
- " 210 145.3 MiB 0.0 MiB if len(gram_matrices) == 0:\n",
- " 211 print('all gram matrices are ignored, no results obtained.')\n",
- " 212 str_fw += '\\nall gram matrices are ignored, no results obtained.\\n\\n'\n",
- " 213 else:\n",
- " 214 # save gram matrices to file.\n",
- " 215 145.4 MiB 0.1 MiB np.savez(results_dir + '/' + ds_name + '.gm', \n",
- " 216 145.4 MiB 0.0 MiB gms=gram_matrices, params=param_list_pre_revised, y=y, \n",
- " 217 145.4 MiB 0.0 MiB gmtime=gram_matrix_time)\n",
- " 218 \n",
- " 219 145.4 MiB 0.0 MiB print(\n",
- " 220 145.4 MiB 0.0 MiB '3. Fitting and predicting using nested cross validation. This could really take a while...'\n",
- " 221 )\n",
- " 222 \n",
- " 223 # ---- use pool.imap_unordered to parallel and track progress. ----\n",
- " 224 # train_pref = []\n",
- " 225 # val_pref = []\n",
- " 226 # test_pref = []\n",
- " 227 # def func_assign(result, var_to_assign):\n",
- " 228 # for idx, itm in enumerate(var_to_assign):\n",
- " 229 # itm.append(result[idx]) \n",
- " 230 # trial_do_partial = partial(trial_do, param_list_pre_revised, param_list, y, model_type)\n",
- " 231 # \n",
- " 232 # parallel_me(trial_do_partial, range(NUM_TRIALS), func_assign, \n",
- " 233 # [train_pref, val_pref, test_pref], glbv=gram_matrices,\n",
- " 234 # method='imap_unordered', n_jobs=n_jobs, chunksize=1,\n",
- " 235 # itr_desc='cross validation')\n",
- " 236 \n",
- " 237 145.4 MiB 0.0 MiB def init_worker(gms_toshare):\n",
- " 238 global G_gms\n",
- " 239 G_gms = gms_toshare\n",
- " 240 \n",
- " 241 # gram_matrices = np.array(gram_matrices)\n",
- " 242 # gms_shape = gram_matrices.shape\n",
- " 243 # gms_array = Array('d', np.reshape(gram_matrices.copy(), -1, order='C'))\n",
- " 244 # pool = Pool(processes=n_jobs, initializer=init_worker, initargs=(gms_array, gms_shape))\n",
- " 245 145.4 MiB 0.0 MiB pool = Pool(processes=n_jobs, initializer=init_worker, initargs=(gram_matrices,))\n",
- " 246 145.4 MiB 0.0 MiB trial_do_partial = partial(parallel_trial_do, param_list_pre_revised, param_list, y, model_type)\n",
- " 247 145.4 MiB 0.0 MiB train_pref = []\n",
- " 248 145.4 MiB 0.0 MiB val_pref = []\n",
- " 249 145.4 MiB 0.0 MiB test_pref = []\n",
- " 250 # if NUM_TRIALS < 1000 * n_jobs:\n",
- " 251 # chunksize = int(NUM_TRIALS / n_jobs) + 1\n",
- " 252 # else:\n",
- " 253 # chunksize = 1000\n",
- " 254 145.4 MiB 0.0 MiB chunksize = 1\n",
- " 255 145.4 MiB 0.0 MiB for o1, o2, o3 in tqdm(pool.imap_unordered(trial_do_partial, range(NUM_TRIALS), chunksize), desc='cross validation', file=sys.stdout):\n",
- " 256 145.4 MiB 0.0 MiB train_pref.append(o1)\n",
- " 257 145.4 MiB 0.0 MiB val_pref.append(o2)\n",
- " 258 145.4 MiB 0.0 MiB test_pref.append(o3)\n",
- " 259 145.4 MiB 0.0 MiB pool.close()\n",
- " 260 145.4 MiB 0.0 MiB pool.join()\n",
- " 261 \n",
- " 262 # # ---- use pool.map to parallel. ----\n",
- " 263 # pool = Pool(n_jobs)\n",
- " 264 # trial_do_partial = partial(trial_do, param_list_pre_revised, param_list, gram_matrices, y[0:250], model_type)\n",
- " 265 # result_perf = pool.map(trial_do_partial, range(NUM_TRIALS))\n",
- " 266 # train_pref = [item[0] for item in result_perf]\n",
- " 267 # val_pref = [item[1] for item in result_perf]\n",
- " 268 # test_pref = [item[2] for item in result_perf]\n",
- " 269 \n",
- " 270 # # ---- direct running, normally use a single CPU core. ----\n",
- " 271 # train_pref = []\n",
- " 272 # val_pref = []\n",
- " 273 # test_pref = []\n",
- " 274 # for i in tqdm(range(NUM_TRIALS), desc='cross validation', file=sys.stdout):\n",
- " 275 # o1, o2, o3 = trial_do(param_list_pre_revised, param_list, gram_matrices, y, model_type, i)\n",
- " 276 # train_pref.append(o1)\n",
- " 277 # val_pref.append(o2)\n",
- " 278 # test_pref.append(o3)\n",
- " 279 # print()\n",
- " 280 \n",
- " 281 145.4 MiB 0.0 MiB print()\n",
- " 282 145.4 MiB 0.0 MiB print('4. Getting final performance...')\n",
- " 283 145.4 MiB 0.0 MiB str_fw += '\\nIII. Performance.\\n\\n'\n",
- " 284 # averages and confidences of performances on outer trials for each combination of parameters\n",
- " 285 145.4 MiB 0.0 MiB average_train_scores = np.mean(train_pref, axis=0)\n",
- " 286 # print('val_pref: ', val_pref[0][0])\n",
- " 287 145.4 MiB 0.0 MiB average_val_scores = np.mean(val_pref, axis=0)\n",
- " 288 # print('test_pref: ', test_pref[0][0])\n",
- " 289 145.4 MiB 0.0 MiB average_perf_scores = np.mean(test_pref, axis=0)\n",
- " 290 # sample std is used here\n",
- " 291 145.4 MiB 0.0 MiB std_train_scores = np.std(train_pref, axis=0, ddof=1)\n",
- " 292 145.4 MiB 0.0 MiB std_val_scores = np.std(val_pref, axis=0, ddof=1)\n",
- " 293 145.4 MiB 0.0 MiB std_perf_scores = np.std(test_pref, axis=0, ddof=1)\n",
- " 294 \n",
- " 295 145.4 MiB 0.0 MiB if model_type == 'regression':\n",
- " 296 145.4 MiB 0.0 MiB best_val_perf = np.amin(average_val_scores)\n",
- " 297 else:\n",
- " 298 best_val_perf = np.amax(average_val_scores)\n",
- " 299 # print('average_val_scores: ', average_val_scores)\n",
- " 300 # print('best_val_perf: ', best_val_perf)\n",
- " 301 # print()\n",
- " 302 145.4 MiB 0.0 MiB best_params_index = np.where(average_val_scores == best_val_perf)\n",
- " 303 # find smallest val std with best val perf.\n",
- " 304 best_val_stds = [\n",
- " 305 145.4 MiB 0.0 MiB std_val_scores[value][best_params_index[1][idx]]\n",
- " 306 145.4 MiB 0.0 MiB for idx, value in enumerate(best_params_index[0])\n",
- " 307 ]\n",
- " 308 145.4 MiB 0.0 MiB min_val_std = np.amin(best_val_stds)\n",
- " 309 145.4 MiB 0.0 MiB best_params_index = np.where(std_val_scores == min_val_std)\n",
- " 310 best_params_out = [\n",
- " 311 145.4 MiB 0.0 MiB param_list_pre_revised[i] for i in best_params_index[0]\n",
- " 312 ]\n",
- " 313 145.4 MiB 0.0 MiB best_params_in = [param_list[i] for i in best_params_index[1]]\n",
- " 314 145.4 MiB 0.0 MiB print('best_params_out: ', best_params_out)\n",
- " 315 145.4 MiB 0.0 MiB print('best_params_in: ', best_params_in)\n",
- " 316 145.4 MiB 0.0 MiB print()\n",
- " 317 145.4 MiB 0.0 MiB print('best_val_perf: ', best_val_perf)\n",
- " 318 145.4 MiB 0.0 MiB print('best_val_std: ', min_val_std)\n",
- " 319 145.4 MiB 0.0 MiB str_fw += 'best settings of hyper-params to build gram matrix: %s\\n' % best_params_out\n",
- " 320 145.4 MiB 0.0 MiB str_fw += 'best settings of other hyper-params: %s\\n\\n' % best_params_in\n",
- " 321 145.4 MiB 0.0 MiB str_fw += 'best_val_perf: %s\\n' % best_val_perf\n",
- " 322 145.4 MiB 0.0 MiB str_fw += 'best_val_std: %s\\n' % min_val_std\n",
- " 323 \n",
- " 324 # print(best_params_index)\n",
- " 325 # print(best_params_index[0])\n",
- " 326 # print(average_perf_scores)\n",
- " 327 final_performance = [\n",
- " 328 145.4 MiB 0.0 MiB average_perf_scores[value][best_params_index[1][idx]]\n",
- " 329 145.4 MiB 0.0 MiB for idx, value in enumerate(best_params_index[0])\n",
- " 330 ]\n",
- " 331 final_confidence = [\n",
- " 332 145.4 MiB 0.0 MiB std_perf_scores[value][best_params_index[1][idx]]\n",
- " 333 145.4 MiB 0.0 MiB for idx, value in enumerate(best_params_index[0])\n",
- " 334 ]\n",
- " 335 145.4 MiB 0.0 MiB print('final_performance: ', final_performance)\n",
- " 336 145.4 MiB 0.0 MiB print('final_confidence: ', final_confidence)\n",
- " 337 145.4 MiB 0.0 MiB str_fw += 'final_performance: %s\\n' % final_performance\n",
- " 338 145.4 MiB 0.0 MiB str_fw += 'final_confidence: %s\\n' % final_confidence\n",
- " 339 train_performance = [\n",
- " 340 145.4 MiB 0.0 MiB average_train_scores[value][best_params_index[1][idx]]\n",
- " 341 145.4 MiB 0.0 MiB for idx, value in enumerate(best_params_index[0])\n",
- " 342 ]\n",
- " 343 train_std = [\n",
- " 344 145.4 MiB 0.0 MiB std_train_scores[value][best_params_index[1][idx]]\n",
- " 345 145.4 MiB 0.0 MiB for idx, value in enumerate(best_params_index[0])\n",
- " 346 ]\n",
- " 347 145.4 MiB 0.0 MiB print('train_performance: %s' % train_performance)\n",
- " 348 145.4 MiB 0.0 MiB print('train_std: ', train_std)\n",
- " 349 145.4 MiB 0.0 MiB str_fw += 'train_performance: %s\\n' % train_performance\n",
- " 350 145.4 MiB 0.0 MiB str_fw += 'train_std: %s\\n\\n' % train_std\n",
- " 351 \n",
- " 352 145.4 MiB 0.0 MiB print()\n",
- " 353 145.4 MiB 0.0 MiB tt_total = time.time() - tts # training time for all hyper-parameters\n",
- " 354 145.4 MiB 0.0 MiB average_gram_matrix_time = np.mean(gram_matrix_time)\n",
- " 355 145.4 MiB 0.0 MiB std_gram_matrix_time = np.std(gram_matrix_time, ddof=1)\n",
- " 356 best_gram_matrix_time = [\n",
- " 357 145.4 MiB 0.0 MiB gram_matrix_time[i] for i in best_params_index[0]\n",
- " 358 ]\n",
- " 359 145.4 MiB 0.0 MiB ave_bgmt = np.mean(best_gram_matrix_time)\n",
- " 360 145.4 MiB 0.0 MiB std_bgmt = np.std(best_gram_matrix_time, ddof=1)\n",
- " 361 145.4 MiB 0.0 MiB print(\n",
- " 362 145.4 MiB 0.0 MiB 'time to calculate gram matrix with different hyper-params: {:.2f}±{:.2f}s'\n",
- " 363 145.4 MiB 0.0 MiB .format(average_gram_matrix_time, std_gram_matrix_time))\n",
- " 364 145.4 MiB 0.0 MiB print('time to calculate best gram matrix: {:.2f}±{:.2f}s'.format(\n",
- " 365 145.4 MiB 0.0 MiB ave_bgmt, std_bgmt))\n",
- " 366 145.4 MiB 0.0 MiB print(\n",
- " 367 145.4 MiB 0.0 MiB 'total training time with all hyper-param choices: {:.2f}s'.format(\n",
- " 368 145.4 MiB 0.0 MiB tt_total))\n",
- " 369 145.4 MiB 0.0 MiB str_fw += 'time to calculate gram matrix with different hyper-params: {:.2f}±{:.2f}s\\n'.format(average_gram_matrix_time, std_gram_matrix_time)\n",
- " 370 145.4 MiB 0.0 MiB str_fw += 'time to calculate best gram matrix: {:.2f}±{:.2f}s\\n'.format(ave_bgmt, std_bgmt)\n",
- " 371 145.4 MiB 0.0 MiB str_fw += 'total training time with all hyper-param choices: {:.2f}s\\n\\n'.format(tt_total)\n",
- " 372 \n",
- " 373 # # save results to file\n",
- " 374 # np.savetxt(results_name_pre + 'average_train_scores.dt',\n",
- " 375 # average_train_scores)\n",
- " 376 # np.savetxt(results_name_pre + 'average_val_scores', average_val_scores)\n",
- " 377 # np.savetxt(results_name_pre + 'average_perf_scores.dt',\n",
- " 378 # average_perf_scores)\n",
- " 379 # np.savetxt(results_name_pre + 'std_train_scores.dt', std_train_scores)\n",
- " 380 # np.savetxt(results_name_pre + 'std_val_scores.dt', std_val_scores)\n",
- " 381 # np.savetxt(results_name_pre + 'std_perf_scores.dt', std_perf_scores)\n",
- " 382 \n",
- " 383 # np.save(results_name_pre + 'best_params_index', best_params_index)\n",
- " 384 # np.save(results_name_pre + 'best_params_pre.dt', best_params_out)\n",
- " 385 # np.save(results_name_pre + 'best_params_in.dt', best_params_in)\n",
- " 386 # np.save(results_name_pre + 'best_val_perf.dt', best_val_perf)\n",
- " 387 # np.save(results_name_pre + 'best_val_std.dt', best_val_std)\n",
- " 388 # np.save(results_name_pre + 'final_performance.dt', final_performance)\n",
- " 389 # np.save(results_name_pre + 'final_confidence.dt', final_confidence)\n",
- " 390 # np.save(results_name_pre + 'train_performance.dt', train_performance)\n",
- " 391 # np.save(results_name_pre + 'train_std.dt', train_std)\n",
- " 392 \n",
- " 393 # np.save(results_name_pre + 'gram_matrix_time.dt', gram_matrix_time)\n",
- " 394 # np.save(results_name_pre + 'average_gram_matrix_time.dt',\n",
- " 395 # average_gram_matrix_time)\n",
- " 396 # np.save(results_name_pre + 'std_gram_matrix_time.dt',\n",
- " 397 # std_gram_matrix_time)\n",
- " 398 # np.save(results_name_pre + 'best_gram_matrix_time.dt',\n",
- " 399 # best_gram_matrix_time)\n",
- " 400 \n",
- " 401 # print out as table.\n",
- " 402 145.4 MiB 0.0 MiB from collections import OrderedDict\n",
- " 403 145.4 MiB 0.0 MiB from tabulate import tabulate\n",
- " 404 145.4 MiB 0.0 MiB table_dict = {}\n",
- " 405 145.4 MiB 0.0 MiB if model_type == 'regression':\n",
- " 406 145.6 MiB 0.0 MiB for param_in in param_list:\n",
- " 407 145.6 MiB 0.2 MiB param_in['alpha'] = '{:.2e}'.format(param_in['alpha'])\n",
- " 408 else:\n",
- " 409 for param_in in param_list:\n",
- " 410 param_in['C'] = '{:.2e}'.format(param_in['C'])\n",
- " 411 145.6 MiB 0.0 MiB table_dict['params'] = [{**param_out, **param_in}\n",
- " 412 145.6 MiB 0.0 MiB for param_in in param_list for param_out in param_list_pre_revised]\n",
- " 413 table_dict['gram_matrix_time'] = [\n",
- " 414 145.6 MiB 0.0 MiB '{:.2f}'.format(gram_matrix_time[index_out])\n",
- " 415 145.6 MiB 0.0 MiB for param_in in param_list\n",
- " 416 145.6 MiB 0.0 MiB for index_out, _ in enumerate(param_list_pre_revised)\n",
- " 417 ]\n",
- " 418 table_dict['valid_perf'] = [\n",
- " 419 145.6 MiB 0.0 MiB '{:.2f}±{:.2f}'.format(average_val_scores[index_out][index_in],\n",
- " 420 std_val_scores[index_out][index_in])\n",
- " 421 145.6 MiB 0.0 MiB for index_in, _ in enumerate(param_list)\n",
- " 422 145.6 MiB 0.0 MiB for index_out, _ in enumerate(param_list_pre_revised)\n",
- " 423 ]\n",
- " 424 table_dict['test_perf'] = [\n",
- " 425 145.6 MiB 0.0 MiB '{:.2f}±{:.2f}'.format(average_perf_scores[index_out][index_in],\n",
- " 426 std_perf_scores[index_out][index_in])\n",
- " 427 145.6 MiB 0.0 MiB for index_in, _ in enumerate(param_list)\n",
- " 428 145.6 MiB 0.0 MiB for index_out, _ in enumerate(param_list_pre_revised)\n",
- " 429 ]\n",
- " 430 table_dict['train_perf'] = [\n",
- " 431 145.6 MiB 0.0 MiB '{:.2f}±{:.2f}'.format(average_train_scores[index_out][index_in],\n",
- " 432 std_train_scores[index_out][index_in])\n",
- " 433 145.6 MiB 0.0 MiB for index_in, _ in enumerate(param_list)\n",
- " 434 145.6 MiB 0.0 MiB for index_out, _ in enumerate(param_list_pre_revised)\n",
- " 435 ]\n",
- " 436 keyorder = [\n",
- " 437 145.6 MiB 0.0 MiB 'params', 'train_perf', 'valid_perf', 'test_perf',\n",
- " 438 145.6 MiB 0.0 MiB 'gram_matrix_time'\n",
- " 439 ]\n",
- " 440 145.6 MiB 0.0 MiB print()\n",
- " 441 145.6 MiB 0.0 MiB tb_print = tabulate(\n",
- " 442 145.6 MiB 0.0 MiB OrderedDict(\n",
- " 443 145.6 MiB 0.0 MiB sorted(table_dict.items(),\n",
- " 444 145.6 MiB 0.0 MiB key=lambda i: keyorder.index(i[0]))),\n",
- " 445 145.6 MiB 0.0 MiB headers='keys')\n",
- " 446 # print(tb_print)\n",
- " 447 145.6 MiB 0.0 MiB str_fw += 'table of performance v.s. hyper-params:\\n\\n%s\\n\\n' % tb_print\n",
- " 448 \n",
- " 449 # read gram matrices from file.\n",
- " 450 else: \n",
- " 451 # Grid of parameters with a discrete number of values for each.\n",
- " 452 # param_list_precomputed = list(ParameterGrid(param_grid_precomputed))\n",
- " 453 param_list = list(ParameterGrid(param_grid))\n",
- " 454 \n",
- " 455 # read gram matrices from file.\n",
- " 456 print()\n",
- " 457 print('2. Reading gram matrices from file...')\n",
- " 458 str_fw += '\\nII. Gram matrices.\\n\\nGram matrices are read from file, see last log for detail.\\n'\n",
- " 459 gmfile = np.load(results_dir + '/' + ds_name + '.gm.npz')\n",
- " 460 gram_matrices = gmfile['gms'] # a list to store gram matrices for all param_grid_precomputed\n",
- " 461 gram_matrix_time = gmfile['gmtime'] # time used to compute the gram matrices\n",
- " 462 param_list_pre_revised = gmfile['params'] # list to store param grids precomputed ignoring the useless ones\n",
- " 463 y = gmfile['y'].tolist()\n",
- " 464 \n",
- " 465 tts = time.time() # start training time\n",
- " 466 # nb_gm_ignore = 0 # the number of gram matrices those should not be considered, as they may contain elements that are not numbers (NaN) \n",
- " 467 print(\n",
- " 468 '3. Fitting and predicting using nested cross validation. This could really take a while...'\n",
- " 469 )\n",
- " 470 \n",
- " 471 # ---- use pool.imap_unordered to parallel and track progress. ----\n",
- " 472 def init_worker(gms_toshare):\n",
- " 473 global G_gms\n",
- " 474 G_gms = gms_toshare\n",
- " 475 \n",
- " 476 pool = Pool(processes=n_jobs, initializer=init_worker, initargs=(gram_matrices,))\n",
- " 477 trial_do_partial = partial(parallel_trial_do, param_list_pre_revised, param_list, y, model_type)\n",
- " 478 train_pref = []\n",
- " 479 val_pref = []\n",
- " 480 test_pref = []\n",
- " 481 chunksize = 1\n",
- " 482 for o1, o2, o3 in tqdm(pool.imap_unordered(trial_do_partial, range(NUM_TRIALS), chunksize), desc='cross validation', file=sys.stdout):\n",
- " 483 train_pref.append(o1)\n",
- " 484 val_pref.append(o2)\n",
- " 485 test_pref.append(o3)\n",
- " 486 pool.close()\n",
- " 487 pool.join()\n",
- " 488 \n",
- " 489 # # ---- use pool.map to parallel. ----\n",
- " 490 # result_perf = pool.map(trial_do_partial, range(NUM_TRIALS))\n",
- " 491 # train_pref = [item[0] for item in result_perf]\n",
- " 492 # val_pref = [item[1] for item in result_perf]\n",
- " 493 # test_pref = [item[2] for item in result_perf]\n",
- " 494 \n",
- " 495 # # ---- use joblib.Parallel to parallel and track progress. ----\n",
- " 496 # trial_do_partial = partial(trial_do, param_list_pre_revised, param_list, gram_matrices, y, model_type)\n",
- " 497 # result_perf = Parallel(n_jobs=n_jobs, verbose=10)(delayed(trial_do_partial)(trial) for trial in range(NUM_TRIALS))\n",
- " 498 # train_pref = [item[0] for item in result_perf]\n",
- " 499 # val_pref = [item[1] for item in result_perf]\n",
- " 500 # test_pref = [item[2] for item in result_perf]\n",
- " 501 \n",
- " 502 # # ---- direct running, normally use a single CPU core. ----\n",
- " 503 # train_pref = []\n",
- " 504 # val_pref = []\n",
- " 505 # test_pref = []\n",
- " 506 # for i in tqdm(range(NUM_TRIALS), desc='cross validation', file=sys.stdout):\n",
- " 507 # o1, o2, o3 = trial_do(param_list_pre_revised, param_list, gram_matrices, y, model_type, i)\n",
- " 508 # train_pref.append(o1)\n",
- " 509 # val_pref.append(o2)\n",
- " 510 # test_pref.append(o3)\n",
- " 511 \n",
- " 512 print()\n",
- " 513 print('4. Getting final performance...')\n",
- " 514 str_fw += '\\nIII. Performance.\\n\\n'\n",
- " 515 # averages and confidences of performances on outer trials for each combination of parameters\n",
- " 516 average_train_scores = np.mean(train_pref, axis=0)\n",
- " 517 average_val_scores = np.mean(val_pref, axis=0)\n",
- " 518 average_perf_scores = np.mean(test_pref, axis=0)\n",
- " 519 # sample std is used here\n",
- " 520 std_train_scores = np.std(train_pref, axis=0, ddof=1)\n",
- " 521 std_val_scores = np.std(val_pref, axis=0, ddof=1)\n",
- " 522 std_perf_scores = np.std(test_pref, axis=0, ddof=1)\n",
- " 523 \n",
- " 524 if model_type == 'regression':\n",
- " 525 best_val_perf = np.amin(average_val_scores)\n",
- " 526 else:\n",
- " 527 best_val_perf = np.amax(average_val_scores)\n",
- " 528 best_params_index = np.where(average_val_scores == best_val_perf)\n",
- " 529 # find smallest val std with best val perf.\n",
- " 530 best_val_stds = [\n",
- " 531 std_val_scores[value][best_params_index[1][idx]]\n",
- " 532 for idx, value in enumerate(best_params_index[0])\n",
- " 533 ]\n",
- " 534 min_val_std = np.amin(best_val_stds)\n",
- " 535 best_params_index = np.where(std_val_scores == min_val_std)\n",
- " 536 best_params_out = [\n",
- " 537 param_list_pre_revised[i] for i in best_params_index[0]\n",
- " 538 ]\n",
- " 539 best_params_in = [param_list[i] for i in best_params_index[1]]\n",
- " 540 print('best_params_out: ', best_params_out)\n",
- " 541 print('best_params_in: ', best_params_in)\n",
- " 542 print()\n",
- " 543 print('best_val_perf: ', best_val_perf)\n",
- " 544 print('best_val_std: ', min_val_std)\n",
- " 545 str_fw += 'best settings of hyper-params to build gram matrix: %s\\n' % best_params_out\n",
- " 546 str_fw += 'best settings of other hyper-params: %s\\n\\n' % best_params_in\n",
- " 547 str_fw += 'best_val_perf: %s\\n' % best_val_perf\n",
- " 548 str_fw += 'best_val_std: %s\\n' % min_val_std\n",
- " 549 \n",
- " 550 final_performance = [\n",
- " 551 average_perf_scores[value][best_params_index[1][idx]]\n",
- " 552 for idx, value in enumerate(best_params_index[0])\n",
- " 553 ]\n",
- " 554 final_confidence = [\n",
- " 555 std_perf_scores[value][best_params_index[1][idx]]\n",
- " 556 for idx, value in enumerate(best_params_index[0])\n",
- " 557 ]\n",
- " 558 print('final_performance: ', final_performance)\n",
- " 559 print('final_confidence: ', final_confidence)\n",
- " 560 str_fw += 'final_performance: %s\\n' % final_performance\n",
- " 561 str_fw += 'final_confidence: %s\\n' % final_confidence\n",
- " 562 train_performance = [\n",
- " 563 average_train_scores[value][best_params_index[1][idx]]\n",
- " 564 for idx, value in enumerate(best_params_index[0])\n",
- " 565 ]\n",
- " 566 train_std = [\n",
- " 567 std_train_scores[value][best_params_index[1][idx]]\n",
- " 568 for idx, value in enumerate(best_params_index[0])\n",
- " 569 ]\n",
- " 570 print('train_performance: %s' % train_performance)\n",
- " 571 print('train_std: ', train_std)\n",
- " 572 str_fw += 'train_performance: %s\\n' % train_performance\n",
- " 573 str_fw += 'train_std: %s\\n\\n' % train_std\n",
- " 574 \n",
- " 575 print()\n",
- " 576 average_gram_matrix_time = np.mean(gram_matrix_time)\n",
- " 577 std_gram_matrix_time = np.std(gram_matrix_time, ddof=1)\n",
- " 578 best_gram_matrix_time = [\n",
- " 579 gram_matrix_time[i] for i in best_params_index[0]\n",
- " 580 ]\n",
- " 581 ave_bgmt = np.mean(best_gram_matrix_time)\n",
- " 582 std_bgmt = np.std(best_gram_matrix_time, ddof=1)\n",
- " 583 print(\n",
- " 584 'time to calculate gram matrix with different hyper-params: {:.2f}±{:.2f}s'\n",
- " 585 .format(average_gram_matrix_time, std_gram_matrix_time))\n",
- " 586 print('time to calculate best gram matrix: {:.2f}±{:.2f}s'.format(\n",
- " 587 ave_bgmt, std_bgmt))\n",
- " 588 tt_poster = time.time() - tts # training time with hyper-param choices who did not participate in calculation of gram matrices\n",
- " 589 print(\n",
- " 590 'training time with hyper-param choices who did not participate in calculation of gram matrices: {:.2f}s'.format(\n",
- " 591 tt_poster))\n",
- " 592 print('total training time with all hyper-param choices: {:.2f}s'.format(\n",
- " 593 tt_poster + np.sum(gram_matrix_time)))\n",
- " 594 # str_fw += 'time to calculate gram matrix with different hyper-params: {:.2f}±{:.2f}s\\n'.format(average_gram_matrix_time, std_gram_matrix_time)\n",
- " 595 # str_fw += 'time to calculate best gram matrix: {:.2f}±{:.2f}s\\n'.format(ave_bgmt, std_bgmt)\n",
- " 596 str_fw += 'training time with hyper-param choices who did not participate in calculation of gram matrices: {:.2f}s\\n\\n'.format(tt_poster)\n",
- " 597 \n",
- " 598 # print out as table.\n",
- " 599 from collections import OrderedDict\n",
- " 600 from tabulate import tabulate\n",
- " 601 table_dict = {}\n",
- " 602 if model_type == 'regression':\n",
- " 603 for param_in in param_list:\n",
- " 604 param_in['alpha'] = '{:.2e}'.format(param_in['alpha'])\n",
- " 605 else:\n",
- " 606 for param_in in param_list:\n",
- " 607 param_in['C'] = '{:.2e}'.format(param_in['C'])\n",
- " 608 table_dict['params'] = [{**param_out, **param_in}\n",
- " 609 for param_in in param_list for param_out in param_list_pre_revised]\n",
- " 610 # table_dict['gram_matrix_time'] = [\n",
- " 611 # '{:.2f}'.format(gram_matrix_time[index_out])\n",
- " 612 # for param_in in param_list\n",
- " 613 # for index_out, _ in enumerate(param_list_pre_revised)\n",
- " 614 # ]\n",
- " 615 table_dict['valid_perf'] = [\n",
- " 616 '{:.2f}±{:.2f}'.format(average_val_scores[index_out][index_in],\n",
- " 617 std_val_scores[index_out][index_in])\n",
- " 618 for index_in, _ in enumerate(param_list)\n",
- " 619 for index_out, _ in enumerate(param_list_pre_revised)\n",
- " 620 ]\n",
- " 621 table_dict['test_perf'] = [\n",
- " 622 '{:.2f}±{:.2f}'.format(average_perf_scores[index_out][index_in],\n",
- " 623 std_perf_scores[index_out][index_in])\n",
- " 624 for index_in, _ in enumerate(param_list)\n",
- " 625 for index_out, _ in enumerate(param_list_pre_revised)\n",
- " 626 ]\n",
- " 627 table_dict['train_perf'] = [\n",
- " 628 '{:.2f}±{:.2f}'.format(average_train_scores[index_out][index_in],\n",
- " 629 std_train_scores[index_out][index_in])\n",
- " 630 for index_in, _ in enumerate(param_list)\n",
- " 631 for index_out, _ in enumerate(param_list_pre_revised)\n",
- " 632 ]\n",
- " 633 keyorder = [\n",
- " 634 'params', 'train_perf', 'valid_perf', 'test_perf'\n",
- " 635 ]\n",
- " 636 print()\n",
- " 637 tb_print = tabulate(\n",
- " 638 OrderedDict(\n",
- " 639 sorted(table_dict.items(),\n",
- " 640 key=lambda i: keyorder.index(i[0]))),\n",
- " 641 headers='keys')\n",
- " 642 # print(tb_print)\n",
- " 643 str_fw += 'table of performance v.s. hyper-params:\\n\\n%s\\n\\n' % tb_print\n",
- " 644 \n",
- " 645 # open file to save all results for this dataset.\n",
- " 646 if not os.path.exists(results_dir):\n",
- " 647 os.makedirs(results_dir)\n",
- " 648 \n",
- " 649 # open file to save all results for this dataset.\n",
- " 650 145.6 MiB 0.0 MiB if not os.path.exists(results_dir + '/' + ds_name + '.output.txt'):\n",
- " 651 with open(results_dir + '/' + ds_name + '.output.txt', 'w') as f:\n",
- " 652 f.write(str_fw)\n",
- " 653 else:\n",
- " 654 145.6 MiB 0.0 MiB with open(results_dir + '/' + ds_name + '.output.txt', 'r+') as f:\n",
- " 655 145.6 MiB 0.0 MiB content = f.read()\n",
- " 656 145.6 MiB 0.0 MiB f.seek(0, 0)\n",
- " 657 145.6 MiB 0.0 MiB f.write(str_fw + '\\n\\n\\n' + content)\n",
- "\n",
- "\n",
- "\n"
- ]
- }
- ],
- "source": [
- "import functools\n",
- "import sys\n",
- "sys.path.insert(0, \"../\")\n",
- "sys.path.insert(0, \"../../\")\n",
- "from libs import *\n",
- "import multiprocessing\n",
- "\n",
- "from pygraph.kernels.spKernel import spkernel\n",
- "from pygraph.utils.kernels import deltakernel, gaussiankernel, kernelproduct\n",
- "#from pygraph.utils.model_selection_precomputed import trial_do\n",
- "\n",
- "dslist = [\n",
- " {'name': 'Acyclic', 'dataset': '../../datasets/acyclic/dataset_bps.ds',\n",
- " 'task': 'regression'}, # node symb\n",
- "# {'name': 'Alkane', 'dataset': '../../datasets/Alkane/dataset.ds', 'task': 'regression',\n",
- "# 'dataset_y': '../../datasets/Alkane/dataset_boiling_point_names.txt', }, \n",
- "# # contains single node graph, node symb\n",
- "# {'name': 'MAO', 'dataset': '../../datasets/MAO/dataset.ds', }, # node/edge symb\n",
- "# {'name': 'PAH', 'dataset': '../../datasets/PAH/dataset.ds', }, # unlabeled\n",
- "# {'name': 'MUTAG', 'dataset': '../../datasets/MUTAG/MUTAG.mat',\n",
- "# 'extra_params': {'am_sp_al_nl_el': [0, 0, 3, 1, 2]}}, # node/edge symb\n",
- "# {'name': 'Letter-med', 'dataset': '../../datasets/Letter-med/Letter-med_A.txt'},\n",
- "# # node nsymb\n",
- "# {'name': 'ENZYMES', 'dataset': '../../datasets/ENZYMES_txt/ENZYMES_A_sparse.txt'},\n",
- "# # node symb/nsymb\n",
- "# {'name': 'Mutagenicity', 'dataset': '../../datasets/Mutagenicity/Mutagenicity_A.txt'},\n",
- "# # node/edge symb\n",
- "# {'name': 'D&D', 'dataset': '../../datasets/D&D/DD.mat',\n",
- "# 'extra_params': {'am_sp_al_nl_el': [0, 1, 2, 1, -1]}}, # node symb\n",
- "\n",
- " # {'name': 'COIL-DEL', 'dataset': '../../datasets/COIL-DEL/COIL-DEL_A.txt'}, # edge symb, node nsymb\n",
- " # # # {'name': 'BZR', 'dataset': '../../datasets/BZR_txt/BZR_A_sparse.txt'}, # node symb/nsymb\n",
- " # # # {'name': 'COX2', 'dataset': '../../datasets/COX2_txt/COX2_A_sparse.txt'}, # node symb/nsymb\n",
- " # {'name': 'Fingerprint', 'dataset': '../../datasets/Fingerprint/Fingerprint_A.txt'},\n",
- " #\n",
- " # # {'name': 'DHFR', 'dataset': '../../datasets/DHFR_txt/DHFR_A_sparse.txt'}, # node symb/nsymb\n",
- " # # {'name': 'SYNTHETIC', 'dataset': '../../datasets/SYNTHETIC_txt/SYNTHETIC_A_sparse.txt'}, # node symb/nsymb\n",
- " # # {'name': 'MSRC9', 'dataset': '../../datasets/MSRC_9_txt/MSRC_9_A.txt'}, # node symb\n",
- " # # {'name': 'MSRC21', 'dataset': '../../datasets/MSRC_21_txt/MSRC_21_A.txt'}, # node symb\n",
- " # # {'name': 'FIRSTMM_DB', 'dataset': '../../datasets/FIRSTMM_DB/FIRSTMM_DB_A.txt'}, # node symb/nsymb ,edge nsymb\n",
- "\n",
- " # # {'name': 'PROTEINS', 'dataset': '../../datasets/PROTEINS_txt/PROTEINS_A_sparse.txt'}, # node symb/nsymb\n",
- " # # {'name': 'PROTEINS_full', 'dataset': '../../datasets/PROTEINS_full_txt/PROTEINS_full_A_sparse.txt'}, # node symb/nsymb\n",
- " # # {'name': 'AIDS', 'dataset': '../../datasets/AIDS/AIDS_A.txt'}, # node symb/nsymb, edge symb\n",
- " # {'name': 'NCI1', 'dataset': '../../datasets/NCI1/NCI1.mat',\n",
- " # 'extra_params': {'am_sp_al_nl_el': [1, 1, 2, 0, -1]}}, # node symb\n",
- " # {'name': 'NCI109', 'dataset': '../../datasets/NCI109/NCI109.mat',\n",
- " # 'extra_params': {'am_sp_al_nl_el': [1, 1, 2, 0, -1]}}, # node symb\n",
- " # {'name': 'NCI-HIV', 'dataset': '../../datasets/NCI-HIV/AIDO99SD.sdf',\n",
- " # 'dataset_y': '../../datasets/NCI-HIV/aids_conc_may04.txt',}, # node/edge symb\n",
- "\n",
- " # # not working below\n",
- " # {'name': 'PTC_FM', 'dataset': '../../datasets/PTC/Train/FM.ds',},\n",
- " # {'name': 'PTC_FR', 'dataset': '../../datasets/PTC/Train/FR.ds',},\n",
- " # {'name': 'PTC_MM', 'dataset': '../../datasets/PTC/Train/MM.ds',},\n",
- " # {'name': 'PTC_MR', 'dataset': '../../datasets/PTC/Train/MR.ds',},\n",
- "]\n",
- "estimator = spkernel\n",
- "mixkernel = functools.partial(kernelproduct, deltakernel, gaussiankernel)\n",
- "param_grid_precomputed = {'node_kernels': [\n",
- " {'symb': deltakernel, 'nsymb': gaussiankernel, 'mix': mixkernel}]}\n",
- "param_grid = [{'C': np.logspace(-10, 10, num=41, base=10)},\n",
- " {'alpha': np.logspace(-10, 10, num=41, base=10)}]\n",
- "\n",
- "for ds in dslist:\n",
- " print()\n",
- " print(ds['name'])\n",
- " model_selection_for_precomputed_kernel(\n",
- " ds['dataset'],\n",
- " estimator,\n",
- " param_grid_precomputed,\n",
- " (param_grid[1] if ('task' in ds and ds['task']\n",
- " == 'regression') else param_grid[0]),\n",
- " (ds['task'] if 'task' in ds else 'classification'),\n",
- " NUM_TRIALS=30,\n",
- " datafile_y=(ds['dataset_y'] if 'dataset_y' in ds else None),\n",
- " extra_params=(ds['extra_params'] if 'extra_params' in ds else None),\n",
- " ds_name=ds['name'],\n",
- " n_jobs=multiprocessing.cpu_count(),\n",
- " read_gm_from_file=False)\n",
- " print()"
- ]
- }
- ],
- "metadata": {
- "kernelspec": {
- "display_name": "Python 3",
- "language": "python",
- "name": "python3"
- },
- "language_info": {
- "codemirror_mode": {
- "name": "ipython",
- "version": 3
- },
- "file_extension": ".py",
- "mimetype": "text/x-python",
- "name": "python",
- "nbconvert_exporter": "python",
- "pygments_lexer": "ipython3",
- "version": "3.6.7"
- }
- },
- "nbformat": 4,
- "nbformat_minor": 2
- }
|
