OpenI
/
graphkit-learn
Not watched
1
0
Fork 0
Code
Releases 2 Wiki Activity
Issues 0 Pull Requests 0 Datasets Model Cloudbrain
You can not select more than 25 topics Topics must start with a chinese character,a letter or number, can include dashes ('-') and can be up to 35 characters long.
Tree: 7b75c1fc97
Branches Tags
${ item.name }
Create branch ${ searchTerm }
from '7b75c1fc97'
${ noResults }
graphkit-learn/pygraph/utils/model_selection_precomputed.py

977 lines
48 kB
Raw Blame History 

  1. import numpy as np

  2. import matplotlib

  3. matplotlib.use('Agg')

  4. from matplotlib import pyplot as plt

  5. from sklearn.kernel_ridge import KernelRidge

  6. from sklearn.svm import SVC

  7. from sklearn.metrics import accuracy_score, mean_squared_error

  8. from sklearn.model_selection import KFold, train_test_split, ParameterGrid

  9. 

  10. #from joblib import Parallel, delayed

  11. from multiprocessing import Pool, Array

  12. from functools import partial

  13. import sys

  14. sys.path.insert(0, "../")

  15. import os

  16. import time

  17. import datetime

  18. #from os.path import basename, splitext

  19. from pygraph.utils.graphfiles import loadDataset

  20. from tqdm import tqdm

  21. 

  22. #from memory_profiler import profile

  23. 

  24. #@profile

  25. def model_selection_for_precomputed_kernel(datafile,

  26.                                            estimator,

  27.                                            param_grid_precomputed,

  28.                                            param_grid,

  29.                                            model_type,

  30.                                            NUM_TRIALS=30,

  31.                                            datafile_y=None,

  32.                                            extra_params=None,

  33.                                            ds_name='ds-unknown',

  34.                                            n_jobs=1,

  35.                                            read_gm_from_file=False,

  36.                                            verbose=True):

  37.     """Perform model selection, fitting and testing for precomputed kernels using nested cv. Print out neccessary data during the process then finally the results.

  38. 

  39.     Parameters

  40.     ----------

  41.     datafile : string

  42.         Path of dataset file.

  43.     estimator : function

  44.         kernel function used to estimate. This function needs to return a gram matrix.

  45.     param_grid_precomputed : dictionary

  46.         Dictionary with names (string) of parameters used to calculate gram matrices as keys and lists of parameter settings to try as values. This enables searching over any sequence of parameter settings. Params with length 1 will be omitted.

  47.     param_grid : dictionary

  48.         Dictionary with names (string) of parameters used as penelties as keys and lists of parameter settings to try as values. This enables searching over any sequence of parameter settings. Params with length 1 will be omitted.

  49.     model_type : string

  50.         Typr of the problem, can be regression or classification.

  51.     NUM_TRIALS : integer

  52.         Number of random trials of outer cv loop. The default is 30.

  53.     datafile_y : string

  54.         Path of file storing y data. This parameter is optional depending on the given dataset file.

  55.     read_gm_from_file : boolean

  56.         Whether gram matrices are loaded from file.

  57. 

  58.     Examples

  59.     --------

  60.     >>> import numpy as np

  61.     >>> import sys

  62.     >>> sys.path.insert(0, "../")

  63.     >>> from pygraph.utils.model_selection_precomputed import model_selection_for_precomputed_kernel

  64.     >>> from pygraph.kernels.weisfeilerLehmanKernel import weisfeilerlehmankernel

  65.     >>>

  66.     >>> datafile = '../../../../datasets/acyclic/Acyclic/dataset_bps.ds'

  67.     >>> estimator = weisfeilerlehmankernel

  68.     >>> param_grid_precomputed = {'height': [0,1,2,3,4,5,6,7,8,9,10], 'base_kernel': ['subtree']}

  69.     >>> param_grid = {"alpha": np.logspace(-2, 2, num = 10, base = 10)}

  70.     >>>

  71.     >>> model_selection_for_precomputed_kernel(datafile, estimator, param_grid_precomputed, param_grid, 'regression')

  72.     """

  73.     tqdm.monitor_interval = 0

  74. 

  75.     results_dir = '../notebooks/results/' + estimator.__name__

  76.     if not os.path.exists(results_dir):

  77.         os.makedirs(results_dir)

  78.     # a string to save all the results.

  79.     str_fw = '###################### log time: ' + datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S") + '. ######################\n\n'

  80.     str_fw += '# This file contains results of ' + estimator.__name__ + ' on dataset ' + ds_name + ',\n# including gram matrices, serial numbers for gram matrix figures and performance.\n\n'

  81. 

  82.     # setup the model type

  83.     model_type = model_type.lower()

  84.     if model_type != 'regression' and model_type != 'classification':

  85.         raise Exception(

  86.             'The model type is incorrect! Please choose from regression or classification.'

  87.         )

  88.     if verbose:

  89.         print()

  90.         print('--- This is a %s problem ---' % model_type)

  91.     str_fw += 'This is a %s problem.\n' % model_type

  92.     

  93.     # calculate gram matrices rather than read them from file.

  94.     if read_gm_from_file == False:

  95.         # Load the dataset

  96.         if verbose:

  97.             print()

  98.             print('\n1. Loading dataset from file...')

  99.         if isinstance(datafile, str):

  100.             dataset, y_all = loadDataset(

  101.                     datafile, filename_y=datafile_y, extra_params=extra_params)

  102.         else: # load data directly from variable.

  103.             dataset = datafile

  104.             y_all = datafile_y                

  105. 

  106.         #     import matplotlib.pyplot as plt

  107.         #     import networkx as nx

  108.         #     nx.draw_networkx(dataset[30])

  109.         #     plt.show()

  110.     

  111.         # Grid of parameters with a discrete number of values for each.

  112.         param_list_precomputed = list(ParameterGrid(param_grid_precomputed))

  113.         param_list = list(ParameterGrid(param_grid))

  114.     

  115.         gram_matrices = [

  116.         ]  # a list to store gram matrices for all param_grid_precomputed

  117.         gram_matrix_time = [

  118.         ]  # a list to store time to calculate gram matrices

  119.         param_list_pre_revised = [

  120.         ]  # list to store param grids precomputed ignoring the useless ones

  121.     

  122.         # calculate all gram matrices

  123.         if verbose:

  124.             print()

  125.             print('2. Calculating gram matrices. This could take a while...')

  126.         str_fw += '\nII. Gram matrices.\n\n'

  127.         tts = time.time()  # start training time

  128.         nb_gm_ignore = 0  # the number of gram matrices those should not be considered, as they may contain elements that are not numbers (NaN)

  129.         for idx, params_out in enumerate(param_list_precomputed):

  130.             y = y_all[:]

  131.             params_out['n_jobs'] = n_jobs

  132.             params_out['verbose'] = verbose

  133. #            print(dataset)

  134. #            import networkx as nx

  135. #            nx.draw_networkx(dataset[1])

  136. #            plt.show()

  137.             rtn_data = estimator(dataset[:], **params_out)

  138.             Kmatrix = rtn_data[0]

  139.             current_run_time = rtn_data[1]

  140.             # for some kernels, some graphs in datasets may not meet the 

  141.             # kernels' requirements for graph structure. These graphs are trimmed. 

  142.             if len(rtn_data) == 3:

  143.                 idx_trim = rtn_data[2]  # the index of trimmed graph list

  144.                 y = [y[idxt] for idxt in idx_trim] # trim y accordingly

  145. #            Kmatrix = np.random.rand(2250, 2250)

  146. #            current_run_time = 0.1

  147.             

  148.             # remove graphs whose kernels with themselves are zeros

  149.             Kmatrix_diag = Kmatrix.diagonal().copy()

  150.             nb_g_ignore = 0

  151.             for idxk, diag in enumerate(Kmatrix_diag):

  152.                 if diag == 0:

  153.                     Kmatrix = np.delete(Kmatrix, (idxk - nb_g_ignore), axis=0)

  154.                     Kmatrix = np.delete(Kmatrix, (idxk - nb_g_ignore), axis=1)

  155.                     nb_g_ignore += 1

  156.             # normalization

  157.             Kmatrix_diag = Kmatrix.diagonal().copy()

  158.             for i in range(len(Kmatrix)):

  159.                 for j in range(i, len(Kmatrix)):

  160.                     Kmatrix[i][j] /= np.sqrt(Kmatrix_diag[i] * Kmatrix_diag[j])

  161.                     Kmatrix[j][i] = Kmatrix[i][j]

  162.             if verbose:

  163.                 print()

  164.             if params_out == {}:

  165.                 if verbose:

  166.                     print('the gram matrix is: ')

  167.                 str_fw += 'the gram matrix is:\n\n'

  168.             else:

  169.                 if verbose:

  170.                     print('the gram matrix with parameters', params_out, 'is: \n\n')

  171.                 str_fw += 'the gram matrix with parameters %s is:\n\n' % params_out

  172.             if len(Kmatrix) < 2:

  173.                 nb_gm_ignore += 1

  174.                 if verbose:

  175.                     print('ignored, as at most only one of all its diagonal value is non-zero.')

  176.                 str_fw += 'ignored, as at most only one of all its diagonal value is non-zero.\n\n'

  177.             else:                

  178.                 if np.isnan(Kmatrix).any(

  179.                 ):  # if the matrix contains elements that are not numbers

  180.                     nb_gm_ignore += 1

  181.                     if verbose:

  182.                         print('ignored, as it contains elements that are not numbers.')

  183.                     str_fw += 'ignored, as it contains elements that are not numbers.\n\n'

  184.                 else:

  185. #                    print(Kmatrix)

  186.                     str_fw += np.array2string(

  187.                             Kmatrix,

  188.                             separator=',') + '\n\n'

  189. #                            separator=',',

  190. #                            threshold=np.inf,

  191. #                            floatmode='unique') + '\n\n'

  192. 

  193.                     fig_file_name = results_dir + '/GM[ds]' + ds_name

  194.                     if params_out != {}:

  195.                         fig_file_name += '[params]' + str(idx)

  196.                     plt.imshow(Kmatrix)

  197.                     plt.colorbar()

  198.                     plt.savefig(fig_file_name + '.eps', format='eps', dpi=300)

  199. #                    plt.show()

  200.                     plt.clf()

  201.                     gram_matrices.append(Kmatrix)

  202.                     gram_matrix_time.append(current_run_time)

  203.                     param_list_pre_revised.append(params_out)

  204.                     if nb_g_ignore > 0:

  205.                         if verbose:

  206.                             print(', where %d graphs are ignored as their graph kernels with themselves are zeros.' % nb_g_ignore)

  207.                         str_fw += ', where %d graphs are ignored as their graph kernels with themselves are zeros.' % nb_g_ignore

  208.         if verbose:

  209.             print()

  210.             print(

  211.             '{} gram matrices are calculated, {} of which are ignored.'.format(

  212.                 len(param_list_precomputed), nb_gm_ignore))

  213.         str_fw += '{} gram matrices are calculated, {} of which are ignored.\n\n'.format(len(param_list_precomputed), nb_gm_ignore)

  214.         str_fw += 'serial numbers of gram matrix figures and their corresponding parameters settings:\n\n'

  215.         str_fw += ''.join([

  216.             '{}: {}\n'.format(idx, params_out)

  217.             for idx, params_out in enumerate(param_list_precomputed)

  218.         ])

  219. 

  220.         if verbose:

  221.             print()

  222.         if len(gram_matrices) == 0:

  223.             if verbose:

  224.                 print('all gram matrices are ignored, no results obtained.')

  225.             str_fw += '\nall gram matrices are ignored, no results obtained.\n\n'

  226.         else:

  227.             # save gram matrices to file.

  228.             np.savez(results_dir + '/' + ds_name + '.gm', 

  229.                      gms=gram_matrices, params=param_list_pre_revised, y=y, 

  230.                      gmtime=gram_matrix_time)

  231.             if verbose:

  232.                 print(

  233.                 '3. Fitting and predicting using nested cross validation. This could really take a while...'

  234.                 )

  235.             

  236.             # ---- use pool.imap_unordered to parallel and track progress. ----

  237. #            train_pref = []

  238. #            val_pref = []

  239. #            test_pref = []

  240. #            def func_assign(result, var_to_assign):

  241. #                for idx, itm in enumerate(var_to_assign):

  242. #                    itm.append(result[idx])                

  243. #            trial_do_partial = partial(trial_do, param_list_pre_revised, param_list, y, model_type)

  244. #                      

  245. #            parallel_me(trial_do_partial, range(NUM_TRIALS), func_assign, 

  246. #                        [train_pref, val_pref, test_pref], glbv=gram_matrices,

  247. #                        method='imap_unordered', n_jobs=n_jobs, chunksize=1,

  248. #                        itr_desc='cross validation')

  249.             

  250.             def init_worker(gms_toshare):

  251.                 global G_gms

  252.                 G_gms = gms_toshare

  253.             

  254. #            gram_matrices = np.array(gram_matrices)

  255. #            gms_shape = gram_matrices.shape

  256. #            gms_array = Array('d', np.reshape(gram_matrices.copy(), -1, order='C'))

  257. #            pool = Pool(processes=n_jobs, initializer=init_worker, initargs=(gms_array, gms_shape))

  258.             pool = Pool(processes=n_jobs, initializer=init_worker, initargs=(gram_matrices,))

  259.             trial_do_partial = partial(parallel_trial_do, param_list_pre_revised, param_list, y, model_type)

  260.             train_pref = []

  261.             val_pref = []

  262.             test_pref = []

  263. #            if NUM_TRIALS < 1000 * n_jobs:

  264. #                chunksize = int(NUM_TRIALS / n_jobs) + 1

  265. #            else:

  266. #                chunksize = 1000

  267.             chunksize = 1

  268.             if verbose:

  269.                 iterator = tqdm(pool.imap_unordered(trial_do_partial, 

  270.                         range(NUM_TRIALS), chunksize), desc='cross validation', file=sys.stdout)

  271.             else:

  272.                 iterator = pool.imap_unordered(trial_do_partial, range(NUM_TRIALS), chunksize)

  273.             for o1, o2, o3 in iterator:

  274.                 train_pref.append(o1)

  275.                 val_pref.append(o2)

  276.                 test_pref.append(o3)

  277.             pool.close()

  278.             pool.join()

  279.     

  280. #            # ---- use pool.map to parallel. ----

  281. #            pool =  Pool(n_jobs)

  282. #            trial_do_partial = partial(trial_do, param_list_pre_revised, param_list, gram_matrices, y[0:250], model_type)

  283. #            result_perf = pool.map(trial_do_partial, range(NUM_TRIALS))

  284. #            train_pref = [item[0] for item in result_perf]

  285. #            val_pref = [item[1] for item in result_perf]

  286. #            test_pref = [item[2] for item in result_perf]

  287.     

  288. #            # ---- direct running, normally use a single CPU core. ----

  289. #            train_pref = []

  290. #            val_pref = []

  291. #            test_pref = []

  292. #            for i in tqdm(range(NUM_TRIALS), desc='cross validation', file=sys.stdout):

  293. #                o1, o2, o3 = trial_do(param_list_pre_revised, param_list, gram_matrices, y, model_type, i)

  294. #                train_pref.append(o1)

  295. #                val_pref.append(o2)

  296. #                test_pref.append(o3)

  297. #            print()

  298.     

  299.             if verbose:

  300.                 print()

  301.                 print('4. Getting final performance...')

  302.             str_fw += '\nIII. Performance.\n\n'

  303.             # averages and confidences of performances on outer trials for each combination of parameters

  304.             average_train_scores = np.mean(train_pref, axis=0)

  305. #            print('val_pref: ', val_pref[0][0])

  306.             average_val_scores = np.mean(val_pref, axis=0)

  307. #            print('test_pref: ', test_pref[0][0])

  308.             average_perf_scores = np.mean(test_pref, axis=0)

  309.             # sample std is used here

  310.             std_train_scores = np.std(train_pref, axis=0, ddof=1)

  311.             std_val_scores = np.std(val_pref, axis=0, ddof=1)

  312.             std_perf_scores = np.std(test_pref, axis=0, ddof=1)

  313.     

  314.             if model_type == 'regression':

  315.                 best_val_perf = np.amin(average_val_scores)

  316.             else:

  317.                 best_val_perf = np.amax(average_val_scores)

  318. #            print('average_val_scores: ', average_val_scores)

  319. #            print('best_val_perf: ', best_val_perf)

  320. #            print()

  321.             best_params_index = np.where(average_val_scores == best_val_perf)

  322.             # find smallest val std with best val perf.

  323.             best_val_stds = [

  324.                 std_val_scores[value][best_params_index[1][idx]]

  325.                 for idx, value in enumerate(best_params_index[0])

  326.             ]

  327.             min_val_std = np.amin(best_val_stds)

  328.             best_params_index = np.where(std_val_scores == min_val_std)

  329.             best_params_out = [

  330.                 param_list_pre_revised[i] for i in best_params_index[0]

  331.             ]

  332.             best_params_in = [param_list[i] for i in best_params_index[1]]

  333.             if verbose:

  334.                 print('best_params_out: ', best_params_out)

  335.                 print('best_params_in: ', best_params_in)

  336.                 print()

  337.                 print('best_val_perf: ', best_val_perf)

  338.                 print('best_val_std: ', min_val_std)

  339.             str_fw += 'best settings of hyper-params to build gram matrix: %s\n' % best_params_out

  340.             str_fw += 'best settings of other hyper-params: %s\n\n' % best_params_in

  341.             str_fw += 'best_val_perf: %s\n' % best_val_perf

  342.             str_fw += 'best_val_std: %s\n' % min_val_std

  343.     

  344. #            print(best_params_index)

  345. #            print(best_params_index[0])

  346. #            print(average_perf_scores)

  347.             final_performance = [

  348.                 average_perf_scores[value][best_params_index[1][idx]]

  349.                 for idx, value in enumerate(best_params_index[0])

  350.             ]

  351.             final_confidence = [

  352.                 std_perf_scores[value][best_params_index[1][idx]]

  353.                 for idx, value in enumerate(best_params_index[0])

  354.             ]

  355.             if verbose:

  356.                 print('final_performance: ', final_performance)

  357.                 print('final_confidence: ', final_confidence)

  358.             str_fw += 'final_performance: %s\n' % final_performance

  359.             str_fw += 'final_confidence: %s\n' % final_confidence

  360.             train_performance = [

  361.                 average_train_scores[value][best_params_index[1][idx]]

  362.                 for idx, value in enumerate(best_params_index[0])

  363.             ]

  364.             train_std = [

  365.                 std_train_scores[value][best_params_index[1][idx]]

  366.                 for idx, value in enumerate(best_params_index[0])

  367.             ]

  368.             if verbose:

  369.                 print('train_performance: %s' % train_performance)

  370.                 print('train_std: ', train_std)

  371.             str_fw += 'train_performance: %s\n' % train_performance

  372.             str_fw += 'train_std: %s\n\n' % train_std

  373. 

  374.             if verbose:

  375.                 print()

  376.             tt_total = time.time() - tts  # training time for all hyper-parameters

  377.             average_gram_matrix_time = np.mean(gram_matrix_time)

  378.             std_gram_matrix_time = np.std(gram_matrix_time, ddof=1) if len(gram_matrix_time) > 1 else 0

  379.             best_gram_matrix_time = [

  380.                 gram_matrix_time[i] for i in best_params_index[0]

  381.             ]

  382.             ave_bgmt = np.mean(best_gram_matrix_time)

  383.             std_bgmt = np.std(best_gram_matrix_time, ddof=1) if len(best_gram_matrix_time) > 1 else 0

  384.             if verbose:

  385.                 print('time to calculate gram matrix with different hyper-params: {:.2f}±{:.2f}s'

  386.                       .format(average_gram_matrix_time, std_gram_matrix_time))

  387.                 print('time to calculate best gram matrix: {:.2f}±{:.2f}s'.format(

  388.                         ave_bgmt, std_bgmt))

  389.                 print('total training time with all hyper-param choices: {:.2f}s'.format(

  390.                         tt_total))

  391.             str_fw += 'time to calculate gram matrix with different hyper-params: {:.2f}±{:.2f}s\n'.format(average_gram_matrix_time, std_gram_matrix_time)

  392.             str_fw += 'time to calculate best gram matrix: {:.2f}±{:.2f}s\n'.format(ave_bgmt, std_bgmt)

  393.             str_fw += 'total training time with all hyper-param choices: {:.2f}s\n\n'.format(tt_total)

  394.     

  395.             # # save results to file

  396.             # np.savetxt(results_name_pre + 'average_train_scores.dt',

  397.             #            average_train_scores)

  398.             # np.savetxt(results_name_pre + 'average_val_scores', average_val_scores)

  399.             # np.savetxt(results_name_pre + 'average_perf_scores.dt',

  400.             #            average_perf_scores)

  401.             # np.savetxt(results_name_pre + 'std_train_scores.dt', std_train_scores)

  402.             # np.savetxt(results_name_pre + 'std_val_scores.dt', std_val_scores)

  403.             # np.savetxt(results_name_pre + 'std_perf_scores.dt', std_perf_scores)

  404.     

  405.             # np.save(results_name_pre + 'best_params_index', best_params_index)

  406.             # np.save(results_name_pre + 'best_params_pre.dt', best_params_out)

  407.             # np.save(results_name_pre + 'best_params_in.dt', best_params_in)

  408.             # np.save(results_name_pre + 'best_val_perf.dt', best_val_perf)

  409.             # np.save(results_name_pre + 'best_val_std.dt', best_val_std)

  410.             # np.save(results_name_pre + 'final_performance.dt', final_performance)

  411.             # np.save(results_name_pre + 'final_confidence.dt', final_confidence)

  412.             # np.save(results_name_pre + 'train_performance.dt', train_performance)

  413.             # np.save(results_name_pre + 'train_std.dt', train_std)

  414.     

  415.             # np.save(results_name_pre + 'gram_matrix_time.dt', gram_matrix_time)

  416.             # np.save(results_name_pre + 'average_gram_matrix_time.dt',

  417.             #         average_gram_matrix_time)

  418.             # np.save(results_name_pre + 'std_gram_matrix_time.dt',

  419.             #         std_gram_matrix_time)

  420.             # np.save(results_name_pre + 'best_gram_matrix_time.dt',

  421.             #         best_gram_matrix_time)

  422.     

  423.             # print out as table.

  424.             from collections import OrderedDict

  425.             from tabulate import tabulate

  426.             table_dict = {}

  427.             if model_type == 'regression':

  428.                 for param_in in param_list:

  429.                     param_in['alpha'] = '{:.2e}'.format(param_in['alpha'])

  430.             else:

  431.                 for param_in in param_list:

  432.                     param_in['C'] = '{:.2e}'.format(param_in['C'])

  433.             table_dict['params'] = [{**param_out, **param_in}

  434.                                     for param_in in param_list for param_out in param_list_pre_revised]

  435.             table_dict['gram_matrix_time'] = [

  436.                 '{:.2f}'.format(gram_matrix_time[index_out])

  437.                 for param_in in param_list

  438.                 for index_out, _ in enumerate(param_list_pre_revised)

  439.             ]

  440.             table_dict['valid_perf'] = [

  441.                 '{:.2f}±{:.2f}'.format(average_val_scores[index_out][index_in],

  442.                                        std_val_scores[index_out][index_in])

  443.                 for index_in, _ in enumerate(param_list)

  444.                 for index_out, _ in enumerate(param_list_pre_revised)

  445.             ]

  446.             table_dict['test_perf'] = [

  447.                 '{:.2f}±{:.2f}'.format(average_perf_scores[index_out][index_in],

  448.                                        std_perf_scores[index_out][index_in])

  449.                 for index_in, _ in enumerate(param_list)

  450.                 for index_out, _ in enumerate(param_list_pre_revised)

  451.             ]

  452.             table_dict['train_perf'] = [

  453.                 '{:.2f}±{:.2f}'.format(average_train_scores[index_out][index_in],

  454.                                        std_train_scores[index_out][index_in])

  455.                 for index_in, _ in enumerate(param_list)

  456.                 for index_out, _ in enumerate(param_list_pre_revised)

  457.             ]

  458.             keyorder = [

  459.                 'params', 'train_perf', 'valid_perf', 'test_perf',

  460.                 'gram_matrix_time'

  461.             ]

  462.             if verbose:

  463.                 print()

  464.             tb_print = tabulate(

  465.                 OrderedDict(

  466.                     sorted(table_dict.items(),

  467.                            key=lambda i: keyorder.index(i[0]))),

  468.                 headers='keys')

  469. #            print(tb_print)

  470.             str_fw += 'table of performance v.s. hyper-params:\n\n%s\n\n' % tb_print

  471.     

  472.     # read gram matrices from file.

  473.     else:    

  474.         # Grid of parameters with a discrete number of values for each.

  475. #        param_list_precomputed = list(ParameterGrid(param_grid_precomputed))

  476.         param_list = list(ParameterGrid(param_grid))

  477.     

  478.         # read gram matrices from file.

  479.         if verbose:

  480.             print()

  481.             print('2. Reading gram matrices from file...')

  482.         str_fw += '\nII. Gram matrices.\n\nGram matrices are read from file, see last log for detail.\n'

  483.         gmfile = np.load(results_dir + '/' + ds_name + '.gm.npz')

  484.         gram_matrices = gmfile['gms'] # a list to store gram matrices for all param_grid_precomputed

  485.         gram_matrix_time = gmfile['gmtime'] # time used to compute the gram matrices

  486.         param_list_pre_revised = gmfile['params'] # list to store param grids precomputed ignoring the useless ones

  487.         y = gmfile['y'].tolist()

  488.         

  489.         tts = time.time()  # start training time

  490. #        nb_gm_ignore = 0  # the number of gram matrices those should not be considered, as they may contain elements that are not numbers (NaN)            

  491.         if verbose:

  492.             print(

  493.                     '3. Fitting and predicting using nested cross validation. This could really take a while...'

  494.                     )

  495.  

  496.         # ---- use pool.imap_unordered to parallel and track progress. ----

  497.         def init_worker(gms_toshare):

  498.             global G_gms

  499.             G_gms = gms_toshare

  500. 

  501.         pool = Pool(processes=n_jobs, initializer=init_worker, initargs=(gram_matrices,))

  502.         trial_do_partial = partial(parallel_trial_do, param_list_pre_revised, param_list, y, model_type)

  503.         train_pref = []

  504.         val_pref = []

  505.         test_pref = []

  506.         chunksize = 1

  507.         if verbose:

  508.             iterator = tqdm(pool.imap_unordered(trial_do_partial, 

  509.                     range(NUM_TRIALS), chunksize), desc='cross validation', file=sys.stdout)

  510.         else:

  511.             iterator = pool.imap_unordered(trial_do_partial, range(NUM_TRIALS), chunksize)

  512.         for o1, o2, o3 in iterator:

  513.             train_pref.append(o1)

  514.             val_pref.append(o2)

  515.             test_pref.append(o3)

  516.         pool.close()

  517.         pool.join()

  518.         

  519.         # # ---- use pool.map to parallel. ----

  520.         # result_perf = pool.map(trial_do_partial, range(NUM_TRIALS))

  521.         # train_pref = [item[0] for item in result_perf]

  522.         # val_pref = [item[1] for item in result_perf]

  523.         # test_pref = [item[2] for item in result_perf]

  524. 

  525.         # # ---- use joblib.Parallel to parallel and track progress. ----

  526.         # trial_do_partial = partial(trial_do, param_list_pre_revised, param_list, gram_matrices, y, model_type)

  527.         # result_perf = Parallel(n_jobs=n_jobs, verbose=10)(delayed(trial_do_partial)(trial) for trial in range(NUM_TRIALS))

  528.         # train_pref = [item[0] for item in result_perf]

  529.         # val_pref = [item[1] for item in result_perf]

  530.         # test_pref = [item[2] for item in result_perf]

  531. 

  532. #        # ---- direct running, normally use a single CPU core. ----

  533. #        train_pref = []

  534. #        val_pref = []

  535. #        test_pref = []

  536. #        for i in tqdm(range(NUM_TRIALS), desc='cross validation', file=sys.stdout):

  537. #            o1, o2, o3 = trial_do(param_list_pre_revised, param_list, gram_matrices, y, model_type, i)

  538. #            train_pref.append(o1)

  539. #            val_pref.append(o2)

  540. #            test_pref.append(o3)

  541. 

  542.         if verbose:

  543.             print()

  544.             print('4. Getting final performance...')

  545.         str_fw += '\nIII. Performance.\n\n'

  546.         # averages and confidences of performances on outer trials for each combination of parameters

  547.         average_train_scores = np.mean(train_pref, axis=0)

  548.         average_val_scores = np.mean(val_pref, axis=0)

  549.         average_perf_scores = np.mean(test_pref, axis=0)

  550.         # sample std is used here

  551.         std_train_scores = np.std(train_pref, axis=0, ddof=1)

  552.         std_val_scores = np.std(val_pref, axis=0, ddof=1)

  553.         std_perf_scores = np.std(test_pref, axis=0, ddof=1)

  554. 

  555.         if model_type == 'regression':

  556.             best_val_perf = np.amin(average_val_scores)

  557.         else:

  558.             best_val_perf = np.amax(average_val_scores)

  559.         best_params_index = np.where(average_val_scores == best_val_perf)

  560.         # find smallest val std with best val perf.

  561.         best_val_stds = [

  562.             std_val_scores[value][best_params_index[1][idx]]

  563.             for idx, value in enumerate(best_params_index[0])

  564.         ]

  565.         min_val_std = np.amin(best_val_stds)

  566.         best_params_index = np.where(std_val_scores == min_val_std)

  567.         best_params_out = [

  568.             param_list_pre_revised[i] for i in best_params_index[0]

  569.         ]

  570.         best_params_in = [param_list[i] for i in best_params_index[1]]

  571.         if verbose:

  572.             print('best_params_out: ', best_params_out)

  573.             print('best_params_in: ', best_params_in)

  574.             print()

  575.             print('best_val_perf: ', best_val_perf)

  576.             print('best_val_std: ', min_val_std)

  577.         str_fw += 'best settings of hyper-params to build gram matrix: %s\n' % best_params_out

  578.         str_fw += 'best settings of other hyper-params: %s\n\n' % best_params_in

  579.         str_fw += 'best_val_perf: %s\n' % best_val_perf

  580.         str_fw += 'best_val_std: %s\n' % min_val_std

  581. 

  582.         final_performance = [

  583.             average_perf_scores[value][best_params_index[1][idx]]

  584.             for idx, value in enumerate(best_params_index[0])

  585.         ]

  586.         final_confidence = [

  587.             std_perf_scores[value][best_params_index[1][idx]]

  588.             for idx, value in enumerate(best_params_index[0])

  589.         ]

  590.         if verbose:

  591.             print('final_performance: ', final_performance)

  592.             print('final_confidence: ', final_confidence)

  593.         str_fw += 'final_performance: %s\n' % final_performance

  594.         str_fw += 'final_confidence: %s\n' % final_confidence

  595.         train_performance = [

  596.             average_train_scores[value][best_params_index[1][idx]]

  597.             for idx, value in enumerate(best_params_index[0])

  598.         ]

  599.         train_std = [

  600.             std_train_scores[value][best_params_index[1][idx]]

  601.             for idx, value in enumerate(best_params_index[0])

  602.         ]

  603.         if verbose:

  604.             print('train_performance: %s' % train_performance)

  605.             print('train_std: ', train_std)

  606.         str_fw += 'train_performance: %s\n' % train_performance

  607.         str_fw += 'train_std: %s\n\n' % train_std

  608. 

  609.         if verbose:

  610.             print()

  611.         average_gram_matrix_time = np.mean(gram_matrix_time)

  612.         std_gram_matrix_time = np.std(gram_matrix_time, ddof=1) if len(gram_matrix_time) > 1 else 0

  613.         best_gram_matrix_time = [

  614.             gram_matrix_time[i] for i in best_params_index[0]

  615.         ]

  616.         ave_bgmt = np.mean(best_gram_matrix_time)

  617.         std_bgmt = np.std(best_gram_matrix_time, ddof=1) if len(best_gram_matrix_time) > 1 else 0

  618.         if verbose:        

  619.             print(

  620.                     'time to calculate gram matrix with different hyper-params: {:.2f}±{:.2f}s'

  621.                     .format(average_gram_matrix_time, std_gram_matrix_time))

  622.             print('time to calculate best gram matrix: {:.2f}±{:.2f}s'.format(

  623.                     ave_bgmt, std_bgmt))

  624.         tt_poster = time.time() - tts  # training time with hyper-param choices who did not participate in calculation of gram matrices

  625.         if verbose:

  626.             print(

  627.                     'training time with hyper-param choices who did not participate in calculation of gram matrices: {:.2f}s'.format(

  628.                             tt_poster))

  629.             print('total training time with all hyper-param choices: {:.2f}s'.format(

  630.                     tt_poster + np.sum(gram_matrix_time)))

  631. #        str_fw += 'time to calculate gram matrix with different hyper-params: {:.2f}±{:.2f}s\n'.format(average_gram_matrix_time, std_gram_matrix_time)

  632. #        str_fw += 'time to calculate best gram matrix: {:.2f}±{:.2f}s\n'.format(ave_bgmt, std_bgmt)

  633.         str_fw += 'training time with hyper-param choices who did not participate in calculation of gram matrices: {:.2f}s\n\n'.format(tt_poster)

  634. 

  635.         # print out as table.

  636.         from collections import OrderedDict

  637.         from tabulate import tabulate

  638.         table_dict = {}

  639.         if model_type == 'regression':

  640.             for param_in in param_list:

  641.                 param_in['alpha'] = '{:.2e}'.format(param_in['alpha'])

  642.         else:

  643.             for param_in in param_list:

  644.                 param_in['C'] = '{:.2e}'.format(param_in['C'])

  645.         table_dict['params'] = [{**param_out, **param_in}

  646.                                 for param_in in param_list for param_out in param_list_pre_revised]

  647. #        table_dict['gram_matrix_time'] = [

  648. #            '{:.2f}'.format(gram_matrix_time[index_out])

  649. #            for param_in in param_list

  650. #            for index_out, _ in enumerate(param_list_pre_revised)

  651. #        ]

  652.         table_dict['valid_perf'] = [

  653.             '{:.2f}±{:.2f}'.format(average_val_scores[index_out][index_in],

  654.                                    std_val_scores[index_out][index_in])

  655.             for index_in, _ in enumerate(param_list)

  656.             for index_out, _ in enumerate(param_list_pre_revised)

  657.         ]

  658.         table_dict['test_perf'] = [

  659.             '{:.2f}±{:.2f}'.format(average_perf_scores[index_out][index_in],

  660.                                    std_perf_scores[index_out][index_in])

  661.             for index_in, _ in enumerate(param_list)

  662.             for index_out, _ in enumerate(param_list_pre_revised)

  663.         ]

  664.         table_dict['train_perf'] = [

  665.             '{:.2f}±{:.2f}'.format(average_train_scores[index_out][index_in],

  666.                                    std_train_scores[index_out][index_in])

  667.             for index_in, _ in enumerate(param_list)

  668.             for index_out, _ in enumerate(param_list_pre_revised)

  669.         ]

  670.         keyorder = [

  671.             'params', 'train_perf', 'valid_perf', 'test_perf'

  672.         ]

  673.         if verbose:

  674.             print()

  675.         tb_print = tabulate(

  676.             OrderedDict(

  677.                 sorted(table_dict.items(),

  678.                        key=lambda i: keyorder.index(i[0]))),

  679.             headers='keys')

  680. #        print(tb_print)

  681.         str_fw += 'table of performance v.s. hyper-params:\n\n%s\n\n' % tb_print

  682. 

  683.         # open file to save all results for this dataset.

  684.         if not os.path.exists(results_dir):

  685.             os.makedirs(results_dir)

  686.             

  687.     # open file to save all results for this dataset.

  688.     if not os.path.exists(results_dir + '/' + ds_name + '.output.txt'):

  689.         with open(results_dir + '/' + ds_name + '.output.txt', 'w') as f:

  690.             f.write(str_fw)

  691.     else:

  692.         with open(results_dir + '/' + ds_name + '.output.txt', 'r+') as f:

  693.             content = f.read()

  694.             f.seek(0, 0)

  695.             f.write(str_fw + '\n\n\n' + content)

  696. 

  697. 

  698. def trial_do(param_list_pre_revised, param_list, gram_matrices, y, model_type, trial): # Test set level

  699. 

  700. #    # get gram matrices from global variables.

  701. #    gram_matrices = np.reshape(G_gms.copy(), G_gms_shape, order='C')

  702.     

  703.     # Arrays to store scores

  704.     train_pref = np.zeros((len(param_list_pre_revised), len(param_list)))

  705.     val_pref = np.zeros((len(param_list_pre_revised), len(param_list)))

  706.     test_pref = np.zeros((len(param_list_pre_revised), len(param_list)))

  707. 

  708.     # randomness added to seeds of split function below. "high" is "size" times

  709.     # 10 so that at least 10 different random output will be yielded. Remove

  710.     # these lines if identical outputs is required.

  711.     rdm_out = np.random.RandomState(seed=None)

  712.     rdm_seed_out_l = rdm_out.uniform(high=len(param_list_pre_revised) * 10, 

  713.                                    size=len(param_list_pre_revised))

  714. #    print(trial, rdm_seed_out_l)

  715. #    print()

  716.     # loop for each outer param tuple

  717.     for index_out, params_out in enumerate(param_list_pre_revised):

  718.         # get gram matrices from global variables.

  719. #        gm_now = G_gms[index_out * G_gms_shape[1] * G_gms_shape[2]:(index_out + 1) * G_gms_shape[1] * G_gms_shape[2]]

  720. #        gm_now = np.reshape(gm_now.copy(), (G_gms_shape[1], G_gms_shape[2]), order='C')

  721.         gm_now = gram_matrices[index_out].copy()

  722.     

  723.         # split gram matrix and y to app and test sets.

  724.         indices = range(len(y))

  725.         # The argument "random_state" in function "train_test_split" can not be

  726.         # set to None, because it will use RandomState instance used by 

  727.         # np.random, which is possible for multiple subprocesses to inherit the

  728.         # same seed if they forked at the same time, leading to identical 

  729.         # random variates for different subprocesses. Instead, we use "trial" 

  730.         # and "index_out" parameters to generate different seeds for different 

  731.         # trials/subprocesses and outer loops. "rdm_seed_out_l" is used to add 

  732.         # randomness into seeds, so that it yields a different output every 

  733.         # time the program is run. To yield identical outputs every time,

  734.         # remove the second line below. Same method is used to the "KFold"

  735.         # function in the inner loop.

  736.         rdm_seed_out = (trial + 1) * (index_out + 1)

  737.         rdm_seed_out = (rdm_seed_out + int(rdm_seed_out_l[index_out])) % (2 ** 32 - 1)

  738. #        print(trial, rdm_seed_out)

  739.         X_app, X_test, y_app, y_test, idx_app, idx_test = train_test_split(

  740.             gm_now, y, indices, test_size=0.1, 

  741.             random_state=rdm_seed_out, shuffle=True)

  742. #        print(trial, idx_app, idx_test)

  743. #        print()

  744.         X_app = X_app[:, idx_app]

  745.         X_test = X_test[:, idx_app]

  746.         y_app = np.array(y_app)

  747.         y_test = np.array(y_test)

  748. 

  749.         rdm_seed_in_l = rdm_out.uniform(high=len(param_list) * 10, 

  750.                                    size=len(param_list))

  751.         # loop for each inner param tuple

  752.         for index_in, params_in in enumerate(param_list):

  753. #            if trial == 0:

  754. #                print(index_out, index_in)

  755. #                print('params_in: ', params_in)

  756. #            st = time.time()

  757.             rdm_seed_in = (trial + 1) * (index_out + 1) * (index_in + 1)

  758. #            print("rdm_seed_in1: ", trial, index_in, rdm_seed_in)

  759.             rdm_seed_in = (rdm_seed_in + int(rdm_seed_in_l[index_in])) % (2 ** 32 - 1)

  760. #            print("rdm_seed_in2: ", trial, index_in, rdm_seed_in)

  761.             inner_cv = KFold(n_splits=10, shuffle=True, random_state=rdm_seed_in)

  762.             current_train_perf = []

  763.             current_valid_perf = []

  764.             current_test_perf = [] 

  765. 

  766.             # For regression use the Kernel Ridge method

  767. #            try:

  768.             if model_type == 'regression':

  769.                 kr = KernelRidge(kernel='precomputed', **params_in)

  770.                 # loop for each split on validation set level

  771.                 # validation set level

  772.                 for train_index, valid_index in inner_cv.split(X_app):

  773. #                    print("train_index, valid_index: ", trial, index_in, train_index, valid_index)

  774. #                    if trial == 0:

  775. #                        print('train_index: ', train_index)

  776. #                        print('valid_index: ', valid_index)

  777. #                        print('idx_test: ', idx_test)

  778. #                        print('y_app[train_index]: ', y_app[train_index])

  779. #                        print('X_app[train_index, :][:, train_index]: ', X_app[train_index, :][:, train_index])

  780. #                        print('X_app[valid_index, :][:, train_index]: ', X_app[valid_index, :][:, train_index])

  781.                     kr.fit(X_app[train_index, :][:, train_index],

  782.                            y_app[train_index])

  783. 

  784.                     # predict on the train, validation and test set

  785.                     y_pred_train = kr.predict(

  786.                         X_app[train_index, :][:, train_index])

  787.                     y_pred_valid = kr.predict(

  788.                         X_app[valid_index, :][:, train_index])

  789. #                    if trial == 0:     

  790. #                        print('y_pred_valid: ', y_pred_valid)

  791. #                        print()

  792.                     y_pred_test = kr.predict(

  793.                         X_test[:, train_index])

  794. 

  795.                     # root mean squared errors

  796.                     current_train_perf.append(

  797.                         np.sqrt(

  798.                             mean_squared_error(

  799.                                 y_app[train_index], y_pred_train)))

  800.                     current_valid_perf.append(

  801.                         np.sqrt(

  802.                             mean_squared_error(

  803.                                 y_app[valid_index], y_pred_valid)))

  804. #                    if trial == 0:

  805. #                        print(mean_squared_error(

  806. #                                y_app[valid_index], y_pred_valid))

  807.                     current_test_perf.append(

  808.                         np.sqrt(

  809.                             mean_squared_error(

  810.                                 y_test, y_pred_test)))

  811.             # For clcassification use SVM

  812.             else:

  813.                 svc = SVC(kernel='precomputed', cache_size=200, 

  814.                           verbose=False, **params_in)

  815.                 # loop for each split on validation set level

  816.                 # validation set level

  817.                 for train_index, valid_index in inner_cv.split(X_app):

  818. #                        np.savez("bug.npy",X_app[train_index, :][:, train_index],y_app[train_index])

  819. #                    if trial == 0:

  820. #                        print('train_index: ', train_index)

  821. #                        print('valid_index: ', valid_index)

  822. #                        print('idx_test: ', idx_test)

  823. #                        print('y_app[train_index]: ', y_app[train_index])

  824. #                        print('X_app[train_index, :][:, train_index]: ', X_app[train_index, :][:, train_index])

  825. #                        print('X_app[valid_index, :][:, train_index]: ', X_app[valid_index, :][:, train_index])

  826.                     svc.fit(X_app[train_index, :][:, train_index],

  827.                            y_app[train_index])

  828.                     

  829.                     # predict on the train, validation and test set

  830.                     y_pred_train = svc.predict(

  831.                         X_app[train_index, :][:, train_index])

  832.                     y_pred_valid = svc.predict(

  833.                         X_app[valid_index, :][:, train_index])

  834.                     y_pred_test = svc.predict(

  835.                         X_test[:, train_index])

  836. 

  837.                     # root mean squared errors

  838.                     current_train_perf.append(

  839.                         accuracy_score(y_app[train_index],

  840.                                        y_pred_train))

  841.                     current_valid_perf.append(

  842.                         accuracy_score(y_app[valid_index],

  843.                                        y_pred_valid))

  844.                     current_test_perf.append(

  845.                         accuracy_score(y_test, y_pred_test))

  846. #            except ValueError:

  847. #                print(sys.exc_info()[0])

  848. #                print(params_out, params_in)

  849. 

  850.             # average performance on inner splits

  851.             train_pref[index_out][index_in] = np.mean(

  852.                 current_train_perf)

  853.             val_pref[index_out][index_in] = np.mean(

  854.                 current_valid_perf)

  855.             test_pref[index_out][index_in] = np.mean(

  856.                 current_test_perf)

  857. #            print(time.time() - st)

  858. #    if trial == 0:

  859. #        print('val_pref: ', val_pref)

  860. #        print('test_pref: ', test_pref)

  861. 

  862.     return train_pref, val_pref, test_pref

  863. 

  864. def parallel_trial_do(param_list_pre_revised, param_list, y, model_type, trial):

  865.     train_pref, val_pref, test_pref = trial_do(param_list_pre_revised, 

  866.                                                param_list, G_gms, y, 

  867.                                                model_type, trial)

  868.     return train_pref, val_pref, test_pref

  869. 

  870. 

  871. def compute_gram_matrices(dataset, y, estimator, param_list_precomputed, 

  872.                           results_dir, ds_name,

  873.                           n_jobs=1, str_fw='', verbose=True):

  874.     gram_matrices = [

  875.         ]  # a list to store gram matrices for all param_grid_precomputed

  876.     gram_matrix_time = [

  877.         ]  # a list to store time to calculate gram matrices

  878.     param_list_pre_revised = [

  879.         ]  # list to store param grids precomputed ignoring the useless ones

  880.     

  881.     nb_gm_ignore = 0  # the number of gram matrices those should not be considered, as they may contain elements that are not numbers (NaN)

  882.     for idx, params_out in enumerate(param_list_precomputed):

  883.         params_out['n_jobs'] = n_jobs

  884. #            print(dataset)

  885. #            import networkx as nx

  886. #            nx.draw_networkx(dataset[1])

  887. #            plt.show()

  888.         rtn_data = estimator(dataset[:], **params_out)

  889.         Kmatrix = rtn_data[0]

  890.         current_run_time = rtn_data[1]

  891.         # for some kernels, some graphs in datasets may not meet the 

  892.         # kernels' requirements for graph structure. These graphs are trimmed. 

  893.         if len(rtn_data) == 3:

  894.             idx_trim = rtn_data[2]  # the index of trimmed graph list

  895.             y = [y[idxt] for idxt in idx_trim] # trim y accordingly

  896. 

  897.         Kmatrix_diag = Kmatrix.diagonal().copy()

  898.         # remove graphs whose kernels with themselves are zeros

  899.         nb_g_ignore = 0

  900.         for idxk, diag in enumerate(Kmatrix_diag):

  901.             if diag == 0:

  902.                 Kmatrix = np.delete(Kmatrix, (idxk - nb_g_ignore), axis=0)

  903.                 Kmatrix = np.delete(Kmatrix, (idxk - nb_g_ignore), axis=1)

  904.                 nb_g_ignore += 1

  905.         # normalization

  906.         for i in range(len(Kmatrix)):

  907.             for j in range(i, len(Kmatrix)):

  908.                 Kmatrix[i][j] /= np.sqrt(Kmatrix_diag[i] * Kmatrix_diag[j])

  909.                 Kmatrix[j][i] = Kmatrix[i][j]

  910. 

  911.         if verbose:

  912.             print()

  913.         if params_out == {}:

  914.             if verbose:

  915.                 print('the gram matrix is: ')

  916.             str_fw += 'the gram matrix is:\n\n'

  917.         else:

  918.             if verbose:

  919.                 print('the gram matrix with parameters', params_out, 'is: ')

  920.             str_fw += 'the gram matrix with parameters %s is:\n\n' % params_out

  921.         if len(Kmatrix) < 2:

  922.             nb_gm_ignore += 1

  923.             if verbose:

  924.                 print('ignored, as at most only one of all its diagonal value is non-zero.')

  925.             str_fw += 'ignored, as at most only one of all its diagonal value is non-zero.\n\n'

  926.         else:                

  927.             if np.isnan(Kmatrix).any(

  928.             ):  # if the matrix contains elements that are not numbers

  929.                 nb_gm_ignore += 1

  930.                 if verbose:

  931.                     print('ignored, as it contains elements that are not numbers.')

  932.                 str_fw += 'ignored, as it contains elements that are not numbers.\n\n'

  933.             else:

  934. #                    print(Kmatrix)

  935.                 str_fw += np.array2string(

  936.                         Kmatrix,

  937.                         separator=',') + '\n\n'

  938. #                            separator=',',

  939. #                            threshold=np.inf,

  940. #                            floatmode='unique') + '\n\n'

  941. 

  942.                 fig_file_name = results_dir + '/GM[ds]' + ds_name

  943.                 if params_out != {}:

  944.                     fig_file_name += '[params]' + str(idx)

  945.                 plt.imshow(Kmatrix)

  946.                 plt.colorbar()

  947.                 plt.savefig(fig_file_name + '.eps', format='eps', dpi=300)

  948. #                    plt.show()

  949.                 plt.clf()

  950.                 gram_matrices.append(Kmatrix)

  951.                 gram_matrix_time.append(current_run_time)

  952.                 param_list_pre_revised.append(params_out)

  953.                 if nb_g_ignore > 0:

  954.                     if verbose:

  955.                         print(', where %d graphs are ignored as their graph kernels with themselves are zeros.' % nb_g_ignore)

  956.                     str_fw += ', where %d graphs are ignored as their graph kernels with themselves are zeros.' % nb_g_ignore

  957.     if verbose:

  958.         print()

  959.         print(

  960.             '{} gram matrices are calculated, {} of which are ignored.'.format(

  961.                 len(param_list_precomputed), nb_gm_ignore))

  962.     str_fw += '{} gram matrices are calculated, {} of which are ignored.\n\n'.format(len(param_list_precomputed), nb_gm_ignore)

  963.     str_fw += 'serial numbers of gram matrix figures and their corresponding parameters settings:\n\n'

  964.     str_fw += ''.join([

  965.         '{}: {}\n'.format(idx, params_out)

  966.         for idx, params_out in enumerate(param_list_precomputed)

  967.     ])

  968.             

  969.     return gram_matrices, gram_matrix_time, param_list_pre_revised, y, str_fw

  970. 

  971. 

  972. def read_gram_matrices_from_file(results_dir, ds_name):

  973.     gmfile = np.load(results_dir + '/' + ds_name + '.gm.npz')

  974.     gram_matrices = gmfile['gms'] # a list to store gram matrices for all param_grid_precomputed

  975.     param_list_pre_revised = gmfile['params'] # list to store param grids precomputed ignoring the useless ones

  976.     y = gmfile['y'].tolist()

  977.     return gram_matrices, param_list_pre_revised, y