graphkit-learn/gklearn/preimage/utils.py

#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
Created on Thu Oct 17 19:05:07 2019

Useful functions.
@author: ljia
"""
#import networkx as nx

import multiprocessing
import numpy as np

from gklearn.kernels.marginalizedKernel import marginalizedkernel
from gklearn.kernels.untilHPathKernel import untilhpathkernel
from gklearn.kernels.spKernel import spkernel
import functools
from gklearn.utils.kernels import deltakernel, gaussiankernel, kernelproduct, polynomialkernel
from gklearn.kernels.structuralspKernel import structuralspkernel
from gklearn.kernels.treeletKernel import treeletkernel
from gklearn.kernels.weisfeilerLehmanKernel import weisfeilerlehmankernel
from gklearn.utils import Dataset
import csv
import matplotlib.pyplot as plt
import networkx as nx


def generate_median_preimage_by_class(ds_name, mpg_options, kernel_options, ged_options, mge_options, save_results=True, save_medians=True, plot_medians=True, dir_save='', ):
	from gklearn.preimage import MedianPreimageGenerator
	from gklearn.utils import split_dataset_by_target
	from gklearn.utils.graphfiles import saveGXL
	
	# 1. get dataset.
	print('getting dataset...')
	dataset_all = Dataset()
	dataset_all.load_predefined_dataset(ds_name)
	datasets = split_dataset_by_target(dataset_all)
# 	dataset.cut_graphs(range(0, 10))

	if save_results:
		# create result files.
		print('creating output files...')
		fn_output_detail, fn_output_summary = __init_output_file(ds_name, kernel_options['name'], mpg_options['fit_method'], dir_save)
		
	sod_sm_list = []
	sod_gm_list = []
	dis_k_sm_list = []
	dis_k_gm_list = []
	dis_k_gi_min_list = []
	time_precompute_gm_list = []
	time_optimize_ec_list = []
	time_generate_list = []
	time_total_list = []
	itrs_list = []
	converged_list = []
	num_updates_ecc_list = []
	nb_sod_sm2gm = [0, 0, 0]
	nb_dis_k_sm2gm = [0, 0, 0]
	nb_dis_k_gi2sm = [0, 0, 0]
	nb_dis_k_gi2gm = [0, 0, 0]
# 	repeats_better_sod_sm2gm = []
# 	repeats_better_dis_k_sm2gm = []
# 	repeats_better_dis_k_gi2sm = []
# 	repeats_better_dis_k_gi2gm = []
		
		
	print('start generating preimage for each class of target...')
	for dataset in datasets:
		print('\ntarget =', dataset.targets[0], '\n')
		num_graphs = len(dataset.graphs)
		
		if num_graphs < 2:
			print('\nnumber of graphs = ', num_graphs, ', skip.\n')
			continue
			
		# 2. set parameters.
		print('1. initializing mpg and setting parameters...')
		mpg = MedianPreimageGenerator()
		mpg.dataset = dataset
		mpg.set_options(**mpg_options.copy())
		mpg.kernel_options = kernel_options.copy()
		mpg.ged_options = ged_options.copy()
		mpg.mge_options = mge_options.copy()

		# 3. compute median preimage.
		print('2. computing median preimage...')
		mpg.run()
		results = mpg.get_results()
		
		# write result detail.
		if save_results:
			print('writing results to files...')
			sod_sm2gm = get_relations(np.sign(results['sod_gen_median'] - results['sod_set_median']))
			dis_k_sm2gm = get_relations(np.sign(results['k_dis_gen_median'] - results['k_dis_set_median']))
			dis_k_gi2sm = get_relations(np.sign(results['k_dis_set_median'] - results['k_dis_dataset']))
			dis_k_gi2gm = get_relations(np.sign(results['k_dis_gen_median'] - results['k_dis_dataset']))

			f_detail = open(dir_save + fn_output_detail, 'a')
			csv.writer(f_detail).writerow([ds_name, kernel_options['name'], 
					  ged_options['edit_cost'], ged_options['method'], 
					  ged_options['attr_distance'], mpg_options['fit_method'], 
					  num_graphs, dataset.targets[0], 1, 
					  results['sod_set_median'], results['sod_gen_median'],
					  results['k_dis_set_median'], results['k_dis_gen_median'], 
					  results['k_dis_dataset'], sod_sm2gm, dis_k_sm2gm, 
					  dis_k_gi2sm, dis_k_gi2gm, 	results['edit_cost_constants'],
					  results['runtime_precompute_gm'], results['runtime_optimize_ec'], 
					  results['runtime_generate_preimage'], results['runtime_total'],
					  results['itrs'], results['converged'],
					  results['num_updates_ecc']])
			f_detail.close()
		
			# compute result summary.
			sod_sm_list.append(results['sod_set_median'])
			sod_gm_list.append(results['sod_gen_median'])
			dis_k_sm_list.append(results['k_dis_set_median'])
			dis_k_gm_list.append(results['k_dis_gen_median'])
			dis_k_gi_min_list.append(results['k_dis_dataset'])
			time_precompute_gm_list.append(results['runtime_precompute_gm'])
			time_optimize_ec_list.append(results['runtime_optimize_ec'])
			time_generate_list.append(results['runtime_generate_preimage'])
			time_total_list.append(results['runtime_total'])
			itrs_list.append(results['itrs'])
			converged_list.append(results['converged'])
			num_updates_ecc_list.append(results['num_updates_ecc'])
			# # SOD SM -> GM
			if results['sod_set_median'] > results['sod_gen_median']:
				nb_sod_sm2gm[0] += 1
	# 			repeats_better_sod_sm2gm.append(1)
			elif results['sod_set_median'] == results['sod_gen_median']:
				nb_sod_sm2gm[1] += 1
			elif results['sod_set_median'] < results['sod_gen_median']:
				nb_sod_sm2gm[2] += 1
			# # dis_k SM -> GM
			if results['k_dis_set_median'] > results['k_dis_gen_median']:
				nb_dis_k_sm2gm[0] += 1
	# 			repeats_better_dis_k_sm2gm.append(1)
			elif results['k_dis_set_median'] == results['k_dis_gen_median']:
				nb_dis_k_sm2gm[1] += 1
			elif results['k_dis_set_median'] < results['k_dis_gen_median']:
				nb_dis_k_sm2gm[2] += 1
			# # dis_k gi -> SM
			if results['k_dis_dataset'] > results['k_dis_set_median']:
				nb_dis_k_gi2sm[0] += 1
	# 			repeats_better_dis_k_gi2sm.append(1)
			elif results['k_dis_dataset'] == results['k_dis_set_median']:
				nb_dis_k_gi2sm[1] += 1
			elif results['k_dis_dataset'] < results['k_dis_set_median']:
				nb_dis_k_gi2sm[2] += 1
			# # dis_k gi -> GM
			if results['k_dis_dataset'] > results['k_dis_gen_median']:
				nb_dis_k_gi2gm[0] += 1
	# 			repeats_better_dis_k_gi2gm.append(1)
			elif results['k_dis_dataset'] == results['k_dis_gen_median']:
				nb_dis_k_gi2gm[1] += 1
			elif results['k_dis_dataset'] < results['k_dis_gen_median']:
				nb_dis_k_gi2gm[2] += 1

			# write result summary for each letter.
			f_summary = open(dir_save + fn_output_summary, 'a')
			csv.writer(f_summary).writerow([ds_name, kernel_options['name'], 
					  ged_options['edit_cost'], ged_options['method'], 
					  ged_options['attr_distance'], mpg_options['fit_method'], 
					  num_graphs, dataset.targets[0],
					  results['sod_set_median'], results['sod_gen_median'],
					  results['k_dis_set_median'], results['k_dis_gen_median'], 
					  results['k_dis_dataset'], sod_sm2gm, dis_k_sm2gm, 
					  dis_k_gi2sm, dis_k_gi2gm, 
					  results['runtime_precompute_gm'], results['runtime_optimize_ec'], 
					  results['runtime_generate_preimage'], results['runtime_total'],
					  results['itrs'], results['converged'],
					  results['num_updates_ecc'], nb_sod_sm2gm, 
 					  nb_dis_k_sm2gm, nb_dis_k_gi2sm, nb_dis_k_gi2gm])
			f_summary.close()
			 
		# save median graphs.
		if save_medians:
			fn_pre_sm = dir_save + 'medians/set_median.' + mpg_options['fit_method'] + '.k' + str(num_graphs) + '.y' + str(dataset.targets[0]) + '.repeat' + str(1)
			saveGXL(mpg.set_median, fn_pre_sm + '.gxl', method='default', 
				    node_labels=dataset.node_labels, edge_labels=dataset.edge_labels, 	
					node_attrs=dataset.node_attrs, edge_attrs=dataset.edge_attrs)
			fn_pre_gm = dir_save + 'medians/gen_median.' + mpg_options['fit_method'] + '.k' + str(num_graphs) + '.y' + str(dataset.targets[0]) + '.repeat' + str(1)
			saveGXL(mpg.gen_median, fn_pre_gm + '.gxl', method='default',
		   		    node_labels=dataset.node_labels, edge_labels=dataset.edge_labels, 	
					node_attrs=dataset.node_attrs, edge_attrs=dataset.edge_attrs)
			fn_best_dataset = dir_save + 'medians/g_best_dataset.' + mpg_options['fit_method'] + '.k' + str(num_graphs) + '.y' + str(dataset.targets[0]) + '.repeat' + str(1)
			saveGXL(mpg.best_from_dataset, fn_best_dataset + '.gxl', method='default',
		   		    node_labels=dataset.node_labels, edge_labels=dataset.edge_labels, 	
					node_attrs=dataset.node_attrs, edge_attrs=dataset.edge_attrs)
		
		# plot median graphs.
		if plot_medians and save_medians:
			if ds_name == 'Letter-high' or ds_name == 'Letter-med' or ds_name == 'Letter-low':			
				draw_Letter_graph(mpg.set_median, fn_pre_sm)
				draw_Letter_graph(mpg.gen_median, fn_pre_gm)
				draw_Letter_graph(mpg.best_from_dataset, fn_best_dataset)
		

	# write result summary for each letter. 
	if save_results:
		sod_sm_mean = np.mean(sod_sm_list)
		sod_gm_mean = np.mean(sod_gm_list)
		dis_k_sm_mean = np.mean(dis_k_sm_list)
		dis_k_gm_mean = np.mean(dis_k_gm_list)
		dis_k_gi_min_mean = np.mean(dis_k_gi_min_list)
		time_precompute_gm_mean = np.mean(time_precompute_gm_list)
		time_optimize_ec_mean = np.mean(time_optimize_ec_list)
		time_generate_mean = np.mean(time_generate_list)
		time_total_mean = np.mean(time_total_list)
		itrs_mean = np.mean(itrs_list)
		num_converged = np.sum(converged_list)
		num_updates_ecc_mean = np.mean(num_updates_ecc_list)
		sod_sm2gm_mean = get_relations(np.sign(sod_gm_mean - sod_sm_mean))
		dis_k_sm2gm_mean = get_relations(np.sign(dis_k_gm_mean - dis_k_sm_mean))
		dis_k_gi2sm_mean = get_relations(np.sign(dis_k_sm_mean - dis_k_gi_min_mean))
		dis_k_gi2gm_mean = get_relations(np.sign(dis_k_gm_mean - dis_k_gi_min_mean))
		f_summary = open(dir_save + fn_output_summary, 'a')
		csv.writer(f_summary).writerow([ds_name, kernel_options['name'], 
				  ged_options['edit_cost'], ged_options['method'], 
				  ged_options['attr_distance'], mpg_options['fit_method'], 
				  num_graphs, 'all',
				  sod_sm_mean, sod_gm_mean, dis_k_sm_mean, dis_k_gm_mean,
				  dis_k_gi_min_mean, sod_sm2gm_mean, dis_k_sm2gm_mean, 
				  dis_k_gi2sm_mean, dis_k_gi2gm_mean,
				  time_precompute_gm_mean, time_optimize_ec_mean,
				  time_generate_mean, time_total_mean, itrs_mean, 
				  num_converged, num_updates_ecc_mean])
		f_summary.close()
		
	print('\ncomplete.')	

	
def __init_output_file(ds_name, gkernel, fit_method, dir_output):
#	fn_output_detail = 'results_detail.' + ds_name + '.' + gkernel + '.' + fit_method + '.csv'
	fn_output_detail = 'results_detail.' + ds_name + '.' + gkernel + '.csv'
	f_detail = open(dir_output + fn_output_detail, 'a')
	csv.writer(f_detail).writerow(['dataset', 'graph kernel', 'edit cost', 
			  'GED method', 'attr distance', 'fit method', 'k', 
			  'target', 'repeat', 'SOD SM', 'SOD GM', 'dis_k SM', 'dis_k GM',
			  'min dis_k gi', 'SOD SM -> GM', 'dis_k SM -> GM', 'dis_k gi -> SM', 
			  'dis_k gi -> GM', 'edit cost constants', 'time precompute gm',
			  'time optimize ec', 'time generate preimage', 'time total',
			  'itrs', 'converged', 'num updates ecc'])
	f_detail.close()
	
#	fn_output_summary = 'results_summary.' + ds_name + '.' + gkernel + '.' + fit_method + '.csv'
	fn_output_summary = 'results_summary.' + ds_name + '.' + gkernel + '.csv'
	f_summary = open(dir_output + fn_output_summary, 'a')
	csv.writer(f_summary).writerow(['dataset', 'graph kernel', 'edit cost', 
			  'GED method', 'attr distance', 'fit method', 'k', 
			  'target', 'SOD SM', 'SOD GM', 'dis_k SM', 'dis_k GM',
			  'min dis_k gi', 'SOD SM -> GM', 'dis_k SM -> GM', 'dis_k gi -> SM', 
			  'dis_k gi -> GM', 'time precompute gm', 'time optimize ec', 
			  'time generate preimage', 'time total', 'itrs', 'num converged', 
			  'num updates ecc', '# SOD SM -> GM', '# dis_k SM -> GM', 
			  '# dis_k gi -> SM', '# dis_k gi -> GM'])
# 			   'repeats better SOD SM -> GM', 
# 			  'repeats better dis_k SM -> GM', 'repeats better dis_k gi -> SM', 
# 			  'repeats better dis_k gi -> GM'])
	f_summary.close()
	
	return fn_output_detail, fn_output_summary


def get_relations(sign):
    if sign == -1:
        return 'better'
    elif sign == 0:
        return 'same'
    elif sign == 1:
        return 'worse'
	
	
#Dessin median courrant
def draw_Letter_graph(graph, file_prefix):
    plt.figure()
    pos = {}
    for n in graph.nodes:
        pos[n] = np.array([float(graph.node[n]['x']),float(graph.node[n]['y'])])
    nx.draw_networkx(graph, pos)
    plt.savefig(file_prefix + '.eps', format='eps', dpi=300)
#    plt.show()
    plt.clf()


def remove_edges(Gn):
	for G in Gn:
		for _, _, attrs in G.edges(data=True):
			attrs.clear()
			
def dis_gstar(idx_g, idx_gi, alpha, Kmatrix, term3=0, withterm3=True):
	term1 = Kmatrix[idx_g, idx_g]
	term2 = 0
	for i, a in enumerate(alpha):
		term2 += a * Kmatrix[idx_g, idx_gi[i]]
	term2 *= 2
	if withterm3 == False:
		for i1, a1 in enumerate(alpha):
			for i2, a2 in enumerate(alpha):
				term3 += a1 * a2 * Kmatrix[idx_gi[i1], idx_gi[i2]]
	return np.sqrt(term1 - term2 + term3)


def compute_k_dis(idx_g, idx_gi, alpha, Kmatrix, term3=0, withterm3=True):
	term1 = Kmatrix[idx_g, idx_g]
	term2 = 0
	for i, a in enumerate(alpha):
		term2 += a * Kmatrix[idx_g, idx_gi[i]]
	term2 *= 2
	if withterm3 == False:
		for i1, a1 in enumerate(alpha):
			for i2, a2 in enumerate(alpha):
				term3 += a1 * a2 * Kmatrix[idx_gi[i1], idx_gi[i2]]
	return np.sqrt(term1 - term2 + term3)


def compute_kernel(Gn, graph_kernel, node_label, edge_label, verbose, parallel='imap_unordered'):
	if graph_kernel == 'marginalizedkernel':
		Kmatrix, _ = marginalizedkernel(Gn, node_label=node_label, edge_label=edge_label,
								  p_quit=0.03, n_iteration=10, remove_totters=False,
								  n_jobs=multiprocessing.cpu_count(), verbose=verbose)
	elif graph_kernel == 'untilhpathkernel':
		Kmatrix, _ = untilhpathkernel(Gn, node_label=node_label, edge_label=edge_label,
								  depth=7, k_func='MinMax', compute_method='trie',
								  parallel=parallel,
								  n_jobs=multiprocessing.cpu_count(), verbose=verbose)
	elif graph_kernel == 'spkernel':
		mixkernel = functools.partial(kernelproduct, deltakernel, gaussiankernel)
		Kmatrix = np.empty((len(Gn), len(Gn)))
#		Kmatrix[:] = np.nan
		Kmatrix, _, idx = spkernel(Gn, node_label=node_label, node_kernels=
							  {'symb': deltakernel, 'nsymb': gaussiankernel, 'mix': mixkernel},
							  n_jobs=multiprocessing.cpu_count(), verbose=verbose)
#		for i, row in enumerate(idx):
#			for j, col in enumerate(idx):
#				Kmatrix[row, col] = Kmatrix_tmp[i, j]
	elif graph_kernel == 'structuralspkernel':
		mixkernel = functools.partial(kernelproduct, deltakernel, gaussiankernel)
		sub_kernels = {'symb': deltakernel, 'nsymb': gaussiankernel, 'mix': mixkernel}
		Kmatrix, _ = structuralspkernel(Gn, node_label=node_label, 
							  edge_label=edge_label, node_kernels=sub_kernels,
							  edge_kernels=sub_kernels,
							  parallel=parallel, n_jobs=multiprocessing.cpu_count(), 
							  verbose=verbose)
	elif graph_kernel == 'treeletkernel':
		pkernel = functools.partial(polynomialkernel, d=2, c=1e5)
#		pkernel = functools.partial(gaussiankernel, gamma=1e-6)
		mixkernel = functools.partial(kernelproduct, deltakernel, gaussiankernel)
		Kmatrix, _ = treeletkernel(Gn, node_label=node_label, edge_label=edge_label,
								   sub_kernel=pkernel, parallel=parallel,
								   n_jobs=multiprocessing.cpu_count(), verbose=verbose)
	elif graph_kernel == 'weisfeilerlehmankernel':
		Kmatrix, _ = weisfeilerlehmankernel(Gn, node_label=node_label, edge_label=edge_label,
								   height=4, base_kernel='subtree', parallel=None,
								   n_jobs=multiprocessing.cpu_count(), verbose=verbose)
		
	# normalization
	Kmatrix_diag = Kmatrix.diagonal().copy()
	for i in range(len(Kmatrix)):
		for j in range(i, len(Kmatrix)):
			Kmatrix[i][j] /= np.sqrt(Kmatrix_diag[i] * Kmatrix_diag[j])
			Kmatrix[j][i] = Kmatrix[i][j]
	return Kmatrix
			

def gram2distances(Kmatrix):
	dmatrix = np.zeros((len(Kmatrix), len(Kmatrix)))
	for i1 in range(len(Kmatrix)):
		for i2 in range(len(Kmatrix)):
			dmatrix[i1, i2] = Kmatrix[i1, i1] + Kmatrix[i2, i2] - 2 * Kmatrix[i1, i2]
	dmatrix = np.sqrt(dmatrix)
	return dmatrix


def kernel_distance_matrix(Gn, node_label, edge_label, Kmatrix=None, 
						   gkernel=None, verbose=True):
	dis_mat = np.empty((len(Gn), len(Gn)))
	if Kmatrix is None:
		Kmatrix = compute_kernel(Gn, gkernel, node_label, edge_label, verbose)
	for i in range(len(Gn)):
		for j in range(i, len(Gn)):
			dis = Kmatrix[i, i] + Kmatrix[j, j] - 2 * Kmatrix[i, j]
			if dis < 0:
				if dis > -1e-10:
					dis = 0
				else:
					raise ValueError('The distance is negative.')
			dis_mat[i, j] = np.sqrt(dis)
			dis_mat[j, i] = dis_mat[i, j]
	dis_max = np.max(np.max(dis_mat))
	dis_min = np.min(np.min(dis_mat[dis_mat != 0]))
	dis_mean = np.mean(np.mean(dis_mat))
	return dis_mat, dis_max, dis_min, dis_mean


def get_same_item_indices(ls):
	"""Get the indices of the same items in a list. Return a dict keyed by items.
	"""
	idx_dict = {}
	for idx, item in enumerate(ls):
		if item in idx_dict:
			idx_dict[item].append(idx)
		else:
			idx_dict[item] = [idx]
	return idx_dict


def k_nearest_neighbors_to_median_in_kernel_space(Gn, Kmatrix=None, gkernel=None,
												  node_label=None, edge_label=None):
	dis_k_all = [] # distance between g_star and each graph.
	alpha = [1 / len(Gn)] * len(Gn)
	if Kmatrix is None:
		Kmatrix = compute_kernel(Gn, gkernel, node_label, edge_label, True)
	term3 = 0
	for i1, a1 in enumerate(alpha):
		for i2, a2 in enumerate(alpha):
			term3 += a1 * a2 * Kmatrix[idx_gi[i1], idx_gi[i2]]
	for ig, g in tqdm(enumerate(Gn_init), desc='computing distances', file=sys.stdout):
		dtemp = dis_gstar(ig, idx_gi, alpha, Kmatrix, term3=term3)
		dis_all.append(dtemp)


def normalize_distance_matrix(D):
	max_value = np.amax(D)
	min_value = np.amin(D)
	return (D - min_value) / (max_value - min_value)