graphkit-learn/gklearn/utils/parallel.py

#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
Created on Tue Dec 11 11:39:46 2018
Parallel aid functions.
@author: ljia
"""
import multiprocessing
from multiprocessing import Pool
import sys
from gklearn.utils import get_iters


def parallel_me(func, func_assign, var_to_assign, itr, len_itr=None, init_worker=None,
				glbv=None, method=None, n_jobs=None, chunksize=None, itr_desc='',
				verbose=True):
	'''
	'''
	if method == 'imap_unordered':
		if glbv: # global varibles required.
#			def init_worker(v_share):
#				global G_var
#				G_var = v_share
			if n_jobs == None:
				n_jobs = multiprocessing.cpu_count()
			with Pool(processes=n_jobs, initializer=init_worker,
					  initargs=glbv) as pool:
				if chunksize is None:
					if len_itr < 100 * n_jobs:
						chunksize = int(len_itr / n_jobs) + 1
					else:
						chunksize = 100

				iterator = get_iters(pool.imap_unordered(func, itr, chunksize),
					desc=itr_desc, file=sys.stdout, length=len_itr,
					verbose=(verbose >= 2))
				for result in iterator:
					func_assign(result, var_to_assign)
			pool.close()
			pool.join()
		else:
			if n_jobs == None:
				n_jobs = multiprocessing.cpu_count()
			with Pool(processes=n_jobs) as pool:
				if chunksize is None:
					if len_itr < 100 * n_jobs:
						chunksize = int(len_itr / n_jobs) + 1
					else:
						chunksize = 100
				iterator = get_iters(pool.imap_unordered(func, itr, chunksize),
					desc=itr_desc, file=sys.stdout, length=len_itr,
					verbose=(verbose >= 2))
				for result in iterator:
					func_assign(result, var_to_assign)
			pool.close()
			pool.join()


def parallel_gm(func, Kmatrix, Gn, init_worker=None, glbv=None,
				method='imap_unordered', n_jobs=None, chunksize=None,
				verbose=True): # @todo: Gn seems not necessary.
	from itertools import combinations_with_replacement
	def func_assign(result, var_to_assign):
		var_to_assign[result[0]][result[1]] = result[2]
		var_to_assign[result[1]][result[0]] = result[2]
	itr = combinations_with_replacement(range(0, len(Gn)), 2)
	len_itr = int(len(Gn) * (len(Gn) + 1) / 2)
	parallel_me(func, func_assign, Kmatrix, itr, len_itr=len_itr,
		init_worker=init_worker, glbv=glbv, method=method, n_jobs=n_jobs,
		chunksize=chunksize, itr_desc='Computing kernels', verbose=verbose)