|
@@ -53,6 +53,26 @@ def partition(ls, n_block):
|
|
|
L=[ls[i:i+1] for i in range (len(ls))]
|
|
|
return L
|
|
|
|
|
|
+# split a two-dimension numpy matrix into n_block numbers of sub-matrices
|
|
|
+def array2d_split(a, n_block):
|
|
|
+ # decompose number of n_jobs to two integers multiply
|
|
|
+ c_tmp=math.floor(math.sqrt(n_block))
|
|
|
+ for i in range (c_tmp,0,-1):
|
|
|
+ if n_block%i==0:
|
|
|
+ c=i
|
|
|
+ r=int(n_block/c)
|
|
|
+ break
|
|
|
+ # split column
|
|
|
+ arr_split_c=np.array_split(a,c,0)
|
|
|
+ arr_split=[]
|
|
|
+ # split row
|
|
|
+ for i in range(c):
|
|
|
+ arr_split_r=np.array_split(arr_split_c[i],r,1)
|
|
|
+ for j in range(r):
|
|
|
+ arr_split.append(arr_split_r[j])
|
|
|
+ return arr_split
|
|
|
+
|
|
|
+
|
|
|
def future_generator(iterable, n_jobs, dict_task):
|
|
|
# iterable is generated by delayed function, after converting to a list, the format is [function, (arg1, arg2, ... ,)]
|
|
|
#print("iterable type is ", type(iterable))
|
|
@@ -91,10 +111,16 @@ def future_generator(iterable, n_jobs, dict_task):
|
|
|
args_split.append([])
|
|
|
# if the array is an numpy array
|
|
|
if type(args[i]) is np.ndarray:
|
|
|
- # split numpy array
|
|
|
- args_split[i]=np.array_split(args[i],n_block)
|
|
|
- # get the length of numpy array
|
|
|
- l_arr.append(args[i].size)
|
|
|
+ # one-dimension matrix
|
|
|
+ if args[i].ndim==1:
|
|
|
+ # split numpy array
|
|
|
+ args_split[i]=np.array_split(args[i],n_block)
|
|
|
+ # get the length of numpy array
|
|
|
+ l_arr.append(args[i].size)
|
|
|
+ # two-dimension matrix
|
|
|
+ elif args[i].ndim==2:
|
|
|
+ # split numpy 2D array
|
|
|
+ args_split[i]=array2d_split(args[i],n_block)
|
|
|
# if the array is a generator
|
|
|
elif isinstance(args[i],types.GeneratorType):
|
|
|
# split generator
|