diff --git a/src/ich/__pycache__/algo.cpython-310.pyc b/src/ich/__pycache__/algo.cpython-310.pyc index 2c7ddafe22fdcf135f9ba2c7fc53c1a3e4ecae6b..361785fd8a34583250cd10f5606ed605b7ecb91b 100644 Binary files a/src/ich/__pycache__/algo.cpython-310.pyc and b/src/ich/__pycache__/algo.cpython-310.pyc differ diff --git a/src/ich/algo.py b/src/ich/algo.py index 8f4a64068fd8a7c3688f96638f7d7a89211aaa8a..c3ac5f42c19df1a76389a6d1811417fe4652d7c7 100644 --- a/src/ich/algo.py +++ b/src/ich/algo.py @@ -219,6 +219,7 @@ def cal_(temp_cor, temp_gedi, htl, htg): result = np.array(result) tempdf = pd.DataFrame(data={'N': result[:, 0], 'S': result[:, 1], 'C': result[:, 2], 'r': result[:, 3], 'rmse': result[:, 4]}) + del result tempdf.dropna(subset=['rmse'], inplace=True) if nn>6: tempdf = tempdf[tempdf['N'] > 3].sort_values(by=['rmse'], ascending=True) @@ -261,6 +262,7 @@ def process_block(i, j, cohArray, lidarArray, initial_ws, htl, htg, parm_): if np.all(np.array(parm) == 0): parm = parm_.copy() + del parm_ if lidarBlock.shape[0]*lidarBlock.shape[1]>initial_ws*initial_ws: # mask[np.shape(mask)[0]//2,np.shape(mask)[1]//2]=1 np.fill_diagonal(mask, 1) @@ -277,6 +279,7 @@ def process_block(i, j, cohArray, lidarArray, initial_ws, htl, htg, parm_): # ht = arc_sinc_fast(gama, parm[2]) * mask # print(lidarBlock.shape) del lidarBlock, cohBlock + gc.collect() return start_i, end_i, start_j, end_j, s_parm, c_parm, rmse_parm, ht, count else: S += 2 @@ -316,8 +319,10 @@ def dynamicWindow(cohArray, lidarArray, initial_ws, htl, htg, batch_size=10): if block_i < rows and block_j < cols: batch_futures.append((block_i, block_j)) - # Submit the batch as a single task - futures.append(executor.submit(process_batch, batch_futures, cohArray, lidarArray, initial_ws, htl, htg, parm_)) + future = executor.submit(process_batch, batch_futures, cohArray, lidarArray, initial_ws, htl, htg, parm_) + + futures.append(future) + del future,batch_futures # Initialize the progress bar with the total number of futures with tqdm(total=len(futures)) as pbar: @@ -336,6 +341,8 @@ def dynamicWindow(cohArray, lidarArray, initial_ws, htl, htg, batch_size=10): pbar.update(1) del results,future gc.collect() + # Clear the futures list to prevent accumulation + futures.clear() return s_parm, c_parm, rmse_parm, ht_, count @@ -346,7 +353,7 @@ def process_batch(batch_futures, cohArray, lidarArray, initial_ws, htl, htg, par start_i, end_i, start_j, end_j, s_p, c_p, r_p, ht, cnt = process_block(block_i, block_j, cohArray, lidarArray, initial_ws, htl, htg, parm_) results.append((start_i, end_i, start_j, end_j, s_p, c_p, r_p, ht, cnt)) # Invoke garbage collection - # gc.collect() + gc.collect() return results