From c540dce3734c5f4f6f9e3443a3a7c05bec3ae5d7 Mon Sep 17 00:00:00 2001
From: Narayana Rao Bhogapurapu <narayanarao.bhogapurapu@gmail.com>
Date: Sat, 12 Oct 2024 22:08:52 -0700
Subject: [PATCH] before profiling

---
 src/ich/__pycache__/algo.cpython-310.pyc | Bin 7409 -> 7484 bytes
 src/ich/algo.py                          |  13 ++++++++++---
 2 files changed, 10 insertions(+), 3 deletions(-)

diff --git a/src/ich/__pycache__/algo.cpython-310.pyc b/src/ich/__pycache__/algo.cpython-310.pyc
index 2c7ddafe22fdcf135f9ba2c7fc53c1a3e4ecae6b..361785fd8a34583250cd10f5606ed605b7ecb91b 100644
GIT binary patch
delta 1016
zcmZ{iO^g&p6vz8j_w+Qq-7`HO^v=vKiyLH}i4rvii0A>MH;}}v2)ky}ndxqJlrb|~
zT`RCu&n_+yIEb4q6B3Bp!@-NE&72H)<ZNOzA;Zms1Og$11j9jNz2PGsv?}%MfBrA^
zD(}61@aDBWnYL}4id=U;9GS1ay^*<P7^gKwStCoe{$IvpfU*9c=HEcNDY0R?+8Uu1
zk+rwMHt~i%3Geg|+9828aWxxjgg7AfIIqJm{gz_^*y2QP_sI7=o!C7kcDrfuV{Qzt
zi{Elz8!Vp`dI~7w)toDq^D~o8ZgJx%Sp(YSc0iu04Bzxbv7&gK-&)wrvpryHGW-Sv
zm3X<(09RZnELvpVAEf<us}=f8kmB$9U0GOkSllV@m?kg|7BRsBm$;CZ+ah_;R0CWv
zq<XTeGFAohGd(A7-p0oGy2P`!T?$*Ccjb|Jk8EDA{Te&W2vS_<gr|zis`{QXw=d2O
zz4X7ly<*ax+2<rC8{q}6aF8*TYuxS`afxS>O}x|tP`agVVd3M2517Slo?V4N{}R4x
z#^oRtS6Bueng5`?Q4&A7XQukR5~#bCxWdbc6R5)sW8D(VC3)=b!VfZYp(j>GU2w#)
z(XH@;xHx(MisFw^I!7Z}24xb1l-~;Lw6>idN9zP)32_K<SYp;})S2J(qED7W%F&Gx
ztBBKxGl(xFX4NM1=t;3rdQtxz&12$b>6QHh{m3F)q~WaAY1HX##&3VtX@x8dyiePW
zXpp91wAf-1?c$YX#0uh+#6XR_&Or4M)a%f*V!Aw~zXZb>ESGn@R?<zP19|8;nw+me
zDo}j7akeICV1h{|?vs^?dzmSEMcgQFfrfZgo>^ujQL$4zJ+0K>KusG?p1zN7KW|U1
zKrf>4f6bvMG5QVS65=xAKMJAWV&FSOf;g*8OZHH1d5#HG>k@NVpd|$sRV5R$6b*)v
z8qo97uk9S9Juhhco;NV+biPxkQAjWJ?XfCABzBGO9l_t2UO}vjQ{zWdADM4iyZ!-{
C9`wNg

delta 941
zcmZ{iO-vI(6vy|?_G4MP{e;pMsDfbH1P+)Of*L}Ci5?VV6oDGkz-|>4ZJBOGaF%cI
zqS83VM1$HMj7NgKYP=ZZfs5CAG9fX>gE7V!4-)59G#+$b<~J|@w{K?NzInFqbC)}<
zX{sc~+L7vWgZtOr?-ga<CP{fR6)XKvz5_%`-_$MO>uS)JCp64`+Ae73d$lI$DGg{T
z0=M{ck43zMcY8bFbt&U@0Vv$^byhuQ&Vp8ytRR2ktB05Tt?v$>@TZ$}=3>f4k_W0Y
z%^=&Jn^pbtOh0!9b_4><Qv_2kR91ta$GZX}P{UUPIUcQf=lCe@oju9zwe80mz(rJW
zL68K&FU&>!V3UKOi1nAnL?i~}4P6#lAFyWKA@F;&3dbX6ubA1xe5v+!e8cjZieYCC
zMwq=uT96M)XAW7uist-FJHUH_L%p6t74sHUvzi5%#6ZMlW@B1WDFR62r^k<*E>mNk
z1uz^pVa2dQhTRIAZdAm+{Z@oU_-1gadyRz+xl^*jEL89sGG<TLPe)8&!C#cbPEQ6T
zL$V25kU_d!)gk2}t4xTm(1!R*T@XV2Rb3whcx{LdmB~51?9|g^V=3K?x6&)3K01N8
ziWoo)3MAB#VN)MXX3tNh=oG5g5Oaum!~$YgAR&*MNjk|lLoE&y#T<8s+m0k`nUP_d
zaO>$ynX#0aGLje4BiXX!Y|hNlRM|;W+1!|!r3_On#57_?pe$#TnX;@SXfi_=`Eq!-
zqXjD6eHw0W+vy}<y>6T6uR<bFvu)U26>Oja`4ZhCv(ZoPW?JN)NFyY8dt_*aIZsG&
z=!=s|&*@QdR6IcM;<5kx4RjKttB8Au`-p#BnLfb4L&PJ*BBFq3{Damgx_N<9SPDKl
uEFngs;%Qd&IMD07yFLszN`v(=fZKeop{E)jC@ofU`9{M<d%xP{+Vc~tn%$KE

diff --git a/src/ich/algo.py b/src/ich/algo.py
index 8f4a640..c3ac5f4 100644
--- a/src/ich/algo.py
+++ b/src/ich/algo.py
@@ -219,6 +219,7 @@ def cal_(temp_cor, temp_gedi, htl, htg):
             result = np.array(result)
             tempdf = pd.DataFrame(data={'N': result[:, 0], 'S': result[:, 1], 'C': result[:, 2],
                                         'r': result[:, 3], 'rmse': result[:, 4]})
+            del result
             tempdf.dropna(subset=['rmse'], inplace=True)
             if nn>6:
                 tempdf = tempdf[tempdf['N'] > 3].sort_values(by=['rmse'], ascending=True)
@@ -261,6 +262,7 @@ def process_block(i, j, cohArray, lidarArray, initial_ws, htl, htg, parm_):
 
             if np.all(np.array(parm) == 0):
                 parm = parm_.copy()
+                del parm_
             if lidarBlock.shape[0]*lidarBlock.shape[1]>initial_ws*initial_ws:
                 # mask[np.shape(mask)[0]//2,np.shape(mask)[1]//2]=1
                 np.fill_diagonal(mask, 1)
@@ -277,6 +279,7 @@ def process_block(i, j, cohArray, lidarArray, initial_ws, htl, htg, parm_):
             # ht = arc_sinc_fast(gama, parm[2]) * mask
             # print(lidarBlock.shape)
             del lidarBlock, cohBlock
+            gc.collect()
             return start_i, end_i, start_j, end_j, s_parm, c_parm, rmse_parm, ht, count
         else:
             S += 2
@@ -316,8 +319,10 @@ def dynamicWindow(cohArray, lidarArray, initial_ws, htl, htg, batch_size=10):
                         if block_i < rows and block_j < cols:
                             batch_futures.append((block_i, block_j))
 
-                # Submit the batch as a single task
-                futures.append(executor.submit(process_batch, batch_futures, cohArray, lidarArray, initial_ws, htl, htg, parm_))
+                future = executor.submit(process_batch, batch_futures, cohArray, lidarArray, initial_ws, htl, htg, parm_)
+
+                futures.append(future)
+                del future,batch_futures
 
         # Initialize the progress bar with the total number of futures
         with tqdm(total=len(futures)) as pbar:
@@ -336,6 +341,8 @@ def dynamicWindow(cohArray, lidarArray, initial_ws, htl, htg, batch_size=10):
                 pbar.update(1)
                 del results,future
                 gc.collect()
+            # Clear the futures list to prevent accumulation
+            futures.clear()
 
     return s_parm, c_parm, rmse_parm, ht_, count
 
@@ -346,7 +353,7 @@ def process_batch(batch_futures, cohArray, lidarArray, initial_ws, htl, htg, par
         start_i, end_i, start_j, end_j, s_p, c_p, r_p, ht, cnt = process_block(block_i, block_j, cohArray, lidarArray, initial_ws, htl, htg, parm_)
         results.append((start_i, end_i, start_j, end_j, s_p, c_p, r_p, ht, cnt))
     # Invoke garbage collection
-    # gc.collect()
+    gc.collect()
     return results
 
 
-- 
GitLab