From c540dce3734c5f4f6f9e3443a3a7c05bec3ae5d7 Mon Sep 17 00:00:00 2001 From: Narayana Rao Bhogapurapu <narayanarao.bhogapurapu@gmail.com> Date: Sat, 12 Oct 2024 22:08:52 -0700 Subject: [PATCH] before profiling --- src/ich/__pycache__/algo.cpython-310.pyc | Bin 7409 -> 7484 bytes src/ich/algo.py | 13 ++++++++++--- 2 files changed, 10 insertions(+), 3 deletions(-) diff --git a/src/ich/__pycache__/algo.cpython-310.pyc b/src/ich/__pycache__/algo.cpython-310.pyc index 2c7ddafe22fdcf135f9ba2c7fc53c1a3e4ecae6b..361785fd8a34583250cd10f5606ed605b7ecb91b 100644 GIT binary patch delta 1016 zcmZ{iO^g&p6vz8j_w+Qq-7`HO^v=vKiyLH}i4rvii0A>MH;}}v2)ky}ndxqJlrb|~ zT`RCu&n_+yIEb4q6B3Bp!@-NE&72H)<ZNOzA;Zms1Og$11j9jNz2PGsv?}%MfBrA^ zD(}61@aDBWnYL}4id=U;9GS1ay^*<P7^gKwStCoe{$IvpfU*9c=HEcNDY0R?+8Uu1 zk+rwMHt~i%3Geg|+9828aWxxjgg7AfIIqJm{gz_^*y2QP_sI7=o!C7kcDrfuV{Qzt zi{Elz8!Vp`dI~7w)toDq^D~o8ZgJx%Sp(YSc0iu04Bzxbv7&gK-&)wrvpryHGW-Sv zm3X<(09RZnELvpVAEf<us}=f8kmB$9U0GOkSllV@m?kg|7BRsBm$;CZ+ah_;R0CWv zq<XTeGFAohGd(A7-p0oGy2P`!T?$*Ccjb|Jk8EDA{Te&W2vS_<gr|zis`{QXw=d2O zz4X7ly<*ax+2<rC8{q}6aF8*TYuxS`afxS>O}x|tP`agVVd3M2517Slo?V4N{}R4x z#^oRtS6Bueng5`?Q4&A7XQukR5~#bCxWdbc6R5)sW8D(VC3)=b!VfZYp(j>GU2w#) z(XH@;xHx(MisFw^I!7Z}24xb1l-~;Lw6>idN9zP)32_K<SYp;})S2J(qED7W%F&Gx ztBBKxGl(xFX4NM1=t;3rdQtxz&12$b>6QHh{m3F)q~WaAY1HX##&3VtX@x8dyiePW zXpp91wAf-1?c$YX#0uh+#6XR_&Or4M)a%f*V!Aw~zXZb>ESGn@R?<zP19|8;nw+me zDo}j7akeICV1h{|?vs^?dzmSEMcgQFfrfZgo>^ujQL$4zJ+0K>KusG?p1zN7KW|U1 zKrf>4f6bvMG5QVS65=xAKMJAWV&FSOf;g*8OZHH1d5#HG>k@NVpd|$sRV5R$6b*)v z8qo97uk9S9Juhhco;NV+biPxkQAjWJ?XfCABzBGO9l_t2UO}vjQ{zWdADM4iyZ!-{ C9`wNg delta 941 zcmZ{iO-vI(6vy|?_G4MP{e;pMsDfbH1P+)Of*L}Ci5?VV6oDGkz-|>4ZJBOGaF%cI zqS83VM1$HMj7NgKYP=ZZfs5CAG9fX>gE7V!4-)59G#+$b<~J|@w{K?NzInFqbC)}< zX{sc~+L7vWgZtOr?-ga<CP{fR6)XKvz5_%`-_$MO>uS)JCp64`+Ae73d$lI$DGg{T z0=M{ck43zMcY8bFbt&U@0Vv$^byhuQ&Vp8ytRR2ktB05Tt?v$>@TZ$}=3>f4k_W0Y z%^=&Jn^pbtOh0!9b_4><Qv_2kR91ta$GZX}P{UUPIUcQf=lCe@oju9zwe80mz(rJW zL68K&FU&>!V3UKOi1nAnL?i~}4P6#lAFyWKA@F;&3dbX6ubA1xe5v+!e8cjZieYCC zMwq=uT96M)XAW7uist-FJHUH_L%p6t74sHUvzi5%#6ZMlW@B1WDFR62r^k<*E>mNk z1uz^pVa2dQhTRIAZdAm+{Z@oU_-1gadyRz+xl^*jEL89sGG<TLPe)8&!C#cbPEQ6T zL$V25kU_d!)gk2}t4xTm(1!R*T@XV2Rb3whcx{LdmB~51?9|g^V=3K?x6&)3K01N8 ziWoo)3MAB#VN)MXX3tNh=oG5g5Oaum!~$YgAR&*MNjk|lLoE&y#T<8s+m0k`nUP_d zaO>$ynX#0aGLje4BiXX!Y|hNlRM|;W+1!|!r3_On#57_?pe$#TnX;@SXfi_=`Eq!- zqXjD6eHw0W+vy}<y>6T6uR<bFvu)U26>Oja`4ZhCv(ZoPW?JN)NFyY8dt_*aIZsG& z=!=s|&*@QdR6IcM;<5kx4RjKttB8Au`-p#BnLfb4L&PJ*BBFq3{Damgx_N<9SPDKl uEFngs;%Qd&IMD07yFLszN`v(=fZKeop{E)jC@ofU`9{M<d%xP{+Vc~tn%$KE diff --git a/src/ich/algo.py b/src/ich/algo.py index 8f4a640..c3ac5f4 100644 --- a/src/ich/algo.py +++ b/src/ich/algo.py @@ -219,6 +219,7 @@ def cal_(temp_cor, temp_gedi, htl, htg): result = np.array(result) tempdf = pd.DataFrame(data={'N': result[:, 0], 'S': result[:, 1], 'C': result[:, 2], 'r': result[:, 3], 'rmse': result[:, 4]}) + del result tempdf.dropna(subset=['rmse'], inplace=True) if nn>6: tempdf = tempdf[tempdf['N'] > 3].sort_values(by=['rmse'], ascending=True) @@ -261,6 +262,7 @@ def process_block(i, j, cohArray, lidarArray, initial_ws, htl, htg, parm_): if np.all(np.array(parm) == 0): parm = parm_.copy() + del parm_ if lidarBlock.shape[0]*lidarBlock.shape[1]>initial_ws*initial_ws: # mask[np.shape(mask)[0]//2,np.shape(mask)[1]//2]=1 np.fill_diagonal(mask, 1) @@ -277,6 +279,7 @@ def process_block(i, j, cohArray, lidarArray, initial_ws, htl, htg, parm_): # ht = arc_sinc_fast(gama, parm[2]) * mask # print(lidarBlock.shape) del lidarBlock, cohBlock + gc.collect() return start_i, end_i, start_j, end_j, s_parm, c_parm, rmse_parm, ht, count else: S += 2 @@ -316,8 +319,10 @@ def dynamicWindow(cohArray, lidarArray, initial_ws, htl, htg, batch_size=10): if block_i < rows and block_j < cols: batch_futures.append((block_i, block_j)) - # Submit the batch as a single task - futures.append(executor.submit(process_batch, batch_futures, cohArray, lidarArray, initial_ws, htl, htg, parm_)) + future = executor.submit(process_batch, batch_futures, cohArray, lidarArray, initial_ws, htl, htg, parm_) + + futures.append(future) + del future,batch_futures # Initialize the progress bar with the total number of futures with tqdm(total=len(futures)) as pbar: @@ -336,6 +341,8 @@ def dynamicWindow(cohArray, lidarArray, initial_ws, htl, htg, batch_size=10): pbar.update(1) del results,future gc.collect() + # Clear the futures list to prevent accumulation + futures.clear() return s_parm, c_parm, rmse_parm, ht_, count @@ -346,7 +353,7 @@ def process_batch(batch_futures, cohArray, lidarArray, initial_ws, htl, htg, par start_i, end_i, start_j, end_j, s_p, c_p, r_p, ht, cnt = process_block(block_i, block_j, cohArray, lidarArray, initial_ws, htl, htg, parm_) results.append((start_i, end_i, start_j, end_j, s_p, c_p, r_p, ht, cnt)) # Invoke garbage collection - # gc.collect() + gc.collect() return results -- GitLab