fixed outfp for when downloading gedi data.

f67e5e1f · Alex Rojas · 28354eaa · f67e5e1f · f67e5e1f · f67e5e1f
Commit f67e5e1f authored 1 year ago by Alex Rojas
--- a/get_gedi_data.py
+++ b/get_gedi_data.py
@@ -26,7 +26,7 @@ def get_gedi_data(url):
        token=credentials['sessionToken']
    )
    basename = os.path.basename(url)
-    outfp = f"output/{basename}.h5"
+    outfp = f"output/{basename}"
    gedi_ds = h5py.File(s3.open(lpdaac_gedi_https_to_s3(url), "rb"), "r")
    with h5py.File(outfp, 'w') as dst:
        for obj in gedi_ds.keys():        

--- a/main.py
+++ b/main.py
@@ -19,7 +19,7 @@ from itertools import repeat
 # Import pgap function from wenge 
 from pgap import GapDS, wf_smooth
 # import custom functions, etc.
-from download_gedi import download_gedi
+# from download_gedi import download_gedi
 from get_gedi_data import get_gedi_data
 ## GET CWD of file to locate path

--- a/notebooks/run-gedi-biomass.ipynb
+++ b/notebooks/run-gedi-biomass.ipynb
@@ -53,7 +53,7 @@
  },
  {
   "cell_type": "code",
-   "execution_count": null,
+   "execution_count": 3,
   "id": "b39e76ac-d765-4b8c-baf1-fe41f3296de9",
   "metadata": {},
   "outputs": [
@@ -63,6 +63,17 @@
     "text": [
      "on file num:  100\r"
     ]
+    },
+    {
+     "ename": "KeyboardInterrupt",
+     "evalue": "",
+     "output_type": "error",
+     "traceback": [
+      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
+      "\u001b[0;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
+      "Cell \u001b[0;32mIn[3], line 35\u001b[0m\n\u001b[1;32m     33\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m counter\u001b[38;5;241m%\u001b[39m\u001b[38;5;241m100\u001b[39m\u001b[38;5;241m==\u001b[39m\u001b[38;5;241m0\u001b[39m:\n\u001b[1;32m     34\u001b[0m     \u001b[38;5;28mprint\u001b[39m(\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mon file num: \u001b[39m\u001b[38;5;124m\"\u001b[39m, counter, end\u001b[38;5;241m=\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;130;01m\\r\u001b[39;00m\u001b[38;5;124m'\u001b[39m)\n\u001b[0;32m---> 35\u001b[0m     \u001b[43mtime\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43msleep\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;241;43m800\u001b[39;49m\u001b[43m)\u001b[49m\n\u001b[1;32m     36\u001b[0m counter\u001b[38;5;241m+\u001b[39m\u001b[38;5;241m=\u001b[39m\u001b[38;5;241m1\u001b[39m\n",
+      "\u001b[0;31mKeyboardInterrupt\u001b[0m: "
+     ]
    }
   ],
   "source": [
@@ -78,7 +89,7 @@
    "# Get matching string pattern and run main.py\n",
    "jobs_list = []\n",
    "counter=1\n",
-    "for l1b_fp in l1b_fpaths[1:]:\n",
+    "for l1b_fp in l1b_fpaths[100:]:\n",
    "    # Get string pattern\n",
    "    str_pattern = re.findall(\"[0-9]{13}\", os.path.basename(l1b_fp))[0] \n",
    "    try:\n",

 %% Cell type:code id:25172537-794b-4135-bb28-63c0fcefbb68 tags:
 ``` python
 import os
 import re
 from maap.maap import MAAP
 import re
 # maap = MAAP(maap_host='api.maap-project.org')
 maap = MAAP(maap_host='api.maap-project.org')
 ```
 %% Output
    Unable to load config file from source maap.cfg
    Unable to load config file from source ./maap.cfg
    Unable to load config file from source /projects/maap.cfg
 %% Cell type:code id:f176e75b-5104-478b-ade6-a84006e966b0 tags:
 ``` python
 import time
 ```
 %% Cell type:code id:f203a6c0-852b-4645-a3a5-4b65c462f4f5 tags:
 ``` python
 ```
 %% Cell type:code id:b0d65f96-5dad-44e9-ae85-9d4086c4b29f tags:
 ``` python
 ```
 %% Cell type:code id:b39e76ac-d765-4b8c-baf1-fe41f3296de9 tags:
 ``` python
 # Change to appropriate filepaths for urls
 l1b_urls_fpath = "/projects/biomass-gedi-conus/data/GEDI-L1B-2022-URLS.txt"
 l2a_urls_fpath = "/projects/biomass-gedi-conus/data/GEDI-L2A-2022-URLS.txt"
 with open(l1b_urls_fpath) as f:
    l1b_fpaths = f.read().splitlines()
 with open(l2a_urls_fpath) as f:
    l2a_fpaths = f.read().splitlines()
 # Get matching string pattern and run main.py
 jobs_list = []
 counter=1
-for l1b_fp in l1b_fpaths[1:]:
+for l1b_fp in l1b_fpaths[100:]:
    # Get string pattern
    str_pattern = re.findall("[0-9]{13}", os.path.basename(l1b_fp))[0]
    try:
        l2a_fp = [s for s in l2a_fpaths if str_pattern in s][0]
    except:
        print("No matching L2 file!", l1b_fp)
        continue
    # print(l1b_fp)
    # print(l2a_fp)
    job = maap.submitJob(identifier="GEDI-biomass-2022",
                   algo_id="arojas_biomass_gedi",
                   version="master",
                   username="arojearthdata",
                   queue="maap-dps-worker-16gb",
                   L1B_URL=l1b_fp,
                   L2A_URL=l2a_fp)
    jobs_list.append(job)
    if counter%100==0:
        print("on file num: ", counter, end='\r')
        time.sleep(800)
    counter+=1
 ```
 %% Output
    on file num:  100
+    ---------------------------------------------------------------------------
+    KeyboardInterrupt                         Traceback (most recent call last)
+Cell     In[3], line 35
+         33 if counter%100==0:
+         34     print("on file num: ", counter, end='\r')
+    ---> 35     time.sleep(800)
+         36 counter+=1
+    KeyboardInterrupt:
 %% Cell type:code id:8f0b9f73-ffb1-40b7-be0e-b31480571a4a tags:
 ``` python
 ```
 %% Cell type:code id:4f6093c5-1f63-4556-abb8-02714f560ad1 tags:
 ``` python
 ```
 %% Cell type:code id:7fe1c6f4-37c5-4fe7-b9ff-51e17b7a5cf0 tags:
 ``` python
 # View job status
 for job in jobs_list:
    print(job.retrieve_status())
    print(job.id)
    break
 ```
 %% Output
    Accepted
    d974c84b-2691-4569-b496-1cc06380ec84
 %% Cell type:code id:2018077f-9640-4e70-bd01-a9d939b3626a tags:
 ``` python
 # View job status
 for job in jobs_list:
    print(job.retrieve_status())
    print(job.id)
    break
 ```
 %% Output
    Succeeded
    6152410f-1e33-4561-965a-83fd364e3767
 %% Cell type:code id:873e9ea7-d6cb-4bb9-b082-5bffb4026615 tags:
 ``` python
 # Retrieve results
 for job in jobs_list:
    print(job.retrieve_result())
    break
 ```
 %% Output
    ['http://maap-ops-workspace.s3-website-us-west-2.amazonaws.com/dataset/triaged_job/arojas_biomass_gedi/master/2023/10/19/edaa25c9-99e8-4b0f-aa52-bf843a6f9ada', 's3://s3-us-west-2.amazonaws.com:80/maap-ops-workspace/dataset/triaged_job/arojas_biomass_gedi/master/2023/10/19/edaa25c9-99e8-4b0f-aa52-bf843a6f9ada', 'https://s3.console.aws.amazon.com/s3/buckets/maap-ops-workspace/dataset/triaged_job/arojas_biomass_gedi/master/2023/10/19/edaa25c9-99e8-4b0f-aa52-bf843a6f9ada/?region=us-east-1&tab=overview', '++ python /app/get_username.py\n+ USERNAME=arojearthdata\n+ DPS_MACHINE_TOKEN=Y6hERQKLKs0oH0h5Kgi3d-gVVyHgW_DR77YeARdzd-a3KJDE-gYldJzhB94s635KySUXi4r9WOtD6Vo_F9cEbfNI_QUhOG8N47heXHVcW1k3vEw3d0StttXYMOnQPtTzSQLAYKQwJLgbtX-R8ug1XGVeMolh97P9nHcW8awN7yjyaR7g\n++ curl -sb -H \'Accept: application/json\' -H \'Content-Type: application/json\' -H \'dps-token: Y6hERQKLKs0oH0h5Kgi3d-gVVyHgW_DR77YeARdzd-a3KJDE-gYldJzhB94s635KySUXi4r9WOtD6Vo_F9cEbfNI_QUhOG8N47heXHVcW1k3vEw3d0StttXYMOnQPtTzSQLAYKQwJLgbtX-R8ug1XGVeMolh97P9nHcW8awN7yjyaR7g\' https://api.maap-project.org/api/members/arojearthdata\n++ jq -r .session_key\n+ export MAAP_PGT=PGT-332-1NdFr4fQXYrHffIoE6zmdrON8fQBjz05-O6ycMomc8i2b7Xt7lZggJXaQaUsQjMs8Cs-8445019b059c\n+ MAAP_PGT=PGT-332-1NdFr4fQXYrHffIoE6zmdrON8fQBjz05-O6ycMomc8i2b7Xt7lZggJXaQaUsQjMs8Cs-8445019b059c\n+ unset DPS_MACHINE_TOKEN\n+ /app/biomass-gedi-conus/run.sh \'https://e4ftl01.cr.usgs.gov//GEDI_L1_L2/GEDI/GEDI01_B.002/2021.05.31/GEDI01_B_2021151223415_O13976_02_T00676_02_005_02_V002.h5\n\' https://e4ftl01.cr.usgs.gov//GEDI_L1_L2/GEDI/GEDI02_A.002/2021.05.31/GEDI02_A_2021151223415_O13976_02_T00676_02_003_02_V002.h5\nTraceback (most recent call last):\n  File "/app/biomass-gedi-conus/main.py", line 197, in <module>\n    out_df.to_csv(outfp, index=False)  \n    ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n  File "/opt/conda/envs/osgeo-env/lib/python3.11/site-packages/pandas/core/generic.py", line 3902, in to_csv\n    return DataFrameRenderer(formatter).to_csv(\n           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n  File "/opt/conda/envs/osgeo-env/lib/python3.11/site-packages/pandas/io/formats/format.py", line 1152, in to_csv\n    csv_formatter.save()\n  File "/opt/conda/envs/osgeo-env/lib/python3.11/site-packages/pandas/io/formats/csvs.py", line 247, in save\n    with get_handle(\n         ^^^^^^^^^^^\n  File "/opt/conda/envs/osgeo-env/lib/python3.11/site-packages/pandas/io/common.py", line 739, in get_handle\n    check_parent_directory(str(handle))\n  File "/opt/conda/envs/osgeo-env/lib/python3.11/site-packages/pandas/io/common.py", line 604, in check_parent_directory\n    raise OSError(rf"Cannot save file into a non-existent directory: \'{parent}\'")\nOSError: Cannot save file into a non-existent directory: \'/data/work/jobs/2023/10/19/03/14/job-arojas_biomass_gedi__master-20231019T030808.292586Z/output/output\'\n\nDuring handling of the above exception, another exception occurred:\n\nTraceback (most recent call last):\n  File "/app/biomass-gedi-conus/main.py", line 199, in <module>\n    print("Couldnt save file: ", l1b_fp)\n                                 ^^^^^^\nNameError: name \'l1b_fp\' is not defined. Did you mean: \'l1b_ds\'?\n+ cp _stderr.txt _alt_traceback.txt\n']
 %% Cell type:code id:e8a40e13-dcdd-4b7f-a3b6-d060bb52eaf5 tags:
 ``` python
 ```
 %% Cell type:code id:988420a0-1145-499c-927c-eec059c0dde9 tags:
 ``` python
 ################
 ###############
 ##############
 ```
 %% Cell type:code id:31626d2a-4edf-4572-81c2-7b68080e2dd3 tags:
 ``` python
 ## Some files downloaded, other didnt (due to limit on download through USGS)
 ```
 %% Cell type:code id:9ecb7ffc-319f-40c9-8f6e-d18b6ea1c890 tags:
 ``` python
 import glob
 indir = "/projects/my-private-bucket/dps_output/arojas_biomass_gedi/master/GEDI-biomass-2022"
 csv_list = []
 for subdir, dirs, files in os.walk(indir):
    for file in files:
        if file.endswith(".csv"):
            # fp = os.path.join(subdir, file)
            # print(fp)
            # csv_list.append(fp)
            csv_list.append(file)
 ```
 %% Cell type:code id:f3ae4e2c-fa90-41ae-8fa2-9e3fd1241157 tags:
 ``` python
 len(csv_list)
 ```
 %% Output
    13
 %% Cell type:code id:5bb204a4-e7d4-4a83-898f-81db03d080aa tags:
 ``` python
 print("done")
 ```
 %% Output
    done
 %% Cell type:code id:bb5b4efc tags:
 ``` python
 l1b_urls_fpath = "../GEDI-L1B-2021-URLS.txt"
 l2a_urls_fpath = "../GEDI-L2A-2021-URLS.txt"
 with open(l1b_urls_fpath) as f:
    l1b_fpaths = f.readlines()
 with open(l2a_urls_fpath) as f:
    l2a_fpaths = f.readlines()
 # Get matching string pattern and run main.py
 jobs_list = []
 counter=1
 for l1b_fp in l1b_fpaths:
    # Get string pattern
    str_pattern = re.findall("[0-9]{13}", os.path.basename(l1b_fp))[0]
    try:
        l2a_fp = [s for s in l2a_fpaths if str_pattern in s][0]
    except:
        print("No matching L2 file!", l1b_fp)
        continue
    # Check if file was already downloaded
    date_str = os.path.basename(l1b_fp).split("_")[2]
    if any(date_str in x for x in csv_list):
        continue
    # print(l1b_fp)
    # print(l2a_fp)
    job = maap.submitJob(identifier="single_file_test_run",
                   algo_id="arojas_biomass_gedi",
                   version="master",
                   username="arojearthdata",
                   queue="maap-dps-worker-16gb",
                   L1B_URL=l1b_fp,
                   L2A_URL=l2a_fp)
    jobs_list.append(job)
    if counter%10==0:
        print("on file num: ", counter, end='\r')
        time.sleep(600)
    counter+=1
 ```
 %% Cell type:code id:6d8ef38d tags:
 ``` python
 ```
 %% Cell type:code id:da773e1f tags:
 ``` python
 ```
 %% Cell type:code id:ee0b897a tags:
 ``` python
 #############
 ## SCRATCH
 #############
 ```
 %% Cell type:code id:b364772a tags:
 ``` python
 # Change to appropriate filepaths for urls
 l1b_urls_fpath = "/projects/biomass-gedi-conus/data/GEDI-L1B-2022-URLS.txt"
 l2a_urls_fpath = "/projects/biomass-gedi-conus/data/GEDI-L2A-2022-URLS.txt"
 with open(l1b_urls_fpath) as f:
    l1b_fpaths = f.read().splitlines()
 with open(l2a_urls_fpath) as f:
    l2a_fpaths = f.read().splitlines()
 # Get matching string pattern and run main.py
 jobs_list = []
 counter=1
 for l1b_fp in l1b_fpaths[1:]:
    # Get string pattern
    str_pattern = re.findall("[0-9]{13}", os.path.basename(l1b_fp))[0]
    try:
        l2a_fp = [s for s in l2a_fpaths if str_pattern in s][0]
    except:
        print("No matching L2 file!", l1b_fp)
        continue
    break
 ```
 %% Cell type:code id:4a6460a7-4d65-4311-901c-e5685daeb2f0 tags:
 ``` python
 print(l1b_fp)
 print(l2a_fp)
 ```
 %% Output
    https://e4ftl01.cr.usgs.gov//GEDI_L1_L2/GEDI/GEDI01_B.002/2022.08.31/GEDI01_B_2022243125140_O21055_03_T02667_02_005_02_V002.h5
    https://e4ftl01.cr.usgs.gov//GEDI_L1_L2/GEDI/GEDI02_A.002/2022.08.31/GEDI02_A_2022243125140_O21055_03_T02667_02_003_02_V002.h5
 %% Cell type:code id:ebb58d7e-c295-4955-a056-58fa8efc596f tags:
 ``` python
 ```