Skip to content
Snippets Groups Projects
Commit b7807941 authored by Alex Rojas's avatar Alex Rojas
Browse files

updated input filepath in main.py file for GEDI data.

parent 44424b0d
No related branches found
No related tags found
No related merge requests found
Hide whitespace changes
Inline Side-by-side
main.py
+ 3
− 2
View file @ b7807941
......@@ -112,8 +112,9 @@ if __name__ == '__main__':
# get level 1 and level 2 data
print(l1b_basename)
print(l2a_basename)
l1b_ds = h5py.File(f"./{l1b_basename}")
l2a_ds = h5py.File(f"./{l2a_basename}")
CWD = os.path.dirname(os.path.abspath(__file__))
l1b_ds = h5py.File(os.path.join(CWD, f"{l1b_basename}"))
l2a_ds = h5py.File(os.path.join(CWD, f"{l2a_basename}"))
except Exception as e:
# Some raw L1B files are corrupt?
print("Corrupt file: ", l1b_basename)
......
notebooks/run-gedi-biomass.ipynb
+ 4
− 4
View file @ b7807941
%% Cell type:code id:25172537-794b-4135-bb28-63c0fcefbb68 tags:
``` python
import os
import re
from maap.maap import MAAP
import re
# maap = MAAP(maap_host='api.maap-project.org')
maap = MAAP(maap_host='api.maap-project.org')
```
%% Output
Unable to load config file from source maap.cfg
Unable to load config file from source ./maap.cfg
Unable to load config file from source /projects/maap.cfg
%% Cell type:code id:f176e75b-5104-478b-ade6-a84006e966b0 tags:
``` python
```
%% Cell type:code id:b39e76ac-d765-4b8c-baf1-fe41f3296de9 tags:
``` python
l1b_urls_fpath = "../GEDI-L1B-2021-URLS.txt"
l2a_urls_fpath = "../GEDI-L2A-2021-URLS.txt"
with open(l1b_urls_fpath) as f:
l1b_fpaths = f.readlines()
with open(l2a_urls_fpath) as f:
l2a_fpaths = f.readlines()
# Get matching string pattern and run main.py
jobs_list = []
counter=1
for l1b_fp in l1b_fpaths:
# Get string pattern
str_pattern = re.findall("[0-9]{13}", os.path.basename(l1b_fp))[0]
try:
l2a_fp = [s for s in l2a_fpaths if str_pattern in s][0]
except:
print("No matching L2 file!", l1b_fp)
continue
print(l1b_fp)
print(l2a_fp)
job = maap.submitJob(identifier="single_file_test_run",
algo_id="arojas_biomass_gedi",
version="master",
username="arojearthdata",
queue="maap-dps-worker-16gb",
L1B_URL=l1b_fp,
L2A_URL=l2a_fp)
jobs_list.append(job)
if counter%20:
print("on file num: ", counter)
counter+=1
break
```
%% Output
https://e4ftl01.cr.usgs.gov//GEDI_L1_L2/GEDI/GEDI01_B.002/2021.05.31/GEDI01_B_2021151223415_O13976_02_T00676_02_005_02_V002.h5
https://e4ftl01.cr.usgs.gov//GEDI_L1_L2/GEDI/GEDI02_A.002/2021.05.31/GEDI02_A_2021151223415_O13976_02_T00676_02_003_02_V002.h5
on file num: 1
%% Cell type:code id:8f0b9f73-ffb1-40b7-be0e-b31480571a4a tags:
``` python
print(jobs_list[:2])
```
%% Output
[{'job_id': '9e4d8c3e-c8e2-47df-82cf-e9f0f4357ce8', 'status': 'Accepted', 'machine_type': None, 'architecture': None, 'machine_memory_size': None, 'directory_size': None, 'operating_system': None, 'job_start_time': None, 'job_end_time': None, 'job_duration_seconds': None, 'cpu_usage': None, 'cache_usage': None, 'mem_usage': None, 'max_mem_usage': None, 'swap_usage': None, 'read_io_stats': None, 'write_io_stats': None, 'sync_io_stats': None, 'async_io_stats': None, 'total_io_stats': None, 'error_details': None, 'response_code': 200, 'outputs': []}]
[{'job_id': '8746412d-a0d2-4b97-b004-337b67c3565d', 'status': 'Accepted', 'machine_type': None, 'architecture': None, 'machine_memory_size': None, 'directory_size': None, 'operating_system': None, 'job_start_time': None, 'job_end_time': None, 'job_duration_seconds': None, 'cpu_usage': None, 'cache_usage': None, 'mem_usage': None, 'max_mem_usage': None, 'swap_usage': None, 'read_io_stats': None, 'write_io_stats': None, 'sync_io_stats': None, 'async_io_stats': None, 'total_io_stats': None, 'error_details': None, 'response_code': 200, 'outputs': []}]
%% Cell type:code id:7fe1c6f4-37c5-4fe7-b9ff-51e17b7a5cf0 tags:
``` python
# View job status
for job in jobs_list:
print(job.retrieve_status())
print(job.id)
break
```
%% Output
Accepted
7c2243dc-d90c-4ac0-8d57-6651f98b1b10
8746412d-a0d2-4b97-b004-337b67c3565d
%% Cell type:code id:2018077f-9640-4e70-bd01-a9d939b3626a tags:
``` python
# View job status
for job in jobs_list:
print(job.retrieve_status())
print(job.id)
break
```
%% Output
Succeeded
7c2243dc-d90c-4ac0-8d57-6651f98b1b10
8746412d-a0d2-4b97-b004-337b67c3565d
%% Cell type:code id:873e9ea7-d6cb-4bb9-b082-5bffb4026615 tags:
``` python
# Retrieve results
for job in jobs_list:
print(job.retrieve_result())
break
```
%% Output
['http://maap-ops-workspace.s3-website-us-west-2.amazonaws.com/arojearthdata/dps_output/arojas_biomass_gedi/master/single_file_test_run/2023/10/19/01/44/37/145570', 's3://s3.us-west-2.amazonaws.com:80/maap-ops-workspace/arojearthdata/dps_output/arojas_biomass_gedi/master/single_file_test_run/2023/10/19/01/44/37/145570', 'https://s3.console.aws.amazon.com/s3/buckets/maap-ops-workspace/arojearthdata/dps_output/arojas_biomass_gedi/master/single_file_test_run/2023/10/19/01/44/37/145570/?region=us-east-1&tab=overview']
%% Cell type:code id:e8a40e13-dcdd-4b7f-a3b6-d060bb52eaf5 tags:
``` python
```
%% Cell type:code id:988420a0-1145-499c-927c-eec059c0dde9 tags:
``` python
```
%% Cell type:code id:31626d2a-4edf-4572-81c2-7b68080e2dd3 tags:
``` python
```
%% Cell type:code id:9ecb7ffc-319f-40c9-8f6e-d18b6ea1c890 tags:
``` python
```
%% Cell type:code id:77a995d1 tags:
``` python
import os
import re
from maap.maap import MAAP
import re
# maap = MAAP(maap_host='api.maap-project.org')
maap = MAAP(maap_host='api.maap-project.org')
```
%% Output
WARNING:maap.maap:Unable to load config file from source maap.cfg
WARNING:maap.maap:Unable to load config file from source ./maap.cfg
WARNING:maap.maap:Unable to load config file from source /projects/maap.cfg
%% Cell type:code id:bb5b4efc tags:
``` python
```
%% Cell type:code id:e598e092 tags:
``` python
l1b_urls_fpath = "../GEDI-L1B-2021-URLS.txt"
l2a_urls_fpath = "../GEDI-L2A-2021-URLS.txt"
with open(l1b_urls_fpath) as f:
l1b_fpaths = f.readlines()
with open(l2a_urls_fpath) as f:
l2a_fpaths = f.readlines()
# Get matching string pattern and run main.py
jobs_list = []
counter=1
for l1b_fp in l1b_fpaths:
# Get string pattern
str_pattern = re.findall("[0-9]{13}", os.path.basename(l1b_fp))[0]
try:
l2a_fp = [s for s in l2a_fpaths if str_pattern in s][0]
except:
print("No matching L2 file!", l1b_fp)
continue
print(l1b_fp)
print(l2a_fp)
job = maap.submitJob(identifier="single_file_test_run",
algo_id="arojas_biomass_gedi_conus",
version="master",
username="arojearthdata",
queue="maap-dps-worker-8gb",
L1B_URL=l1b_fp,
L2A_URL=l2a_fp)
jobs_list.append(job)
if counter%20:
print("on file num: ", counter)
counter+=1
break
```
%% Output
https://e4ftl01.cr.usgs.gov//GEDI_L1_L2/GEDI/GEDI01_B.002/2021.05.31/GEDI01_B_2021151223415_O13976_02_T00676_02_005_02_V002.h5
https://e4ftl01.cr.usgs.gov//GEDI_L1_L2/GEDI/GEDI02_A.002/2021.05.31/GEDI02_A_2021151223415_O13976_02_T00676_02_003_02_V002.h5
on file num: 1
%% Cell type:code id:273c2151 tags:
``` python
print(jobs_list[:2])
```
%% Output
[{'job_id': 'e02adfbd-c1f1-417f-9563-d158eafed27c', 'status': 'Accepted', 'machine_type': None, 'architecture': None, 'machine_memory_size': None, 'directory_size': None, 'operating_system': None, 'job_start_time': None, 'job_end_time': None, 'job_duration_seconds': None, 'cpu_usage': None, 'cache_usage': None, 'mem_usage': None, 'max_mem_usage': None, 'swap_usage': None, 'read_io_stats': None, 'write_io_stats': None, 'sync_io_stats': None, 'async_io_stats': None, 'total_io_stats': None, 'error_details': None, 'response_code': 200, 'outputs': []}]
%% Cell type:code id:27f8a6cd tags:
``` python
# View job status
for job in jobs_list:
print(job.retrieve_status())
print(job.id)
break
```
%% Output
Accepted
e02adfbd-c1f1-417f-9563-d158eafed27c
%% Cell type:code id:758ab8f6 tags:
``` python
# View job status
for job in jobs_list:
print(job.retrieve_status())
print(job.id)
break
```
%% Output
Running
e02adfbd-c1f1-417f-9563-d158eafed27c
%% Cell type:code id:dcfc811e tags:
``` python
# Retrieve results
for job in jobs_list:
print(job.retrieve_result())
break
```
%% Output
['http://maap-ops-workspace.s3-website-us-west-2.amazonaws.com/dataset/triaged_job/arojas_biomass_gedi_conus/master/2023/09/27/4ad5a4ee-4859-47aa-b825-d2a85562129c', 's3://s3-us-west-2.amazonaws.com:80/maap-ops-workspace/dataset/triaged_job/arojas_biomass_gedi_conus/master/2023/09/27/4ad5a4ee-4859-47aa-b825-d2a85562129c', 'https://s3.console.aws.amazon.com/s3/buckets/maap-ops-workspace/dataset/triaged_job/arojas_biomass_gedi_conus/master/2023/09/27/4ad5a4ee-4859-47aa-b825-d2a85562129c/?region=us-east-1&tab=overview', '+ export HOME=/root\n+ HOME=/root\n+ exec /docker-stats-on-exit-shim _docker_stats.json /app/dps_wrapper.sh /app/biomass-gedi-conus/run.sh \'https://e4ftl01.cr.usgs.gov//GEDI_L1_L2/GEDI/GEDI01_B.002/2021.05.31/GEDI01_B_2021151223415_O13976_02_T00676_02_005_02_V002.h5\n\' https://e4ftl01.cr.usgs.gov//GEDI_L1_L2/GEDI/GEDI02_A.002/2021.05.31/GEDI02_A_2021151223415_O13976_02_T00676_02_003_02_V002.h5\n+ /app/biomass-gedi-conus/run.sh \'https://e4ftl01.cr.usgs.gov//GEDI_L1_L2/GEDI/GEDI01_B.002/2021.05.31/GEDI01_B_2021151223415_O13976_02_T00676_02_005_02_V002.h5\n\' https://e4ftl01.cr.usgs.gov//GEDI_L1_L2/GEDI/GEDI02_A.002/2021.05.31/GEDI02_A_2021151223415_O13976_02_T00676_02_003_02_V002.h5\nMatplotlib created a temporary cache directory at /tmp/matplotlib-u_3gfo0a because the default path (/root/.config/matplotlib) is not a writable directory; it is highly recommended to set the MPLCONFIGDIR environment variable to a writable directory, in particular to speed up the import of Matplotlib and to better support multiprocessing.\nFontconfig error: No writable cache directories\nTraceback (most recent call last):\n File "fiona/ogrext.pyx", line 136, in fiona.ogrext.gdal_open_vector\n File "fiona/_err.pyx", line 291, in fiona._err.exc_wrap_pointer\nfiona._err.CPLE_OpenFailedError: /data/work/jobs/2023/09/27/05/17/job-arojas_biomass_gedi_conus__master-20230927T051209.837755Z/NEON_Domains/NEON_Domains.shp: No such file or directory\n\nDuring handling of the above exception, another exception occurred:\n\nTraceback (most recent call last):\n File "/app/biomass-gedi-conus/main.py", line 108, in <module>\n domain_polys = gpd.read_file(domain_poly_fp)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n File "/opt/conda/envs/osgeo-env/lib/python3.11/site-packages/geopandas/io/file.py", line 281, in _read_file\n return _read_file_fiona(\n ^^^^^^^^^^^^^^^^^\n File "/opt/conda/envs/osgeo-env/lib/python3.11/site-packages/geopandas/io/file.py", line 322, in _read_file_fiona\n with reader(path_or_bytes, **kwargs) as features:\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n File "/opt/conda/envs/osgeo-env/lib/python3.11/site-packages/fiona/env.py", line 457, in wrapper\n return f(*args, **kwds)\n ^^^^^^^^^^^^^^^^\n File "/opt/conda/envs/osgeo-env/lib/python3.11/site-packages/fiona/__init__.py", line 292, in open\n colxn = Collection(\n ^^^^^^^^^^^\n File "/opt/conda/envs/osgeo-env/lib/python3.11/site-packages/fiona/collection.py", line 243, in __init__\n self.session.start(self, **kwargs)\n File "fiona/ogrext.pyx", line 588, in fiona.ogrext.Session.start\n File "fiona/ogrext.pyx", line 143, in fiona.ogrext.gdal_open_vector\nfiona.errors.DriverError: /data/work/jobs/2023/09/27/05/17/job-arojas_biomass_gedi_conus__master-20230927T051209.837755Z/NEON_Domains/NEON_Domains.shp: No such file or directory\n+ cp _stderr.txt _alt_traceback.txt\n']
%% Cell type:code id:6d8ef38d tags:
``` python
```
%% Cell type:code id:da773e1f tags:
``` python
```
%% Cell type:code id:ee0b897a tags:
``` python
```
%% Cell type:code id:b364772a tags:
``` python
```
......
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment