Skip to content
Snippets Groups Projects
Commit 2cd4d6ee authored by Alex Rojas's avatar Alex Rojas
Browse files

updated notebook

parent 6ade147b
No related branches found
No related tags found
No related merge requests found
Hide whitespace changes
Inline Side-by-side
notebooks/run-gedi-biomass.ipynb
+ 18
− 8
View file @ 2cd4d6ee
%% Cell type:code id:25172537-794b-4135-bb28-63c0fcefbb68 tags: %% Cell type:code id:25172537-794b-4135-bb28-63c0fcefbb68 tags:
``` python ``` python
import os import os
import re import re
from maap.maap import MAAP from maap.maap import MAAP
# maap = MAAP(maap_host='api.maap-project.org') # maap = MAAP(maap_host='api.maap-project.org')
maap = MAAP(maap_host='api.maap-project.org') maap = MAAP(maap_host='api.maap-project.org')
``` ```
%% Output
Unable to load config file from source maap.cfg
Unable to load config file from source ./maap.cfg
Unable to load config file from source /projects/maap.cfg
%% Cell type:code id:f176e75b-5104-478b-ade6-a84006e966b0 tags: %% Cell type:code id:f176e75b-5104-478b-ade6-a84006e966b0 tags:
``` python ``` python
``` ```
%% Cell type:code id:b39e76ac-d765-4b8c-baf1-fe41f3296de9 tags: %% Cell type:code id:b39e76ac-d765-4b8c-baf1-fe41f3296de9 tags:
``` python ``` python
l1b_urls_fpath = "../GEDI-L1B-2021-URLS.txt" l1b_urls_fpath = "../GEDI-L1B-2021-URLS.txt"
l2a_urls_fpath = "../GEDI-L2A-2021-URLS.txt" l2a_urls_fpath = "../GEDI-L2A-2021-URLS.txt"
with open(l1b_urls_fpath) as f: with open(l1b_urls_fpath) as f:
l1b_fpaths = f.readlines() l1b_fpaths = f.readlines()
with open(l2a_urls_fpath) as f: with open(l2a_urls_fpath) as f:
l2a_fpaths = f.readlines() l2a_fpaths = f.readlines()
# Get matching string pattern and run main.py # Get matching string pattern and run main.py
jobs_list = [] jobs_list = []
counter=1 counter=1
for l1b_fp in l1b_fpaths: for l1b_fp in l1b_fpaths:
# Get string pattern # Get string pattern
str_pattern = re.findall("[0-9]{13}", os.path.basename(l1b_fp))[0] str_pattern = re.findall("[0-9]{13}", os.path.basename(l1b_fp))[0]
try: try:
l2a_fp = [s for s in l2a_fpaths if str_pattern in s][0] l2a_fp = [s for s in l2a_fpaths if str_pattern in s][0]
except: except:
print("No matching L2 file!", l1b_fp) print("No matching L2 file!", l1b_fp)
continue continue
print(l1b_fp) print(l1b_fp)
print(l2a_fp) print(l2a_fp)
job = maap.submitJob(identifier="single_file_test_run", job = maap.submitJob(identifier="single_file_test_run",
algo_id="arojas_biomass_gedi_conus", algo_id="arojas_biomass_gedi_conus",
version="master", version="master",
username="arojearthdata", username="arojearthdata",
queue="maap-dps-worker-8gb", queue="maap-dps-worker-8gb",
L1B_URL=l1b_fp, L1B_URL=l1b_fp,
L2A_URL=l2a_fp) L2A_URL=l2a_fp)
jobs_list.append(job) jobs_list.append(job)
if counter%20: if counter%20:
print("on file num: ", counter) print("on file num: ", counter)
counter+=1 counter+=1
break break
``` ```
%% Output %% Output
https://e4ftl01.cr.usgs.gov//GEDI_L1_L2/GEDI/GEDI01_B.002/2021.05.31/GEDI01_B_2021151223415_O13976_02_T00676_02_005_02_V002.h5 https://e4ftl01.cr.usgs.gov//GEDI_L1_L2/GEDI/GEDI01_B.002/2021.05.31/GEDI01_B_2021151223415_O13976_02_T00676_02_005_02_V002.h5
https://e4ftl01.cr.usgs.gov//GEDI_L1_L2/GEDI/GEDI02_A.002/2021.05.31/GEDI02_A_2021151223415_O13976_02_T00676_02_003_02_V002.h5 https://e4ftl01.cr.usgs.gov//GEDI_L1_L2/GEDI/GEDI02_A.002/2021.05.31/GEDI02_A_2021151223415_O13976_02_T00676_02_003_02_V002.h5
on file num: 1 on file num: 1
%% Cell type:code id:8f0b9f73-ffb1-40b7-be0e-b31480571a4a tags: %% Cell type:code id:8f0b9f73-ffb1-40b7-be0e-b31480571a4a tags:
``` python ``` python
print(jobs_list[:2]) print(jobs_list[:2])
``` ```
%% Output %% Output
[{'job_id': '6910d9a3-4135-484c-b855-e0a498fe6077', 'status': 'Running', 'machine_type': None, 'architecture': None, 'machine_memory_size': None, 'directory_size': None, 'operating_system': None, 'job_start_time': None, 'job_end_time': None, 'job_duration_seconds': None, 'cpu_usage': None, 'cache_usage': None, 'mem_usage': None, 'max_mem_usage': None, 'swap_usage': None, 'read_io_stats': None, 'write_io_stats': None, 'sync_io_stats': None, 'async_io_stats': None, 'total_io_stats': None, 'error_details': None, 'response_code': 200, 'outputs': []}] [{'job_id': '44d040c7-4db6-44af-817a-5c2cc7cd32f0', 'status': 'Running', 'machine_type': None, 'architecture': None, 'machine_memory_size': None, 'directory_size': None, 'operating_system': None, 'job_start_time': None, 'job_end_time': None, 'job_duration_seconds': None, 'cpu_usage': None, 'cache_usage': None, 'mem_usage': None, 'max_mem_usage': None, 'swap_usage': None, 'read_io_stats': None, 'write_io_stats': None, 'sync_io_stats': None, 'async_io_stats': None, 'total_io_stats': None, 'error_details': None, 'response_code': 200, 'outputs': []}]
%% Cell type:code id:7fe1c6f4-37c5-4fe7-b9ff-51e17b7a5cf0 tags: %% Cell type:code id:7fe1c6f4-37c5-4fe7-b9ff-51e17b7a5cf0 tags:
``` python ``` python
# View job status # View job status
for job in jobs_list: for job in jobs_list:
print(job.retrieve_status()) print(job.retrieve_status())
print(job.id) print(job.id)
break break
``` ```
%% Output %% Output
Running Running
6910d9a3-4135-484c-b855-e0a498fe6077 44d040c7-4db6-44af-817a-5c2cc7cd32f0
%% Cell type:code id:2018077f-9640-4e70-bd01-a9d939b3626a tags: %% Cell type:code id:2018077f-9640-4e70-bd01-a9d939b3626a tags:
``` python ``` python
# View job status # View job status
for job in jobs_list: for job in jobs_list:
print(job.retrieve_status()) print(job.retrieve_status())
print(job.id) print(job.id)
break break
``` ```
%% Output %% Output
Failed Failed
6910d9a3-4135-484c-b855-e0a498fe6077 44d040c7-4db6-44af-817a-5c2cc7cd32f0
%% Cell type:code id:873e9ea7-d6cb-4bb9-b082-5bffb4026615 tags: %% Cell type:code id:873e9ea7-d6cb-4bb9-b082-5bffb4026615 tags:
``` python ``` python
# Retrieve results # Retrieve results
for job in jobs_list: for job in jobs_list:
print(job.retrieve_result()) print(job.retrieve_result())
break break
``` ```
%% Output %% Output
['http://maap-ops-workspace.s3-website-us-west-2.amazonaws.com/dataset/triaged_job/arojas_biomass_gedi_conus/master/2023/09/09/6910d9a3-4135-484c-b855-e0a498fe6077', 's3://s3-us-west-2.amazonaws.com:80/maap-ops-workspace/dataset/triaged_job/arojas_biomass_gedi_conus/master/2023/09/09/6910d9a3-4135-484c-b855-e0a498fe6077', 'https://s3.console.aws.amazon.com/s3/buckets/maap-ops-workspace/dataset/triaged_job/arojas_biomass_gedi_conus/master/2023/09/09/6910d9a3-4135-484c-b855-e0a498fe6077/?region=us-east-1&tab=overview', '+ export HOME=/root\n+ HOME=/root\n+ exec /docker-stats-on-exit-shim _docker_stats.json /app/dps_wrapper.sh /app/biomass-gedi-conus/run.sh \'https://e4ftl01.cr.usgs.gov//GEDI_L1_L2/GEDI/GEDI01_B.002/2021.05.31/GEDI01_B_2021151223415_O13976_02_T00676_02_005_02_V002.h5\n\' https://e4ftl01.cr.usgs.gov//GEDI_L1_L2/GEDI/GEDI02_A.002/2021.05.31/GEDI02_A_2021151223415_O13976_02_T00676_02_003_02_V002.h5\n+ /app/biomass-gedi-conus/run.sh \'https://e4ftl01.cr.usgs.gov//GEDI_L1_L2/GEDI/GEDI01_B.002/2021.05.31/GEDI01_B_2021151223415_O13976_02_T00676_02_005_02_V002.h5\n\' https://e4ftl01.cr.usgs.gov//GEDI_L1_L2/GEDI/GEDI02_A.002/2021.05.31/GEDI02_A_2021151223415_O13976_02_T00676_02_003_02_V002.h5\nTraceback (most recent call last):\n File "/app/biomass-gedi-conus/main.py", line 22, in <module>\n from download_gedi import download_gedi\n File "/app/biomass-gedi-conus/download_gedi.py", line 3, in <module>\n from maap.maap import MAAP\nModuleNotFoundError: No module named \'maap\'\n+ cp _stderr.txt _alt_traceback.txt\n'] ['http://maap-ops-workspace.s3-website-us-west-2.amazonaws.com/dataset/triaged_job/arojas_biomass_gedi_conus/master/2023/09/14/44d040c7-4db6-44af-817a-5c2cc7cd32f0', 's3://s3-us-west-2.amazonaws.com:80/maap-ops-workspace/dataset/triaged_job/arojas_biomass_gedi_conus/master/2023/09/14/44d040c7-4db6-44af-817a-5c2cc7cd32f0', 'https://s3.console.aws.amazon.com/s3/buckets/maap-ops-workspace/dataset/triaged_job/arojas_biomass_gedi_conus/master/2023/09/14/44d040c7-4db6-44af-817a-5c2cc7cd32f0/?region=us-east-1&tab=overview', '+ export HOME=/root\n+ HOME=/root\n+ exec /docker-stats-on-exit-shim _docker_stats.json /app/dps_wrapper.sh /app/biomass-gedi-conus/run.sh \'https://e4ftl01.cr.usgs.gov//GEDI_L1_L2/GEDI/GEDI01_B.002/2021.05.31/GEDI01_B_2021151223415_O13976_02_T00676_02_005_02_V002.h5\n\' https://e4ftl01.cr.usgs.gov//GEDI_L1_L2/GEDI/GEDI02_A.002/2021.05.31/GEDI02_A_2021151223415_O13976_02_T00676_02_003_02_V002.h5\n+ /app/biomass-gedi-conus/run.sh \'https://e4ftl01.cr.usgs.gov//GEDI_L1_L2/GEDI/GEDI01_B.002/2021.05.31/GEDI01_B_2021151223415_O13976_02_T00676_02_005_02_V002.h5\n\' https://e4ftl01.cr.usgs.gov//GEDI_L1_L2/GEDI/GEDI02_A.002/2021.05.31/GEDI02_A_2021151223415_O13976_02_T00676_02_003_02_V002.h5\nTraceback (most recent call last):\n File "/app/biomass-gedi-conus/main.py", line 22, in <module>\n from download_gedi import download_gedi\n File "/app/biomass-gedi-conus/download_gedi.py", line 3, in <module>\n from maap.maap import MAAP\nModuleNotFoundError: No module named \'maap\'\n+ cp _stderr.txt _alt_traceback.txt\n']
%% Cell type:code id:e8a40e13-dcdd-4b7f-a3b6-d060bb52eaf5 tags: %% Cell type:code id:e8a40e13-dcdd-4b7f-a3b6-d060bb52eaf5 tags:
``` python ``` python
``` ```
%% Cell type:code id:988420a0-1145-499c-927c-eec059c0dde9 tags: %% Cell type:code id:988420a0-1145-499c-927c-eec059c0dde9 tags:
``` python ``` python
``` ```
%% Cell type:code id:31626d2a-4edf-4572-81c2-7b68080e2dd3 tags: %% Cell type:code id:31626d2a-4edf-4572-81c2-7b68080e2dd3 tags:
``` python ``` python
``` ```
%% Cell type:code id:9ecb7ffc-319f-40c9-8f6e-d18b6ea1c890 tags: %% Cell type:code id:9ecb7ffc-319f-40c9-8f6e-d18b6ea1c890 tags:
``` python ``` python
``` ```
......
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment