diff --git a/main.py b/main.py index 8ef5c8a435cb817d75a4ff05db38869041101514..b7338a20c70fb3d14abd9ed54c5dcf77db59c484 100644 --- a/main.py +++ b/main.py @@ -21,8 +21,8 @@ from pgap import GapDS, wf_smooth # import custom functions, etc. from download_gedi import download_gedi -## GET CONSTANTS -CWD = os.getcwd() +## GET CWD of file to locate path +CWD = os.path.dirname(os.path.abspath(__file__)) ## Function to return beam dataframe with some meta data def get_beam_gdf(beam,l1b_ds,l2a_ds): diff --git a/notebooks/run-gedi-biomass.ipynb b/notebooks/run-gedi-biomass.ipynb index 21f618883f0d6c598d3aeb0d0531a27c03ea219e..7525dcf64a419c3987d6ac81b85b868c4fd88e26 100644 --- a/notebooks/run-gedi-biomass.ipynb +++ b/notebooks/run-gedi-biomass.ipynb @@ -97,7 +97,7 @@ "name": "stdout", "output_type": "stream", "text": [ - "[{'job_id': 'ba87642f-df6e-47f5-b509-0d94ca990b2d', 'status': 'Accepted', 'machine_type': None, 'architecture': None, 'machine_memory_size': None, 'directory_size': None, 'operating_system': None, 'job_start_time': None, 'job_end_time': None, 'job_duration_seconds': None, 'cpu_usage': None, 'cache_usage': None, 'mem_usage': None, 'max_mem_usage': None, 'swap_usage': None, 'read_io_stats': None, 'write_io_stats': None, 'sync_io_stats': None, 'async_io_stats': None, 'total_io_stats': None, 'error_details': None, 'response_code': 200, 'outputs': []}]\n" + "[{'job_id': '4ad5a4ee-4859-47aa-b825-d2a85562129c', 'status': 'Accepted', 'machine_type': None, 'architecture': None, 'machine_memory_size': None, 'directory_size': None, 'operating_system': None, 'job_start_time': None, 'job_end_time': None, 'job_duration_seconds': None, 'cpu_usage': None, 'cache_usage': None, 'mem_usage': None, 'max_mem_usage': None, 'swap_usage': None, 'read_io_stats': None, 'write_io_stats': None, 'sync_io_stats': None, 'async_io_stats': None, 'total_io_stats': None, 'error_details': None, 'response_code': 200, 'outputs': []}]\n" ] } ], @@ -107,7 +107,7 @@ }, { "cell_type": "code", - "execution_count": 4, + "execution_count": 5, "id": "7fe1c6f4-37c5-4fe7-b9ff-51e17b7a5cf0", "metadata": {}, "outputs": [ @@ -115,8 +115,8 @@ "name": "stdout", "output_type": "stream", "text": [ - "Accepted\n", - "ba87642f-df6e-47f5-b509-0d94ca990b2d\n" + "Failed\n", + "4ad5a4ee-4859-47aa-b825-d2a85562129c\n" ] } ], @@ -161,7 +161,7 @@ "name": "stdout", "output_type": "stream", "text": [ - "['http://maap-ops-workspace.s3-website-us-west-2.amazonaws.com/dataset/triaged_job/arojas_biomass_gedi_conus/master/2023/09/27/ba87642f-df6e-47f5-b509-0d94ca990b2d', 's3://s3-us-west-2.amazonaws.com:80/maap-ops-workspace/dataset/triaged_job/arojas_biomass_gedi_conus/master/2023/09/27/ba87642f-df6e-47f5-b509-0d94ca990b2d', 'https://s3.console.aws.amazon.com/s3/buckets/maap-ops-workspace/dataset/triaged_job/arojas_biomass_gedi_conus/master/2023/09/27/ba87642f-df6e-47f5-b509-0d94ca990b2d/?region=us-east-1&tab=overview', '+ export HOME=/root\\n+ HOME=/root\\n+ exec /docker-stats-on-exit-shim _docker_stats.json /app/dps_wrapper.sh /app/biomass-gedi-conus/run.sh \\'https://e4ftl01.cr.usgs.gov//GEDI_L1_L2/GEDI/GEDI01_B.002/2021.05.31/GEDI01_B_2021151223415_O13976_02_T00676_02_005_02_V002.h5\\n\\' https://e4ftl01.cr.usgs.gov//GEDI_L1_L2/GEDI/GEDI02_A.002/2021.05.31/GEDI02_A_2021151223415_O13976_02_T00676_02_003_02_V002.h5\\n+ /app/biomass-gedi-conus/run.sh \\'https://e4ftl01.cr.usgs.gov//GEDI_L1_L2/GEDI/GEDI01_B.002/2021.05.31/GEDI01_B_2021151223415_O13976_02_T00676_02_005_02_V002.h5\\n\\' https://e4ftl01.cr.usgs.gov//GEDI_L1_L2/GEDI/GEDI02_A.002/2021.05.31/GEDI02_A_2021151223415_O13976_02_T00676_02_003_02_V002.h5\\nMatplotlib created a temporary cache directory at /tmp/matplotlib-ej7h32oy because the default path (/root/.config/matplotlib) is not a writable directory; it is highly recommended to set the MPLCONFIGDIR environment variable to a writable directory, in particular to speed up the import of Matplotlib and to better support multiprocessing.\\nFontconfig error: No writable cache directories\\nTraceback (most recent call last):\\n File \"fiona/ogrext.pyx\", line 136, in fiona.ogrext.gdal_open_vector\\n File \"fiona/_err.pyx\", line 291, in fiona._err.exc_wrap_pointer\\nfiona._err.CPLE_OpenFailedError: /data/work/jobs/2023/09/27/04/01/job-arojas_biomass_gedi_conus__master-20230927T035526.152033Z/./NEON_Domains/NEON_Domains.shp: No such file or directory\\n\\nDuring handling of the above exception, another exception occurred:\\n\\nTraceback (most recent call last):\\n File \"/app/biomass-gedi-conus/main.py\", line 110, in <module>\\n domain_polys = gpd.read_file(domain_poly_fp)\\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\\n File \"/opt/conda/envs/osgeo-env/lib/python3.11/site-packages/geopandas/io/file.py\", line 281, in _read_file\\n return _read_file_fiona(\\n ^^^^^^^^^^^^^^^^^\\n File \"/opt/conda/envs/osgeo-env/lib/python3.11/site-packages/geopandas/io/file.py\", line 322, in _read_file_fiona\\n with reader(path_or_bytes, **kwargs) as features:\\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\\n File \"/opt/conda/envs/osgeo-env/lib/python3.11/site-packages/fiona/env.py\", line 457, in wrapper\\n return f(*args, **kwds)\\n ^^^^^^^^^^^^^^^^\\n File \"/opt/conda/envs/osgeo-env/lib/python3.11/site-packages/fiona/__init__.py\", line 292, in open\\n colxn = Collection(\\n ^^^^^^^^^^^\\n File \"/opt/conda/envs/osgeo-env/lib/python3.11/site-packages/fiona/collection.py\", line 243, in __init__\\n self.session.start(self, **kwargs)\\n File \"fiona/ogrext.pyx\", line 588, in fiona.ogrext.Session.start\\n File \"fiona/ogrext.pyx\", line 143, in fiona.ogrext.gdal_open_vector\\nfiona.errors.DriverError: /data/work/jobs/2023/09/27/04/01/job-arojas_biomass_gedi_conus__master-20230927T035526.152033Z/./NEON_Domains/NEON_Domains.shp: No such file or directory\\n+ cp _stderr.txt _alt_traceback.txt\\n']\n" + "['http://maap-ops-workspace.s3-website-us-west-2.amazonaws.com/dataset/triaged_job/arojas_biomass_gedi_conus/master/2023/09/27/4ad5a4ee-4859-47aa-b825-d2a85562129c', 's3://s3-us-west-2.amazonaws.com:80/maap-ops-workspace/dataset/triaged_job/arojas_biomass_gedi_conus/master/2023/09/27/4ad5a4ee-4859-47aa-b825-d2a85562129c', 'https://s3.console.aws.amazon.com/s3/buckets/maap-ops-workspace/dataset/triaged_job/arojas_biomass_gedi_conus/master/2023/09/27/4ad5a4ee-4859-47aa-b825-d2a85562129c/?region=us-east-1&tab=overview', '+ export HOME=/root\\n+ HOME=/root\\n+ exec /docker-stats-on-exit-shim _docker_stats.json /app/dps_wrapper.sh /app/biomass-gedi-conus/run.sh \\'https://e4ftl01.cr.usgs.gov//GEDI_L1_L2/GEDI/GEDI01_B.002/2021.05.31/GEDI01_B_2021151223415_O13976_02_T00676_02_005_02_V002.h5\\n\\' https://e4ftl01.cr.usgs.gov//GEDI_L1_L2/GEDI/GEDI02_A.002/2021.05.31/GEDI02_A_2021151223415_O13976_02_T00676_02_003_02_V002.h5\\n+ /app/biomass-gedi-conus/run.sh \\'https://e4ftl01.cr.usgs.gov//GEDI_L1_L2/GEDI/GEDI01_B.002/2021.05.31/GEDI01_B_2021151223415_O13976_02_T00676_02_005_02_V002.h5\\n\\' https://e4ftl01.cr.usgs.gov//GEDI_L1_L2/GEDI/GEDI02_A.002/2021.05.31/GEDI02_A_2021151223415_O13976_02_T00676_02_003_02_V002.h5\\nMatplotlib created a temporary cache directory at /tmp/matplotlib-u_3gfo0a because the default path (/root/.config/matplotlib) is not a writable directory; it is highly recommended to set the MPLCONFIGDIR environment variable to a writable directory, in particular to speed up the import of Matplotlib and to better support multiprocessing.\\nFontconfig error: No writable cache directories\\nTraceback (most recent call last):\\n File \"fiona/ogrext.pyx\", line 136, in fiona.ogrext.gdal_open_vector\\n File \"fiona/_err.pyx\", line 291, in fiona._err.exc_wrap_pointer\\nfiona._err.CPLE_OpenFailedError: /data/work/jobs/2023/09/27/05/17/job-arojas_biomass_gedi_conus__master-20230927T051209.837755Z/NEON_Domains/NEON_Domains.shp: No such file or directory\\n\\nDuring handling of the above exception, another exception occurred:\\n\\nTraceback (most recent call last):\\n File \"/app/biomass-gedi-conus/main.py\", line 108, in <module>\\n domain_polys = gpd.read_file(domain_poly_fp)\\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\\n File \"/opt/conda/envs/osgeo-env/lib/python3.11/site-packages/geopandas/io/file.py\", line 281, in _read_file\\n return _read_file_fiona(\\n ^^^^^^^^^^^^^^^^^\\n File \"/opt/conda/envs/osgeo-env/lib/python3.11/site-packages/geopandas/io/file.py\", line 322, in _read_file_fiona\\n with reader(path_or_bytes, **kwargs) as features:\\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\\n File \"/opt/conda/envs/osgeo-env/lib/python3.11/site-packages/fiona/env.py\", line 457, in wrapper\\n return f(*args, **kwds)\\n ^^^^^^^^^^^^^^^^\\n File \"/opt/conda/envs/osgeo-env/lib/python3.11/site-packages/fiona/__init__.py\", line 292, in open\\n colxn = Collection(\\n ^^^^^^^^^^^\\n File \"/opt/conda/envs/osgeo-env/lib/python3.11/site-packages/fiona/collection.py\", line 243, in __init__\\n self.session.start(self, **kwargs)\\n File \"fiona/ogrext.pyx\", line 588, in fiona.ogrext.Session.start\\n File \"fiona/ogrext.pyx\", line 143, in fiona.ogrext.gdal_open_vector\\nfiona.errors.DriverError: /data/work/jobs/2023/09/27/05/17/job-arojas_biomass_gedi_conus__master-20230927T051209.837755Z/NEON_Domains/NEON_Domains.shp: No such file or directory\\n+ cp _stderr.txt _alt_traceback.txt\\n']\n" ] } ],