From ba560d8493a3f3844296ab4688ebb1b91292a444 Mon Sep 17 00:00:00 2001 From: Alex Rojas <a.rojas8907@gmail.com> Date: Wed, 27 Sep 2023 02:48:19 +0000 Subject: [PATCH] added some print statements to run.sh and main.py. Added os.getcwd() for full filepath to Domains_poly.shp. --- build-env.sh | 1 - main.py | 11 +++++++++-- notebooks/run-gedi-biomass.ipynb | 14 +++++++------- run.sh | 4 +++- 4 files changed, 19 insertions(+), 11 deletions(-) diff --git a/build-env.sh b/build-env.sh index 60d3113..802ebb2 100644 --- a/build-env.sh +++ b/build-env.sh @@ -4,7 +4,6 @@ basedir=$( cd "$(dirname "$0")" ; pwd -P) echo installing environment... mamba env create --name osgeo-env -f ${basedir}/environment.yml -conda env list # conda create --name osgeo-env --clone base # source activate osgeo-env # mamba install -c conda-forge scipy -y diff --git a/main.py b/main.py index f0e3833..1fae7ec 100644 --- a/main.py +++ b/main.py @@ -21,6 +21,11 @@ from pgap import GapDS, wf_smooth # import custom functions, etc. from download_gedi import download_gedi +## GET CONSTANTS +CWD = os.getcwd() +print("current working directory...") +print(CWD) + ## Function to return beam dataframe with some meta data def get_beam_gdf(beam,l1b_ds,l2a_ds): # Create gdf to filter for GEDI returns @@ -101,8 +106,10 @@ if __name__ == '__main__': l2a_basename = os.path.basename(l2a_url) # Read in domain polys and allom data - domain_polys = gpd.read_file("./NEON_Domains/NEON_Domains.shp") - allom_df = pd.read_csv("./NEON-DOMAINS-HSE-edited.csv") + domain_poly_fp = os.path.join(CWD, "./NEON_Domains/NEON_Domains.shp") + domain_polys = gpd.read_file(domain_poly_fp) + allom_fp = os.path.join(CWD, "./NEON-DOMAINS-HSE-edited.csv") + allom_df = pd.read_csv(allom_fp) try: diff --git a/notebooks/run-gedi-biomass.ipynb b/notebooks/run-gedi-biomass.ipynb index 89f3f09..18131e1 100644 --- a/notebooks/run-gedi-biomass.ipynb +++ b/notebooks/run-gedi-biomass.ipynb @@ -97,7 +97,7 @@ "name": "stdout", "output_type": "stream", "text": [ - "[{'job_id': '57289b00-6800-408a-8d6b-03fb449b5bba', 'status': 'Running', 'machine_type': None, 'architecture': None, 'machine_memory_size': None, 'directory_size': None, 'operating_system': None, 'job_start_time': None, 'job_end_time': None, 'job_duration_seconds': None, 'cpu_usage': None, 'cache_usage': None, 'mem_usage': None, 'max_mem_usage': None, 'swap_usage': None, 'read_io_stats': None, 'write_io_stats': None, 'sync_io_stats': None, 'async_io_stats': None, 'total_io_stats': None, 'error_details': None, 'response_code': 200, 'outputs': []}]\n" + "[{'job_id': '973bf65b-2305-441d-87eb-32b6e91e1c43', 'status': 'Accepted', 'machine_type': None, 'architecture': None, 'machine_memory_size': None, 'directory_size': None, 'operating_system': None, 'job_start_time': None, 'job_end_time': None, 'job_duration_seconds': None, 'cpu_usage': None, 'cache_usage': None, 'mem_usage': None, 'max_mem_usage': None, 'swap_usage': None, 'read_io_stats': None, 'write_io_stats': None, 'sync_io_stats': None, 'async_io_stats': None, 'total_io_stats': None, 'error_details': None, 'response_code': 200, 'outputs': []}]\n" ] } ], @@ -115,8 +115,8 @@ "name": "stdout", "output_type": "stream", "text": [ - "Running\n", - "57289b00-6800-408a-8d6b-03fb449b5bba\n" + "Accepted\n", + "973bf65b-2305-441d-87eb-32b6e91e1c43\n" ] } ], @@ -130,7 +130,7 @@ }, { "cell_type": "code", - "execution_count": 7, + "execution_count": 13, "id": "2018077f-9640-4e70-bd01-a9d939b3626a", "metadata": {}, "outputs": [ @@ -139,7 +139,7 @@ "output_type": "stream", "text": [ "Failed\n", - "57289b00-6800-408a-8d6b-03fb449b5bba\n" + "973bf65b-2305-441d-87eb-32b6e91e1c43\n" ] } ], @@ -153,7 +153,7 @@ }, { "cell_type": "code", - "execution_count": 8, + "execution_count": 14, "id": "873e9ea7-d6cb-4bb9-b082-5bffb4026615", "metadata": {}, "outputs": [ @@ -161,7 +161,7 @@ "name": "stdout", "output_type": "stream", "text": [ - "['http://maap-ops-workspace.s3-website-us-west-2.amazonaws.com/dataset/triaged_job/arojas_biomass_gedi_conus/master/2023/09/20/57289b00-6800-408a-8d6b-03fb449b5bba', 's3://s3-us-west-2.amazonaws.com:80/maap-ops-workspace/dataset/triaged_job/arojas_biomass_gedi_conus/master/2023/09/20/57289b00-6800-408a-8d6b-03fb449b5bba', 'https://s3.console.aws.amazon.com/s3/buckets/maap-ops-workspace/dataset/triaged_job/arojas_biomass_gedi_conus/master/2023/09/20/57289b00-6800-408a-8d6b-03fb449b5bba/?region=us-east-1&tab=overview', '+ export HOME=/root\\n+ HOME=/root\\n+ exec /docker-stats-on-exit-shim _docker_stats.json /app/dps_wrapper.sh /app/biomass-gedi-conus/run.sh \\'https://e4ftl01.cr.usgs.gov//GEDI_L1_L2/GEDI/GEDI01_B.002/2021.05.31/GEDI01_B_2021151223415_O13976_02_T00676_02_005_02_V002.h5\\n\\' https://e4ftl01.cr.usgs.gov//GEDI_L1_L2/GEDI/GEDI02_A.002/2021.05.31/GEDI02_A_2021151223415_O13976_02_T00676_02_003_02_V002.h5\\n+ /app/biomass-gedi-conus/run.sh \\'https://e4ftl01.cr.usgs.gov//GEDI_L1_L2/GEDI/GEDI01_B.002/2021.05.31/GEDI01_B_2021151223415_O13976_02_T00676_02_005_02_V002.h5\\n\\' https://e4ftl01.cr.usgs.gov//GEDI_L1_L2/GEDI/GEDI02_A.002/2021.05.31/GEDI02_A_2021151223415_O13976_02_T00676_02_003_02_V002.h5\\nTraceback (most recent call last):\\n File \"/app/biomass-gedi-conus/main.py\", line 22, in <module>\\n from download_gedi import download_gedi\\n File \"/app/biomass-gedi-conus/download_gedi.py\", line 3, in <module>\\n from maap.maap import MAAP\\nModuleNotFoundError: No module named \\'maap\\'\\n+ cp _stderr.txt _alt_traceback.txt\\n']\n" + "['http://maap-ops-workspace.s3-website-us-west-2.amazonaws.com/dataset/triaged_job/arojas_biomass_gedi_conus/master/2023/09/27/973bf65b-2305-441d-87eb-32b6e91e1c43', 's3://s3-us-west-2.amazonaws.com:80/maap-ops-workspace/dataset/triaged_job/arojas_biomass_gedi_conus/master/2023/09/27/973bf65b-2305-441d-87eb-32b6e91e1c43', 'https://s3.console.aws.amazon.com/s3/buckets/maap-ops-workspace/dataset/triaged_job/arojas_biomass_gedi_conus/master/2023/09/27/973bf65b-2305-441d-87eb-32b6e91e1c43/?region=us-east-1&tab=overview', '+ export HOME=/root\\n+ HOME=/root\\n+ exec /docker-stats-on-exit-shim _docker_stats.json /app/dps_wrapper.sh /app/biomass-gedi-conus/run.sh \\'https://e4ftl01.cr.usgs.gov//GEDI_L1_L2/GEDI/GEDI01_B.002/2021.05.31/GEDI01_B_2021151223415_O13976_02_T00676_02_005_02_V002.h5\\n\\' https://e4ftl01.cr.usgs.gov//GEDI_L1_L2/GEDI/GEDI02_A.002/2021.05.31/GEDI02_A_2021151223415_O13976_02_T00676_02_003_02_V002.h5\\n+ /app/biomass-gedi-conus/run.sh \\'https://e4ftl01.cr.usgs.gov//GEDI_L1_L2/GEDI/GEDI01_B.002/2021.05.31/GEDI01_B_2021151223415_O13976_02_T00676_02_005_02_V002.h5\\n\\' https://e4ftl01.cr.usgs.gov//GEDI_L1_L2/GEDI/GEDI02_A.002/2021.05.31/GEDI02_A_2021151223415_O13976_02_T00676_02_003_02_V002.h5\\nMatplotlib created a temporary cache directory at /tmp/matplotlib-rr0rwtpl because the default path (/root/.config/matplotlib) is not a writable directory; it is highly recommended to set the MPLCONFIGDIR environment variable to a writable directory, in particular to speed up the import of Matplotlib and to better support multiprocessing.\\nFontconfig error: No writable cache directories\\nTraceback (most recent call last):\\n File \"fiona/ogrext.pyx\", line 136, in fiona.ogrext.gdal_open_vector\\n File \"fiona/_err.pyx\", line 291, in fiona._err.exc_wrap_pointer\\nfiona._err.CPLE_OpenFailedError: ./NEON_Domains/NEON_Domains.shp: No such file or directory\\n\\nDuring handling of the above exception, another exception occurred:\\n\\nTraceback (most recent call last):\\n File \"/app/biomass-gedi-conus/main.py\", line 104, in <module>\\n domain_polys = gpd.read_file(\"./NEON_Domains/NEON_Domains.shp\")\\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\\n File \"/opt/conda/envs/osgeo-env/lib/python3.11/site-packages/geopandas/io/file.py\", line 281, in _read_file\\n return _read_file_fiona(\\n ^^^^^^^^^^^^^^^^^\\n File \"/opt/conda/envs/osgeo-env/lib/python3.11/site-packages/geopandas/io/file.py\", line 322, in _read_file_fiona\\n with reader(path_or_bytes, **kwargs) as features:\\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\\n File \"/opt/conda/envs/osgeo-env/lib/python3.11/site-packages/fiona/env.py\", line 457, in wrapper\\n return f(*args, **kwds)\\n ^^^^^^^^^^^^^^^^\\n File \"/opt/conda/envs/osgeo-env/lib/python3.11/site-packages/fiona/__init__.py\", line 292, in open\\n colxn = Collection(\\n ^^^^^^^^^^^\\n File \"/opt/conda/envs/osgeo-env/lib/python3.11/site-packages/fiona/collection.py\", line 243, in __init__\\n self.session.start(self, **kwargs)\\n File \"fiona/ogrext.pyx\", line 588, in fiona.ogrext.Session.start\\n File \"fiona/ogrext.pyx\", line 143, in fiona.ogrext.gdal_open_vector\\nfiona.errors.DriverError: ./NEON_Domains/NEON_Domains.shp: No such file or directory\\n+ cp _stderr.txt _alt_traceback.txt\\n']\n" ] } ], diff --git a/run.sh b/run.sh index 623a043..dc438ff 100644 --- a/run.sh +++ b/run.sh @@ -5,7 +5,9 @@ basedir=$( cd "$(dirname "$0")" ; pwd -P) # Activate environment that was created in the builed-env.sh file source activate osgeo-env - +# view contents of where thi file is +echo viewing contents of run.sh folder... +ls # Create output dir OUTPUTDIR="${PWD}/output" mkdir -p ${OUTPUTDIR} -- GitLab