small updates

5e40f3a9 · Alex Rojas · 639c7f4d · 5e40f3a9
Commit 5e40f3a9 authored 1 year ago by Alex Rojas
--- a/notebooks/run-gedi-biomass.ipynb
+++ b/notebooks/run-gedi-biomass.ipynb
@@ -7,17 +7,21 @@
   "metadata": {},
   "outputs": [
    {
-     "name": "stdout",
+     "name": "stderr",
     "output_type": "stream",
     "text": [
-      "/projects/biomass-gedi-conus/src/biomass-gedi-conus/notebooks\n"
+      "Unable to load config file from source maap.cfg \n",
+      "Unable to load config file from source ./maap.cfg \n",
+      "Unable to load config file from source /projects/maap.cfg \n"
     ]
    }
   ],
   "source": [
    "import os\n",
    "import re\n",
-    "print(os.getcwd())"
+    "from maap.maap import MAAP\n",
+    "# maap = MAAP(maap_host='api.maap-project.org')\n",
+    "maap = MAAP(maap_host='api.maap-project.org')"
   ]
  },
  {
@@ -32,7 +36,8 @@
     "text": [
      "https://e4ftl01.cr.usgs.gov//GEDI_L1_L2/GEDI/GEDI01_B.002/2021.05.31/GEDI01_B_2021151223415_O13976_02_T00676_02_005_02_V002.h5\n",
      "\n",
-      "https://e4ftl01.cr.usgs.gov//GEDI_L1_L2/GEDI/GEDI02_A.002/2021.05.31/GEDI02_A_2021151223415_O13976_02_T00676_02_003_02_V002.h5\n"
+      "https://e4ftl01.cr.usgs.gov//GEDI_L1_L2/GEDI/GEDI02_A.002/2021.05.31/GEDI02_A_2021151223415_O13976_02_T00676_02_003_02_V002.h5\n",
+      "on file num:  1\n"
     ]
    }
   ],
@@ -46,6 +51,8 @@
    "    l2a_fpaths = f.readlines()\n",
    "\n",
    "# Get matching string pattern and run main.py\n",
+    "jobs_list = []\n",
+    "counter=1\n",
    "for l1b_fp in l1b_fpaths:\n",
    "    # Get string pattern\n",
    "    str_pattern = re.findall(\"[0-9]{13}\", os.path.basename(l1b_fp))[0] \n",
@@ -57,24 +64,88 @@
    "    \n",
    "    print(l1b_fp)\n",
    "    print(l2a_fp)\n",
+    "    job = maap.submitJob(identifier=\"single file test run\",\n",
+    "                   algo_id=\"arojas_biomass_gedi_conus\",\n",
+    "                   version=\"master\",\n",
+    "                   username=\"arojearthdata\",\n",
+    "                   queue=\"null\",\n",
+    "                   L1B_URL=l1b_fp,\n",
+    "                   L2A_URL=l2a_fp)\n",
+    "    jobs_list.append(job)\n",
+    "    \n",
+    "    if counter%20:\n",
+    "        print(\"on file num: \", counter)\n",
+    "    counter+=1\n",
    "    break"
   ]
  },
  {
   "cell_type": "code",
-   "execution_count": null,
+   "execution_count": 3,
+   "id": "8f0b9f73-ffb1-40b7-be0e-b31480571a4a",
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "[{'job_id': '8c3396a7-500b-4b55-ad25-ca8c0fcfe1da', 'status': 'Accepted', 'machine_type': None, 'architecture': None, 'machine_memory_size': None, 'directory_size': None, 'operating_system': None, 'job_start_time': None, 'job_end_time': None, 'job_duration_seconds': None, 'cpu_usage': None, 'cache_usage': None, 'mem_usage': None, 'max_mem_usage': None, 'swap_usage': None, 'read_io_stats': None, 'write_io_stats': None, 'sync_io_stats': None, 'async_io_stats': None, 'total_io_stats': None, 'error_details': None, 'response_code': 200, 'outputs': []}]\n"
+     ]
+    }
+   ],
+   "source": [
+    "print(jobs_list[:2])"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 15,
   "id": "7fe1c6f4-37c5-4fe7-b9ff-51e17b7a5cf0",
   "metadata": {},
-   "outputs": [],
-   "source": []
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Accepted\n",
+      "8c3396a7-500b-4b55-ad25-ca8c0fcfe1da\n"
+     ]
+    }
+   ],
+   "source": [
+    "# View job status\n",
+    "for job in jobs_list:\n",
+    "    print(job.retrieve_status())\n",
+    "    print(job.id)\n",
+    "    break"
+   ]
  },
  {
   "cell_type": "code",
-   "execution_count": null,
+   "execution_count": 12,
   "id": "873e9ea7-d6cb-4bb9-b082-5bffb4026615",
   "metadata": {},
-   "outputs": [],
-   "source": []
+   "outputs": [
+    {
+     "ename": "RuntimeError",
+     "evalue": "response is not 200 or 201. code: 500. details: b'<ows:ExceptionReport xmlns:wps=\"http://www.opengis.net/wps/2.0\" xmlns:xsi=\"http://www.w3.org/2001/XMLSchema-instance\" xmlns:schemaLocation=\"http://schemas.opengis.net/wps/2.0/wps.xsd\" xmlns:ows=\"http://www.opengis.net/ows/2.0\"><ows:Exception exceptionCode=\"FailedGetResult\" locator=\"GetResult\"><ows:ExceptionText>Failed to get job result of job with id: 8c3396a7-500b-4b55-ad25-ca8c0fcfe1da. Aborting retrieving information of job because status is job-queued. If you don\\'t see expected results, please contact administrator of DPS</ows:ExceptionText></ows:Exception></ows:ExceptionReport>'",
+     "output_type": "error",
+     "traceback": [
+      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
+      "\u001b[0;31mRuntimeError\u001b[0m                              Traceback (most recent call last)",
+      "Cell \u001b[0;32mIn[12], line 3\u001b[0m\n\u001b[1;32m      1\u001b[0m \u001b[38;5;66;03m# Retrieve results\u001b[39;00m\n\u001b[1;32m      2\u001b[0m \u001b[38;5;28;01mfor\u001b[39;00m job \u001b[38;5;129;01min\u001b[39;00m jobs_list:\n\u001b[0;32m----> 3\u001b[0m     \u001b[38;5;28mprint\u001b[39m(\u001b[43mjob\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mretrieve_result\u001b[49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m)\n\u001b[1;32m      4\u001b[0m     \u001b[38;5;28;01mbreak\u001b[39;00m\n",
+      "File \u001b[0;32m/maap-py/maap/dps/dps_job.py:98\u001b[0m, in \u001b[0;36mDPSJob.retrieve_result\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m     92\u001b[0m logger\u001b[38;5;241m.\u001b[39mdebug(headers)\n\u001b[1;32m     93\u001b[0m response \u001b[38;5;241m=\u001b[39m requests\u001b[38;5;241m.\u001b[39mget(\n\u001b[1;32m     94\u001b[0m     url\u001b[38;5;241m=\u001b[39murl,\n\u001b[1;32m     95\u001b[0m     verify\u001b[38;5;241m=\u001b[39m\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m__not_self_signed,\n\u001b[1;32m     96\u001b[0m     headers\u001b[38;5;241m=\u001b[39mheaders\n\u001b[1;32m     97\u001b[0m )\n\u001b[0;32m---> 98\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mset_job_results_result(\u001b[43mRequestsUtils\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mcheck_response\u001b[49m\u001b[43m(\u001b[49m\u001b[43mresponse\u001b[49m\u001b[43m)\u001b[49m)\n\u001b[1;32m     99\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39moutputs\n",
+      "File \u001b[0;32m/maap-py/maap/utils/requests_utils.py:25\u001b[0m, in \u001b[0;36mRequestsUtils.check_response\u001b[0;34m(dps_response)\u001b[0m\n\u001b[1;32m     22\u001b[0m \u001b[38;5;129m@staticmethod\u001b[39m\n\u001b[1;32m     23\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21mcheck_response\u001b[39m(dps_response):\n\u001b[1;32m     24\u001b[0m     \u001b[38;5;28;01mif\u001b[39;00m dps_response\u001b[38;5;241m.\u001b[39mstatus_code \u001b[38;5;129;01mnot\u001b[39;00m \u001b[38;5;129;01min\u001b[39;00m [\u001b[38;5;241m200\u001b[39m, \u001b[38;5;241m201\u001b[39m]:\n\u001b[0;32m---> 25\u001b[0m         \u001b[38;5;28;01mraise\u001b[39;00m \u001b[38;5;167;01mRuntimeError\u001b[39;00m(\u001b[38;5;124m'\u001b[39m\u001b[38;5;124mresponse is not 200 or 201. code: \u001b[39m\u001b[38;5;132;01m{}\u001b[39;00m\u001b[38;5;124m. details: \u001b[39m\u001b[38;5;132;01m{}\u001b[39;00m\u001b[38;5;124m'\u001b[39m\u001b[38;5;241m.\u001b[39mformat(dps_response\u001b[38;5;241m.\u001b[39mstatus_code,\n\u001b[1;32m     26\u001b[0m                                                                                       dps_response\u001b[38;5;241m.\u001b[39mcontent))\n\u001b[1;32m     27\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m dps_response\u001b[38;5;241m.\u001b[39mcontent\u001b[38;5;241m.\u001b[39mdecode(\u001b[38;5;124m'\u001b[39m\u001b[38;5;124mUTF-8\u001b[39m\u001b[38;5;124m'\u001b[39m)\n",
+      "\u001b[0;31mRuntimeError\u001b[0m: response is not 200 or 201. code: 500. details: b'<ows:ExceptionReport xmlns:wps=\"http://www.opengis.net/wps/2.0\" xmlns:xsi=\"http://www.w3.org/2001/XMLSchema-instance\" xmlns:schemaLocation=\"http://schemas.opengis.net/wps/2.0/wps.xsd\" xmlns:ows=\"http://www.opengis.net/ows/2.0\"><ows:Exception exceptionCode=\"FailedGetResult\" locator=\"GetResult\"><ows:ExceptionText>Failed to get job result of job with id: 8c3396a7-500b-4b55-ad25-ca8c0fcfe1da. Aborting retrieving information of job because status is job-queued. If you don\\'t see expected results, please contact administrator of DPS</ows:ExceptionText></ows:Exception></ows:ExceptionReport>'"
+     ]
+    }
+   ],
+   "source": [
+    "# Retrieve results\n",
+    "for job in jobs_list:\n",
+    "    print(job.retrieve_result())\n",
+    "    break"
+   ]
  },
  {
   "cell_type": "code",
@@ -111,9 +182,9 @@
 ],
 "metadata": {
  "kernelspec": {
-   "display_name": "osgeo-env-v1",
+   "display_name": "Python 3 (ipykernel)",
   "language": "python",
-   "name": "osgeo-env-v1"
+   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
@@ -125,7 +196,7 @@
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
-   "version": "3.11.4"
+   "version": "3.10.8"
  }
 },
 "nbformat": 4,

 %% Cell type:code id:25172537-794b-4135-bb28-63c0fcefbb68 tags:

 ``` python
 import os
 import re
-print(os.getcwd())
+from maap.maap import MAAP
+# maap = MAAP(maap_host='api.maap-project.org')
+maap = MAAP(maap_host='api.maap-project.org')
 ```

 %% Output

-    /projects/biomass-gedi-conus/src/biomass-gedi-conus/notebooks
+    Unable to load config file from source maap.cfg
+    Unable to load config file from source ./maap.cfg
+    Unable to load config file from source /projects/maap.cfg

 %% Cell type:code id:b39e76ac-d765-4b8c-baf1-fe41f3296de9 tags:

 ``` python
 l1b_urls_fpath = "../GEDI-L1B-2021-URLS.txt"
 l2a_urls_fpath = "../GEDI-L2A-2021-URLS.txt"

 with open(l1b_urls_fpath) as f:
    l1b_fpaths = f.readlines()
 with open(l2a_urls_fpath) as f:
    l2a_fpaths = f.readlines()

 # Get matching string pattern and run main.py
+jobs_list = []
+counter=1
 for l1b_fp in l1b_fpaths:
    # Get string pattern
    str_pattern = re.findall("[0-9]{13}", os.path.basename(l1b_fp))[0]
    try:
        l2a_fp = [s for s in l2a_fpaths if str_pattern in s][0]
    except:
        print("No matching L2 file!", l1b_fp)
        continue

    print(l1b_fp)
    print(l2a_fp)
+    job = maap.submitJob(identifier="single file test run",
+                   algo_id="arojas_biomass_gedi_conus",
+                   version="master",
+                   username="arojearthdata",
+                   queue="null",
+                   L1B_URL=l1b_fp,
+                   L2A_URL=l2a_fp)
+    jobs_list.append(job)
+
+    if counter%20:
+        print("on file num: ", counter)
+    counter+=1
    break
 ```

 %% Output

    https://e4ftl01.cr.usgs.gov//GEDI_L1_L2/GEDI/GEDI01_B.002/2021.05.31/GEDI01_B_2021151223415_O13976_02_T00676_02_005_02_V002.h5
    
    https://e4ftl01.cr.usgs.gov//GEDI_L1_L2/GEDI/GEDI02_A.002/2021.05.31/GEDI02_A_2021151223415_O13976_02_T00676_02_003_02_V002.h5
+    on file num:  1
+
+%% Cell type:code id:8f0b9f73-ffb1-40b7-be0e-b31480571a4a tags:
+
+``` python
+print(jobs_list[:2])
+```
+
+%% Output
+
+    [{'job_id': '8c3396a7-500b-4b55-ad25-ca8c0fcfe1da', 'status': 'Accepted', 'machine_type': None, 'architecture': None, 'machine_memory_size': None, 'directory_size': None, 'operating_system': None, 'job_start_time': None, 'job_end_time': None, 'job_duration_seconds': None, 'cpu_usage': None, 'cache_usage': None, 'mem_usage': None, 'max_mem_usage': None, 'swap_usage': None, 'read_io_stats': None, 'write_io_stats': None, 'sync_io_stats': None, 'async_io_stats': None, 'total_io_stats': None, 'error_details': None, 'response_code': 200, 'outputs': []}]

 %% Cell type:code id:7fe1c6f4-37c5-4fe7-b9ff-51e17b7a5cf0 tags:

 ``` python
+# View job status
+for job in jobs_list:
+    print(job.retrieve_status())
+    print(job.id)
+    break
 ```

+%% Output
+
+    Accepted
+    8c3396a7-500b-4b55-ad25-ca8c0fcfe1da
+
 %% Cell type:code id:873e9ea7-d6cb-4bb9-b082-5bffb4026615 tags:

 ``` python
+# Retrieve results
+for job in jobs_list:
+    print(job.retrieve_result())
+    break
 ```

+%% Output
+
+    ---------------------------------------------------------------------------
+    RuntimeError                              Traceback (most recent call last)
+Cell     In[12], line 3
+          1 # Retrieve results
+          2 for job in jobs_list:
+    ----> 3     print(job.retrieve_result())
+          4     break
+File     /maap-py/maap/dps/dps_job.py:98, in DPSJob.retrieve_result(self)
+         92 logger.debug(headers)
+         93 response = requests.get(
+         94     url=url,
+         95     verify=self.__not_self_signed,
+         96     headers=headers
+         97 )
+    ---> 98 self.set_job_results_result(RequestsUtils.check_response(response))
+         99 return self.outputs
+File     /maap-py/maap/utils/requests_utils.py:25, in RequestsUtils.check_response(dps_response)
+         22 @staticmethod
+         23 def check_response(dps_response):
+         24     if dps_response.status_code not in [200, 201]:
+    ---> 25         raise RuntimeError('response is not 200 or 201. code: {}. details: {}'.format(dps_response.status_code,
+         26                                                                                       dps_response.content))
+         27     return dps_response.content.decode('UTF-8')
+    RuntimeError: response is not 200 or 201. code: 500. details: b'<ows:ExceptionReport xmlns:wps="http://www.opengis.net/wps/2.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xmlns:schemaLocation="http://schemas.opengis.net/wps/2.0/wps.xsd" xmlns:ows="http://www.opengis.net/ows/2.0"><ows:Exception exceptionCode="FailedGetResult" locator="GetResult"><ows:ExceptionText>Failed to get job result of job with id: 8c3396a7-500b-4b55-ad25-ca8c0fcfe1da. Aborting retrieving information of job because status is job-queued. If you don\'t see expected results, please contact administrator of DPS</ows:ExceptionText></ows:Exception></ows:ExceptionReport>'
+
 %% Cell type:code id:e8a40e13-dcdd-4b7f-a3b6-d060bb52eaf5 tags:

 ``` python
 ```

 %% Cell type:code id:988420a0-1145-499c-927c-eec059c0dde9 tags:

 ``` python
 ```

 %% Cell type:code id:31626d2a-4edf-4572-81c2-7b68080e2dd3 tags:

 ``` python
 ```

 %% Cell type:code id:9ecb7ffc-319f-40c9-8f6e-d18b6ea1c890 tags:

 ``` python
 ```