From 96f3fe5294a00d432e89fa202006591aa72ef7d8 Mon Sep 17 00:00:00 2001 From: nehajo88 <nehajo88@gmail.com> Date: Wed, 9 Nov 2022 16:15:10 +0000 Subject: [PATCH] modified alg yaml --- .../algorithm_config-checkpoint.yaml | 2 +- .../algorithm_config_template-checkpoint.yaml | 25 +++ .../run_country_summaries-checkpoint.ipynb | 145 ++++++++++++------ country_summaries/dps/algorithm_config.yaml | 2 +- .../dps/algorithm_config_template.yaml | 25 +++ .../dps/run_country_summaries.ipynb | 145 ++++++++++++------ 6 files changed, 242 insertions(+), 102 deletions(-) create mode 100644 country_summaries/dps/.ipynb_checkpoints/algorithm_config_template-checkpoint.yaml create mode 100644 country_summaries/dps/algorithm_config_template.yaml diff --git a/country_summaries/dps/.ipynb_checkpoints/algorithm_config-checkpoint.yaml b/country_summaries/dps/.ipynb_checkpoints/algorithm_config-checkpoint.yaml index 22dfd08..a74c67b 100644 --- a/country_summaries/dps/.ipynb_checkpoints/algorithm_config-checkpoint.yaml +++ b/country_summaries/dps/.ipynb_checkpoints/algorithm_config-checkpoint.yaml @@ -1,6 +1,6 @@ # DO NOT DELETE # THIS CONFIG IS AUTO-GENERATED BY ADE UI -algo_name: country_summaries +algo_name: run_country_summaries version: master environment: ubuntu repository_url: https://repo.ops.maap-project.org/lduncanson/biomass_harmonization.git #note username and token is not needed diff --git a/country_summaries/dps/.ipynb_checkpoints/algorithm_config_template-checkpoint.yaml b/country_summaries/dps/.ipynb_checkpoints/algorithm_config_template-checkpoint.yaml new file mode 100644 index 0000000..ce6a010 --- /dev/null +++ b/country_summaries/dps/.ipynb_checkpoints/algorithm_config_template-checkpoint.yaml @@ -0,0 +1,25 @@ +# DO NOT DELETE +# THIS CONFIG IS AUTO-GENERATED BY ADE UI +algo_name: run_country_summaries +version: master +environment: ubuntu +repository_url: https://nehajo88:Bv1WEYgk3mQoFAxm8shR@repo.ops.maap-project.org/lduncanson/biomass_harmonization.git +docker_url: mas.maap-project.org:5000/root/ade-base-images/r:latest +# queue chosen when registering +queue: 15GB + +# fill out these fields +# explain what this algorithm does +description: +# path to the wrapper script for running the algorithm +run_command: /projects/biomass_harmonization/country_summaries/dps/run_country_summaries.sh +# set a storage value in GB or MB or KB, e.g. "100GB", "20MB", "10KB" +disk_space: 10GB +inputs: +# remove below this line if no inputs +# rename and set algorithm input names accordingly + - name: path-number + download: False + - name: file-to-copy-in + download: True + diff --git a/country_summaries/dps/.ipynb_checkpoints/run_country_summaries-checkpoint.ipynb b/country_summaries/dps/.ipynb_checkpoints/run_country_summaries-checkpoint.ipynb index 1cad846..a97ca1a 100644 --- a/country_summaries/dps/.ipynb_checkpoints/run_country_summaries-checkpoint.ipynb +++ b/country_summaries/dps/.ipynb_checkpoints/run_country_summaries-checkpoint.ipynb @@ -2,8 +2,8 @@ "cells": [ { "cell_type": "code", - "execution_count": 1, - "id": "deadly-thong", + "execution_count": 9, + "id": "liked-ridge", "metadata": {}, "outputs": [], "source": [ @@ -13,7 +13,7 @@ }, { "cell_type": "markdown", - "id": "golden-limitation", + "id": "retained-dairy", "metadata": {}, "source": [ "# Launch DPS for Country_summaries_DPS.R" @@ -21,10 +21,44 @@ }, { "cell_type": "code", - "execution_count": 6, - "id": "incorrect-corrections", + "execution_count": 10, + "id": "conditional-appearance", "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Requirement already satisfied: geopandas in /opt/conda/lib/python3.7/site-packages (0.10.2)\n", + "Requirement already satisfied: shapely>=1.6 in /opt/conda/lib/python3.7/site-packages (from geopandas) (1.8.5.post1)\n", + "Requirement already satisfied: fiona>=1.8 in /opt/conda/lib/python3.7/site-packages (from geopandas) (1.8.22)\n", + "Requirement already satisfied: pyproj>=2.2.0 in /opt/conda/lib/python3.7/site-packages (from geopandas) (3.2.1)\n", + "Requirement already satisfied: pandas>=0.25.0 in /opt/conda/lib/python3.7/site-packages (from geopandas) (1.1.4)\n", + "Requirement already satisfied: attrs>=17 in /opt/conda/lib/python3.7/site-packages (from fiona>=1.8->geopandas) (20.3.0)\n", + "Requirement already satisfied: click>=4.0 in /opt/conda/lib/python3.7/site-packages (from fiona>=1.8->geopandas) (8.0.1)\n", + "Requirement already satisfied: click-plugins>=1.0 in /opt/conda/lib/python3.7/site-packages (from fiona>=1.8->geopandas) (1.1.1)\n", + "Requirement already satisfied: setuptools in /opt/conda/lib/python3.7/site-packages (from fiona>=1.8->geopandas) (41.4.0)\n", + "Requirement already satisfied: munch in /opt/conda/lib/python3.7/site-packages (from fiona>=1.8->geopandas) (2.5.0)\n", + "Requirement already satisfied: six>=1.7 in /opt/conda/lib/python3.7/site-packages (from fiona>=1.8->geopandas) (1.12.0)\n", + "Requirement already satisfied: cligj>=0.5 in /opt/conda/lib/python3.7/site-packages (from fiona>=1.8->geopandas) (0.7.2)\n", + "Requirement already satisfied: certifi in /opt/conda/lib/python3.7/site-packages (from fiona>=1.8->geopandas) (2021.5.30)\n", + "Requirement already satisfied: python-dateutil>=2.7.3 in /opt/conda/lib/python3.7/site-packages (from pandas>=0.25.0->geopandas) (2.8.1)\n", + "Requirement already satisfied: pytz>=2017.2 in /opt/conda/lib/python3.7/site-packages (from pandas>=0.25.0->geopandas) (2021.1)\n", + "Requirement already satisfied: numpy>=1.15.4 in /opt/conda/lib/python3.7/site-packages (from pandas>=0.25.0->geopandas) (1.19.4)\n", + "Requirement already satisfied: importlib-metadata in /opt/conda/lib/python3.7/site-packages (from click>=4.0->fiona>=1.8->geopandas) (3.7.3)\n", + "Requirement already satisfied: typing-extensions>=3.6.4 in /opt/conda/lib/python3.7/site-packages (from importlib-metadata->click>=4.0->fiona>=1.8->geopandas) (3.7.4.3)\n", + "Requirement already satisfied: zipp>=0.5 in /opt/conda/lib/python3.7/site-packages (from importlib-metadata->click>=4.0->fiona>=1.8->geopandas) (3.4.1)\n", + "\u001b[33mWARNING: Running pip as the 'root' user can result in broken permissions and conflicting behaviour with the system package manager. It is recommended to use a virtual environment instead: https://pip.pypa.io/warnings/venv\u001b[0m\u001b[33m\n", + "\u001b[0m\u001b[33mWARNING: You are using pip version 22.0.3; however, version 22.3 is available.\n", + "You should consider upgrading via the '/opt/conda/bin/python -m pip install --upgrade pip' command.\u001b[0m\u001b[33m\n", + "\u001b[0mRequirement already satisfied: xmltodict in /opt/conda/lib/python3.7/site-packages (0.13.0)\n", + "\u001b[33mWARNING: Running pip as the 'root' user can result in broken permissions and conflicting behaviour with the system package manager. It is recommended to use a virtual environment instead: https://pip.pypa.io/warnings/venv\u001b[0m\u001b[33m\n", + "\u001b[0m\u001b[33mWARNING: You are using pip version 22.0.3; however, version 22.3 is available.\n", + "You should consider upgrading via the '/opt/conda/bin/python -m pip install --upgrade pip' command.\u001b[0m\u001b[33m\n", + "\u001b[0m" + ] + } + ], "source": [ "!pip install geopandas\n", "import os\n", @@ -48,7 +82,7 @@ }, { "cell_type": "markdown", - "id": "given-mixture", + "id": "norwegian-survival", "metadata": {}, "source": [ "## Test from Terminal" @@ -56,7 +90,7 @@ }, { "cell_type": "markdown", - "id": "timely-liberia", + "id": "separate-pharmaceutical", "metadata": {}, "source": [ "## Commit with Tag for running\n", @@ -65,17 +99,17 @@ "2) follow git instructions (every time!!):\n", "git add changes\n", "git commit -m 'message'\n", - "git tag -f country_summary_analysis_v1\n", + "git tag -f country_summaries\n", "\n", "git push\n", - "git push origin -f country_summary_analysis_v1\n", + "git push origin -f country_summaries\n", "\n", "3) if it looks weird check git log to make sure tag is at same place as origin and dps\n" ] }, { "cell_type": "markdown", - "id": "thorough-litigation", + "id": "identified-energy", "metadata": {}, "source": [ "## register via terminal\n", @@ -84,7 +118,7 @@ }, { "cell_type": "markdown", - "id": "subsequent-cartoon", + "id": "complicated-guess", "metadata": {}, "source": [ "## BUILD Rscript ARGUMENTS\n", @@ -93,7 +127,7 @@ }, { "cell_type": "markdown", - "id": "elementary-vocabulary", + "id": "consecutive-omega", "metadata": {}, "source": [ "## Run a DPS job\n" @@ -101,8 +135,8 @@ }, { "cell_type": "code", - "execution_count": 8, - "id": "nervous-design", + "execution_count": 22, + "id": "capital-pathology", "metadata": {}, "outputs": [ { @@ -110,9 +144,13 @@ "output_type": "stream", "text": [ "# of input tiles for DPS: 4\n", - "Current time:\t202211071746\n", - "CPU times: user 45.1 ms, sys: 6.58 ms, total: 51.7 ms\n", - "Wall time: 500 ms\n" + "https://maap-ops-workspace.s3.amazonaws.com/shared/nehajo88/Data/Country_boundaries/GEDI_countries_Dubayah_individual/ne_10m_a_1_SEN.gpkg\n", + "https://maap-ops-workspace.s3.amazonaws.com/shared/nehajo88/Data/Country_boundaries/GEDI_countries_Dubayah_individual/ne_10m_a_1_MEX.gpkg\n", + "https://maap-ops-workspace.s3.amazonaws.com/shared/nehajo88/Data/Country_boundaries/GEDI_countries_Dubayah_individual/ne_10m_a_1_GHA.gpkg\n", + "https://maap-ops-workspace.s3.amazonaws.com/shared/nehajo88/Data/Country_boundaries/GEDI_countries_Dubayah_individual/ne_10m_a_1_ECU.gpkg\n", + "Current time:\t202211091601\n", + "CPU times: user 50.6 ms, sys: 0 ns, total: 50.6 ms\n", + "Wall time: 302 ms\n" ] }, { @@ -156,8 +194,8 @@ " <td></td>\n", " <td>1</td>\n", " <td>SEN</td>\n", - " <td>2022-11-07 17:46:33.218372</td>\n", - " <td>17</td>\n", + " <td>2022-11-09 16:01:27.268162</td>\n", + " <td>16</td>\n", " <td>country_summaries_ubuntu</td>\n", " <td>nehajo88</td>\n", " <td>maap-dps-worker-32gb</td>\n", @@ -169,8 +207,8 @@ " <td></td>\n", " <td>2</td>\n", " <td>MEX</td>\n", - " <td>2022-11-07 17:46:33.242768</td>\n", - " <td>17</td>\n", + " <td>2022-11-09 16:01:27.294090</td>\n", + " <td>16</td>\n", " <td>country_summaries_ubuntu</td>\n", " <td>nehajo88</td>\n", " <td>maap-dps-worker-32gb</td>\n", @@ -182,8 +220,8 @@ " <td></td>\n", " <td>3</td>\n", " <td>GHA</td>\n", - " <td>2022-11-07 17:46:33.267152</td>\n", - " <td>17</td>\n", + " <td>2022-11-09 16:01:27.318182</td>\n", + " <td>16</td>\n", " <td>country_summaries_ubuntu</td>\n", " <td>nehajo88</td>\n", " <td>maap-dps-worker-32gb</td>\n", @@ -195,8 +233,8 @@ " <td></td>\n", " <td>4</td>\n", " <td>ECU</td>\n", - " <td>2022-11-07 17:46:33.291207</td>\n", - " <td>17</td>\n", + " <td>2022-11-09 16:01:27.342796</td>\n", + " <td>16</td>\n", " <td>country_summaries_ubuntu</td>\n", " <td>nehajo88</td>\n", " <td>maap-dps-worker-32gb</td>\n", @@ -213,10 +251,10 @@ "0 failed 500 4 ECU \n", "\n", " submit_time dbs_job_hour algo_id \\\n", - "0 2022-11-07 17:46:33.218372 17 country_summaries_ubuntu \n", - "0 2022-11-07 17:46:33.242768 17 country_summaries_ubuntu \n", - "0 2022-11-07 17:46:33.267152 17 country_summaries_ubuntu \n", - "0 2022-11-07 17:46:33.291207 17 country_summaries_ubuntu \n", + "0 2022-11-09 16:01:27.268162 16 country_summaries_ubuntu \n", + "0 2022-11-09 16:01:27.294090 16 country_summaries_ubuntu \n", + "0 2022-11-09 16:01:27.318182 16 country_summaries_ubuntu \n", + "0 2022-11-09 16:01:27.342796 16 country_summaries_ubuntu \n", "\n", " user worker_type \n", "0 nehajo88 maap-dps-worker-32gb \n", @@ -225,7 +263,7 @@ "0 nehajo88 maap-dps-worker-32gb " ] }, - "execution_count": 8, + "execution_count": 22, "metadata": {}, "output_type": "execute_result" } @@ -245,8 +283,9 @@ "\n", "for i, INPUT_TILE_NUM in enumerate(INPUT_FILE_LIST):\n", " INPUT_Country_shapefile = 'https://maap-ops-workspace.s3.amazonaws.com/shared/nehajo88/Data/Country_boundaries/GEDI_countries_Dubayah_individual/ne_10m_a_1_' + INPUT_TILE_NUM + '.gpkg'\n", + " print(INPUT_Country_shapefile)\n", " DPS_num = i+1\n", - " IDENTIFIER = 'country_summaries'\n", + " IDENTIFIER = 'run_country_summaries'\n", " ALGO_ID = f'{IDENTIFIER}_ubuntu'\n", " USER = 'nehajo88'\n", " WORKER_TYPE = 'maap-dps-worker-32gb'\n", @@ -259,13 +298,13 @@ " 'Shapefile_url' : INPUT_Country_shapefile,\n", " 'Base_folder': f\"output/\",\n", " 'CCI_file' : f\"input/{os.path.basename(CCI_file_location)}\",\n", - " 'CCI_file_url' : CCI_file_location\n", + " 'CCI_file_url' : CCI_file_location,\n", " 'JPL_file' : f\"input/{os.path.basename(JPL_file_location)}\",\n", - " 'JPL_file_url' : JPL_file_location\n", + " 'JPL_file_url' : JPL_file_location,\n", " 'JPL_CORRECTED_file' : f\"input/{os.path.basename(JPL_CORRECTED_file_location)}\",\n", - " 'JPL_CORRECTED_file_url' : JPL_CORRECTED_file_location\n", + " 'JPL_CORRECTED_file_url' : JPL_CORRECTED_file_location,\n", " 'GEDI_file' : f\"input/{os.path.basename(GEDI_file_location)}\",\n", - " 'GEDI_file_url' : GEDI_file_location\n", + " 'GEDI_file_url' : GEDI_file_location,\n", " 'NCEO_file' : f\"input/{os.path.basename(NCEO_file_location)}\",\n", " 'NCEO_file_url' : NCEO_file_location\n", "# 'Functions_file' : f\"input/{os.path.basename(Functions_file_location)}\",\n", @@ -275,7 +314,7 @@ " submit_result = maap.submitJob(\n", " identifier=IDENTIFIER,\n", " algo_id=ALGO_ID,\n", - " version='country_summary_analysis_v1',\n", + " version='master',\n", " username=USER, # username needs to be the same as whoever created the workspace\n", " queue=WORKER_TYPE,\n", " **in_param_dict\n", @@ -303,7 +342,7 @@ }, { "cell_type": "markdown", - "id": "improving-implement", + "id": "attended-parcel", "metadata": {}, "source": [ "## Assess DPS Results" @@ -311,24 +350,30 @@ }, { "cell_type": "code", - "execution_count": 17, - "id": "selected-country", + "execution_count": 20, + "id": "signed-solution", "metadata": {}, "outputs": [ { - "ename": "KeyError", - "evalue": "'wps:JobID'", + "name": "stdout", + "output_type": "stream", + "text": [ + "Count total jobs:\t4\n", + "Count pending jobs:\t0\n", + "Count running jobs:\t4\n", + "Count succeeded jobs:\t0\n", + "Count failed jobs:\t0\n" + ] + }, + { + "ename": "ZeroDivisionError", + "evalue": "division by zero", "output_type": "error", "traceback": [ "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m", - "\u001b[0;31mKeyError\u001b[0m Traceback (most recent call last)", + "\u001b[0;31mZeroDivisionError\u001b[0m Traceback (most recent call last)", "\u001b[0;32m<timed exec>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n", - "\u001b[0;32m/opt/conda/lib/python3.7/site-packages/pandas/core/frame.py\u001b[0m in \u001b[0;36mmerge\u001b[0;34m(self, right, how, on, left_on, right_on, left_index, right_index, sort, suffixes, copy, indicator, validate)\u001b[0m\n\u001b[1;32m 7961\u001b[0m \u001b[0mcopy\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mcopy\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 7962\u001b[0m \u001b[0mindicator\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mindicator\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 7963\u001b[0;31m \u001b[0mvalidate\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mvalidate\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 7964\u001b[0m )\n\u001b[1;32m 7965\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n", - "\u001b[0;32m/opt/conda/lib/python3.7/site-packages/pandas/core/reshape/merge.py\u001b[0m in \u001b[0;36mmerge\u001b[0;34m(left, right, how, on, left_on, right_on, left_index, right_index, sort, suffixes, copy, indicator, validate)\u001b[0m\n\u001b[1;32m 85\u001b[0m \u001b[0mcopy\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mcopy\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 86\u001b[0m \u001b[0mindicator\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mindicator\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 87\u001b[0;31m \u001b[0mvalidate\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mvalidate\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 88\u001b[0m )\n\u001b[1;32m 89\u001b[0m \u001b[0;32mreturn\u001b[0m \u001b[0mop\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mget_result\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", - "\u001b[0;32m/opt/conda/lib/python3.7/site-packages/pandas/core/reshape/merge.py\u001b[0m in \u001b[0;36m__init__\u001b[0;34m(self, left, right, how, on, left_on, right_on, axis, left_index, right_index, sort, suffixes, copy, indicator, validate)\u001b[0m\n\u001b[1;32m 650\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mright_join_keys\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 651\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mjoin_names\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 652\u001b[0;31m ) = self._get_merge_keys()\n\u001b[0m\u001b[1;32m 653\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 654\u001b[0m \u001b[0;31m# validate the merge keys dtypes. We may need to coerce\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", - "\u001b[0;32m/opt/conda/lib/python3.7/site-packages/pandas/core/reshape/merge.py\u001b[0m in \u001b[0;36m_get_merge_keys\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m 1003\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0mis_rkey\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mrk\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 1004\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0mrk\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1005\u001b[0;31m \u001b[0mright_keys\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mappend\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mright\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_get_label_or_level_values\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mrk\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 1006\u001b[0m \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 1007\u001b[0m \u001b[0;31m# work-around for merge_asof(right_index=True)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", - "\u001b[0;32m/opt/conda/lib/python3.7/site-packages/pandas/core/generic.py\u001b[0m in \u001b[0;36m_get_label_or_level_values\u001b[0;34m(self, key, axis)\u001b[0m\n\u001b[1;32m 1561\u001b[0m \u001b[0mvalues\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0maxes\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0maxis\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mget_level_values\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mkey\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_values\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 1562\u001b[0m \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1563\u001b[0;31m \u001b[0;32mraise\u001b[0m \u001b[0mKeyError\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mkey\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 1564\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 1565\u001b[0m \u001b[0;31m# Check for duplicates\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", - "\u001b[0;31mKeyError\u001b[0m: 'wps:JobID'" + "\u001b[0;31mZeroDivisionError\u001b[0m: division by zero" ] } ], @@ -359,7 +404,7 @@ { "cell_type": "code", "execution_count": null, - "id": "enormous-exhibition", + "id": "affiliated-allergy", "metadata": {}, "outputs": [], "source": [] diff --git a/country_summaries/dps/algorithm_config.yaml b/country_summaries/dps/algorithm_config.yaml index 22dfd08..a74c67b 100644 --- a/country_summaries/dps/algorithm_config.yaml +++ b/country_summaries/dps/algorithm_config.yaml @@ -1,6 +1,6 @@ # DO NOT DELETE # THIS CONFIG IS AUTO-GENERATED BY ADE UI -algo_name: country_summaries +algo_name: run_country_summaries version: master environment: ubuntu repository_url: https://repo.ops.maap-project.org/lduncanson/biomass_harmonization.git #note username and token is not needed diff --git a/country_summaries/dps/algorithm_config_template.yaml b/country_summaries/dps/algorithm_config_template.yaml new file mode 100644 index 0000000..ce6a010 --- /dev/null +++ b/country_summaries/dps/algorithm_config_template.yaml @@ -0,0 +1,25 @@ +# DO NOT DELETE +# THIS CONFIG IS AUTO-GENERATED BY ADE UI +algo_name: run_country_summaries +version: master +environment: ubuntu +repository_url: https://nehajo88:Bv1WEYgk3mQoFAxm8shR@repo.ops.maap-project.org/lduncanson/biomass_harmonization.git +docker_url: mas.maap-project.org:5000/root/ade-base-images/r:latest +# queue chosen when registering +queue: 15GB + +# fill out these fields +# explain what this algorithm does +description: +# path to the wrapper script for running the algorithm +run_command: /projects/biomass_harmonization/country_summaries/dps/run_country_summaries.sh +# set a storage value in GB or MB or KB, e.g. "100GB", "20MB", "10KB" +disk_space: 10GB +inputs: +# remove below this line if no inputs +# rename and set algorithm input names accordingly + - name: path-number + download: False + - name: file-to-copy-in + download: True + diff --git a/country_summaries/dps/run_country_summaries.ipynb b/country_summaries/dps/run_country_summaries.ipynb index 1cad846..a97ca1a 100644 --- a/country_summaries/dps/run_country_summaries.ipynb +++ b/country_summaries/dps/run_country_summaries.ipynb @@ -2,8 +2,8 @@ "cells": [ { "cell_type": "code", - "execution_count": 1, - "id": "deadly-thong", + "execution_count": 9, + "id": "liked-ridge", "metadata": {}, "outputs": [], "source": [ @@ -13,7 +13,7 @@ }, { "cell_type": "markdown", - "id": "golden-limitation", + "id": "retained-dairy", "metadata": {}, "source": [ "# Launch DPS for Country_summaries_DPS.R" @@ -21,10 +21,44 @@ }, { "cell_type": "code", - "execution_count": 6, - "id": "incorrect-corrections", + "execution_count": 10, + "id": "conditional-appearance", "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Requirement already satisfied: geopandas in /opt/conda/lib/python3.7/site-packages (0.10.2)\n", + "Requirement already satisfied: shapely>=1.6 in /opt/conda/lib/python3.7/site-packages (from geopandas) (1.8.5.post1)\n", + "Requirement already satisfied: fiona>=1.8 in /opt/conda/lib/python3.7/site-packages (from geopandas) (1.8.22)\n", + "Requirement already satisfied: pyproj>=2.2.0 in /opt/conda/lib/python3.7/site-packages (from geopandas) (3.2.1)\n", + "Requirement already satisfied: pandas>=0.25.0 in /opt/conda/lib/python3.7/site-packages (from geopandas) (1.1.4)\n", + "Requirement already satisfied: attrs>=17 in /opt/conda/lib/python3.7/site-packages (from fiona>=1.8->geopandas) (20.3.0)\n", + "Requirement already satisfied: click>=4.0 in /opt/conda/lib/python3.7/site-packages (from fiona>=1.8->geopandas) (8.0.1)\n", + "Requirement already satisfied: click-plugins>=1.0 in /opt/conda/lib/python3.7/site-packages (from fiona>=1.8->geopandas) (1.1.1)\n", + "Requirement already satisfied: setuptools in /opt/conda/lib/python3.7/site-packages (from fiona>=1.8->geopandas) (41.4.0)\n", + "Requirement already satisfied: munch in /opt/conda/lib/python3.7/site-packages (from fiona>=1.8->geopandas) (2.5.0)\n", + "Requirement already satisfied: six>=1.7 in /opt/conda/lib/python3.7/site-packages (from fiona>=1.8->geopandas) (1.12.0)\n", + "Requirement already satisfied: cligj>=0.5 in /opt/conda/lib/python3.7/site-packages (from fiona>=1.8->geopandas) (0.7.2)\n", + "Requirement already satisfied: certifi in /opt/conda/lib/python3.7/site-packages (from fiona>=1.8->geopandas) (2021.5.30)\n", + "Requirement already satisfied: python-dateutil>=2.7.3 in /opt/conda/lib/python3.7/site-packages (from pandas>=0.25.0->geopandas) (2.8.1)\n", + "Requirement already satisfied: pytz>=2017.2 in /opt/conda/lib/python3.7/site-packages (from pandas>=0.25.0->geopandas) (2021.1)\n", + "Requirement already satisfied: numpy>=1.15.4 in /opt/conda/lib/python3.7/site-packages (from pandas>=0.25.0->geopandas) (1.19.4)\n", + "Requirement already satisfied: importlib-metadata in /opt/conda/lib/python3.7/site-packages (from click>=4.0->fiona>=1.8->geopandas) (3.7.3)\n", + "Requirement already satisfied: typing-extensions>=3.6.4 in /opt/conda/lib/python3.7/site-packages (from importlib-metadata->click>=4.0->fiona>=1.8->geopandas) (3.7.4.3)\n", + "Requirement already satisfied: zipp>=0.5 in /opt/conda/lib/python3.7/site-packages (from importlib-metadata->click>=4.0->fiona>=1.8->geopandas) (3.4.1)\n", + "\u001b[33mWARNING: Running pip as the 'root' user can result in broken permissions and conflicting behaviour with the system package manager. It is recommended to use a virtual environment instead: https://pip.pypa.io/warnings/venv\u001b[0m\u001b[33m\n", + "\u001b[0m\u001b[33mWARNING: You are using pip version 22.0.3; however, version 22.3 is available.\n", + "You should consider upgrading via the '/opt/conda/bin/python -m pip install --upgrade pip' command.\u001b[0m\u001b[33m\n", + "\u001b[0mRequirement already satisfied: xmltodict in /opt/conda/lib/python3.7/site-packages (0.13.0)\n", + "\u001b[33mWARNING: Running pip as the 'root' user can result in broken permissions and conflicting behaviour with the system package manager. It is recommended to use a virtual environment instead: https://pip.pypa.io/warnings/venv\u001b[0m\u001b[33m\n", + "\u001b[0m\u001b[33mWARNING: You are using pip version 22.0.3; however, version 22.3 is available.\n", + "You should consider upgrading via the '/opt/conda/bin/python -m pip install --upgrade pip' command.\u001b[0m\u001b[33m\n", + "\u001b[0m" + ] + } + ], "source": [ "!pip install geopandas\n", "import os\n", @@ -48,7 +82,7 @@ }, { "cell_type": "markdown", - "id": "given-mixture", + "id": "norwegian-survival", "metadata": {}, "source": [ "## Test from Terminal" @@ -56,7 +90,7 @@ }, { "cell_type": "markdown", - "id": "timely-liberia", + "id": "separate-pharmaceutical", "metadata": {}, "source": [ "## Commit with Tag for running\n", @@ -65,17 +99,17 @@ "2) follow git instructions (every time!!):\n", "git add changes\n", "git commit -m 'message'\n", - "git tag -f country_summary_analysis_v1\n", + "git tag -f country_summaries\n", "\n", "git push\n", - "git push origin -f country_summary_analysis_v1\n", + "git push origin -f country_summaries\n", "\n", "3) if it looks weird check git log to make sure tag is at same place as origin and dps\n" ] }, { "cell_type": "markdown", - "id": "thorough-litigation", + "id": "identified-energy", "metadata": {}, "source": [ "## register via terminal\n", @@ -84,7 +118,7 @@ }, { "cell_type": "markdown", - "id": "subsequent-cartoon", + "id": "complicated-guess", "metadata": {}, "source": [ "## BUILD Rscript ARGUMENTS\n", @@ -93,7 +127,7 @@ }, { "cell_type": "markdown", - "id": "elementary-vocabulary", + "id": "consecutive-omega", "metadata": {}, "source": [ "## Run a DPS job\n" @@ -101,8 +135,8 @@ }, { "cell_type": "code", - "execution_count": 8, - "id": "nervous-design", + "execution_count": 22, + "id": "capital-pathology", "metadata": {}, "outputs": [ { @@ -110,9 +144,13 @@ "output_type": "stream", "text": [ "# of input tiles for DPS: 4\n", - "Current time:\t202211071746\n", - "CPU times: user 45.1 ms, sys: 6.58 ms, total: 51.7 ms\n", - "Wall time: 500 ms\n" + "https://maap-ops-workspace.s3.amazonaws.com/shared/nehajo88/Data/Country_boundaries/GEDI_countries_Dubayah_individual/ne_10m_a_1_SEN.gpkg\n", + "https://maap-ops-workspace.s3.amazonaws.com/shared/nehajo88/Data/Country_boundaries/GEDI_countries_Dubayah_individual/ne_10m_a_1_MEX.gpkg\n", + "https://maap-ops-workspace.s3.amazonaws.com/shared/nehajo88/Data/Country_boundaries/GEDI_countries_Dubayah_individual/ne_10m_a_1_GHA.gpkg\n", + "https://maap-ops-workspace.s3.amazonaws.com/shared/nehajo88/Data/Country_boundaries/GEDI_countries_Dubayah_individual/ne_10m_a_1_ECU.gpkg\n", + "Current time:\t202211091601\n", + "CPU times: user 50.6 ms, sys: 0 ns, total: 50.6 ms\n", + "Wall time: 302 ms\n" ] }, { @@ -156,8 +194,8 @@ " <td></td>\n", " <td>1</td>\n", " <td>SEN</td>\n", - " <td>2022-11-07 17:46:33.218372</td>\n", - " <td>17</td>\n", + " <td>2022-11-09 16:01:27.268162</td>\n", + " <td>16</td>\n", " <td>country_summaries_ubuntu</td>\n", " <td>nehajo88</td>\n", " <td>maap-dps-worker-32gb</td>\n", @@ -169,8 +207,8 @@ " <td></td>\n", " <td>2</td>\n", " <td>MEX</td>\n", - " <td>2022-11-07 17:46:33.242768</td>\n", - " <td>17</td>\n", + " <td>2022-11-09 16:01:27.294090</td>\n", + " <td>16</td>\n", " <td>country_summaries_ubuntu</td>\n", " <td>nehajo88</td>\n", " <td>maap-dps-worker-32gb</td>\n", @@ -182,8 +220,8 @@ " <td></td>\n", " <td>3</td>\n", " <td>GHA</td>\n", - " <td>2022-11-07 17:46:33.267152</td>\n", - " <td>17</td>\n", + " <td>2022-11-09 16:01:27.318182</td>\n", + " <td>16</td>\n", " <td>country_summaries_ubuntu</td>\n", " <td>nehajo88</td>\n", " <td>maap-dps-worker-32gb</td>\n", @@ -195,8 +233,8 @@ " <td></td>\n", " <td>4</td>\n", " <td>ECU</td>\n", - " <td>2022-11-07 17:46:33.291207</td>\n", - " <td>17</td>\n", + " <td>2022-11-09 16:01:27.342796</td>\n", + " <td>16</td>\n", " <td>country_summaries_ubuntu</td>\n", " <td>nehajo88</td>\n", " <td>maap-dps-worker-32gb</td>\n", @@ -213,10 +251,10 @@ "0 failed 500 4 ECU \n", "\n", " submit_time dbs_job_hour algo_id \\\n", - "0 2022-11-07 17:46:33.218372 17 country_summaries_ubuntu \n", - "0 2022-11-07 17:46:33.242768 17 country_summaries_ubuntu \n", - "0 2022-11-07 17:46:33.267152 17 country_summaries_ubuntu \n", - "0 2022-11-07 17:46:33.291207 17 country_summaries_ubuntu \n", + "0 2022-11-09 16:01:27.268162 16 country_summaries_ubuntu \n", + "0 2022-11-09 16:01:27.294090 16 country_summaries_ubuntu \n", + "0 2022-11-09 16:01:27.318182 16 country_summaries_ubuntu \n", + "0 2022-11-09 16:01:27.342796 16 country_summaries_ubuntu \n", "\n", " user worker_type \n", "0 nehajo88 maap-dps-worker-32gb \n", @@ -225,7 +263,7 @@ "0 nehajo88 maap-dps-worker-32gb " ] }, - "execution_count": 8, + "execution_count": 22, "metadata": {}, "output_type": "execute_result" } @@ -245,8 +283,9 @@ "\n", "for i, INPUT_TILE_NUM in enumerate(INPUT_FILE_LIST):\n", " INPUT_Country_shapefile = 'https://maap-ops-workspace.s3.amazonaws.com/shared/nehajo88/Data/Country_boundaries/GEDI_countries_Dubayah_individual/ne_10m_a_1_' + INPUT_TILE_NUM + '.gpkg'\n", + " print(INPUT_Country_shapefile)\n", " DPS_num = i+1\n", - " IDENTIFIER = 'country_summaries'\n", + " IDENTIFIER = 'run_country_summaries'\n", " ALGO_ID = f'{IDENTIFIER}_ubuntu'\n", " USER = 'nehajo88'\n", " WORKER_TYPE = 'maap-dps-worker-32gb'\n", @@ -259,13 +298,13 @@ " 'Shapefile_url' : INPUT_Country_shapefile,\n", " 'Base_folder': f\"output/\",\n", " 'CCI_file' : f\"input/{os.path.basename(CCI_file_location)}\",\n", - " 'CCI_file_url' : CCI_file_location\n", + " 'CCI_file_url' : CCI_file_location,\n", " 'JPL_file' : f\"input/{os.path.basename(JPL_file_location)}\",\n", - " 'JPL_file_url' : JPL_file_location\n", + " 'JPL_file_url' : JPL_file_location,\n", " 'JPL_CORRECTED_file' : f\"input/{os.path.basename(JPL_CORRECTED_file_location)}\",\n", - " 'JPL_CORRECTED_file_url' : JPL_CORRECTED_file_location\n", + " 'JPL_CORRECTED_file_url' : JPL_CORRECTED_file_location,\n", " 'GEDI_file' : f\"input/{os.path.basename(GEDI_file_location)}\",\n", - " 'GEDI_file_url' : GEDI_file_location\n", + " 'GEDI_file_url' : GEDI_file_location,\n", " 'NCEO_file' : f\"input/{os.path.basename(NCEO_file_location)}\",\n", " 'NCEO_file_url' : NCEO_file_location\n", "# 'Functions_file' : f\"input/{os.path.basename(Functions_file_location)}\",\n", @@ -275,7 +314,7 @@ " submit_result = maap.submitJob(\n", " identifier=IDENTIFIER,\n", " algo_id=ALGO_ID,\n", - " version='country_summary_analysis_v1',\n", + " version='master',\n", " username=USER, # username needs to be the same as whoever created the workspace\n", " queue=WORKER_TYPE,\n", " **in_param_dict\n", @@ -303,7 +342,7 @@ }, { "cell_type": "markdown", - "id": "improving-implement", + "id": "attended-parcel", "metadata": {}, "source": [ "## Assess DPS Results" @@ -311,24 +350,30 @@ }, { "cell_type": "code", - "execution_count": 17, - "id": "selected-country", + "execution_count": 20, + "id": "signed-solution", "metadata": {}, "outputs": [ { - "ename": "KeyError", - "evalue": "'wps:JobID'", + "name": "stdout", + "output_type": "stream", + "text": [ + "Count total jobs:\t4\n", + "Count pending jobs:\t0\n", + "Count running jobs:\t4\n", + "Count succeeded jobs:\t0\n", + "Count failed jobs:\t0\n" + ] + }, + { + "ename": "ZeroDivisionError", + "evalue": "division by zero", "output_type": "error", "traceback": [ "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m", - "\u001b[0;31mKeyError\u001b[0m Traceback (most recent call last)", + "\u001b[0;31mZeroDivisionError\u001b[0m Traceback (most recent call last)", "\u001b[0;32m<timed exec>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n", - "\u001b[0;32m/opt/conda/lib/python3.7/site-packages/pandas/core/frame.py\u001b[0m in \u001b[0;36mmerge\u001b[0;34m(self, right, how, on, left_on, right_on, left_index, right_index, sort, suffixes, copy, indicator, validate)\u001b[0m\n\u001b[1;32m 7961\u001b[0m \u001b[0mcopy\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mcopy\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 7962\u001b[0m \u001b[0mindicator\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mindicator\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 7963\u001b[0;31m \u001b[0mvalidate\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mvalidate\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 7964\u001b[0m )\n\u001b[1;32m 7965\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n", - "\u001b[0;32m/opt/conda/lib/python3.7/site-packages/pandas/core/reshape/merge.py\u001b[0m in \u001b[0;36mmerge\u001b[0;34m(left, right, how, on, left_on, right_on, left_index, right_index, sort, suffixes, copy, indicator, validate)\u001b[0m\n\u001b[1;32m 85\u001b[0m \u001b[0mcopy\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mcopy\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 86\u001b[0m \u001b[0mindicator\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mindicator\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 87\u001b[0;31m \u001b[0mvalidate\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mvalidate\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 88\u001b[0m )\n\u001b[1;32m 89\u001b[0m \u001b[0;32mreturn\u001b[0m \u001b[0mop\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mget_result\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", - "\u001b[0;32m/opt/conda/lib/python3.7/site-packages/pandas/core/reshape/merge.py\u001b[0m in \u001b[0;36m__init__\u001b[0;34m(self, left, right, how, on, left_on, right_on, axis, left_index, right_index, sort, suffixes, copy, indicator, validate)\u001b[0m\n\u001b[1;32m 650\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mright_join_keys\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 651\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mjoin_names\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 652\u001b[0;31m ) = self._get_merge_keys()\n\u001b[0m\u001b[1;32m 653\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 654\u001b[0m \u001b[0;31m# validate the merge keys dtypes. We may need to coerce\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", - "\u001b[0;32m/opt/conda/lib/python3.7/site-packages/pandas/core/reshape/merge.py\u001b[0m in \u001b[0;36m_get_merge_keys\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m 1003\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0mis_rkey\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mrk\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 1004\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0mrk\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1005\u001b[0;31m \u001b[0mright_keys\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mappend\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mright\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_get_label_or_level_values\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mrk\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 1006\u001b[0m \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 1007\u001b[0m \u001b[0;31m# work-around for merge_asof(right_index=True)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", - "\u001b[0;32m/opt/conda/lib/python3.7/site-packages/pandas/core/generic.py\u001b[0m in \u001b[0;36m_get_label_or_level_values\u001b[0;34m(self, key, axis)\u001b[0m\n\u001b[1;32m 1561\u001b[0m \u001b[0mvalues\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0maxes\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0maxis\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mget_level_values\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mkey\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_values\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 1562\u001b[0m \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1563\u001b[0;31m \u001b[0;32mraise\u001b[0m \u001b[0mKeyError\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mkey\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 1564\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 1565\u001b[0m \u001b[0;31m# Check for duplicates\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", - "\u001b[0;31mKeyError\u001b[0m: 'wps:JobID'" + "\u001b[0;31mZeroDivisionError\u001b[0m: division by zero" ] } ], @@ -359,7 +404,7 @@ { "cell_type": "code", "execution_count": null, - "id": "enormous-exhibition", + "id": "affiliated-allergy", "metadata": {}, "outputs": [], "source": [] -- GitLab