From cd2d511354b1fcf5a0fe2f316ed929f4bd7ce7ff Mon Sep 17 00:00:00 2001 From: Sujen Shah <sujen.shah@jpl.nasa.gov> Date: Mon, 24 Mar 2025 20:43:38 +0000 Subject: [PATCH 1/7] Add dps run file --- run.sh | 9 +++++++++ 1 file changed, 9 insertions(+) create mode 100644 run.sh diff --git a/run.sh b/run.sh new file mode 100644 index 0000000..1a0dbc2 --- /dev/null +++ b/run.sh @@ -0,0 +1,9 @@ +#!/bin/bash + +basedir=$( cd "$(dirname "$0")" ; pwd -P) + +OUTPUTDIR="${PWD}/output" + +ANALYSIS_DATE=$1 + +conda run --live-stream --name mytobac /root/aos_test/src/Tracking_Paper_Plots-Lorenzo-aws.py ${ANALYSIS_DATE} \ No newline at end of file -- GitLab From 24940b59f8b2d846d4da387780614c9f5c2a9826 Mon Sep 17 00:00:00 2001 From: Sujen Shah <sujen.shah@jpl.nasa.gov> Date: Tue, 25 Mar 2025 19:01:15 +0000 Subject: [PATCH 2/7] Update run.sh to add python executable to conda run --- run.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/run.sh b/run.sh index 1a0dbc2..4c5480d 100644 --- a/run.sh +++ b/run.sh @@ -6,4 +6,4 @@ OUTPUTDIR="${PWD}/output" ANALYSIS_DATE=$1 -conda run --live-stream --name mytobac /root/aos_test/src/Tracking_Paper_Plots-Lorenzo-aws.py ${ANALYSIS_DATE} \ No newline at end of file +conda run --live-stream --name mytobac python /root/aos_test/src/Tracking_Paper_Plots-Lorenzo-aws.py ${ANALYSIS_DATE} \ No newline at end of file -- GitLab From 69cc0c02c03f6939279e6f1697943bcddd0b8895 Mon Sep 17 00:00:00 2001 From: Sujen Shah <sujen.shah@jpl.nasa.gov> Date: Tue, 25 Mar 2025 19:05:53 +0000 Subject: [PATCH 3/7] Add algorithm config --- algorithm_config.yaml | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) create mode 100644 algorithm_config.yaml diff --git a/algorithm_config.yaml b/algorithm_config.yaml new file mode 100644 index 0000000..9030d55 --- /dev/null +++ b/algorithm_config.yaml @@ -0,0 +1,17 @@ +algorithm_description: Run AOS Tracking paper plots on DPS +algorithm_name: aos_maap_dps +algorithm_version: main +build_command: '' +disk_space: 20GB +docker_container_url: mas.maap-project.org/david.m.giles/aos_test/aos_dps:develop +inputs: + config: [] + file: [] + positional: + - default: '' + description: '' + name: analysis_date + required: false +queue: maap-dps-sandbox +repository_url: https://repo.maap-project.org/sshah/aos_maap_dps.git +run_command: aos_maap_dps/run.sh \ No newline at end of file -- GitLab From dcbc1b102887fbf8b8a5247d557a67d431d2a185 Mon Sep 17 00:00:00 2001 From: Chuck Daniels <chuck@developmentseed.org> Date: Fri, 28 Mar 2025 19:03:40 -0400 Subject: [PATCH 4/7] Get Earthdata Login creds from secrets --- build.sh | 5 +++++ run.sh | 21 +++++++++++++++++---- 2 files changed, 22 insertions(+), 4 deletions(-) create mode 100644 build.sh diff --git a/build.sh b/build.sh new file mode 100644 index 0000000..e25c16e --- /dev/null +++ b/build.sh @@ -0,0 +1,5 @@ +#!/bin/bash +set -euo pipefail + +# Install maap-py so we can use MAAP Secrets in run.sh +conda run --no-capture-output --name mytobac python -m pip install maap-py diff --git a/run.sh b/run.sh index 4c5480d..c252ddd 100644 --- a/run.sh +++ b/run.sh @@ -1,9 +1,22 @@ #!/bin/bash -basedir=$( cd "$(dirname "$0")" ; pwd -P) +readarray -d " " -t credentials <<<"$( + python -c ' +from maap.maap import MAAP +maap = MAAP() +username = maap.secrets.get_secret("EARTHDATA_USERNAME") +password = maap.secrets.get_secret("EARTHDATA_PASSWORD") +print(username, password) +' +)" -OUTPUTDIR="${PWD}/output" +EARTHDATA_USERNAME=$(echo -e "${credentials[0]}" | tr -d '[:space:]') +EARTHDATA_PASSWORD=$(echo -e "${credentials[1]}" | tr -d '[:space:]') +export EARTHDATA_USERNAME +export EARTHDATA_PASSWORD -ANALYSIS_DATE=$1 +outputdir="${PWD}/output" -conda run --live-stream --name mytobac python /root/aos_test/src/Tracking_Paper_Plots-Lorenzo-aws.py ${ANALYSIS_DATE} \ No newline at end of file +analysis_date=$1 + +conda run --no-capture-output --name mytobac python /root/aos_test/src/Tracking_Paper_Plots-Lorenzo-aws.py "${analysis_date}" -- GitLab From 0a58b357e606660671c0670b5618400992dd1bbc Mon Sep 17 00:00:00 2001 From: Sujen Shah <sujen.shah@jpl.nasa.gov> Date: Mon, 31 Mar 2025 22:31:23 +0000 Subject: [PATCH 5/7] Update run.sh to use mytobac when import maap secrets --- run.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/run.sh b/run.sh index c252ddd..c693d72 100644 --- a/run.sh +++ b/run.sh @@ -1,7 +1,7 @@ #!/bin/bash readarray -d " " -t credentials <<<"$( - python -c ' + conda run --no-capture-output --name mytobac python -c ' from maap.maap import MAAP maap = MAAP() username = maap.secrets.get_secret("EARTHDATA_USERNAME") -- GitLab From 253c4eaa670b6b4e498a94d483a06d162b2c9c38 Mon Sep 17 00:00:00 2001 From: Chuck Daniels <chuck@developmentseed.org> Date: Tue, 15 Apr 2025 17:03:05 -0400 Subject: [PATCH 6/7] Call new scripts --- build.sh | 2 ++ run.sh | 10 ++++++++-- 2 files changed, 10 insertions(+), 2 deletions(-) diff --git a/build.sh b/build.sh index e25c16e..75d8632 100644 --- a/build.sh +++ b/build.sh @@ -3,3 +3,5 @@ set -euo pipefail # Install maap-py so we can use MAAP Secrets in run.sh conda run --no-capture-output --name mytobac python -m pip install maap-py + +wget https://www.hdfeos.org/software/h4cflib/bin/linux/v1.4/CentOS7/h4tonccf_nc4 -O "${HOME}/local/bin/h4tonccr_nc4" diff --git a/run.sh b/run.sh index c693d72..2740ed7 100644 --- a/run.sh +++ b/run.sh @@ -1,5 +1,7 @@ #!/bin/bash +set -euo pipefail + readarray -d " " -t credentials <<<"$( conda run --no-capture-output --name mytobac python -c ' from maap.maap import MAAP @@ -15,8 +17,12 @@ EARTHDATA_PASSWORD=$(echo -e "${credentials[1]}" | tr -d '[:space:]') export EARTHDATA_USERNAME export EARTHDATA_PASSWORD -outputdir="${PWD}/output" +# All of the scripts below will use AOS_WORKING_DIR as the root output directory. +AOS_WORKING_DIR="${PWD}/output" +export AOS_WORKING_DIR analysis_date=$1 -conda run --no-capture-output --name mytobac python /root/aos_test/src/Tracking_Paper_Plots-Lorenzo-aws.py "${analysis_date}" +conda run --no-capture-output --name mytobac python /root/aos_test/src/acquire_and_convert_calipso.py "${analysis_date}" +conda run --no-capture-output --name mytobac python /root/aos_test/src/acquire_goes.py "${analysis_date}" +conda run --no-capture-output --name mytobac python /root/aos_test/src/process_object_tracking.py "${analysis_date}" -- GitLab From 89432e2c94f8bc1c32dd51efed5afdf18fbf4239 Mon Sep 17 00:00:00 2001 From: Administrator <admin@example.com> Date: Wed, 16 Apr 2025 16:57:50 +0000 Subject: [PATCH 7/7] Update build.sh to create local bin directory --- build.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/build.sh b/build.sh index 75d8632..55e24c5 100644 --- a/build.sh +++ b/build.sh @@ -4,4 +4,5 @@ set -euo pipefail # Install maap-py so we can use MAAP Secrets in run.sh conda run --no-capture-output --name mytobac python -m pip install maap-py +mkdir -p "${HOME}/local/bin/" wget https://www.hdfeos.org/software/h4cflib/bin/linux/v1.4/CentOS7/h4tonccf_nc4 -O "${HOME}/local/bin/h4tonccr_nc4" -- GitLab