From 2d4c1215a83ca17f5ea2fe8733b84d8f08f3156b Mon Sep 17 00:00:00 2001
From: Chuck Daniels <chuck@developmentseed.org>
Date: Tue, 29 Apr 2025 13:49:55 -0400
Subject: [PATCH 1/5] Ignore common files
---
.gitignore | 422 +++++++++++++++++++++++++++++++++++++++++++++++++++++
1 file changed, 422 insertions(+)
create mode 100644 .gitignore
diff --git a/.gitignore b/.gitignore
new file mode 100644
index 0000000..6c8f7a6
--- /dev/null
+++ b/.gitignore
@@ -0,0 +1,422 @@
+# Created by https://www.toptal.com/developers/gitignore/api/python
+# Edit at https://www.toptal.com/developers/gitignore?templates=python
+
+### Python ###
+# Byte-compiled / optimized / DLL files
+__pycache__/
+*.py[cod]
+*$py.class
+
+# C extensions
+*.so
+
+# Distribution / packaging
+.Python
+build/
+develop-eggs/
+dist/
+downloads/
+eggs/
+.eggs/
+lib/
+lib64/
+parts/
+sdist/
+var/
+wheels/
+share/python-wheels/
+*.egg-info/
+.installed.cfg
+*.egg
+MANIFEST
+
+# PyInstaller
+# Usually these files are written by a python script from a template
+# before PyInstaller builds the exe, so as to inject date/other infos into it.
+*.manifest
+*.spec
+
+# Installer logs
+pip-log.txt
+pip-delete-this-directory.txt
+
+# Unit test / coverage reports
+htmlcov/
+.tox/
+.nox/
+.coverage
+.coverage.*
+.cache
+nosetests.xml
+coverage.xml
+*.cover
+*.py,cover
+.hypothesis/
+.pytest_cache/
+cover/
+
+# Translations
+*.mo
+*.pot
+
+# Django stuff:
+*.log
+local_settings.py
+db.sqlite3
+db.sqlite3-journal
+
+# Flask stuff:
+instance/
+.webassets-cache
+
+# Scrapy stuff:
+.scrapy
+
+# Sphinx documentation
+docs/_build/
+
+# PyBuilder
+.pybuilder/
+target/
+
+# Jupyter Notebook
+.ipynb_checkpoints
+
+# IPython
+profile_default/
+ipython_config.py
+
+# pyenv
+# For a library or package, you might want to ignore these files since the code is
+# intended to run in multiple environments; otherwise, check them in:
+# .python-version
+
+# pipenv
+# According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control.
+# However, in case of collaboration, if having platform-specific dependencies or dependencies
+# having no cross-platform support, pipenv may install dependencies that don't work, or not
+# install all needed dependencies.
+#Pipfile.lock
+
+# poetry
+# Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control.
+# This is especially recommended for binary packages to ensure reproducibility, and is more
+# commonly ignored for libraries.
+# https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control
+#poetry.lock
+
+# pdm
+# Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control.
+#pdm.lock
+# pdm stores project-wide configurations in .pdm.toml, but it is recommended to not include it
+# in version control.
+# https://pdm.fming.dev/#use-with-ide
+.pdm.toml
+
+# PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm
+__pypackages__/
+
+# Celery stuff
+celerybeat-schedule
+celerybeat.pid
+
+# SageMath parsed files
+*.sage.py
+
+# Environments
+.env
+.venv
+env/
+venv/
+ENV/
+env.bak/
+venv.bak/
+
+# Spyder project settings
+.spyderproject
+.spyproject
+
+# Rope project settings
+.ropeproject
+
+# mkdocs documentation
+/site
+
+# mypy
+.mypy_cache/
+.dmypy.json
+dmypy.json
+
+# Pyre type checker
+.pyre/
+
+# pytype static type analyzer
+.pytype/
+
+# Cython debug symbols
+cython_debug/
+
+# PyCharm
+# JetBrains specific template is maintained in a separate JetBrains.gitignore that can
+# be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore
+# and can be added to the global gitignore or merged into this file. For a more nuclear
+# option (not recommended) you can uncomment the following to ignore the entire idea folder.
+#.idea/
+
+### Python Patch ###
+# Poetry local configuration file - https://python-poetry.org/docs/configuration/#local-configuration
+poetry.toml
+
+# ruff
+.ruff_cache/
+
+# LSP config files
+pyrightconfig.json
+
+# End of https://www.toptal.com/developers/gitignore/api/python
+# Created by https://www.toptal.com/developers/gitignore/api/macos
+# Edit at https://www.toptal.com/developers/gitignore?templates=macos
+
+### macOS ###
+# General
+.DS_Store
+.AppleDouble
+.LSOverride
+
+# Icon must end with two \r
+Icon
+
+
+# Thumbnails
+._*
+
+# Files that might appear in the root of a volume
+.DocumentRevisions-V100
+.fseventsd
+.Spotlight-V100
+.TemporaryItems
+.Trashes
+.VolumeIcon.icns
+.com.apple.timemachine.donotpresent
+
+# Directories potentially created on remote AFP share
+.AppleDB
+.AppleDesktop
+Network Trash Folder
+Temporary Items
+.apdisk
+
+### macOS Patch ###
+# iCloud generated files
+*.icloud
+
+# End of https://www.toptal.com/developers/gitignore/api/macos
+# Created by https://www.toptal.com/developers/gitignore/api/linux
+# Edit at https://www.toptal.com/developers/gitignore?templates=linux
+
+### Linux ###
+*~
+
+# temporary files which can be created if a process still has a handle open of a deleted file
+.fuse_hidden*
+
+# KDE directory preferences
+.directory
+
+# Linux trash folder which might appear on any partition or disk
+.Trash-*
+
+# .nfs files are created when an open file is removed but is still being accessed
+.nfs*
+
+# End of https://www.toptal.com/developers/gitignore/api/linux
+# Created by https://www.toptal.com/developers/gitignore/api/windows
+# Edit at https://www.toptal.com/developers/gitignore?templates=windows
+
+### Windows ###
+# Windows thumbnail cache files
+Thumbs.db
+Thumbs.db:encryptable
+ehthumbs.db
+ehthumbs_vista.db
+
+# Dump file
+*.stackdump
+
+# Folder config file
+[Dd]esktop.ini
+
+# Recycle Bin used on file shares
+$RECYCLE.BIN/
+
+# Windows Installer files
+*.cab
+*.msi
+*.msix
+*.msm
+*.msp
+
+# Windows shortcuts
+*.lnk
+
+# End of https://www.toptal.com/developers/gitignore/api/windows
+# Created by https://www.toptal.com/developers/gitignore/api/visualstudiocode
+# Edit at https://www.toptal.com/developers/gitignore?templates=visualstudiocode
+
+### VisualStudioCode ###
+.vscode/
+
+# Local History for Visual Studio Code
+.history/
+
+# Built Visual Studio Code Extensions
+*.vsix
+
+### VisualStudioCode Patch ###
+# Ignore all local history of files
+.history
+.ionide
+
+# End of https://www.toptal.com/developers/gitignore/api/visualstudiocode
+# Created by https://www.toptal.com/developers/gitignore/api/jetbrains
+# Edit at https://www.toptal.com/developers/gitignore?templates=jetbrains
+
+### JetBrains ###
+# Covers JetBrains IDEs: IntelliJ, RubyMine, PhpStorm, AppCode, PyCharm, CLion, Android Studio, WebStorm and Rider
+# Reference: https://intellij-support.jetbrains.com/hc/en-us/articles/206544839
+
+# User-specific stuff
+.idea/**/workspace.xml
+.idea/**/tasks.xml
+.idea/**/usage.statistics.xml
+.idea/**/dictionaries
+.idea/**/shelf
+
+# AWS User-specific
+.idea/**/aws.xml
+
+# Generated files
+.idea/**/contentModel.xml
+
+# Sensitive or high-churn files
+.idea/**/dataSources/
+.idea/**/dataSources.ids
+.idea/**/dataSources.local.xml
+.idea/**/sqlDataSources.xml
+.idea/**/dynamic.xml
+.idea/**/uiDesigner.xml
+.idea/**/dbnavigator.xml
+
+# Gradle
+.idea/**/gradle.xml
+.idea/**/libraries
+
+# Gradle and Maven with auto-import
+# When using Gradle or Maven with auto-import, you should exclude module files,
+# since they will be recreated, and may cause churn. Uncomment if using
+# auto-import.
+# .idea/artifacts
+# .idea/compiler.xml
+# .idea/jarRepositories.xml
+# .idea/modules.xml
+# .idea/*.iml
+# .idea/modules
+# *.iml
+# *.ipr
+
+# CMake
+cmake-build-*/
+
+# Mongo Explorer plugin
+.idea/**/mongoSettings.xml
+
+# File-based project format
+*.iws
+
+# IntelliJ
+out/
+
+# mpeltonen/sbt-idea plugin
+.idea_modules/
+
+# JIRA plugin
+atlassian-ide-plugin.xml
+
+# Cursive Clojure plugin
+.idea/replstate.xml
+
+# SonarLint plugin
+.idea/sonarlint/
+
+# Crashlytics plugin (for Android Studio and IntelliJ)
+com_crashlytics_export_strings.xml
+crashlytics.properties
+crashlytics-build.properties
+fabric.properties
+
+# Editor-based Rest Client
+.idea/httpRequests
+
+# Android studio 3.1+ serialized cache file
+.idea/caches/build_file_checksums.ser
+
+### JetBrains Patch ###
+# Comment Reason: https://github.com/joeblau/gitignore.io/issues/186#issuecomment-215987721
+
+# *.iml
+# modules.xml
+# .idea/misc.xml
+# *.ipr
+
+# Sonarlint plugin
+# https://plugins.jetbrains.com/plugin/7973-sonarlint
+.idea/**/sonarlint/
+
+# SonarQube Plugin
+# https://plugins.jetbrains.com/plugin/7238-sonarqube-community-plugin
+.idea/**/sonarIssues.xml
+
+# Markdown Navigator plugin
+# https://plugins.jetbrains.com/plugin/7896-markdown-navigator-enhanced
+.idea/**/markdown-navigator.xml
+.idea/**/markdown-navigator-enh.xml
+.idea/**/markdown-navigator/
+
+# Cache file creation bug
+# See https://youtrack.jetbrains.com/issue/JBR-2257
+.idea/$CACHE_FILE$
+
+# CodeStream plugin
+# https://plugins.jetbrains.com/plugin/12206-codestream
+.idea/codestream.xml
+
+# Azure Toolkit for IntelliJ plugin
+# https://plugins.jetbrains.com/plugin/8053-azure-toolkit-for-intellij
+.idea/**/azureSettings.xml
+
+# End of https://www.toptal.com/developers/gitignore/api/jetbrains
+# Created by https://www.toptal.com/developers/gitignore/api/vim
+# Edit at https://www.toptal.com/developers/gitignore?templates=vim
+
+### Vim ###
+# Swap
+[._]*.s[a-v][a-z]
+!*.svg # comment out if you don't need vector files
+[._]*.sw[a-p]
+[._]s[a-rt-v][a-z]
+[._]ss[a-gi-z]
+[._]sw[a-p]
+
+# Session
+Session.vim
+Sessionx.vim
+
+# Temporary
+.netrwhist
+*~
+# Auto-generated tag files
+tags
+# Persistent undo
+[._]*.un~
+
+# End of https://www.toptal.com/developers/gitignore/api/vim
--
GitLab
From 858c65200f15bfe2c3d8eb4a6ba080ed344f739a Mon Sep 17 00:00:00 2001
From: Chuck Daniels <chuck@developmentseed.org>
Date: Wed, 30 Apr 2025 08:46:06 -0700
Subject: [PATCH 2/5] Bump disk space to 40GB
---
algorithm_config.yaml | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/algorithm_config.yaml b/algorithm_config.yaml
index ae9cb6b..6d96f5f 100644
--- a/algorithm_config.yaml
+++ b/algorithm_config.yaml
@@ -2,7 +2,7 @@ algorithm_description: Run AOS Tracking paper plots on DPS
algorithm_name: aos_maap_dps
algorithm_version: main
build_command: aos_maap_dps/build.sh
-disk_space: 20GB
+disk_space: 40GB
docker_container_url: mas.maap-project.org/david.m.giles/aos_test/aos_dps:develop
inputs:
config: []
--
GitLab
From b81c742d91c1f9fabd99e4c2455f612d7c4a87cc Mon Sep 17 00:00:00 2001
From: Chuck Daniels <chuck@developmentseed.org>
Date: Wed, 30 Apr 2025 08:46:24 -0700
Subject: [PATCH 3/5] Add helper notebooks
---
notebooks/register-algorithm.ipynb | 110 +++++++++++++
notebooks/set-secrets.ipynb | 245 +++++++++++++++++++++++++++++
notebooks/submit-job.ipynb | 132 ++++++++++++++++
3 files changed, 487 insertions(+)
create mode 100644 notebooks/register-algorithm.ipynb
create mode 100644 notebooks/set-secrets.ipynb
create mode 100644 notebooks/submit-job.ipynb
diff --git a/notebooks/register-algorithm.ipynb b/notebooks/register-algorithm.ipynb
new file mode 100644
index 0000000..8bd3c2b
--- /dev/null
+++ b/notebooks/register-algorithm.ipynb
@@ -0,0 +1,110 @@
+{
+ "cells": [
+ {
+ "cell_type": "markdown",
+ "id": "8a2d8d86-2187-44d7-9f44-d5e6583622b5",
+ "metadata": {},
+ "source": [
+ "# Register the Algorithm"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "3561be3b-2600-4075-b177-0d79c78e334a",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "import os\n",
+ "import os.path\n",
+ "import sys\n",
+ "import yaml\n",
+ "\n",
+ "from maap.maap import MAAP"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "id": "eb2e6f45-47cb-48c5-9fe2-feccc07d5c17",
+ "metadata": {},
+ "source": [
+ "## Create MAAP Instance for Initiating Registration"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "38f966fd-9f1e-4f57-a930-14c0f471ea19",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "maap = MAAP()"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "id": "ca846531-802e-4f23-8e5d-df84b812a590",
+ "metadata": {},
+ "source": [
+ "## Initiate a Registration Process\n",
+ "\n",
+ "Running the following cell will _initiate_ a registration process to register the algorithm configured in the file `algorith_config.yaml` at the root of this repository. If _initiation_ succeeds, the URL of the initiated registration _process_ will appear in the cell output, otherwise an error message will appear.\n",
+ "\n",
+ "If an error appears in the cell output, it likely indicates that there is a problem with the DPS itself that is preventing the initiation of any registration processes, and you should use Slack to request assistance.\n",
+ "\n",
+ "If a registration process URL appears in the cell output. Open a browser to the given URL to check the status of the regsitration process. This process will likely take 5-6 minutes to complete successfully, but might fail much more quickly than that.\n",
+ "\n",
+ "When the registration process completes successfully will you be able to run the new version of the registered algorithm.\n",
+ "\n",
+ "When the registration process fails, it might be a transient problem, so you may simply need to rerun the following cell to get past the transient error. If failure of registration processes persists after 3 or 4 attempts, use Slack to request assistance, as there may be a problem with DPS."
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "eb743993-1098-497e-bc17-94bcc81b1eaa",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "base_dir = os.path.dirname(os.getcwd())\n",
+ "algorithm_config_yaml = os.path.join(base_dir, \"algorithm_config.yaml\")\n",
+ "\n",
+ "with open(algorithm_config_yaml) as f:\n",
+ " algorithm_config = yaml.safe_load(f)\n",
+ "\n",
+ "algorithm_name = algorithm_config[\"algorithm_name\"]\n",
+ "algorithm_version = algorithm_config[\"algorithm_version\"]\n",
+ "\n",
+ "if response := maap.register_algorithm_from_yaml_file(algorithm_config_yaml):\n",
+ " job_web_url = response.json()[\"message\"][\"job_web_url\"]\n",
+ " print(\n",
+ " f\"Initiated algorithm registration for {algorithm_name}:{algorithm_version}.\"\n",
+ " f\" Check progress at {job_web_url}.\"\n",
+ " )\n",
+ "else:\n",
+ " print(\"ERROR:\", response.json()[\"message\"], file=sys.stderr)"
+ ]
+ }
+ ],
+ "metadata": {
+ "kernelspec": {
+ "display_name": "Python 3 (ipykernel)",
+ "language": "python",
+ "name": "python3"
+ },
+ "language_info": {
+ "codemirror_mode": {
+ "name": "ipython",
+ "version": 3
+ },
+ "file_extension": ".py",
+ "mimetype": "text/x-python",
+ "name": "python",
+ "nbconvert_exporter": "python",
+ "pygments_lexer": "ipython3",
+ "version": "3.10.13"
+ }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 5
+}
diff --git a/notebooks/set-secrets.ipynb b/notebooks/set-secrets.ipynb
new file mode 100644
index 0000000..6719bca
--- /dev/null
+++ b/notebooks/set-secrets.ipynb
@@ -0,0 +1,245 @@
+{
+ "cells": [
+ {
+ "cell_type": "markdown",
+ "id": "331918b2-c03c-4d14-9737-d65b94ed7e73",
+ "metadata": {},
+ "source": [
+ "# Add MAAP Secrets for Earthdata Login Credentials\n",
+ "\n",
+ "Earthdata Login credentials are required for downloading files from NASA DAACs. In order for a DPS job to be able to use such credentials in a safe manner, it looks for them in the MAAP Secrets store. Therefore, you must add the necessary secrets before a DPS job that requires them is submitted, otherwise the job will fail due to the missing secrets."
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "cbdb9c92-6219-4d58-b9f4-206ee507ce51",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "from maap.maap import MAAP"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "e2f0c041-646c-47e3-9c9f-4937ed9f7907",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "def mask(value: str, *, show_last: int = 4) -> str:\n",
+ " \"\"\"Mask a value.\n",
+ " \n",
+ " Parameters\n",
+ " ----------\n",
+ " value:\n",
+ " Value to mask.\n",
+ " show_last:\n",
+ " Number of trailing characters to leave unmasked.\n",
+ " \n",
+ " Returns\n",
+ " -------\n",
+ " Masked value, where all but `show_last` characters of the specified value are\n",
+ " replaced with an asterisk (`*`). If necessary, the value is further left-padded\n",
+ " with asterisks to ensure there are at least as many asterisks as there are\n",
+ " unmasked trailing characters.\n",
+ " \"\"\"\n",
+ "\n",
+ " # Make sure the returned value contains at least the same number of masked\n",
+ " # characters as unmasked characters for a bit of added \"security.\"\n",
+ " n_masked = max(len(value), 2 * show_last) - show_last\n",
+ " \n",
+ " return f\"{'*' * n_masked}{value[-show_last:]}\"\n",
+ "\n",
+ "\n",
+ "def prompt_user(\n",
+ " prompt: str,\n",
+ " *,\n",
+ " value: str | None = None,\n",
+ " sensitive: bool = True,\n",
+ ") -> str | None:\n",
+ " \"\"\"Prompt user for a possibly sensitive input value.\n",
+ "\n",
+ " Parameters\n",
+ " ----------\n",
+ " prompt:\n",
+ " Human-readable prompt to present to the user, which is combined with `value`\n",
+ " when prompting the user.\n",
+ " value:\n",
+ " Current value (if any) of what to prompt the user for, which is combined with\n",
+ " `prompt` when prompting the user, and masked, if `senstive` is `True`.\n",
+ " sensitive:\n",
+ " If `True`, mask the current value (if any) when prompting the user, and also\n",
+ " avoid echoing user input. Otherwise, the current value is left unmasked and\n",
+ " user input is echoed.\n",
+ "\n",
+ " Returns\n",
+ " -------\n",
+ " Value entered by the user at the prompt, or `None` if either the user pressed the\n",
+ " Return or Enter key without supplying any input, or a KeyboardInterrupt occurred.\n",
+ " \"\"\"\n",
+ " from getpass import getpass\n",
+ " from IPython.display import clear_output\n",
+ "\n",
+ " masked_value = mask(value) if sensitive else value\n",
+ " get_user_input = getpass if sensitive else input\n",
+ "\n",
+ " try:\n",
+ " return get_user_input(f\"{prompt} [{masked_value}]:\") or None\n",
+ " except KeyboardInterrupt:\n",
+ " clear_output() # Remove prompt from cell output\n",
+ "\n",
+ " \n",
+ "def get_secret(maap: MAAP, name: str) -> str | None:\n",
+ " \"\"\"Get the value of a MAAP secret, or `None` if the secret does not exist.\n",
+ " \n",
+ " Parameters\n",
+ " ----------\n",
+ " maap:\n",
+ " MAAP instance to use for secrets management.\n",
+ " name:\n",
+ " Name of the secret to get the value of.\n",
+ "\n",
+ " Returns\n",
+ " -------\n",
+ " Value of the secret, if it exists; otherwise `None`.\n",
+ " \"\"\"\n",
+ "\n",
+ " # Calling maap.secrets.get_secret returns the value of the requested secret, if it\n",
+ " # exists, but oddly returns an object with an error code if it does not exist,\n",
+ " # rather than simply (and conveniently) returning `None`.\n",
+ " \n",
+ " return value if isinstance(value := maap.secrets.get_secret(name), str) else None\n",
+ "\n",
+ "\n",
+ "def set_secret_interactively(\n",
+ " maap: MAAP,\n",
+ " *,\n",
+ " prompt: str,\n",
+ " name: str,\n",
+ " sensitive: bool = True,\n",
+ ") -> str | None:\n",
+ " \"\"\"Set the value of a secret by prompting the user for a value.\n",
+ "\n",
+ " If the secret already exists, the user prompt will include the existing value,\n",
+ " which will be masked if `sensitive` is `True` (showing only the last 4\n",
+ " characters of the value).\n",
+ "\n",
+ " If an empty input is provided (i.e., either the Enter or Return key is pressed at\n",
+ " the prompt, without any other input, or a KeyboardInterrupt occurs), the secret is\n",
+ " not set and `None` is returned. Otherwise, the secret is set to the user-supplied\n",
+ " value and that value is returned.\n",
+ "\n",
+ " Parameters\n",
+ " ----------\n",
+ " maap:\n",
+ " MAAP instance to use for secrets management.\n",
+ " prompt:\n",
+ " Human-readable prompt to display to user when prompting for input.\n",
+ " name:\n",
+ " Name of the secret.\n",
+ " sensitive:\n",
+ " If `True`, the prompt will mask the current secret value, if there is one.\n",
+ " Further, user input will not be echoed. Otherwise, the prompt will show the\n",
+ " full secret value, if there is one, and user input will be echoed.\n",
+ "\n",
+ " Returns\n",
+ " -------\n",
+ " User-supplied input provided at the prompt, or `None` if the Enter or Return key\n",
+ " was pressed without input.\n",
+ " \"\"\"\n",
+ " if (value := prompt_user(prompt, value=get_secret(maap, name), sensitive=sensitive)):\n",
+ " maap.secrets.add_secret(name, value)\n",
+ " masked_value = mask(value) if sensitive else value\n",
+ " print(f\"The secret {name} was set to {masked_value}.\")\n",
+ " return value\n",
+ "\n",
+ " print(f\"The value for secret {name} was left unchanged.\")"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "id": "97611ae6-837e-4021-9e4e-bee447749d5e",
+ "metadata": {},
+ "source": [
+ "## Create MAAP Instance for Managing Secrets"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "10612951-3392-4e32-983e-6e9a85c8c938",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "maap = MAAP()"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "id": "98d52db0-7187-4aa3-a24a-7f3cbfce63c5",
+ "metadata": {},
+ "source": [
+ "## Store Earthdata Login Username as a Secret"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "986a47cb-0586-4f62-827b-6df6fcb609d4",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "set_secret_interactively(\n",
+ " maap,\n",
+ " prompt=\"Earthdata Login username\",\n",
+ " name=\"EARTHDATA_USERNAME\",\n",
+ " sensitive=False\n",
+ ")"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "id": "d6fee256-c3dc-4cb7-a4f1-8f6edab3491c",
+ "metadata": {},
+ "source": [
+ "## Store Earthdata Login Password as a Secret"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "b41558e4-1c1a-40f9-87e3-f3d9b9ffc98e",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "set_secret_interactively(\n",
+ " maap,\n",
+ " prompt=\"Earthdata Login password\",\n",
+ " name=\"EARTHDATA_PASSWORD\"\n",
+ ")"
+ ]
+ }
+ ],
+ "metadata": {
+ "kernelspec": {
+ "display_name": "Python 3 (ipykernel)",
+ "language": "python",
+ "name": "python3"
+ },
+ "language_info": {
+ "codemirror_mode": {
+ "name": "ipython",
+ "version": 3
+ },
+ "file_extension": ".py",
+ "mimetype": "text/x-python",
+ "name": "python",
+ "nbconvert_exporter": "python",
+ "pygments_lexer": "ipython3",
+ "version": "3.10.13"
+ }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 5
+}
diff --git a/notebooks/submit-job.ipynb b/notebooks/submit-job.ipynb
new file mode 100644
index 0000000..48b93af
--- /dev/null
+++ b/notebooks/submit-job.ipynb
@@ -0,0 +1,132 @@
+{
+ "cells": [
+ {
+ "cell_type": "markdown",
+ "id": "778cecaf-495e-4ab9-b8a7-7329b74fb49b",
+ "metadata": {},
+ "source": [
+ "# Submit an AOS Job"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "2af023f9-90f5-4749-add8-bc81d566014a",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "from datetime import date\n",
+ "\n",
+ "import ipywidgets as widgets\n",
+ "from maap.maap import MAAP"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "987727e8-0cdb-4613-a9ef-071a7872abd4",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "maap = MAAP()"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "abd6f889-bfda-4592-b096-1b6bff1e120a",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "def submit_aos_job(analysis_date: date) -> str | None:\n",
+ " \"\"\"Submit an AOS job for an analysis date.\n",
+ "\n",
+ " Returns\n",
+ " -------\n",
+ " A MAAP job ID, if a job was successfully submitted; otherwise `None`.\n",
+ " \"\"\"\n",
+ " import re\n",
+ " import sys\n",
+ "\n",
+ " analysis_date_str = analysis_date.strftime(\"%Y-%m-%d\")\n",
+ " job = maap.submitJob(\n",
+ " analysis_date_str, # arbitrary job tag/label\n",
+ " \"aos_maap_dps\", # algorithm name\n",
+ " \"main\", # algorithm version\n",
+ " \"maap-dps-sandbox\", # job queue\n",
+ " analysis_date=analysis_date_str,\n",
+ " )\n",
+ "\n",
+ " if job.id:\n",
+ " print(f\"Submitted job with job ID {job.id}\", file=sys.stderr)\n",
+ " return job.id\n",
+ "\n",
+ " match = re.search(\n",
+ " r\"<ows:ExceptionText>(?P<msg>.*)</ows:ExceptionText>\", job.error_details\n",
+ " )\n",
+ " print(match[\"msg\"] if match else job.error_details, file=sys.stderr)\n",
+ "\n",
+ " return None"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "id": "bb0a6014-8fcd-474f-94c7-bbeb5f35f50c",
+ "metadata": {},
+ "source": [
+ "## Pick an Analysis Date and Submit a Job"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 19,
+ "id": "689c9988-ff4b-41a0-aa2b-427e96d2a16e",
+ "metadata": {},
+ "outputs": [
+ {
+ "data": {
+ "application/vnd.jupyter.widget-view+json": {
+ "model_id": "a12b061a9b7441499c0824f67537d3cc",
+ "version_major": 2,
+ "version_minor": 0
+ },
+ "text/plain": [
+ "interactive(children=(DatePicker(value=None, description='Analysis Date', step=1, style=DescriptionStyle(descr…"
+ ]
+ },
+ "metadata": {},
+ "output_type": "display_data"
+ }
+ ],
+ "source": [
+ "date_picker = widgets.DatePicker(\n",
+ " description=\"Analysis Date\",\n",
+ " style=dict(description_width=\"initial\"),\n",
+ ")\n",
+ "interactively = widgets.interact_manual.options(manual_name=\"Submit AOS Job\")\n",
+ "interactively(submit_aos_job, analysis_date=date_picker);"
+ ]
+ }
+ ],
+ "metadata": {
+ "kernelspec": {
+ "display_name": "Python 3 (ipykernel)",
+ "language": "python",
+ "name": "python3"
+ },
+ "language_info": {
+ "codemirror_mode": {
+ "name": "ipython",
+ "version": 3
+ },
+ "file_extension": ".py",
+ "mimetype": "text/x-python",
+ "name": "python",
+ "nbconvert_exporter": "python",
+ "pygments_lexer": "ipython3",
+ "version": "3.10.13"
+ }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 5
+}
--
GitLab
From 7b436ed7d3f8442d49e36d5bacca7c76053f16b7 Mon Sep 17 00:00:00 2001
From: Chuck Daniels <chuck@developmentseed.org>
Date: Wed, 30 Apr 2025 13:50:55 -0700
Subject: [PATCH 4/5] Combine prompts into same cell
---
notebooks/set-secrets.ipynb | 31 +++++--------------------------
1 file changed, 5 insertions(+), 26 deletions(-)
diff --git a/notebooks/set-secrets.ipynb b/notebooks/set-secrets.ipynb
index 6719bca..d1be3fe 100644
--- a/notebooks/set-secrets.ipynb
+++ b/notebooks/set-secrets.ipynb
@@ -88,6 +88,7 @@
" return get_user_input(f\"{prompt} [{masked_value}]:\") or None\n",
" except KeyboardInterrupt:\n",
" clear_output() # Remove prompt from cell output\n",
+ " raise\n",
"\n",
" \n",
"def get_secret(maap: MAAP, name: str) -> str | None:\n",
@@ -180,7 +181,7 @@
"id": "98d52db0-7187-4aa3-a24a-7f3cbfce63c5",
"metadata": {},
"source": [
- "## Store Earthdata Login Username as a Secret"
+ "## Store Earthdata Login Credentials as MAAP Secrets"
]
},
{
@@ -191,32 +192,10 @@
"outputs": [],
"source": [
"set_secret_interactively(\n",
- " maap,\n",
- " prompt=\"Earthdata Login username\",\n",
- " name=\"EARTHDATA_USERNAME\",\n",
- " sensitive=False\n",
- ")"
- ]
- },
- {
- "cell_type": "markdown",
- "id": "d6fee256-c3dc-4cb7-a4f1-8f6edab3491c",
- "metadata": {},
- "source": [
- "## Store Earthdata Login Password as a Secret"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": null,
- "id": "b41558e4-1c1a-40f9-87e3-f3d9b9ffc98e",
- "metadata": {},
- "outputs": [],
- "source": [
+ " maap, prompt=\"Earthdata Login username\", name=\"EARTHDATA_USERNAME\", sensitive=False\n",
+ ")\n",
"set_secret_interactively(\n",
- " maap,\n",
- " prompt=\"Earthdata Login password\",\n",
- " name=\"EARTHDATA_PASSWORD\"\n",
+ " maap, prompt=\"Earthdata Login password\", name=\"EARTHDATA_PASSWORD\"\n",
")"
]
}
--
GitLab
From 67dc72a416a208a381631a74d275e0e2aea50c61 Mon Sep 17 00:00:00 2001
From: Chuck Daniels <chuck@developmentseed.org>
Date: Wed, 30 Apr 2025 13:52:10 -0700
Subject: [PATCH 5/5] Add queue dropdown; default to AOS queue
---
notebooks/submit-job.ipynb | 91 +++++++++++++++++++++++++++-----------
1 file changed, 65 insertions(+), 26 deletions(-)
diff --git a/notebooks/submit-job.ipynb b/notebooks/submit-job.ipynb
index 48b93af..092333e 100644
--- a/notebooks/submit-job.ipynb
+++ b/notebooks/submit-job.ipynb
@@ -10,7 +10,7 @@
},
{
"cell_type": "code",
- "execution_count": null,
+ "execution_count": 1,
"id": "2af023f9-90f5-4749-add8-bc81d566014a",
"metadata": {},
"outputs": [],
@@ -18,27 +18,37 @@
"from datetime import date\n",
"\n",
"import ipywidgets as widgets\n",
- "from maap.maap import MAAP"
+ "from maap.maap import MAAP\n",
+ "\n",
+ "DEFAULT_QUEUE = \"maap-dps-aos-worker-32gb\""
]
},
{
"cell_type": "code",
- "execution_count": null,
- "id": "987727e8-0cdb-4613-a9ef-071a7872abd4",
+ "execution_count": 2,
+ "id": "50a503fc-8115-4b98-a5dc-4bd438b0f254",
"metadata": {},
"outputs": [],
"source": [
- "maap = MAAP()"
+ "def get_job_queues(maap: MAAP) -> list[str]:\n",
+ " response = maap.getQueues()\n",
+ " response.raise_for_status()\n",
+ "\n",
+ " return response.json()[\"queues\"]"
]
},
{
"cell_type": "code",
- "execution_count": null,
+ "execution_count": 3,
"id": "abd6f889-bfda-4592-b096-1b6bff1e120a",
"metadata": {},
"outputs": [],
"source": [
- "def submit_aos_job(analysis_date: date) -> str | None:\n",
+ "def submit_aos_job(\n",
+ " maap: MAAP,\n",
+ " analysis_date: date,\n",
+ " queue: str = DEFAULT_QUEUE,\n",
+ ") -> str | None:\n",
" \"\"\"Submit an AOS job for an analysis date.\n",
"\n",
" Returns\n",
@@ -48,25 +58,30 @@
" import re\n",
" import sys\n",
"\n",
+ " if analysis_date is None:\n",
+ " print(\"Specify a date\", file=sys.stderr)\n",
+ " return None\n",
+ "\n",
" analysis_date_str = analysis_date.strftime(\"%Y-%m-%d\")\n",
" job = maap.submitJob(\n",
- " analysis_date_str, # arbitrary job tag/label\n",
- " \"aos_maap_dps\", # algorithm name\n",
- " \"main\", # algorithm version\n",
- " \"maap-dps-sandbox\", # job queue\n",
+ " identifier=analysis_date_str, # custom job tag/label\n",
+ " algo_id=\"aos_maap_dps\", # algorithm name\n",
+ " version=\"main\", # algorithm version\n",
+ " queue=queue, # job queue\n",
+ " username=maap.profile.account_info()[\"username\"],\n",
+ " # Algorithm inputs:\n",
" analysis_date=analysis_date_str,\n",
" )\n",
"\n",
- " if job.id:\n",
- " print(f\"Submitted job with job ID {job.id}\", file=sys.stderr)\n",
- " return job.id\n",
+ " if not job.id:\n",
+ " match = re.search(\n",
+ " r\"<ows:ExceptionText>(?P<msg>.*)</ows:ExceptionText>\", job.error_details\n",
+ " )\n",
+ " print(match[\"msg\"] if match else job.error_details, file=sys.stderr)\n",
+ " return None\n",
"\n",
- " match = re.search(\n",
- " r\"<ows:ExceptionText>(?P<msg>.*)</ows:ExceptionText>\", job.error_details\n",
- " )\n",
- " print(match[\"msg\"] if match else job.error_details, file=sys.stderr)\n",
- "\n",
- " return None"
+ " print(f\"Submitted job with job ID {job.id}\", file=sys.stderr)\n",
+ " return job.id"
]
},
{
@@ -74,24 +89,34 @@
"id": "bb0a6014-8fcd-474f-94c7-bbeb5f35f50c",
"metadata": {},
"source": [
- "## Pick an Analysis Date and Submit a Job"
+ "## Specify Inputs and Submit a Job"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 4,
+ "id": "987727e8-0cdb-4613-a9ef-071a7872abd4",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "maap = MAAP()"
]
},
{
"cell_type": "code",
- "execution_count": 19,
+ "execution_count": 5,
"id": "689c9988-ff4b-41a0-aa2b-427e96d2a16e",
"metadata": {},
"outputs": [
{
"data": {
"application/vnd.jupyter.widget-view+json": {
- "model_id": "a12b061a9b7441499c0824f67537d3cc",
+ "model_id": "d01fe462138b43f3ba546af09d0ef2ef",
"version_major": 2,
"version_minor": 0
},
"text/plain": [
- "interactive(children=(DatePicker(value=None, description='Analysis Date', step=1, style=DescriptionStyle(descr…"
+ "interactive(children=(DatePicker(value=None, description='Analysis Date:', step=1, style=DescriptionStyle(desc…"
]
},
"metadata": {},
@@ -100,11 +125,25 @@
],
"source": [
"date_picker = widgets.DatePicker(\n",
- " description=\"Analysis Date\",\n",
+ " description=\"Analysis Date:\",\n",
" style=dict(description_width=\"initial\"),\n",
")\n",
+ "\n",
+ "queues = get_job_queues(maap)\n",
+ "queue_picker = widgets.Dropdown(\n",
+ " description=\"Job Queue:\",\n",
+ " options=queues,\n",
+ " value=DEFAULT_QUEUE if DEFAULT_QUEUE in queues else None,\n",
+ " style=dict(description_width=\"initial\"),\n",
+ ")\n",
+ "\n",
"interactively = widgets.interact_manual.options(manual_name=\"Submit AOS Job\")\n",
- "interactively(submit_aos_job, analysis_date=date_picker);"
+ "interactively(\n",
+ " submit_aos_job,\n",
+ " maap=widgets.fixed(maap),\n",
+ " analysis_date=date_picker,\n",
+ " queue=queue_picker\n",
+ ");"
]
}
],
--
GitLab