diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000000000000000000000000000000000000..6c8f7a65cb0eaa299037c30c775f375396b8ef21 --- /dev/null +++ b/.gitignore @@ -0,0 +1,422 @@ +# Created by https://www.toptal.com/developers/gitignore/api/python +# Edit at https://www.toptal.com/developers/gitignore?templates=python + +### Python ### +# Byte-compiled / optimized / DLL files +__pycache__/ +*.py[cod] +*$py.class + +# C extensions +*.so + +# Distribution / packaging +.Python +build/ +develop-eggs/ +dist/ +downloads/ +eggs/ +.eggs/ +lib/ +lib64/ +parts/ +sdist/ +var/ +wheels/ +share/python-wheels/ +*.egg-info/ +.installed.cfg +*.egg +MANIFEST + +# PyInstaller +# Usually these files are written by a python script from a template +# before PyInstaller builds the exe, so as to inject date/other infos into it. +*.manifest +*.spec + +# Installer logs +pip-log.txt +pip-delete-this-directory.txt + +# Unit test / coverage reports +htmlcov/ +.tox/ +.nox/ +.coverage +.coverage.* +.cache +nosetests.xml +coverage.xml +*.cover +*.py,cover +.hypothesis/ +.pytest_cache/ +cover/ + +# Translations +*.mo +*.pot + +# Django stuff: +*.log +local_settings.py +db.sqlite3 +db.sqlite3-journal + +# Flask stuff: +instance/ +.webassets-cache + +# Scrapy stuff: +.scrapy + +# Sphinx documentation +docs/_build/ + +# PyBuilder +.pybuilder/ +target/ + +# Jupyter Notebook +.ipynb_checkpoints + +# IPython +profile_default/ +ipython_config.py + +# pyenv +# For a library or package, you might want to ignore these files since the code is +# intended to run in multiple environments; otherwise, check them in: +# .python-version + +# pipenv +# According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. +# However, in case of collaboration, if having platform-specific dependencies or dependencies +# having no cross-platform support, pipenv may install dependencies that don't work, or not +# install all needed dependencies. +#Pipfile.lock + +# poetry +# Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control. +# This is especially recommended for binary packages to ensure reproducibility, and is more +# commonly ignored for libraries. +# https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control +#poetry.lock + +# pdm +# Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control. +#pdm.lock +# pdm stores project-wide configurations in .pdm.toml, but it is recommended to not include it +# in version control. +# https://pdm.fming.dev/#use-with-ide +.pdm.toml + +# PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm +__pypackages__/ + +# Celery stuff +celerybeat-schedule +celerybeat.pid + +# SageMath parsed files +*.sage.py + +# Environments +.env +.venv +env/ +venv/ +ENV/ +env.bak/ +venv.bak/ + +# Spyder project settings +.spyderproject +.spyproject + +# Rope project settings +.ropeproject + +# mkdocs documentation +/site + +# mypy +.mypy_cache/ +.dmypy.json +dmypy.json + +# Pyre type checker +.pyre/ + +# pytype static type analyzer +.pytype/ + +# Cython debug symbols +cython_debug/ + +# PyCharm +# JetBrains specific template is maintained in a separate JetBrains.gitignore that can +# be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore +# and can be added to the global gitignore or merged into this file. For a more nuclear +# option (not recommended) you can uncomment the following to ignore the entire idea folder. +#.idea/ + +### Python Patch ### +# Poetry local configuration file - https://python-poetry.org/docs/configuration/#local-configuration +poetry.toml + +# ruff +.ruff_cache/ + +# LSP config files +pyrightconfig.json + +# End of https://www.toptal.com/developers/gitignore/api/python +# Created by https://www.toptal.com/developers/gitignore/api/macos +# Edit at https://www.toptal.com/developers/gitignore?templates=macos + +### macOS ### +# General +.DS_Store +.AppleDouble +.LSOverride + +# Icon must end with two \r +Icon + + +# Thumbnails +._* + +# Files that might appear in the root of a volume +.DocumentRevisions-V100 +.fseventsd +.Spotlight-V100 +.TemporaryItems +.Trashes +.VolumeIcon.icns +.com.apple.timemachine.donotpresent + +# Directories potentially created on remote AFP share +.AppleDB +.AppleDesktop +Network Trash Folder +Temporary Items +.apdisk + +### macOS Patch ### +# iCloud generated files +*.icloud + +# End of https://www.toptal.com/developers/gitignore/api/macos +# Created by https://www.toptal.com/developers/gitignore/api/linux +# Edit at https://www.toptal.com/developers/gitignore?templates=linux + +### Linux ### +*~ + +# temporary files which can be created if a process still has a handle open of a deleted file +.fuse_hidden* + +# KDE directory preferences +.directory + +# Linux trash folder which might appear on any partition or disk +.Trash-* + +# .nfs files are created when an open file is removed but is still being accessed +.nfs* + +# End of https://www.toptal.com/developers/gitignore/api/linux +# Created by https://www.toptal.com/developers/gitignore/api/windows +# Edit at https://www.toptal.com/developers/gitignore?templates=windows + +### Windows ### +# Windows thumbnail cache files +Thumbs.db +Thumbs.db:encryptable +ehthumbs.db +ehthumbs_vista.db + +# Dump file +*.stackdump + +# Folder config file +[Dd]esktop.ini + +# Recycle Bin used on file shares +$RECYCLE.BIN/ + +# Windows Installer files +*.cab +*.msi +*.msix +*.msm +*.msp + +# Windows shortcuts +*.lnk + +# End of https://www.toptal.com/developers/gitignore/api/windows +# Created by https://www.toptal.com/developers/gitignore/api/visualstudiocode +# Edit at https://www.toptal.com/developers/gitignore?templates=visualstudiocode + +### VisualStudioCode ### +.vscode/ + +# Local History for Visual Studio Code +.history/ + +# Built Visual Studio Code Extensions +*.vsix + +### VisualStudioCode Patch ### +# Ignore all local history of files +.history +.ionide + +# End of https://www.toptal.com/developers/gitignore/api/visualstudiocode +# Created by https://www.toptal.com/developers/gitignore/api/jetbrains +# Edit at https://www.toptal.com/developers/gitignore?templates=jetbrains + +### JetBrains ### +# Covers JetBrains IDEs: IntelliJ, RubyMine, PhpStorm, AppCode, PyCharm, CLion, Android Studio, WebStorm and Rider +# Reference: https://intellij-support.jetbrains.com/hc/en-us/articles/206544839 + +# User-specific stuff +.idea/**/workspace.xml +.idea/**/tasks.xml +.idea/**/usage.statistics.xml +.idea/**/dictionaries +.idea/**/shelf + +# AWS User-specific +.idea/**/aws.xml + +# Generated files +.idea/**/contentModel.xml + +# Sensitive or high-churn files +.idea/**/dataSources/ +.idea/**/dataSources.ids +.idea/**/dataSources.local.xml +.idea/**/sqlDataSources.xml +.idea/**/dynamic.xml +.idea/**/uiDesigner.xml +.idea/**/dbnavigator.xml + +# Gradle +.idea/**/gradle.xml +.idea/**/libraries + +# Gradle and Maven with auto-import +# When using Gradle or Maven with auto-import, you should exclude module files, +# since they will be recreated, and may cause churn. Uncomment if using +# auto-import. +# .idea/artifacts +# .idea/compiler.xml +# .idea/jarRepositories.xml +# .idea/modules.xml +# .idea/*.iml +# .idea/modules +# *.iml +# *.ipr + +# CMake +cmake-build-*/ + +# Mongo Explorer plugin +.idea/**/mongoSettings.xml + +# File-based project format +*.iws + +# IntelliJ +out/ + +# mpeltonen/sbt-idea plugin +.idea_modules/ + +# JIRA plugin +atlassian-ide-plugin.xml + +# Cursive Clojure plugin +.idea/replstate.xml + +# SonarLint plugin +.idea/sonarlint/ + +# Crashlytics plugin (for Android Studio and IntelliJ) +com_crashlytics_export_strings.xml +crashlytics.properties +crashlytics-build.properties +fabric.properties + +# Editor-based Rest Client +.idea/httpRequests + +# Android studio 3.1+ serialized cache file +.idea/caches/build_file_checksums.ser + +### JetBrains Patch ### +# Comment Reason: https://github.com/joeblau/gitignore.io/issues/186#issuecomment-215987721 + +# *.iml +# modules.xml +# .idea/misc.xml +# *.ipr + +# Sonarlint plugin +# https://plugins.jetbrains.com/plugin/7973-sonarlint +.idea/**/sonarlint/ + +# SonarQube Plugin +# https://plugins.jetbrains.com/plugin/7238-sonarqube-community-plugin +.idea/**/sonarIssues.xml + +# Markdown Navigator plugin +# https://plugins.jetbrains.com/plugin/7896-markdown-navigator-enhanced +.idea/**/markdown-navigator.xml +.idea/**/markdown-navigator-enh.xml +.idea/**/markdown-navigator/ + +# Cache file creation bug +# See https://youtrack.jetbrains.com/issue/JBR-2257 +.idea/$CACHE_FILE$ + +# CodeStream plugin +# https://plugins.jetbrains.com/plugin/12206-codestream +.idea/codestream.xml + +# Azure Toolkit for IntelliJ plugin +# https://plugins.jetbrains.com/plugin/8053-azure-toolkit-for-intellij +.idea/**/azureSettings.xml + +# End of https://www.toptal.com/developers/gitignore/api/jetbrains +# Created by https://www.toptal.com/developers/gitignore/api/vim +# Edit at https://www.toptal.com/developers/gitignore?templates=vim + +### Vim ### +# Swap +[._]*.s[a-v][a-z] +!*.svg # comment out if you don't need vector files +[._]*.sw[a-p] +[._]s[a-rt-v][a-z] +[._]ss[a-gi-z] +[._]sw[a-p] + +# Session +Session.vim +Sessionx.vim + +# Temporary +.netrwhist +*~ +# Auto-generated tag files +tags +# Persistent undo +[._]*.un~ + +# End of https://www.toptal.com/developers/gitignore/api/vim diff --git a/algorithm_config.yaml b/algorithm_config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..6d96f5f0a4c25af49d86f4cdc19fe242003a8754 --- /dev/null +++ b/algorithm_config.yaml @@ -0,0 +1,17 @@ +algorithm_description: Run AOS Tracking paper plots on DPS +algorithm_name: aos_maap_dps +algorithm_version: main +build_command: aos_maap_dps/build.sh +disk_space: 40GB +docker_container_url: mas.maap-project.org/david.m.giles/aos_test/aos_dps:develop +inputs: + config: [] + file: [] + positional: + - default: '' + description: '' + name: analysis_date + required: false +queue: maap-dps-sandbox +repository_url: https://repo.maap-project.org/sshah/aos_maap_dps.git +run_command: aos_maap_dps/run.sh \ No newline at end of file diff --git a/build.sh b/build.sh new file mode 100644 index 0000000000000000000000000000000000000000..e25c16e59c09d5f0de31353794a1e5e491c17b78 --- /dev/null +++ b/build.sh @@ -0,0 +1,5 @@ +#!/bin/bash +set -euo pipefail + +# Install maap-py so we can use MAAP Secrets in run.sh +conda run --no-capture-output --name mytobac python -m pip install maap-py diff --git a/notebooks/register-algorithm.ipynb b/notebooks/register-algorithm.ipynb new file mode 100644 index 0000000000000000000000000000000000000000..8bd3c2b0c002ea4835e869672da16614e9e21212 --- /dev/null +++ b/notebooks/register-algorithm.ipynb @@ -0,0 +1,110 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "id": "8a2d8d86-2187-44d7-9f44-d5e6583622b5", + "metadata": {}, + "source": [ + "# Register the Algorithm" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "3561be3b-2600-4075-b177-0d79c78e334a", + "metadata": {}, + "outputs": [], + "source": [ + "import os\n", + "import os.path\n", + "import sys\n", + "import yaml\n", + "\n", + "from maap.maap import MAAP" + ] + }, + { + "cell_type": "markdown", + "id": "eb2e6f45-47cb-48c5-9fe2-feccc07d5c17", + "metadata": {}, + "source": [ + "## Create MAAP Instance for Initiating Registration" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "38f966fd-9f1e-4f57-a930-14c0f471ea19", + "metadata": {}, + "outputs": [], + "source": [ + "maap = MAAP()" + ] + }, + { + "cell_type": "markdown", + "id": "ca846531-802e-4f23-8e5d-df84b812a590", + "metadata": {}, + "source": [ + "## Initiate a Registration Process\n", + "\n", + "Running the following cell will _initiate_ a registration process to register the algorithm configured in the file `algorith_config.yaml` at the root of this repository. If _initiation_ succeeds, the URL of the initiated registration _process_ will appear in the cell output, otherwise an error message will appear.\n", + "\n", + "If an error appears in the cell output, it likely indicates that there is a problem with the DPS itself that is preventing the initiation of any registration processes, and you should use Slack to request assistance.\n", + "\n", + "If a registration process URL appears in the cell output. Open a browser to the given URL to check the status of the regsitration process. This process will likely take 5-6 minutes to complete successfully, but might fail much more quickly than that.\n", + "\n", + "When the registration process completes successfully will you be able to run the new version of the registered algorithm.\n", + "\n", + "When the registration process fails, it might be a transient problem, so you may simply need to rerun the following cell to get past the transient error. If failure of registration processes persists after 3 or 4 attempts, use Slack to request assistance, as there may be a problem with DPS." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "eb743993-1098-497e-bc17-94bcc81b1eaa", + "metadata": {}, + "outputs": [], + "source": [ + "base_dir = os.path.dirname(os.getcwd())\n", + "algorithm_config_yaml = os.path.join(base_dir, \"algorithm_config.yaml\")\n", + "\n", + "with open(algorithm_config_yaml) as f:\n", + " algorithm_config = yaml.safe_load(f)\n", + "\n", + "algorithm_name = algorithm_config[\"algorithm_name\"]\n", + "algorithm_version = algorithm_config[\"algorithm_version\"]\n", + "\n", + "if response := maap.register_algorithm_from_yaml_file(algorithm_config_yaml):\n", + " job_web_url = response.json()[\"message\"][\"job_web_url\"]\n", + " print(\n", + " f\"Initiated algorithm registration for {algorithm_name}:{algorithm_version}.\"\n", + " f\" Check progress at {job_web_url}.\"\n", + " )\n", + "else:\n", + " print(\"ERROR:\", response.json()[\"message\"], file=sys.stderr)" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3 (ipykernel)", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.10.13" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/notebooks/set-secrets.ipynb b/notebooks/set-secrets.ipynb new file mode 100644 index 0000000000000000000000000000000000000000..fb7ea5ee6adf4f4d1e0a98d8f0594a77ebf4b0db --- /dev/null +++ b/notebooks/set-secrets.ipynb @@ -0,0 +1,224 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "id": "331918b2-c03c-4d14-9737-d65b94ed7e73", + "metadata": {}, + "source": [ + "# Add MAAP Secrets for Earthdata Login Credentials\n", + "\n", + "Earthdata Login credentials are required for downloading files from NASA DAACs. In order for a DPS job to be able to use such credentials in a safe manner, it looks for them in the MAAP Secrets store. Therefore, you must add the necessary secrets before a DPS job that requires them is submitted, otherwise the job will fail due to the missing secrets." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "cbdb9c92-6219-4d58-b9f4-206ee507ce51", + "metadata": {}, + "outputs": [], + "source": [ + "from maap.maap import MAAP" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "e2f0c041-646c-47e3-9c9f-4937ed9f7907", + "metadata": {}, + "outputs": [], + "source": [ + "def mask(value: str, *, show_last: int = 4) -> str:\n", + " \"\"\"Mask a value.\n", + " \n", + " Parameters\n", + " ----------\n", + " value:\n", + " Value to mask.\n", + " show_last:\n", + " Number of trailing characters to leave unmasked.\n", + " \n", + " Returns\n", + " -------\n", + " Masked value, where all but `show_last` characters of the specified value are\n", + " replaced with an asterisk (`*`). If necessary, the value is further left-padded\n", + " with asterisks to ensure there are at least as many asterisks as there are\n", + " unmasked trailing characters.\n", + " \"\"\"\n", + "\n", + " # Make sure the returned value contains at least the same number of masked\n", + " # characters as unmasked characters for a bit of added \"security.\"\n", + " n_masked = max(len(value), 2 * show_last) - show_last\n", + " \n", + " return f\"{'*' * n_masked}{value[-show_last:]}\"\n", + "\n", + "\n", + "def prompt_user(\n", + " prompt: str,\n", + " *,\n", + " value: str | None = None,\n", + " sensitive: bool = True,\n", + ") -> str | None:\n", + " \"\"\"Prompt user for a possibly sensitive input value.\n", + "\n", + " Parameters\n", + " ----------\n", + " prompt:\n", + " Human-readable prompt to present to the user, which is combined with `value`\n", + " when prompting the user.\n", + " value:\n", + " Current value (if any) of what to prompt the user for, which is combined with\n", + " `prompt` when prompting the user, and masked, if `senstive` is `True`.\n", + " sensitive:\n", + " If `True`, mask the current value (if any) when prompting the user, and also\n", + " avoid echoing user input. Otherwise, the current value is left unmasked and\n", + " user input is echoed.\n", + "\n", + " Returns\n", + " -------\n", + " Value entered by the user at the prompt, or `None` if either the user pressed the\n", + " Return or Enter key without supplying any input, or a KeyboardInterrupt occurred.\n", + " \"\"\"\n", + " from getpass import getpass\n", + " from IPython.display import clear_output\n", + "\n", + " masked_value = mask(value) if value and sensitive else value\n", + " get_user_input = getpass if sensitive else input\n", + "\n", + " try:\n", + " return get_user_input(f\"{prompt} [{masked_value}]:\") or None\n", + " except KeyboardInterrupt:\n", + " clear_output() # Remove prompt from cell output\n", + " raise\n", + "\n", + " \n", + "def get_secret(maap: MAAP, name: str) -> str | None:\n", + " \"\"\"Get the value of a MAAP secret, or `None` if the secret does not exist.\n", + " \n", + " Parameters\n", + " ----------\n", + " maap:\n", + " MAAP instance to use for secrets management.\n", + " name:\n", + " Name of the secret to get the value of.\n", + "\n", + " Returns\n", + " -------\n", + " Value of the secret, if it exists; otherwise `None`.\n", + " \"\"\"\n", + "\n", + " # Calling maap.secrets.get_secret returns the value of the requested secret, if it\n", + " # exists, but oddly returns an object with an error code if it does not exist,\n", + " # rather than simply (and conveniently) returning `None`.\n", + " \n", + " return value if isinstance(value := maap.secrets.get_secret(name), str) else None\n", + "\n", + "\n", + "def set_secret_interactively(\n", + " maap: MAAP,\n", + " *,\n", + " prompt: str,\n", + " name: str,\n", + " sensitive: bool = True,\n", + ") -> str | None:\n", + " \"\"\"Set the value of a secret by prompting the user for a value.\n", + "\n", + " If the secret already exists, the user prompt will include the existing value,\n", + " which will be masked if `sensitive` is `True` (showing only the last 4\n", + " characters of the value).\n", + "\n", + " If an empty input is provided (i.e., either the Enter or Return key is pressed at\n", + " the prompt, without any other input, or a KeyboardInterrupt occurs), the secret is\n", + " not set and `None` is returned. Otherwise, the secret is set to the user-supplied\n", + " value and that value is returned.\n", + "\n", + " Parameters\n", + " ----------\n", + " maap:\n", + " MAAP instance to use for secrets management.\n", + " prompt:\n", + " Human-readable prompt to display to user when prompting for input.\n", + " name:\n", + " Name of the secret.\n", + " sensitive:\n", + " If `True`, the prompt will mask the current secret value, if there is one.\n", + " Further, user input will not be echoed. Otherwise, the prompt will show the\n", + " full secret value, if there is one, and user input will be echoed.\n", + "\n", + " Returns\n", + " -------\n", + " User-supplied input provided at the prompt, or `None` if the Enter or Return key\n", + " was pressed without input.\n", + " \"\"\"\n", + " if (value := prompt_user(prompt, value=get_secret(maap, name), sensitive=sensitive)):\n", + " maap.secrets.add_secret(name, value)\n", + " masked_value = mask(value) if sensitive else value\n", + " print(f\"The secret {name} was set to {masked_value}.\")\n", + " return value\n", + "\n", + " print(f\"The value for secret {name} was left unchanged.\")" + ] + }, + { + "cell_type": "markdown", + "id": "97611ae6-837e-4021-9e4e-bee447749d5e", + "metadata": {}, + "source": [ + "## Create MAAP Instance for Managing Secrets" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "10612951-3392-4e32-983e-6e9a85c8c938", + "metadata": {}, + "outputs": [], + "source": [ + "maap = MAAP()" + ] + }, + { + "cell_type": "markdown", + "id": "98d52db0-7187-4aa3-a24a-7f3cbfce63c5", + "metadata": {}, + "source": [ + "## Store Earthdata Login Credentials as MAAP Secrets" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "986a47cb-0586-4f62-827b-6df6fcb609d4", + "metadata": {}, + "outputs": [], + "source": [ + "set_secret_interactively(\n", + " maap, prompt=\"Earthdata Login username\", name=\"EARTHDATA_USERNAME\", sensitive=False\n", + ")\n", + "set_secret_interactively(\n", + " maap, prompt=\"Earthdata Login password\", name=\"EARTHDATA_PASSWORD\"\n", + ");" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3 (ipykernel)", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.10.13" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/notebooks/submit-job.ipynb b/notebooks/submit-job.ipynb new file mode 100644 index 0000000000000000000000000000000000000000..092333e17326fc5ce0aba86b269a853d0cb72383 --- /dev/null +++ b/notebooks/submit-job.ipynb @@ -0,0 +1,171 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "id": "778cecaf-495e-4ab9-b8a7-7329b74fb49b", + "metadata": {}, + "source": [ + "# Submit an AOS Job" + ] + }, + { + "cell_type": "code", + "execution_count": 1, + "id": "2af023f9-90f5-4749-add8-bc81d566014a", + "metadata": {}, + "outputs": [], + "source": [ + "from datetime import date\n", + "\n", + "import ipywidgets as widgets\n", + "from maap.maap import MAAP\n", + "\n", + "DEFAULT_QUEUE = \"maap-dps-aos-worker-32gb\"" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "id": "50a503fc-8115-4b98-a5dc-4bd438b0f254", + "metadata": {}, + "outputs": [], + "source": [ + "def get_job_queues(maap: MAAP) -> list[str]:\n", + " response = maap.getQueues()\n", + " response.raise_for_status()\n", + "\n", + " return response.json()[\"queues\"]" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "id": "abd6f889-bfda-4592-b096-1b6bff1e120a", + "metadata": {}, + "outputs": [], + "source": [ + "def submit_aos_job(\n", + " maap: MAAP,\n", + " analysis_date: date,\n", + " queue: str = DEFAULT_QUEUE,\n", + ") -> str | None:\n", + " \"\"\"Submit an AOS job for an analysis date.\n", + "\n", + " Returns\n", + " -------\n", + " A MAAP job ID, if a job was successfully submitted; otherwise `None`.\n", + " \"\"\"\n", + " import re\n", + " import sys\n", + "\n", + " if analysis_date is None:\n", + " print(\"Specify a date\", file=sys.stderr)\n", + " return None\n", + "\n", + " analysis_date_str = analysis_date.strftime(\"%Y-%m-%d\")\n", + " job = maap.submitJob(\n", + " identifier=analysis_date_str, # custom job tag/label\n", + " algo_id=\"aos_maap_dps\", # algorithm name\n", + " version=\"main\", # algorithm version\n", + " queue=queue, # job queue\n", + " username=maap.profile.account_info()[\"username\"],\n", + " # Algorithm inputs:\n", + " analysis_date=analysis_date_str,\n", + " )\n", + "\n", + " if not job.id:\n", + " match = re.search(\n", + " r\"<ows:ExceptionText>(?P<msg>.*)</ows:ExceptionText>\", job.error_details\n", + " )\n", + " print(match[\"msg\"] if match else job.error_details, file=sys.stderr)\n", + " return None\n", + "\n", + " print(f\"Submitted job with job ID {job.id}\", file=sys.stderr)\n", + " return job.id" + ] + }, + { + "cell_type": "markdown", + "id": "bb0a6014-8fcd-474f-94c7-bbeb5f35f50c", + "metadata": {}, + "source": [ + "## Specify Inputs and Submit a Job" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "id": "987727e8-0cdb-4613-a9ef-071a7872abd4", + "metadata": {}, + "outputs": [], + "source": [ + "maap = MAAP()" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "id": "689c9988-ff4b-41a0-aa2b-427e96d2a16e", + "metadata": {}, + "outputs": [ + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "d01fe462138b43f3ba546af09d0ef2ef", + "version_major": 2, + "version_minor": 0 + }, + "text/plain": [ + "interactive(children=(DatePicker(value=None, description='Analysis Date:', step=1, style=DescriptionStyle(desc…" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "date_picker = widgets.DatePicker(\n", + " description=\"Analysis Date:\",\n", + " style=dict(description_width=\"initial\"),\n", + ")\n", + "\n", + "queues = get_job_queues(maap)\n", + "queue_picker = widgets.Dropdown(\n", + " description=\"Job Queue:\",\n", + " options=queues,\n", + " value=DEFAULT_QUEUE if DEFAULT_QUEUE in queues else None,\n", + " style=dict(description_width=\"initial\"),\n", + ")\n", + "\n", + "interactively = widgets.interact_manual.options(manual_name=\"Submit AOS Job\")\n", + "interactively(\n", + " submit_aos_job,\n", + " maap=widgets.fixed(maap),\n", + " analysis_date=date_picker,\n", + " queue=queue_picker\n", + ");" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3 (ipykernel)", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.10.13" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/run.sh b/run.sh new file mode 100644 index 0000000000000000000000000000000000000000..2740ed73365d06719d92155c008d5b0d993c8811 --- /dev/null +++ b/run.sh @@ -0,0 +1,28 @@ +#!/bin/bash + +set -euo pipefail + +readarray -d " " -t credentials <<<"$( + conda run --no-capture-output --name mytobac python -c ' +from maap.maap import MAAP +maap = MAAP() +username = maap.secrets.get_secret("EARTHDATA_USERNAME") +password = maap.secrets.get_secret("EARTHDATA_PASSWORD") +print(username, password) +' +)" + +EARTHDATA_USERNAME=$(echo -e "${credentials[0]}" | tr -d '[:space:]') +EARTHDATA_PASSWORD=$(echo -e "${credentials[1]}" | tr -d '[:space:]') +export EARTHDATA_USERNAME +export EARTHDATA_PASSWORD + +# All of the scripts below will use AOS_WORKING_DIR as the root output directory. +AOS_WORKING_DIR="${PWD}/output" +export AOS_WORKING_DIR + +analysis_date=$1 + +conda run --no-capture-output --name mytobac python /root/aos_test/src/acquire_and_convert_calipso.py "${analysis_date}" +conda run --no-capture-output --name mytobac python /root/aos_test/src/acquire_goes.py "${analysis_date}" +conda run --no-capture-output --name mytobac python /root/aos_test/src/process_object_tracking.py "${analysis_date}"