Skip to content
Snippets Groups Projects
Commit 7c963f17 authored by gcorradini's avatar gcorradini
Browse files

Merge branch 'gc/gh_actions_v2_vs_v3' into 'conus-dps'

Gc/gh actions v2 vs v3

See merge request !89
parents 9f3730c1 ba1d04a1
No related branches found
No related tags found
1 merge request!89Gc/gh actions v2 vs v3
# .github/actions/run-dps-job/action.yml
name: 'Run DPS Job V2'
description: 'Builds and runs a Docker image to kick off DPS jobs'
inputs:
algo_name:
description: 'Algorithm name'
required: true
github_ref:
description: 'Branch name or tag'
required: true
username:
description: 'Username'
required: true
queue:
description: 'Queue'
required: true
maap_image_env:
description: 'ubuntu'
required: true
runs:
using: "composite"
steps:
- shell: bash
run: |
docker run \
--env MAAP_PGT=${{ secrets.MAAP_PGT }} \
--env MAAP_CONF=/maap-py \
--pull=always -u ${UID}:${DOCKER_GID} -v ${PWD}:${PWD} -w ${PWD} \
--rm mas.maap-project.org/root/maap-workspaces/base_images/vanilla:v3.1.4 \
/opt/conda/envs/vanilla/bin/python maap_runtime/submit-dps-job.py \
${{ inputs.algo_name }} \
${{ inputs.github_ref }} \
${{ inputs.username }} \
${{ inputs.queue }} \
${{ inputs.maap_image_env }}
# .github/actions/run-dps-job/action.yml
name: 'Run DPS Job V3'
description: 'Builds and runs a Docker image to kick off DPS jobs'
inputs:
algo_name:
description: 'Algorithm name'
required: true
github_ref:
description: 'Branch name or tag'
required: true
username:
description: 'Username'
required: true
queue:
description: 'Queue'
required: true
maap_image_env:
description: 'MAAP image environment'
required: true
json_params:
description: 'json encoded params to pass to the job'
required: true
runs:
using: "composite"
steps:
- shell: bash
run: |
docker run \
--env MAAP_PGT=${{ secrets.MAAP_PGT }} \
--env MAAP_CONF=/maap-py \
--pull=always -u ${UID}:${DOCKER_GID} -v ${PWD}:${PWD} -w ${PWD} \
--rm mas.maap-project.org/root/maap-workspaces/base_images/vanilla:v3.1.4 \
/opt/conda/envs/vanilla/bin/python maap_runtime/submit-dps-job.py \
${{ inputs.algo_name }} \
${{ inputs.github_ref }} \
${{ inputs.username }} \
${{ inputs.queue }} \
${{ inputs.maap_image_env }} \
--params ${{ inputs.json_params }}
name: main
on:
workflow_dispatch:
jobs:
test:
if: contains('["ranchodeluxe"]', github.actor)
runs-on: ubuntu-latest
steps:
- name: Checkout repository
uses: actions/checkout@v2
with:
ref: conus-dps
- name: echo
run: |
echo 'hello world!'
name: conus nrt v3
on:
workflow_dispatch:
schedule:
- cron: '0 */11 * * *'
jobs:
run_dps_job_v3:
if: contains('["ranchodeluxe"]', github.actor)
runs-on: ubuntu-latest
environment: production
steps:
- name: checkout repository
uses: actions/checkout@v2
with:
ref: primarykeyv2
- name: pip install fireatlas
run: |
pip install .
- name: get docker GID and set DOCKER_GID environment variable
run: |
echo "DOCKER_GID=$(getent group docker | cut -d: -f3)" >> $GITHUB_ENV
- name: get current user UID and set UID environment variable
run: |
echo "UID=$(id -u)" >> $GITHUB_ENV
- name: print DOCKER_GID and UID
run: |
echo "Docker GID: $DOCKER_GID"
echo "User UID: $UID"
- name: set ted now
run: |
FULL_YEAR=$(date +"%Y")
MONTH_INT=$(date +"%-m") # Remove zero padding for month
DAY_INT=$(date +"%-d") # Remove zero padding for day
HOUR=$(date +"%I") # Use 12-hour format
MERIDIAN=$(date +"%p") # Get AM/PM
# NOTE: this has to be JSON encoded
echo "NOW_TED=\"[$FULL_YEAR, $MONTH_INT, $DAY_INT, \\\"$MERIDIAN\\\"]\"" >> $GITHUB_ENV
echo "##########################"
echo $NOW_TED
- name: kick off the DPS job
uses: Earth-Information-System/fireatlas/.github/actions/run-dps-job-v3@conus-dps
with:
algo_name: eis-feds-dask-coordinator-v3
github_ref: 1.0.0
username: gcorradini
queue: maap-dps-eis-worker-64gb
maap_image_env: ubuntu
json_params: '{"regnm": "CONUS", "bbox": "[-126.401171875,24.071240929282325,-61.36210937500001,49.40003415463647]", "tst": "[2024,1,1,\"AM\"]", "ted": $NOW_TED, "operation": "--coordinate-all"}'
name: data update checker v2
on:
workflow_dispatch:
schedule:
- cron: '0 */4 * * *'
jobs:
run_dps_job_v2:
if: contains('["ranchodeluxe"]', github.actor)
runs-on: ubuntu-latest
environment: production
steps:
- name: checkout repository
uses: actions/checkout@v2
with:
ref: conus-dps
- name: get docker GID and set DOCKER_GID environment variable
run: |
echo "DOCKER_GID=$(getent group docker | cut -d: -f3)" >> $GITHUB_ENV
- name: get current user UID and set UID environment variable
run: |
echo "UID=$(id -u)" >> $GITHUB_ENV
- name: print DOCKER_GID and UID
run: |
echo "Docker GID: $DOCKER_GID"
echo "User UID: $UID"
- name: kick off the DPS job
uses: Earth-Information-System/fireatlas/.github/actions/run-dps-job-v2@conus-dps
with:
algo_name: eis-fire-feds-dps-data-checker-v2
github_ref: conus-dps
username: gcorradini
queue: maap-dps-eis-worker-32gb
maap_image_env: ubuntu
name: data update checker name: data update checker v3
on: on:
workflow_dispatch: workflow_dispatch:
...@@ -6,7 +6,7 @@ on: ...@@ -6,7 +6,7 @@ on:
- cron: '0 */4 * * *' - cron: '0 */4 * * *'
jobs: jobs:
data_update_checker: run_dps_job_v3:
if: contains('["ranchodeluxe"]', github.actor) if: contains('["ranchodeluxe"]', github.actor)
runs-on: ubuntu-latest runs-on: ubuntu-latest
environment: production environment: production
...@@ -34,18 +34,13 @@ jobs: ...@@ -34,18 +34,13 @@ jobs:
echo "User UID: $UID" echo "User UID: $UID"
- name: kick off the DPS job - name: kick off the DPS job
run: | uses: Earth-Information-System/fireatlas/.github/actions/run-dps-job-v3@conus-dps
docker run \ with:
--env MAAP_PGT=${{ secrets.MAAP_PGT }} \ algo_name: eis-feds-dask-coordinator-v3
--env MAAP_CONF=/maap-py \ github_ref: 1.0.0
--pull=always -u ${UID}:${DOCKER_GID} -v ${PWD}:${PWD} -w ${PWD} \ username: gcorradini
--rm mas.maap-project.org/root/maap-workspaces/base_images/vanilla:v3.1.4 \ queue: maap-dps-eis-worker-64gb
/opt/conda/envs/vanilla/bin/python maap_runtime/submit-dps-job.py \ maap_image_env: ubuntu
eis-feds-dask-coordinator-v3 \ json_params '{"regnm": "", "bbox": "[]", "tst": "[]", "ted": "[]", "operation": "--data-update"}'
1.0.0 \
gcorradini \
maap-dps-eis-worker-64gb \
ubuntu \
--params '{"regnm": "", "bbox": "[]", "tst": "[]", "ted": "[]", "operation": "--data-update"}'
...@@ -6,7 +6,7 @@ from maap.maap import MAAP ...@@ -6,7 +6,7 @@ from maap.maap import MAAP
parser = argparse.ArgumentParser() parser = argparse.ArgumentParser()
parser.add_argument("algo", type=str, help="the name of the registered algorithm in MAAP") parser.add_argument("algo", type=str, help="the name of the registered algorithm in MAAP")
parser.add_argument("version", type=str, help="the git branch to run against") parser.add_argument("github_ref", type=str, help="the git branch or tag to run against")
parser.add_argument("username", type=str, help="the username who submits the job") parser.add_argument("username", type=str, help="the username who submits the job")
parser.add_argument("queue", type=str, help="the queue where we want the job to run") parser.add_argument("queue", type=str, help="the queue where we want the job to run")
parser.add_argument("maap_environment", type=str, help="the MAAP image environment to run the job inside of") parser.add_argument("maap_environment", type=str, help="the MAAP image environment to run the job inside of")
...@@ -15,12 +15,12 @@ parser.add_argument("--params", type=str, help="an optional json serialized stri ...@@ -15,12 +15,12 @@ parser.add_argument("--params", type=str, help="an optional json serialized stri
default="{}") default="{}")
def submit_job(algo_id, version, username, queue, maap_environment, params={}): def submit_job(algo_id, github_ref, username, queue, maap_environment, params={}):
maap = MAAP(maap_host='api.maap-project.org') maap = MAAP(maap_host='api.maap-project.org')
return maap.submitJob( return maap.submitJob(
identifier=f"job-{algo_id}:{version}", identifier=f"job-{algo_id}:{github_ref}",
algo_id=f"{algo_id}", # MAAP seems to expect `submitJob` to identify things this way algo_id=f"{algo_id}", # MAAP seems to expect `submitJob` to identify things this way
version=version, version=github_ref,
username=username, username=username,
queue=queue, queue=queue,
**params **params
...@@ -30,7 +30,7 @@ def submit_job(algo_id, version, username, queue, maap_environment, params={}): ...@@ -30,7 +30,7 @@ def submit_job(algo_id, version, username, queue, maap_environment, params={}):
if __name__ == '__main__': if __name__ == '__main__':
args = parser.parse_args() args = parser.parse_args()
deserialized_params = json.loads(args.params) deserialized_params = json.loads(args.params)
pargs = [args.algo, args.version, args.username, args.queue, args.maap_environment] pargs = [args.algo, args.github_ref, args.username, args.queue, args.maap_environment]
print(f"[ ARGS ]: {pargs}") print(f"[ ARGS ]: {pargs}")
response = submit_job(*pargs, params=deserialized_params) response = submit_job(*pargs, params=deserialized_params)
print(response) print(response)
......
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment