Skip to content
Snippets Groups Projects
Commit 0b31a1a5 authored by ranchodeluxe's avatar ranchodeluxe
Browse files

small changes to run independently

parent 29a469f0
No related branches found
No related tags found
1 merge request!61Additions: primary keys, speed ups (preprocessing and postprocessing), tests, scalene, reorg entire fireatlas
import json
import argparse
import fireatlas
from dask import delayed
from fireatlas.utils import timed
from fireatlas import FireRunDaskCoordinator
from fireatlas import settings
from fireatlas import FireLog
from fireatlas.FireTime import t_generator
def validate_json(s):
......@@ -16,9 +18,15 @@ def validate_json(s):
@timed
def Run(region: fireatlas.FireType.Region, tst: fireatlas.FireType.TimeStep):
FireLog.logger.info(f"Running preprocessing code for {region[0]} at {tst=} with source {settings.FIRE_SOURCE}")
FireRunDaskCoordinator.job_preprocess_region_t([None,], region, tst)
def Run(region: fireatlas.FireType.Region, tst: fireatlas.FireType.TimeStep, ted: fireatlas.FireType.TimeStep):
FireLog.logger.info(f"Running preprocess region-at-t code for {region[0]} at {tst=} with source {settings.FIRE_SOURCE}")
list_of_timesteps = list(t_generator(tst, ted))
region_and_t_results = [
FireRunDaskCoordinator.job_preprocess_region_t([None,], region, t)
for t in list_of_timesteps
]
dag = delayed(lambda x: x)(region_and_t_results)
dag.compute()
if __name__ == "__main__":
......@@ -31,5 +39,6 @@ if __name__ == "__main__":
parser = argparse.ArgumentParser()
parser.add_argument("--regnm", type=str)
parser.add_argument("--tst", type=validate_json)
parser.add_argument("--ted", type=validate_json)
args = parser.parse_args()
Run([args.regnm, None], args.tst)
import json
import argparse
import glob
from dask import delayed
from fireatlas.utils import timed
from fireatlas import FireRunDaskCoordinator
from fireatlas import settings
@timed
......@@ -14,7 +16,14 @@ def Run():
:return: None
"""
FireRunDaskCoordinator.job_data_update_checker()
data_update_results = FireRunDaskCoordinator.job_data_update_checker()
data_upload_results = [
FireRunDaskCoordinator.concurrent_copy_inputs_from_local_to_s3([data_update_results,], local_filepath)
for local_filepath in glob.glob(f"{settings.LOCAL_PATH}/{settings.PREPROCESSED_DIR}/*/*.txt")
]
dag = delayed(lambda x: x)(data_upload_results)
dag.compute()
if __name__ == "__main__":
......
import json
import glob
import os
import argparse
import fireatlas
from dask import delayed
from itertools import chain
from fireatlas.utils import timed
from fireatlas import FireRunDaskCoordinator
from fireatlas import settings
def validate_json(s):
......@@ -15,8 +21,21 @@ def validate_json(s):
@timed
def Run(region: fireatlas.FireTypes.Region, tst: fireatlas.FireTypes.TimeStep, ted: fireatlas.FireTypes.TimeStep):
FireRunDaskCoordinator.job_fire_forward([None, ], region, tst, ted)
"""
"""
fire_forward_results = FireRunDaskCoordinator.job_fire_forward([None, ], region, tst, ted)
# take all fire forward output and upload all snapshots/largefire outputs in parallel
data_dir = os.path.join(settings.LOCAL_PATH, settings.OUTPUT_DIR, region[0], str(tst[0]))
fgb_upload_results = [
FireRunDaskCoordinator.concurrent_copy_outputs_from_local_to_s3(fire_forward_results, local_filepath)
for local_filepath in list(chain(
glob.glob(os.path.join(data_dir, "Snapshot", "*", "*.fgb")),
glob.glob(os.path.join(data_dir, "Largefire", "*", "*.fgb"))
))
]
dag = delayed(lambda x: x)(fgb_upload_results)
dag.compute()
if __name__ == "__main__":
""" The main code to run time forwarding for a time period
......
......@@ -15,7 +15,8 @@ def validate_json(s):
@timed
def Run(region: fireatlas.FireTypes.Region):
FireRunDaskCoordinator.job_preprocess_region(region)
dag = FireRunDaskCoordinator.job_preprocess_region(region)
dag.compute()
if __name__ == "__main__":
......
......@@ -108,7 +108,7 @@ pushd "$basedir"
--outfile "${output_dir}/profile.html" --- FireRunPreprocessRegion.py --regnm=$regnm --bbox="$bbox"
elif [[ $selected_flag == "preprocess-region-t" ]]; then
scalene --cli --no-browser --reduced-profile --html --column-width 180 \
--outfile "${output_dir}/profile.html" --- FireRunByRegionAndT.py --regnm=$regnm --tst="$tst"
--outfile "${output_dir}/profile.html" --- FireRunByRegionAndT.py --regnm=$regnm --tst="$tst" --ted="$ted"
elif [[ $selected_flag == "fire-forward" ]]; then
scalene --cli --no-browser --reduced-profile --html --column-width 180 \
--outfile "${output_dir}/profile.html" --- FireRunFireForward.py --regnm=$regnm --tst="$tst" --ted="$ted"
......
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment