Skip to content

Commit 1bfc0bc

Browse files
julietcohenjulietcohen
committed
scripts modified for full high ice run on Delta
1 parent 574d7fc commit 1bfc0bc

7 files changed

+49
-42
lines changed

‎IN_PROGRESS_VIZ_WORKFLOW.py‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -73,7 +73,7 @@ def main():
7373

7474
# (optionally) Comment out steps you don't need 😁
7575
# todo: sync footprints to nodes.
76-
step0_staging()
76+
# step0_staging()
7777
# todo: rsync staging to /scratch
7878
# todo: merge staged files in /scratch # ./merge_staged_vector_tiles.py
7979
# DO NOT RUN 3d-tiling UNTIL WORKFLOW CAN ACCOMODATE FILE HIERARCHY:step1_3d_tiles() # default parameter batch_size = 300
@@ -83,7 +83,7 @@ def main():
8383
# step3_raster_lower(batch_size_geotiffs=100) # rasterize all LOWER Z levels
8484
# todo: immediately after initiating above step, start rsync script to continuously sync geotiff files,
8585
# or immediately after the above step is done, rsync all files at once if there is time left in job
86-
# step4_webtiles(batch_size_web_tiles=250) # convert to web tiles.
86+
step4_webtiles(batch_size_web_tiles=250) # convert to web tiles.
8787

8888
# mem_testing = False
8989
# if mem_testing:

‎PRODUCTION_IWP_CONFIG.py‎

Lines changed: 6 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -8,11 +8,11 @@
88
# define user on Delta, avoid writing files to other user's dir
99
user = subprocess.check_output("whoami").strip().decode("ascii")
1010
#head_node = 'cn___/'
11-
head_node = 'gpub___/'
11+
head_node = 'gpub020/'
1212
# define desired location for output files within user dir
1313
# ensures a new subfolder every run as long as new run is not started within same day as last run
1414
# following path is the output subdir for test run, using just on subdir of the alaska files that is only ~8% of the Alaska dir, 23.5 GB
15-
output_subdir = 'IWP/output/iwp_testRun'
15+
output_subdir = 'IWP/output/iwp_high_20230516'
1616
#output_subdir = datetime.now().strftime("%b-%d-%y")
1717
# don't use subprocess to retrieve date for subdir because runs might span over 2 days if they go overnight
1818

@@ -23,7 +23,8 @@
2323
# input path
2424
#INPUT = '/scratch/bbou/julietcohen/IWP/input/2023-01-19/iwp_files/high/' # The output data of MAPLE. Which is the input data for STAGING.
2525
#INPUT = '/scratch/bbou/julietcohen/IWP/input/2023-01-19/iwp_files/high/russia/226_227_iwp/'
26-
INPUT = '/scratch/bbou/julietcohen/IWP/input/few_adjacent_russia/iwp/'
26+
#INPUT = '/scratch/bbou/julietcohen/IWP/input/few_adjacent_russia/iwp/'
27+
INPUT = '/scratch/bbou/julietcohen/IWP/input/2023-01-19/iwp_files/high/'
2728

2829
# output path
2930
OUTPUT = f'/scratch/bbou/{user}/{output_subdir}/' # Dir for results. High I/O is good.
@@ -32,7 +33,8 @@
3233
FOOTPRINTS_LOCAL = '/tmp/staged_footprints/'
3334
#FOOTPRINTS_REMOTE = '/scratch/bbou/julietcohen/IWP/footprint_files_with_date_20230119/high/'
3435
#FOOTPRINTS_REMOTE = '/scratch/bbou/julietcohen/IWP/footprint_files_with_date_20230119/high/russia/226_227_iwp/'
35-
FOOTPRINTS_REMOTE = '/scratch/bbou/julietcohen/IWP/input/few_adjacent_russia/footprints/'
36+
#FOOTPRINTS_REMOTE = '/scratch/bbou/julietcohen/IWP/input/few_adjacent_russia/footprints/'
37+
FOOTPRINTS_REMOTE = '/scratch/bbou/julietcohen/IWP/footprint_files_with_date_20230119/high/'
3638

3739
# staging paths
3840
STAGING_LOCAL = '/tmp/staged/'

‎merge_staged_vector_tiles.py‎

Lines changed: 27 additions & 24 deletions
Original file line numberDiff line numberDiff line change
@@ -40,33 +40,38 @@
4040
def main():
4141
'''
4242
Usage Instructions:
43-
- Set the list of paths to the vector tiles `staged_dir_paths_list`
44-
- Choose one of the paths from the last step, remove the head node
45-
from the `staged_dir_paths_list` and set it as the `merged_dir_path`
43+
- Enter the HEAD node (lowest # node) at the end of the path for `merged_dir_path`
44+
- Enter all worker nodes (all other nodes) at the end of the paths that make
45+
up `staged_dir_paths_list`
4646
'''
4747

4848
#######################
4949
#### Change me 😁 ####
5050
#######################
5151
# todo -- get files from dirs automatically, using os.lsdir().
5252
#BASE_DIR = '/scratch/bbou/julietcohen/IWP/output/...'
53-
merged_dir_path = f"{IWP_CONFIG['dir_staged']}cn___" # this path SHOULD NOT be in the `staged_dir_paths_list`
53+
merged_dir_path = f"{IWP_CONFIG['dir_staged']}gpub020" # this path SHOULD NOT be in the `staged_dir_paths_list`
5454
staged_dir_paths_list = [
55-
f"{IWP_CONFIG['dir_staged']}cn___",
56-
f"{IWP_CONFIG['dir_staged']}cn___",
57-
f"{IWP_CONFIG['dir_staged']}cn___",
58-
# f"{IWP_CONFIG['dir_staged']}cn___",
59-
# f"{IWP_CONFIG['dir_staged']}cn___",
60-
# f"{IWP_CONFIG['dir_staged']}cn___",
61-
# f"{IWP_CONFIG['dir_staged']}cn___",
62-
# f"{IWP_CONFIG['dir_staged']}cn___",
63-
# f"{IWP_CONFIG['dir_staged']}cn___",
64-
# f"{IWP_CONFIG['dir_staged']}cn___",
65-
# f"{IWP_CONFIG['dir_staged']}cn___",
66-
# f"{IWP_CONFIG['dir_staged']}cn___",
67-
# f"{IWP_CONFIG['dir_staged']}cn___",
68-
# f"{IWP_CONFIG['dir_staged']}cn___",
69-
# f"{IWP_CONFIG['dir_staged']}cn___",
55+
f"{IWP_CONFIG['dir_staged']}gpub041",
56+
#f"{IWP_CONFIG['dir_staged']}gpub041",
57+
#f"{IWP_CONFIG['dir_staged']}gpub041",
58+
#f"{IWP_CONFIG['dir_staged']}gpub038",
59+
#f"{IWP_CONFIG['dir_staged']}gpub027",
60+
#f"{IWP_CONFIG['dir_staged']}gpub028",
61+
#f"{IWP_CONFIG['dir_staged']}gpub029",
62+
#f"{IWP_CONFIG['dir_staged']}gpub030",
63+
#f"{IWP_CONFIG['dir_staged']}gpub031",
64+
#f"{IWP_CONFIG['dir_staged']}gpub032",
65+
#f"{IWP_CONFIG['dir_staged']}gpub033",
66+
#f"{IWP_CONFIG['dir_staged']}gpub034",
67+
#f"{IWP_CONFIG['dir_staged']}gpub035",
68+
#f"{IWP_CONFIG['dir_staged']}gpub036",
69+
#f"{IWP_CONFIG['dir_staged']}gpub037",
70+
#f"{IWP_CONFIG['dir_staged']}gpub038",
71+
#f"{IWP_CONFIG['dir_staged']}gpub039",
72+
#f"{IWP_CONFIG['dir_staged']}gpub040",
73+
#f"{IWP_CONFIG['dir_staged']}gpub041",
74+
#f"{IWP_CONFIG['dir_staged']}gpub___",
7075
]
7176
##############################
7277
#### END OF Change me 😁 ####
@@ -338,11 +343,9 @@ def merge_tile(incoming_tile_in_path, incoming_tile_out_path, isDestructive, sta
338343
dedup_method = stager.config.get_deduplication_method()
339344
if dedup_method is not None:
340345
mode = 'w'
341-
## concatenate the geodataframes:
342-
## for IWP data, we do not set the config to dedup at staging
343-
## so combine_and_deduplicate() does NOT deduplicate here,
344-
## we are just ensuring that the tiles within the head node
345-
## contains all the polygons for that tile, regardless of which node
346+
## concatenate the geodataframes and identify duplicates:
347+
## ensuring that the tiles within the head node contains
348+
## all the polygons for that tile, regardless of which node
346349
## processed each polygon
347350
incoming_gdf = stager.combine_and_deduplicate(
348351
incoming_gdf, incoming_tile_out_path)

‎rsync_log_to_scratch.py‎

Lines changed: 4 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -6,18 +6,19 @@
66
import pprint
77
import PRODUCTION_IWP_CONFIG
88
IWP_CONFIG = PRODUCTION_IWP_CONFIG.IWP_CONFIG
9+
IWP_CONFIG2 = IWP_CONFIG.copy()
910

1011
#import lake_change_config
1112
#IWP_CONFIG = lake_change_config.IWP_CONFIG
1213

1314
# set config properties for current context
1415
#IWP_CONFIG['dir_staged'] = IWP_CONFIG['dir_staged_local']
1516
SOURCE = '/tmp/' + 'log.log'
16-
IWP_CONFIG['dir_staged'] = IWP_CONFIG['dir_staged_remote']
17-
DESTINATION = IWP_CONFIG['dir_staged']
17+
IWP_CONFIG2['dir_staged'] = IWP_CONFIG2['dir_staged_remote']
18+
DESTINATION = IWP_CONFIG2['dir_staged']
1819

1920
print("Using config: ")
20-
pprint.pprint(IWP_CONFIG)
21+
pprint.pprint(IWP_CONFIG2)
2122

2223
# define user on Delta, avoid writing files to other user's dir
2324
user = subprocess.check_output("whoami").strip().decode("ascii")

‎rsync_staging_to_scratch.py‎

Lines changed: 6 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -7,18 +7,19 @@
77
import pprint
88
import PRODUCTION_IWP_CONFIG
99
IWP_CONFIG = PRODUCTION_IWP_CONFIG.IWP_CONFIG
10+
IWP_CONFIG2 = IWP_CONFIG.copy()
1011

1112
#import lake_change_config
1213
#IWP_CONFIG = lake_change_config.IWP_CONFIG
1314

1415
# set config properties for current context
15-
IWP_CONFIG['dir_staged'] = IWP_CONFIG['dir_staged_local']
16-
SOURCE = IWP_CONFIG['dir_staged']
17-
IWP_CONFIG['dir_staged'] = IWP_CONFIG['dir_staged_remote']
18-
DESTINATION = IWP_CONFIG['dir_staged']
16+
IWP_CONFIG2['dir_staged'] = IWP_CONFIG2['dir_staged_local']
17+
SOURCE = IWP_CONFIG2['dir_staged']
18+
IWP_CONFIG2['dir_staged'] = IWP_CONFIG2['dir_staged_remote']
19+
DESTINATION = IWP_CONFIG2['dir_staged']
1920

2021
print("Using config: ")
21-
pprint.pprint(IWP_CONFIG)
22+
pprint.pprint(IWP_CONFIG2)
2223

2324
# define user on Delta, avoid writing files to other user's dir
2425
user = subprocess.check_output("whoami").strip().decode("ascii")

‎slurm/BEST-v2_gpu_ray.slurm‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -3,11 +3,11 @@
33
#SBATCH --job-name=pdg_viz
44
#SBATCH --partition=gpuA40x4
55
#SBATCH --account=
6-
#SBATCH --time=04:00:00
6+
#SBATCH --time=17:00:00
77

88
#SBATCH --export=ALL,RAY_worker_register_timeout_seconds=120
99

10-
#SBATCH --nodes=1
10+
#SBATCH --nodes=20
1111
#SBATCH --gpus-per-node=1
1212
#SBATCH --mem=0
1313
#SBATCH --exclusive

‎slurm/BEST_cpu_ray_double_srun.slurm‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -3,11 +3,11 @@
33
#SBATCH --job-name=pdg_viz
44
#SBATCH --partition=cpu
55
#SBATCH --account=
6-
#SBATCH --time=01:00:00
6+
#SBATCH --time=24:00:00
77

88
#SBATCH --export=ALL,RAY_worker_register_timeout_seconds=120
99

10-
#SBATCH --nodes=1
10+
#SBATCH --nodes=20
1111
#SBATCH --mem=0
1212
#SBATCH --exclusive
1313

0 commit comments

Comments
 (0)