fix formatting

Author: priyakasimbeg

scoring/score_submissions.py

@@ -160,6 +160,7 @@ def compute_leaderboard_score(df, normalize=True):
 def main(_):
   results = {}
   os.makedirs(FLAGS.output_dir, exist_ok=True)
+  logging.info(f"Scoring submissions in {FLAGS.submission_directory}")
 
   # Optionally read results to filename
   if FLAGS.load_results_from_filename:
@@ -172,8 +173,7 @@ def main(_):
       print(submission)
       if submission in FLAGS.exclude_submissions.split(','):
         continue
-      experiment_path = os.path.join(FLAGS.submission_directory,
-                                      submission)
+      experiment_path = os.path.join(FLAGS.submission_directory, submission)
       df = scoring_utils.get_experiment_df(experiment_path)
       results[submission] = df
       summary_df = get_submission_summary(df)

scoring/utils/slurm/make_job_config.py

@@ -1,15 +1,16 @@
 import json
-from absl import flags
+import os
+
 from absl import app
+from absl import flags
 import jax
-import os
 
 SUBMISSION_PATH = '/submissions_algorithms/external_tuning/shampoo_submission/submission.py'
 TUNING_SEARCH_SPACE = '/submissions_algorithms/external_tuning/shampoo_submission/tuning_search_space.json'
 EXPERIMENT_DIR = 'submissions/rolling_leaderboard/external_tuning/shampoo'
 FRAMEWORK = 'pytorch'
 
-flags.DEFINE_string('submission_path', 
+flags.DEFINE_string('submission_path',
                     SUBMISSION_PATH,
                     'Path to submission module.')
 flags.DEFINE_string('tuning_search_space',
@@ -18,63 +19,47 @@
 flags.DEFINE_string('experiment_dir',
                     EXPERIMENT_DIR,
                     'Path to experiment dir where logs will be saved.')
-flags.DEFINE_enum('framework',
-                  FRAMEWORK,
-                  enum_values=['jax', 'pytorch'],
-                  help='Can be either pytorch or jax.')  
-flags.DEFINE_integer('seed',
-                     0, 
-                     'RNG seed to to generate study seeds from.')
-flags.DEFINE_enum('tuning_ruleset',
-                  'external',
-                  enum_values=['external', 'self'],
-                  help='Which tuning ruleset to score this submission on. Can be external or self.'
-                 )
+flags.DEFINE_enum(
+    'framework',
+    FRAMEWORK,
+    enum_values=['jax', 'pytorch'],
+    help='Can be either pytorch or jax.')
+flags.DEFINE_integer('seed', 0, 'RNG seed to to generate study seeds from.')
+flags.DEFINE_enum(
+    'tuning_ruleset',
+    'external',
+    enum_values=['external', 'self'],
+    help='Which tuning ruleset to score this submission on. Can be external or self.'
+)
 
 FLAGS = flags.FLAGS
 
-
 MIN_INT = -2**(31)
 MAX_INT = 2**(31) - 1
 NUM_TUNING_TRIALS = 5  # For external tuning ruleset
 NUM_STUDIES = 3
 
 WORKLOADS = {
-  "imagenet_resnet": {
-    "dataset": "imagenet"
-  },
-  "imagenet_vit": {
-    "dataset": "imagenet"
-  },
-  "fastmri": {
-    "dataset": "fastmri"
-  },
-  "ogbg": {
-    "dataset": "ogbg"
-  },
-  "wmt": {
-    "dataset": "wmt"
-  },
-  "librispeech_deepspeech": {
-    "dataset": "librispeech"
-  },
-  "criteo1tb": {
-    "dataset": "criteo1tb"
-  },
-  "librispeech_conformer": {
-    "dataset": "librispeech"
-  }
+    "imagenet_resnet": {"dataset": "imagenet"},
+    "imagenet_vit": {"dataset": "imagenet"},
+    "fastmri": {"dataset": "fastmri"},
+    "ogbg": {"dataset": "ogbg"},
+    "wmt": {"dataset": "wmt"},
+    "librispeech_deepspeech": {"dataset": "librispeech"},
+    "criteo1tb": {"dataset": "criteo1tb"},
+    "librispeech_conformer": {"dataset": "librispeech"}
 }
 
+
 def main(_):
-  workloads=WORKLOADS.keys()
+  workloads = WORKLOADS.keys()
   key = jax.random.key(FLAGS.seed)
 
   jobs = []
 
   for workload in workloads:
     # Fold in hash(workload) mod(max(uint32))
-    workload_key = jax.random.fold_in(key, hash(workload) % (2**32-1))
+    workload_key = jax.random.fold_in(key, hash(workload) % (2**32 - 1))
     for study_index in range(NUM_STUDIES):
       study_key = jax.random.fold_in(workload_key, study_index)
       for hparam_index in range(NUM_TUNING_TRIALS):
@@ -99,7 +84,6 @@ def main(_):
         jobs.append(job)
         print(job)
 
-
   # Convert job array to dict with job indices
   job_dict = {}
   for i, job in enumerate(jobs):
@@ -110,4 +94,4 @@ def main(_):
 
 
 if __name__ == '__main__':
-  app.run(main)
+  app.run(main)