Upgrade Modal

Author: mryab

.github/workflows/run-tests-on-modal.yml

@@ -39,7 +39,7 @@ jobs:
       - name: Install build dependencies
         run: |
           python -m pip install --upgrade pip
-          pip install modal==0.73.32
+          pip install modal==1.1.0
 
       - name: Run tests
         run: |
@@ -74,7 +74,7 @@ jobs:
       - name: Install build dependencies
         run: |
           python -m pip install --upgrade pip
-          pip install modal==0.73.32
+          pip install modal==1.1.0
 
       - name: Measure and upload coverage
         run: |
@@ -105,7 +105,7 @@ jobs:
       - name: Install build dependencies
         run: |
           python -m pip install --upgrade pip
-          pip install modal==0.73.32
+          pip install modal==1.1.0
 
       - name: Run p2pd tests
         run: |

modal_ci.py

@@ -15,13 +15,6 @@
             "cd bitsandbytes && cmake -DCOMPUTE_BACKEND=cpu -S . && make && pip --no-cache install . ",
         ]
     )
-    .add_local_dir("hivemind", remote_path="/root/hivemind/hivemind")
-    .add_local_file("requirements.txt", remote_path="/root/hivemind/requirements.txt")
-    .add_local_file("requirements-dev.txt", remote_path="/root/hivemind/requirements-dev.txt")
-    .add_local_file("requirements-docs.txt", remote_path="/root/hivemind/requirements-docs.txt")
-    .add_local_file("setup.py", remote_path="/root/hivemind/setup.py")
-    .add_local_file("pyproject.toml", remote_path="/root/hivemind/pyproject.toml")
-    .add_local_dir("tests", remote_path="/root/hivemind/tests")
 )
 
 # Create an image with golang and other system dependencies
@@ -41,13 +34,6 @@
             "cd bitsandbytes && cmake -DCOMPUTE_BACKEND=cpu -S . && make && pip --no-cache install . ",
         ]
     )
-    .add_local_dir("hivemind", remote_path="/root/hivemind/hivemind")
-    .add_local_file("requirements.txt", remote_path="/root/hivemind/requirements.txt")
-    .add_local_file("requirements-dev.txt", remote_path="/root/hivemind/requirements-dev.txt")
-    .add_local_file("requirements-docs.txt", remote_path="/root/hivemind/requirements-docs.txt")
-    .add_local_file("setup.py", remote_path="/root/hivemind/setup.py")
-    .add_local_file("pyproject.toml", remote_path="/root/hivemind/pyproject.toml")
-    .add_local_dir("tests", remote_path="/root/hivemind/tests")
 )
 
 
@@ -85,7 +71,12 @@ def setup_environment(*, build_p2pd=False):
     return environment
 
 
-@app.function(image=image, timeout=600, cpu=8, memory=8192)
+@app.function(
+    image=image.add_local_dir(".", remote_path="/root/hivemind"),
+    timeout=1200,
+    cpu=8,
+    memory=8192,
+)
 def run_tests():
     environment = setup_environment(build_p2pd=False)
 
@@ -104,7 +95,13 @@ def run_tests():
     )
 
 
-@app.function(image=image, timeout=900, cpu=8, memory=8192, secrets=[codecov_secret])
+@app.function(
+    image=image.add_local_dir(".", remote_path="/root/hivemind"),
+    timeout=900,
+    cpu=8,
+    memory=8192,
+    secrets=[codecov_secret],
+)
 def run_codecov():
     environment = setup_environment(build_p2pd=False)
 
@@ -145,7 +142,12 @@ def run_codecov():
     )
 
 
-@app.function(image=image_with_golang, timeout=600, cpu=1, memory=4096)
+@app.function(
+    image=image_with_golang.add_local_dir(".", remote_path="/root/hivemind"),
+    timeout=600,
+    cpu=1,
+    memory=4096,
+)
 def build_and_test_p2pd():
     environment = setup_environment(build_p2pd=True)
 

setup.py

@@ -11,6 +11,7 @@
 from setuptools import setup
 from setuptools.command.build_py import build_py
 from setuptools.command.develop import develop
+from setuptools.command.egg_info import egg_info
 
 P2PD_VERSION = "v0.5.0.hivemind1"
 
@@ -126,7 +127,7 @@ def finalize_options(self):
         super().finalize_options()
         # Convert string to boolean if needed
         if isinstance(self.buildgo, str):
-            self.buildgo = self.buildgo.lower() in ('true', '1', 'yes', 'on')
+            self.buildgo = self.buildgo.lower() in ("true", "1", "yes", "on")
 
     def run(self):
         if self.buildgo:
@@ -146,6 +147,13 @@ def run(self):
         super().run()
 
 
+class EggInfo(egg_info):
+    def run(self):
+        # Generate protobuf files early in the build process
+        proto_compile(os.path.join(here, "hivemind", "proto"))
+        super().run()
+
+
 setup(
-    cmdclass={"build_py": BuildPy, "develop": Develop},
+    cmdclass={"build_py": BuildPy, "develop": Develop, "egg_info": EggInfo},
 )