Disable use_cuda for local_sgd_integ_tests

Author: H-Huang

torchft/local_sgd_integ_test.py

@@ -16,7 +16,7 @@
 from torchft.local_sgd import DiLoCo, LocalSGD
 from torchft.manager import Manager
 from torchft.manager_integ_test import FailureInjector, MyModel, Runner
-from torchft.process_group import ProcessGroupBabyNCCL, ProcessGroupGloo
+from torchft.process_group import ProcessGroupGloo, ProcessGroupBabyNCCL
 
 logger: logging.Logger = logging.getLogger(__name__)
 
@@ -197,9 +197,11 @@ def state_dict() -> Dict[str, Dict[str, object]]:  # pyre-ignore[53]
 
 
 class LocalSGDIntegTest(TestCase):
+    # TODO: race condition due to using NCCL in threads causes manager allreduce to sometimes not be correct
+    # Because of that the test is disabled for cuda
     @parameterized.expand(
         [
-            (True,),
+            # (True,),
             (False,),
         ]
     )
@@ -259,7 +261,7 @@ def test_local_sgd_recovery(self, use_cuda: bool) -> None:
 
     @parameterized.expand(
         [
-            (True,),
+            # (True,),
             (False,),
         ]
     )
@@ -319,7 +321,7 @@ def test_diloco_healthy(self, use_cuda: bool) -> None:
 
     @parameterized.expand(
         [
-            (True,),
+            # (True,),
             (False,),
         ]
     )