remove negative index bug in multicontrolnet, also support both regular and union controlnets for multi

Author: christopher-beckham

src/diffusers/models/controlnet_flux.py

@@ -479,13 +479,20 @@ def forward(
         # Regular Multi-ControlNets
         # load all ControlNets into memories
         else:
+            
             for i, (image, mode, scale, controlnet) in enumerate(
                 zip(controlnet_cond, controlnet_mode, conditioning_scale, self.nets)
             ):
+                if mode is not None:
+                    mode_ = torch.LongTensor([[mode]]).to(device=image.device)
+                else:
+                    mode_ = mode
+                if controlnet.union and mode_ is None:
+                    raise ValueError(f"controlnet #{i} in `self.nets` is a union model, controlnet_mode[{i}] cannot be `None`")
                 block_samples, single_block_samples = controlnet(
                     hidden_states=hidden_states,
                     controlnet_cond=image,
-                    controlnet_mode=mode[:, None],
+                    controlnet_mode=mode_,
                     conditioning_scale=scale,
                     timestep=timestep,
                     guidance=guidance,

src/diffusers/pipelines/flux/pipeline_flux_controlnet.py

@@ -749,6 +749,7 @@ def __call__(
 
             # set control mode
             orig_mode_type = type(control_mode)
+            
             if control_mode is not None:
                 control_mode = torch.tensor(control_mode).to(device, dtype=torch.long).view(-1,1)
                 if orig_mode_type == int:
@@ -788,16 +789,7 @@ def __call__(
             control_image = control_images
 
             # set control mode
-            control_mode_ = []
-            if isinstance(control_mode, list):
-                for cmode in control_mode:
-                    if cmode is None:
-                        control_mode_.append(-1)
-                    else:
-                        control_mode_.append(cmode)
-                control_mode = torch.tensor(control_mode_).to(device, dtype=torch.long)
-                control_mode = control_mode.view(-1, 1)
-            else:
+            if not isinstance(control_mode, list):
                 raise ValueError("For multi-controlnet, control_mode should be a list")
 
         # 4. Prepare latent variables