feat(ui): pull bbox into functionality for control/ip adapters

Author: psychedelicious

invokeai/frontend/web/public/locales/en.json

@@ -1782,6 +1782,8 @@
         "flipVertical": "Flip Vertical",
         "stagingOnCanvas": "Staging images on",
         "replaceLayer": "Replace Layer",
+        "pullBboxIntoLayer": "Pull Bbox into Layer",
+        "pullBboxIntoIPAdapter": "Pull Bbox into IP Adapter",
         "fill": {
             "fillColor": "Fill Color",
             "fillStyle": "Fill Style",

invokeai/frontend/web/src/features/controlLayers/components/ControlLayer/ControlLayerControlAdapter.tsx

@@ -6,6 +6,7 @@ import { Weight } from 'features/controlLayers/components/common/Weight';
 import { ControlLayerControlAdapterControlMode } from 'features/controlLayers/components/ControlLayer/ControlLayerControlAdapterControlMode';
 import { ControlLayerControlAdapterModel } from 'features/controlLayers/components/ControlLayer/ControlLayerControlAdapterModel';
 import { useEntityIdentifierContext } from 'features/controlLayers/contexts/EntityIdentifierContext';
+import { useIsSavingCanvas, usePullBboxIntoLayer } from 'features/controlLayers/hooks/saveCanvasHooks';
 import { useEntityFilter } from 'features/controlLayers/hooks/useEntityFilter';
 import {
   controlLayerBeginEndStepPctChanged,
@@ -17,7 +18,7 @@ import { selectCanvasSlice, selectEntityOrThrow } from 'features/controlLayers/s
 import type { CanvasEntityIdentifier, ControlModeV2 } from 'features/controlLayers/store/types';
 import { memo, useCallback, useMemo } from 'react';
 import { useTranslation } from 'react-i18next';
-import { PiShootingStarBold } from 'react-icons/pi';
+import { PiBoundingBoxBold, PiShootingStarBold } from 'react-icons/pi';
 import type { ControlNetModelConfig, T2IAdapterModelConfig } from 'services/api/types';
 
 const useControlLayerControlAdapter = (entityIdentifier: CanvasEntityIdentifier<'control_layer'>) => {
@@ -68,6 +69,9 @@ export const ControlLayerControlAdapter = memo(() => {
     [dispatch, entityIdentifier]
   );
 
+  const pullBboxIntoLayer = usePullBboxIntoLayer(entityIdentifier);
+  const isSaving = useIsSavingCanvas();
+
   return (
     <Flex flexDir="column" gap={3} position="relative" w="full">
       <Flex w="full" gap={2}>
@@ -80,6 +84,14 @@ export const ControlLayerControlAdapter = memo(() => {
           tooltip={t('controlLayers.filter.filter')}
           icon={<PiShootingStarBold />}
         />
+        <IconButton
+          onClick={pullBboxIntoLayer}
+          isLoading={isSaving.isTrue}
+          variant="ghost"
+          aria-label={t('controlLayers.pullBboxIntoLayer')}
+          tooltip={t('controlLayers.pullBboxIntoLayer')}
+          icon={<PiBoundingBoxBold />}
+        />
       </Flex>
       <Weight weight={controlAdapter.weight} onChange={onChangeWeight} />
       <BeginEndStepPct beginEndStepPct={controlAdapter.beginEndStepPct} onChange={onChangeBeginEndStepPct} />

invokeai/frontend/web/src/features/controlLayers/components/IPAdapter/IPAdapterSettings.tsx

@@ -1,11 +1,12 @@
-import { Box, Flex } from '@invoke-ai/ui-library';
+import { Box, Flex, IconButton } from '@invoke-ai/ui-library';
 import { createSelector } from '@reduxjs/toolkit';
 import { useAppDispatch, useAppSelector } from 'app/store/storeHooks';
 import { BeginEndStepPct } from 'features/controlLayers/components/common/BeginEndStepPct';
 import { CanvasEntitySettingsWrapper } from 'features/controlLayers/components/common/CanvasEntitySettingsWrapper';
 import { Weight } from 'features/controlLayers/components/common/Weight';
 import { IPAdapterMethod } from 'features/controlLayers/components/IPAdapter/IPAdapterMethod';
 import { useEntityIdentifierContext } from 'features/controlLayers/contexts/EntityIdentifierContext';
+import { useIsSavingCanvas, usePullBboxIntoIPAdapter } from 'features/controlLayers/hooks/saveCanvasHooks';
 import {
   ipaBeginEndStepPctChanged,
   ipaCLIPVisionModelChanged,
@@ -18,12 +19,15 @@ import { selectCanvasSlice, selectEntityOrThrow } from 'features/controlLayers/s
 import type { CLIPVisionModelV2, IPMethodV2 } from 'features/controlLayers/store/types';
 import type { IPAImageDropData } from 'features/dnd/types';
 import { memo, useCallback, useMemo } from 'react';
+import { useTranslation } from 'react-i18next';
+import { PiBoundingBoxBold } from 'react-icons/pi';
 import type { ImageDTO, IPAdapterModelConfig, IPALayerImagePostUploadAction } from 'services/api/types';
 
 import { IPAdapterImagePreview } from './IPAdapterImagePreview';
 import { IPAdapterModel } from './IPAdapterModel';
 
 export const IPAdapterSettings = memo(() => {
+  const { t } = useTranslation();
   const dispatch = useAppDispatch();
   const entityIdentifier = useEntityIdentifierContext('ip_adapter');
   const selectIPAdapter = useMemo(
@@ -82,6 +86,8 @@ export const IPAdapterSettings = memo(() => {
     () => ({ type: 'SET_IPA_IMAGE', id: entityIdentifier.id }),
     [entityIdentifier.id]
   );
+  const pullBboxIntoIPAdapter = usePullBboxIntoIPAdapter(entityIdentifier);
+  const isSaving = useIsSavingCanvas();
 
   return (
     <CanvasEntitySettingsWrapper>
@@ -95,6 +101,14 @@ export const IPAdapterSettings = memo(() => {
               onChangeCLIPVisionModel={onChangeCLIPVisionModel}
             />
           </Box>
+          <IconButton
+            onClick={pullBboxIntoIPAdapter}
+            isLoading={isSaving.isTrue}
+            variant="ghost"
+            aria-label={t('controlLayers.pullBboxIntoIPAdapter')}
+            tooltip={t('controlLayers.pullBboxIntoIPAdapter')}
+            icon={<PiBoundingBoxBold />}
+          />
         </Flex>
         <Flex gap={4} w="full" alignItems="center">
           <Flex flexDir="column" gap={3} w="full">

invokeai/frontend/web/src/features/controlLayers/components/RegionalGuidance/RegionalGuidanceIPAdapterSettings.tsx

@@ -7,6 +7,10 @@ import { IPAdapterImagePreview } from 'features/controlLayers/components/IPAdapt
 import { IPAdapterMethod } from 'features/controlLayers/components/IPAdapter/IPAdapterMethod';
 import { IPAdapterModel } from 'features/controlLayers/components/IPAdapter/IPAdapterModel';
 import { useEntityIdentifierContext } from 'features/controlLayers/contexts/EntityIdentifierContext';
+import {
+  useIsSavingCanvas,
+  usePullBboxIntoRegionalGuidanceIPAdapter,
+} from 'features/controlLayers/hooks/saveCanvasHooks';
 import {
   rgIPAdapterBeginEndStepPctChanged,
   rgIPAdapterCLIPVisionModelChanged,
@@ -20,7 +24,8 @@ import { selectCanvasSlice, selectRegionalGuidanceIPAdapter } from 'features/con
 import type { CLIPVisionModelV2, IPMethodV2 } from 'features/controlLayers/store/types';
 import type { RGIPAdapterImageDropData } from 'features/dnd/types';
 import { memo, useCallback, useMemo } from 'react';
-import { PiTrashSimpleBold } from 'react-icons/pi';
+import { useTranslation } from 'react-i18next';
+import { PiBoundingBoxBold, PiTrashSimpleBold } from 'react-icons/pi';
 import type { ImageDTO, IPAdapterModelConfig, RGIPAdapterImagePostUploadAction } from 'services/api/types';
 import { assert } from 'tsafe';
 
@@ -31,6 +36,7 @@ type Props = {
 
 export const RegionalGuidanceIPAdapterSettings = memo(({ ipAdapterId, ipAdapterNumber }: Props) => {
   const entityIdentifier = useEntityIdentifierContext('regional_guidance');
+  const { t } = useTranslation();
   const dispatch = useAppDispatch();
   const onDeleteIPAdapter = useCallback(() => {
     dispatch(rgIPAdapterDeleted({ entityIdentifier, ipAdapterId }));
@@ -100,6 +106,8 @@ export const RegionalGuidanceIPAdapterSettings = memo(({ ipAdapterId, ipAdapterN
     () => ({ type: 'SET_RG_IP_ADAPTER_IMAGE', id: entityIdentifier.id, ipAdapterId }),
     [entityIdentifier.id, ipAdapterId]
   );
+  const pullBboxIntoIPAdapter = usePullBboxIntoRegionalGuidanceIPAdapter(entityIdentifier, ipAdapterId);
+  const isSaving = useIsSavingCanvas();
 
   return (
     <Flex flexDir="column" gap={3}>
@@ -125,6 +133,14 @@ export const RegionalGuidanceIPAdapterSettings = memo(({ ipAdapterId, ipAdapterN
               onChangeCLIPVisionModel={onChangeCLIPVisionModel}
             />
           </Box>
+          <IconButton
+            onClick={pullBboxIntoIPAdapter}
+            isLoading={isSaving.isTrue}
+            variant="ghost"
+            aria-label={t('controlLayers.pullBboxIntoIPAdapter')}
+            tooltip={t('controlLayers.pullBboxIntoIPAdapter')}
+            icon={<PiBoundingBoxBold />}
+          />
         </Flex>
         <Flex gap={4} w="full" alignItems="center">
           <Flex flexDir="column" gap={3} w="full">

invokeai/frontend/web/src/features/controlLayers/hooks/saveCanvasHooks.ts

@@ -5,9 +5,18 @@ import { isOk, withResultAsync } from 'common/util/result';
 import { useCanvasManager } from 'features/controlLayers/contexts/CanvasManagerProviderGate';
 import { selectDefaultControlAdapter, selectDefaultIPAdapter } from 'features/controlLayers/hooks/addLayerHooks';
 import { getPrefixedId } from 'features/controlLayers/konva/util';
-import { controlLayerAdded, ipaAdded, rasterLayerAdded, rgAdded } from 'features/controlLayers/store/canvasSlice';
+import {
+  controlLayerAdded,
+  entityRasterized,
+  ipaAdded,
+  ipaImageChanged,
+  rasterLayerAdded,
+  rgAdded,
+  rgIPAdapterImageChanged,
+} from 'features/controlLayers/store/canvasSlice';
 import type {
   CanvasControlLayerState,
+  CanvasEntityIdentifier,
   CanvasIPAdapterState,
   CanvasRasterLayerState,
   CanvasRegionalGuidanceState,
@@ -102,7 +111,7 @@ export const useSaveBboxAsRegionalGuidanceIPAdapter = () => {
       dispatch(rgAdded({ overrides, isSelected: true }));
     };
 
-    return { region: 'bbox', saveToGallery: true, onSave };
+    return { region: 'bbox', saveToGallery: false, onSave };
   }, [defaultIPAdapter, dispatch]);
   const saveBboxAsRegionalGuidanceIPAdapter = useSaveCanvas(saveBboxAsRegionalGuidanceIPAdapterArg);
   return saveBboxAsRegionalGuidanceIPAdapter;
@@ -123,7 +132,7 @@ export const useSaveBboxAsGlobalIPAdapter = () => {
       dispatch(ipaAdded({ overrides, isSelected: true }));
     };
 
-    return { region: 'bbox', saveToGallery: true, onSave };
+    return { region: 'bbox', saveToGallery: false, onSave };
   }, [defaultIPAdapter, dispatch]);
   const saveBboxAsIPAdapter = useSaveCanvas(saveBboxAsIPAdapterArg);
   return saveBboxAsIPAdapter;
@@ -140,7 +149,7 @@ export const useSaveBboxAsRasterLayer = () => {
       dispatch(rasterLayerAdded({ overrides, isSelected: true }));
     };
 
-    return { region: 'bbox', saveToGallery: true, onSave };
+    return { region: 'bbox', saveToGallery: false, onSave };
   }, [dispatch]);
   const saveBboxAsRasterLayer = useSaveCanvas(saveBboxAsRasterLayerArg);
   return saveBboxAsRasterLayer;
@@ -160,8 +169,63 @@ export const useSaveBboxAsControlLayer = () => {
       dispatch(controlLayerAdded({ overrides, isSelected: true }));
     };
 
-    return { region: 'bbox', saveToGallery: true, onSave };
+    return { region: 'bbox', saveToGallery: false, onSave };
   }, [defaultControlAdapter, dispatch]);
   const saveBboxAsControlLayer = useSaveCanvas(saveBboxAsControlLayerArg);
   return saveBboxAsControlLayer;
 };
+
+export const usePullBboxIntoLayer = (entityIdentifier: CanvasEntityIdentifier<'control_layer' | 'raster_layer'>) => {
+  const dispatch = useAppDispatch();
+
+  const pullBboxIntoLayerArg = useMemo<UseSaveCanvasArg>(() => {
+    const onSave = (imageDTO: ImageDTO, rect: Rect) => {
+      dispatch(
+        entityRasterized({
+          entityIdentifier,
+          position: { x: rect.x, y: rect.y },
+          imageObject: imageDTOToImageObject(imageDTO),
+          replaceObjects: true,
+        })
+      );
+    };
+
+    return { region: 'bbox', saveToGallery: false, onSave };
+  }, [dispatch, entityIdentifier]);
+
+  const pullBboxIntoLayer = useSaveCanvas(pullBboxIntoLayerArg);
+  return pullBboxIntoLayer;
+};
+
+export const usePullBboxIntoIPAdapter = (entityIdentifier: CanvasEntityIdentifier<'ip_adapter'>) => {
+  const dispatch = useAppDispatch();
+
+  const pullBboxIntoIPAdapterArg = useMemo<UseSaveCanvasArg>(() => {
+    const onSave = (imageDTO: ImageDTO, _: Rect) => {
+      dispatch(ipaImageChanged({ entityIdentifier, imageDTO }));
+    };
+
+    return { region: 'bbox', saveToGallery: false, onSave };
+  }, [dispatch, entityIdentifier]);
+
+  const pullBboxIntoIPAdapter = useSaveCanvas(pullBboxIntoIPAdapterArg);
+  return pullBboxIntoIPAdapter;
+};
+
+export const usePullBboxIntoRegionalGuidanceIPAdapter = (
+  entityIdentifier: CanvasEntityIdentifier<'regional_guidance'>,
+  ipAdapterId: string
+) => {
+  const dispatch = useAppDispatch();
+
+  const pullBboxIntoRegionalGuidanceIPAdapterArg = useMemo<UseSaveCanvasArg>(() => {
+    const onSave = (imageDTO: ImageDTO, _: Rect) => {
+      dispatch(rgIPAdapterImageChanged({ entityIdentifier, ipAdapterId, imageDTO }));
+    };
+
+    return { region: 'bbox', saveToGallery: false, onSave };
+  }, [dispatch, entityIdentifier, ipAdapterId]);
+
+  const pullBboxIntoRegionalGuidanceIPAdapter = useSaveCanvas(pullBboxIntoRegionalGuidanceIPAdapterArg);
+  return pullBboxIntoRegionalGuidanceIPAdapter;
+};