diff --git a/invokeai/frontend/web/public/locales/en.json b/invokeai/frontend/web/public/locales/en.json index e9c3f550234..57d993b27ae 100644 --- a/invokeai/frontend/web/public/locales/en.json +++ b/invokeai/frontend/web/public/locales/en.json @@ -1322,7 +1322,8 @@ "unableToCopyDesc": "Your browser does not support clipboard access. Firefox users may be able to fix this by following ", "unableToCopyDesc_theseSteps": "these steps", "fluxFillIncompatibleWithT2IAndI2I": "FLUX Fill is not compatible with Text to Image or Image to Image. Use other FLUX models for these tasks.", - "imagen3IncompatibleGenerationMode": "Imagen3 only supports Text to Image. Use other models for Image to Image, Inpainting and Outpainting tasks.", + "imagen3IncompatibleGenerationMode": "Google Imagen3 supports Text to Image only. Ensure the bounding box is empty, or use other models for Image to Image, Inpainting and Outpainting tasks.", + "chatGPT4oIncompatibleGenerationMode": "ChatGPT 4o supports Text to Image and Image to Image. Use other models Inpainting and Outpainting tasks.", "problemUnpublishingWorkflow": "Problem Unpublishing Workflow", "problemUnpublishingWorkflowDescription": "There was a problem unpublishing the workflow. Please try again.", "workflowUnpublished": "Workflow Unpublished" diff --git a/invokeai/frontend/web/src/features/controlLayers/components/IPAdapter/GlobalReferenceImageModel.tsx b/invokeai/frontend/web/src/features/controlLayers/components/IPAdapter/GlobalReferenceImageModel.tsx new file mode 100644 index 00000000000..4bf4674bb8d --- /dev/null +++ b/invokeai/frontend/web/src/features/controlLayers/components/IPAdapter/GlobalReferenceImageModel.tsx @@ -0,0 +1,65 @@ +import { Combobox, FormControl, Tooltip } from '@invoke-ai/ui-library'; +import { useAppSelector } from 'app/store/storeHooks'; +import { useGroupedModelCombobox } from 'common/hooks/useGroupedModelCombobox'; +import { selectBase } from 'features/controlLayers/store/paramsSlice'; +import { NavigateToModelManagerButton } from 'features/parameters/components/MainModel/NavigateToModelManagerButton'; +import { memo, useCallback, useMemo } from 'react'; +import { useTranslation } from 'react-i18next'; +import { useGlobalReferenceImageModels } from 'services/api/hooks/modelsByType'; +import type { AnyModelConfig, ApiModelConfig, FLUXReduxModelConfig, IPAdapterModelConfig } from 'services/api/types'; + +type Props = { + modelKey: string | null; + onChangeModel: (modelConfig: IPAdapterModelConfig | FLUXReduxModelConfig | ApiModelConfig) => void; +}; + +export const GlobalReferenceImageModel = memo(({ modelKey, onChangeModel }: Props) => { + const { t } = useTranslation(); + const currentBaseModel = useAppSelector(selectBase); + const [modelConfigs, { isLoading }] = useGlobalReferenceImageModels(); + const selectedModel = useMemo(() => modelConfigs.find((m) => m.key === modelKey), [modelConfigs, modelKey]); + + const _onChangeModel = useCallback( + (modelConfig: IPAdapterModelConfig | FLUXReduxModelConfig | ApiModelConfig | null) => { + if (!modelConfig) { + return; + } + onChangeModel(modelConfig); + }, + [onChangeModel] + ); + + const getIsDisabled = useCallback( + (model: AnyModelConfig): boolean => { + const hasMainModel = Boolean(currentBaseModel); + const hasSameBase = currentBaseModel === model.base; + return !hasMainModel || !hasSameBase; + }, + [currentBaseModel] + ); + + const { options, value, onChange, noOptionsMessage } = useGroupedModelCombobox({ + modelConfigs, + onChange: _onChangeModel, + selectedModel, + getIsDisabled, + isLoading, + }); + + return ( + + + + + + + ); +}); + +GlobalReferenceImageModel.displayName = 'GlobalReferenceImageModel'; diff --git a/invokeai/frontend/web/src/features/controlLayers/components/IPAdapter/IPAdapterImagePreview.tsx b/invokeai/frontend/web/src/features/controlLayers/components/IPAdapter/IPAdapterImagePreview.tsx index 153752e7619..77c5ded10d4 100644 --- a/invokeai/frontend/web/src/features/controlLayers/components/IPAdapter/IPAdapterImagePreview.tsx +++ b/invokeai/frontend/web/src/features/controlLayers/components/IPAdapter/IPAdapterImagePreview.tsx @@ -61,7 +61,7 @@ export const IPAdapterImagePreview = memo( )} {imageDTO && ( <> - + ) => createSelector( @@ -80,7 +80,7 @@ const IPAdapterSettingsContent = memo(() => { ); const onChangeModel = useCallback( - (modelConfig: IPAdapterModelConfig | FLUXReduxModelConfig) => { + (modelConfig: IPAdapterModelConfig | FLUXReduxModelConfig | ApiModelConfig) => { dispatch(referenceImageIPAdapterModelChanged({ entityIdentifier, modelConfig })); }, [dispatch, entityIdentifier] @@ -113,11 +113,7 @@ const IPAdapterSettingsContent = memo(() => { - + {ipAdapter.type === 'ip_adapter' && ( )} diff --git a/invokeai/frontend/web/src/features/controlLayers/components/IPAdapter/IPAdapterModel.tsx b/invokeai/frontend/web/src/features/controlLayers/components/IPAdapter/RegionalReferenceImageModel.tsx similarity index 74% rename from invokeai/frontend/web/src/features/controlLayers/components/IPAdapter/IPAdapterModel.tsx rename to invokeai/frontend/web/src/features/controlLayers/components/IPAdapter/RegionalReferenceImageModel.tsx index 1e0a3b5c0a3..2e9ff0ea33e 100644 --- a/invokeai/frontend/web/src/features/controlLayers/components/IPAdapter/IPAdapterModel.tsx +++ b/invokeai/frontend/web/src/features/controlLayers/components/IPAdapter/RegionalReferenceImageModel.tsx @@ -5,29 +5,26 @@ import { selectBase } from 'features/controlLayers/store/paramsSlice'; import { NavigateToModelManagerButton } from 'features/parameters/components/MainModel/NavigateToModelManagerButton'; import { memo, useCallback, useMemo } from 'react'; import { useTranslation } from 'react-i18next'; -import { useIPAdapterOrFLUXReduxModels } from 'services/api/hooks/modelsByType'; +import { useRegionalReferenceImageModels } from 'services/api/hooks/modelsByType'; import type { AnyModelConfig, FLUXReduxModelConfig, IPAdapterModelConfig } from 'services/api/types'; type Props = { - isRegionalGuidance: boolean; modelKey: string | null; onChangeModel: (modelConfig: IPAdapterModelConfig | FLUXReduxModelConfig) => void; }; -export const IPAdapterModel = memo(({ isRegionalGuidance, modelKey, onChangeModel }: Props) => { +const filter = (config: IPAdapterModelConfig | FLUXReduxModelConfig) => { + // FLUX supports regional guidance for FLUX Redux models only - not IP Adapter models. + if (config.base === 'flux' && config.type === 'ip_adapter') { + return false; + } + return true; +}; + +export const RegionalReferenceImageModel = memo(({ modelKey, onChangeModel }: Props) => { const { t } = useTranslation(); const currentBaseModel = useAppSelector(selectBase); - const filter = useCallback( - (config: IPAdapterModelConfig | FLUXReduxModelConfig) => { - // FLUX supports regional guidance for FLUX Redux models only - not IP Adapter models. - if (isRegionalGuidance && config.base === 'flux' && config.type === 'ip_adapter') { - return false; - } - return true; - }, - [isRegionalGuidance] - ); - const [modelConfigs, { isLoading }] = useIPAdapterOrFLUXReduxModels(filter); + const [modelConfigs, { isLoading }] = useRegionalReferenceImageModels(filter); const selectedModel = useMemo(() => modelConfigs.find((m) => m.key === modelKey), [modelConfigs, modelKey]); const _onChangeModel = useCallback( @@ -73,4 +70,4 @@ export const IPAdapterModel = memo(({ isRegionalGuidance, modelKey, onChangeMode ); }); -IPAdapterModel.displayName = 'IPAdapterModel'; +RegionalReferenceImageModel.displayName = 'RegionalReferenceImageModel'; diff --git a/invokeai/frontend/web/src/features/controlLayers/components/RegionalGuidance/RegionalGuidanceIPAdapterSettings.tsx b/invokeai/frontend/web/src/features/controlLayers/components/RegionalGuidance/RegionalGuidanceIPAdapterSettings.tsx index c539763afc5..17e148a2313 100644 --- a/invokeai/frontend/web/src/features/controlLayers/components/RegionalGuidance/RegionalGuidanceIPAdapterSettings.tsx +++ b/invokeai/frontend/web/src/features/controlLayers/components/RegionalGuidance/RegionalGuidanceIPAdapterSettings.tsx @@ -7,7 +7,7 @@ import { CLIPVisionModel } from 'features/controlLayers/components/IPAdapter/CLI import { FLUXReduxImageInfluence } from 'features/controlLayers/components/IPAdapter/FLUXReduxImageInfluence'; import { IPAdapterImagePreview } from 'features/controlLayers/components/IPAdapter/IPAdapterImagePreview'; import { IPAdapterMethod } from 'features/controlLayers/components/IPAdapter/IPAdapterMethod'; -import { IPAdapterModel } from 'features/controlLayers/components/IPAdapter/IPAdapterModel'; +import { RegionalReferenceImageModel } from 'features/controlLayers/components/IPAdapter/RegionalReferenceImageModel'; import { RegionalGuidanceIPAdapterSettingsEmptyState } from 'features/controlLayers/components/RegionalGuidance/RegionalGuidanceIPAdapterSettingsEmptyState'; import { useEntityIdentifierContext } from 'features/controlLayers/contexts/EntityIdentifierContext'; import { usePullBboxIntoRegionalGuidanceReferenceImage } from 'features/controlLayers/hooks/saveCanvasHooks'; @@ -140,11 +140,7 @@ const RegionalGuidanceIPAdapterSettingsContent = memo(({ referenceImageId }: Pro - + {ipAdapter.type === 'ip_adapter' && ( )} diff --git a/invokeai/frontend/web/src/features/controlLayers/hooks/addLayerHooks.ts b/invokeai/frontend/web/src/features/controlLayers/hooks/addLayerHooks.ts index af7dda98d7e..27b01fe4940 100644 --- a/invokeai/frontend/web/src/features/controlLayers/hooks/addLayerHooks.ts +++ b/invokeai/frontend/web/src/features/controlLayers/hooks/addLayerHooks.ts @@ -17,16 +17,26 @@ import { selectBase } from 'features/controlLayers/store/paramsSlice'; import { selectCanvasSlice, selectEntity } from 'features/controlLayers/store/selectors'; import type { CanvasEntityIdentifier, + CanvasReferenceImageState, CanvasRegionalGuidanceState, ControlLoRAConfig, ControlNetConfig, IPAdapterConfig, T2IAdapterConfig, } from 'features/controlLayers/store/types'; -import { initialControlNet, initialIPAdapter, initialT2IAdapter } from 'features/controlLayers/store/util'; +import { + initialChatGPT4oReferenceImage, + initialControlNet, + initialIPAdapter, + initialT2IAdapter, +} from 'features/controlLayers/store/util'; import { zModelIdentifierField } from 'features/nodes/types/common'; import { useCallback } from 'react'; -import { modelConfigsAdapterSelectors, selectModelConfigsQuery } from 'services/api/endpoints/models'; +import { + modelConfigsAdapterSelectors, + selectMainModelConfig, + selectModelConfigsQuery, +} from 'services/api/endpoints/models'; import type { ControlLoRAModelConfig, ControlNetModelConfig, @@ -64,6 +74,35 @@ export const selectDefaultControlAdapter = createSelector( } ); +export const selectDefaultRefImageConfig = createSelector( + selectMainModelConfig, + selectModelConfigsQuery, + selectBase, + (selectedMainModel, query, base): CanvasReferenceImageState['ipAdapter'] => { + if (selectedMainModel?.base === 'chatgpt-4o') { + const referenceImage = deepClone(initialChatGPT4oReferenceImage); + referenceImage.model = zModelIdentifierField.parse(selectedMainModel); + return referenceImage; + } + + const { data } = query; + let model: IPAdapterModelConfig | null = null; + if (data) { + const modelConfigs = modelConfigsAdapterSelectors.selectAll(data).filter(isIPAdapterModelConfig); + const compatibleModels = modelConfigs.filter((m) => (base ? m.base === base : true)); + model = compatibleModels[0] ?? modelConfigs[0] ?? null; + } + const ipAdapter = deepClone(initialIPAdapter); + if (model) { + ipAdapter.model = zModelIdentifierField.parse(model); + if (model.base === 'flux') { + ipAdapter.clipVisionModel = 'ViT-L'; + } + } + return ipAdapter; + } +); + /** * Selects the default IP adapter configuration based on the model configurations and the base. * @@ -146,11 +185,11 @@ export const useAddRegionalReferenceImage = () => { export const useAddGlobalReferenceImage = () => { const dispatch = useAppDispatch(); - const defaultIPAdapter = useAppSelector(selectDefaultIPAdapter); + const defaultRefImage = useAppSelector(selectDefaultRefImageConfig); const func = useCallback(() => { - const overrides = { ipAdapter: deepClone(defaultIPAdapter) }; + const overrides = { ipAdapter: deepClone(defaultRefImage) }; dispatch(referenceImageAdded({ isSelected: true, overrides })); - }, [defaultIPAdapter, dispatch]); + }, [defaultRefImage, dispatch]); return func; }; diff --git a/invokeai/frontend/web/src/features/controlLayers/hooks/saveCanvasHooks.ts b/invokeai/frontend/web/src/features/controlLayers/hooks/saveCanvasHooks.ts index 3267111fa2d..06f4a0328e7 100644 --- a/invokeai/frontend/web/src/features/controlLayers/hooks/saveCanvasHooks.ts +++ b/invokeai/frontend/web/src/features/controlLayers/hooks/saveCanvasHooks.ts @@ -3,7 +3,7 @@ import { useAppDispatch, useAppSelector, useAppStore } from 'app/store/storeHook import { deepClone } from 'common/util/deepClone'; import { withResultAsync } from 'common/util/result'; import { useCanvasManager } from 'features/controlLayers/contexts/CanvasManagerProviderGate'; -import { selectDefaultIPAdapter } from 'features/controlLayers/hooks/addLayerHooks'; +import { selectDefaultIPAdapter, selectDefaultRefImageConfig } from 'features/controlLayers/hooks/addLayerHooks'; import { getPrefixedId } from 'features/controlLayers/konva/util'; import { controlLayerAdded, @@ -198,7 +198,7 @@ export const useNewRegionalReferenceImageFromBbox = () => { export const useNewGlobalReferenceImageFromBbox = () => { const { t } = useTranslation(); const dispatch = useAppDispatch(); - const defaultIPAdapter = useAppSelector(selectDefaultIPAdapter); + const defaultIPAdapter = useAppSelector(selectDefaultRefImageConfig); const arg = useMemo(() => { const onSave = (imageDTO: ImageDTO) => { diff --git a/invokeai/frontend/web/src/features/controlLayers/hooks/useIsEntityTypeEnabled.ts b/invokeai/frontend/web/src/features/controlLayers/hooks/useIsEntityTypeEnabled.ts index 201ffdb3b97..6ab5b90c36b 100644 --- a/invokeai/frontend/web/src/features/controlLayers/hooks/useIsEntityTypeEnabled.ts +++ b/invokeai/frontend/web/src/features/controlLayers/hooks/useIsEntityTypeEnabled.ts @@ -19,7 +19,7 @@ export const useIsEntityTypeEnabled = (entityType: CanvasEntityType) => { const isEntityTypeEnabled = useMemo(() => { switch (entityType) { case 'reference_image': - return !isSD3 && !isCogView4 && !isImagen3 && !isChatGPT4o; + return !isSD3 && !isCogView4 && !isImagen3; case 'regional_guidance': return !isSD3 && !isCogView4 && !isImagen3 && !isChatGPT4o; case 'control_layer': diff --git a/invokeai/frontend/web/src/features/controlLayers/store/canvasSlice.ts b/invokeai/frontend/web/src/features/controlLayers/store/canvasSlice.ts index fdccaffb72a..1e73699c4e7 100644 --- a/invokeai/frontend/web/src/features/controlLayers/store/canvasSlice.ts +++ b/invokeai/frontend/web/src/features/controlLayers/store/canvasSlice.ts @@ -34,9 +34,10 @@ import { isMainModelBase, zModelIdentifierField } from 'features/nodes/types/com import { ASPECT_RATIO_MAP } from 'features/parameters/components/Bbox/constants'; import { getGridSize, getIsSizeOptimal, getOptimalDimension } from 'features/parameters/util/optimalDimension'; import type { IRect } from 'konva/lib/types'; -import { merge } from 'lodash-es'; +import { isEqual, merge } from 'lodash-es'; import type { UndoableOptions } from 'redux-undo'; import type { + ApiModelConfig, ControlLoRAModelConfig, ControlNetModelConfig, FLUXReduxModelConfig, @@ -76,6 +77,7 @@ import { getReferenceImageState, getRegionalGuidanceState, imageDTOToImageWithDims, + initialChatGPT4oReferenceImage, initialControlLoRA, initialControlNet, initialFLUXRedux, @@ -644,7 +646,10 @@ export const canvasSlice = createSlice({ referenceImageIPAdapterModelChanged: ( state, action: PayloadAction< - EntityIdentifierPayload<{ modelConfig: IPAdapterModelConfig | FLUXReduxModelConfig | null }, 'reference_image'> + EntityIdentifierPayload< + { modelConfig: IPAdapterModelConfig | FLUXReduxModelConfig | ApiModelConfig | null }, + 'reference_image' + > > ) => { const { entityIdentifier, modelConfig } = action.payload; @@ -652,33 +657,51 @@ export const canvasSlice = createSlice({ if (!entity) { return; } + + const oldModel = entity.ipAdapter.model; + + // First set the new model entity.ipAdapter.model = modelConfig ? zModelIdentifierField.parse(modelConfig) : null; if (!entity.ipAdapter.model) { return; } - if (entity.ipAdapter.type === 'ip_adapter' && entity.ipAdapter.model.type === 'flux_redux') { - // Switching from ip_adapter to flux_redux + if (isEqual(oldModel, entity.ipAdapter.model)) { + // Nothing changed, so we don't need to do anything + return; + } + + // The type of ref image depends on the model. When the user switches the model, we rebuild the ref image. + // When we switch the model, we keep the image the same, but change the other parameters. + + if (entity.ipAdapter.model.base === 'chatgpt-4o') { + // Switching to chatgpt-4o ref image entity.ipAdapter = { - ...initialFLUXRedux, + ...initialChatGPT4oReferenceImage, image: entity.ipAdapter.image, model: entity.ipAdapter.model, }; return; } - if (entity.ipAdapter.type === 'flux_redux' && entity.ipAdapter.model.type === 'ip_adapter') { - // Switching from flux_redux to ip_adapter + if (entity.ipAdapter.model.type === 'flux_redux') { + // Switching to flux_redux entity.ipAdapter = { - ...initialIPAdapter, + ...initialFLUXRedux, image: entity.ipAdapter.image, model: entity.ipAdapter.model, }; return; } - if (entity.ipAdapter.type === 'ip_adapter') { + if (entity.ipAdapter.model.type === 'ip_adapter') { + // Switching to ip_adapter + entity.ipAdapter = { + ...initialIPAdapter, + image: entity.ipAdapter.image, + model: entity.ipAdapter.model, + }; // Ensure that the IP Adapter model is compatible with the CLIP Vision model if (entity.ipAdapter.model?.base === 'flux') { entity.ipAdapter.clipVisionModel = 'ViT-L'; @@ -686,6 +709,7 @@ export const canvasSlice = createSlice({ // Fall back to ViT-H (ViT-G would also work) entity.ipAdapter.clipVisionModel = 'ViT-H'; } + return; } }, referenceImageIPAdapterCLIPVisionModelChanged: ( diff --git a/invokeai/frontend/web/src/features/controlLayers/store/types.ts b/invokeai/frontend/web/src/features/controlLayers/store/types.ts index 8afbc434749..753eaff0b1b 100644 --- a/invokeai/frontend/web/src/features/controlLayers/store/types.ts +++ b/invokeai/frontend/web/src/features/controlLayers/store/types.ts @@ -245,6 +245,18 @@ const zFLUXReduxConfig = z.object({ }); export type FLUXReduxConfig = z.infer; +const zChatGPT4oReferenceImageConfig = z.object({ + type: z.literal('chatgpt_4o_reference_image'), + image: zImageWithDims.nullable(), + /** + * TODO(psyche): Technically there is no model for ChatGPT 4o reference images - it's just a field in the API call. + * But we use a model drop down to switch between different ref image types, so there needs to be a model here else + * there will be no way to switch between ref image types. + */ + model: zServerValidatedModelIdentifierField.nullable(), +}); +export type ChatGPT4oReferenceImageConfig = z.infer; + const zCanvasEntityBase = z.object({ id: zId, name: zName, @@ -254,15 +266,19 @@ const zCanvasEntityBase = z.object({ const zCanvasReferenceImageState = zCanvasEntityBase.extend({ type: z.literal('reference_image'), - ipAdapter: z.discriminatedUnion('type', [zIPAdapterConfig, zFLUXReduxConfig]), + // This should be named `referenceImage` but we need to keep it as `ipAdapter` for backwards compatibility + ipAdapter: z.discriminatedUnion('type', [zIPAdapterConfig, zFLUXReduxConfig, zChatGPT4oReferenceImageConfig]), }); export type CanvasReferenceImageState = z.infer; -export const isIPAdapterConfig = (config: IPAdapterConfig | FLUXReduxConfig): config is IPAdapterConfig => +export const isIPAdapterConfig = (config: CanvasReferenceImageState['ipAdapter']): config is IPAdapterConfig => config.type === 'ip_adapter'; -export const isFLUXReduxConfig = (config: IPAdapterConfig | FLUXReduxConfig): config is FLUXReduxConfig => +export const isFLUXReduxConfig = (config: CanvasReferenceImageState['ipAdapter']): config is FLUXReduxConfig => config.type === 'flux_redux'; +export const isChatGPT4oReferenceImageConfig = ( + config: CanvasReferenceImageState['ipAdapter'] +): config is ChatGPT4oReferenceImageConfig => config.type === 'chatgpt_4o_reference_image'; const zFillStyle = z.enum(['solid', 'grid', 'crosshatch', 'diagonal', 'horizontal', 'vertical']); export type FillStyle = z.infer; diff --git a/invokeai/frontend/web/src/features/controlLayers/store/util.ts b/invokeai/frontend/web/src/features/controlLayers/store/util.ts index e9bc8bd990c..c5d23b8c083 100644 --- a/invokeai/frontend/web/src/features/controlLayers/store/util.ts +++ b/invokeai/frontend/web/src/features/controlLayers/store/util.ts @@ -7,6 +7,7 @@ import type { CanvasRasterLayerState, CanvasReferenceImageState, CanvasRegionalGuidanceState, + ChatGPT4oReferenceImageConfig, ControlLoRAConfig, ControlNetConfig, FLUXReduxConfig, @@ -77,6 +78,11 @@ export const initialFLUXRedux: FLUXReduxConfig = { model: null, imageInfluence: 'highest', }; +export const initialChatGPT4oReferenceImage: ChatGPT4oReferenceImageConfig = { + type: 'chatgpt_4o_reference_image', + image: null, + model: null, +}; export const initialT2IAdapter: T2IAdapterConfig = { type: 't2i_adapter', model: null, diff --git a/invokeai/frontend/web/src/features/imageActions/actions.ts b/invokeai/frontend/web/src/features/imageActions/actions.ts index dfb5fec2f2b..d33e9b2d2ee 100644 --- a/invokeai/frontend/web/src/features/imageActions/actions.ts +++ b/invokeai/frontend/web/src/features/imageActions/actions.ts @@ -1,6 +1,6 @@ import type { AppDispatch, RootState } from 'app/store/store'; import { deepClone } from 'common/util/deepClone'; -import { selectDefaultIPAdapter } from 'features/controlLayers/hooks/addLayerHooks'; +import { selectDefaultIPAdapter, selectDefaultRefImageConfig } from 'features/controlLayers/hooks/addLayerHooks'; import { CanvasEntityAdapterBase } from 'features/controlLayers/konva/CanvasEntity/CanvasEntityAdapterBase'; import { getPrefixedId } from 'features/controlLayers/konva/util'; import { canvasReset } from 'features/controlLayers/store/actions'; @@ -116,7 +116,7 @@ export const createNewCanvasEntityFromImage = (arg: { break; } case 'reference_image': { - const ipAdapter = deepClone(selectDefaultIPAdapter(getState())); + const ipAdapter = deepClone(selectDefaultRefImageConfig(getState())); ipAdapter.image = imageDTOToImageWithDims(imageDTO); dispatch(referenceImageAdded({ overrides: { ipAdapter }, isSelected: true })); break; @@ -238,7 +238,7 @@ export const newCanvasFromImage = (arg: { break; } case 'reference_image': { - const ipAdapter = deepClone(selectDefaultIPAdapter(getState())); + const ipAdapter = deepClone(selectDefaultRefImageConfig(getState())); ipAdapter.image = imageDTOToImageWithDims(imageDTO); dispatch(canvasReset()); dispatch(referenceImageAdded({ overrides: { ipAdapter }, isSelected: true })); diff --git a/invokeai/frontend/web/src/features/nodes/util/graph/generation/buildChatGPT4oGraph.ts b/invokeai/frontend/web/src/features/nodes/util/graph/generation/buildChatGPT4oGraph.ts index a63230319d5..7c65c2e0a6d 100644 --- a/invokeai/frontend/web/src/features/nodes/util/graph/generation/buildChatGPT4oGraph.ts +++ b/invokeai/frontend/web/src/features/nodes/util/graph/generation/buildChatGPT4oGraph.ts @@ -4,7 +4,9 @@ import type { CanvasManager } from 'features/controlLayers/konva/CanvasManager'; import { getPrefixedId } from 'features/controlLayers/konva/util'; import { selectCanvasSettingsSlice } from 'features/controlLayers/store/canvasSettingsSlice'; import { selectCanvasSlice } from 'features/controlLayers/store/selectors'; -import { isChatGPT4oAspectRatioID } from 'features/controlLayers/store/types'; +import { isChatGPT4oAspectRatioID, isChatGPT4oReferenceImageConfig } from 'features/controlLayers/store/types'; +import { getGlobalReferenceImageWarnings } from 'features/controlLayers/store/validators'; +import type { ImageField } from 'features/nodes/types/common'; import { Graph } from 'features/nodes/util/graph/generation/Graph'; import { CANVAS_OUTPUT_PREFIX, @@ -13,6 +15,7 @@ import { } from 'features/nodes/util/graph/graphBuilderUtils'; import type { GraphBuilderReturn } from 'features/nodes/util/graph/types'; import { t } from 'i18next'; +import { selectMainModelConfig } from 'services/api/endpoints/models'; import type { Equals } from 'tsafe'; import { assert } from 'tsafe'; @@ -21,21 +24,40 @@ const log = logger('system'); export const buildChatGPT4oGraph = async (state: RootState, manager: CanvasManager): Promise => { const generationMode = await manager.compositor.getGenerationMode(); - assert( - generationMode === 'txt2img' || generationMode === 'img2img', - t('toast.gptImageIncompatibleWithInpaintAndOutpaint') - ); + assert(generationMode === 'txt2img' || generationMode === 'img2img', t('toast.chatGPT4oIncompatibleGenerationMode')); log.debug({ generationMode }, 'Building GPT Image graph'); + const model = selectMainModelConfig(state); + const canvas = selectCanvasSlice(state); const canvasSettings = selectCanvasSettingsSlice(state); const { bbox } = canvas; const { positivePrompt } = selectPresetModifiedPrompts(state); + assert(model, 'No model found in state'); + assert(model.base === 'chatgpt-4o', 'Model is not a FLUX model'); + assert(isChatGPT4oAspectRatioID(bbox.aspectRatio.id), 'ChatGPT 4o does not support this aspect ratio'); + const validRefImages = canvas.referenceImages.entities + .filter((entity) => entity.isEnabled) + .filter((entity) => isChatGPT4oReferenceImageConfig(entity.ipAdapter)) + .filter((entity) => getGlobalReferenceImageWarnings(entity, model).length === 0); + + let reference_images: ImageField[] | undefined = undefined; + + if (validRefImages.length > 0) { + reference_images = []; + for (const entity of validRefImages) { + assert(entity.ipAdapter.image, 'Image is required for reference image'); + reference_images.push({ + image_name: entity.ipAdapter.image.image_name, + }); + } + } + const is_intermediate = canvasSettings.sendToCanvas; const board = canvasSettings.sendToCanvas ? undefined : getBoardField(state); @@ -47,6 +69,7 @@ export const buildChatGPT4oGraph = async (state: RootState, manager: CanvasManag id: getPrefixedId(CANVAS_OUTPUT_PREFIX), positive_prompt: positivePrompt, aspect_ratio: bbox.aspectRatio.id, + reference_images, use_cache: false, is_intermediate, board, @@ -69,7 +92,9 @@ export const buildChatGPT4oGraph = async (state: RootState, manager: CanvasManag type: 'chatgpt_4o_edit_image', id: getPrefixedId(CANVAS_OUTPUT_PREFIX), positive_prompt: positivePrompt, - image: { image_name }, + aspect_ratio: bbox.aspectRatio.id, + base_image: { image_name }, + reference_images, use_cache: false, is_intermediate, board, @@ -80,5 +105,5 @@ export const buildChatGPT4oGraph = async (state: RootState, manager: CanvasManag }; } - assert>(false, 'Invalid generation mode for gpt image'); + assert>(false, 'Invalid generation mode for ChatGPT '); }; diff --git a/invokeai/frontend/web/src/features/nodes/util/graph/generation/buildImagen3Graph.ts b/invokeai/frontend/web/src/features/nodes/util/graph/generation/buildImagen3Graph.ts index 6bb6a1ea699..cbd0d004f96 100644 --- a/invokeai/frontend/web/src/features/nodes/util/graph/generation/buildImagen3Graph.ts +++ b/invokeai/frontend/web/src/features/nodes/util/graph/generation/buildImagen3Graph.ts @@ -58,29 +58,5 @@ export const buildImagen3Graph = async (state: RootState, manager: CanvasManager }; } - if (generationMode === 'img2img') { - const adapters = manager.compositor.getVisibleAdaptersOfType('raster_layer'); - const { image_name } = await manager.compositor.getCompositeImageDTO(adapters, bbox.rect, { - is_intermediate: true, - silent: true, - }); - const g = new Graph(getPrefixedId('imagen3_img2img_graph')); - const imagen3 = g.addNode({ - // @ts-expect-error: These nodes are not available in the OSS application - type: 'google_imagen3_edit_image', - id: getPrefixedId(CANVAS_OUTPUT_PREFIX), - positive_prompt: positivePrompt, - negative_prompt: negativePrompt, - base_image: { image_name }, - is_intermediate, - board, - }); - return { - g, - seedFieldIdentifier: { nodeId: imagen3.id, fieldName: 'seed' }, - positivePromptFieldIdentifier: { nodeId: imagen3.id, fieldName: 'positive_prompt' }, - }; - } - - assert>(false, 'Invalid generation mode for imagen3'); + assert>(false, 'Invalid generation mode for Imagen3'); }; diff --git a/invokeai/frontend/web/src/services/api/hooks/modelsByType.ts b/invokeai/frontend/web/src/services/api/hooks/modelsByType.ts index 24c6da56b33..e34207a9c45 100644 --- a/invokeai/frontend/web/src/services/api/hooks/modelsByType.ts +++ b/invokeai/frontend/web/src/services/api/hooks/modelsByType.ts @@ -9,6 +9,7 @@ import { } from 'services/api/endpoints/models'; import type { AnyModelConfig } from 'services/api/types'; import { + isChatGPT4oModelConfig, isCLIPEmbedModelConfig, isCLIPVisionModelConfig, isCogView4MainModelModelConfig, @@ -81,7 +82,10 @@ export const useFluxVAEModels = (args?: ModelHookArgs) => export const useCLIPVisionModels = buildModelsHook(isCLIPVisionModelConfig); export const useSigLipModels = buildModelsHook(isSigLipModelConfig); export const useFluxReduxModels = buildModelsHook(isFluxReduxModelConfig); -export const useIPAdapterOrFLUXReduxModels = buildModelsHook( +export const useGlobalReferenceImageModels = buildModelsHook( + (config) => isIPAdapterModelConfig(config) || isFluxReduxModelConfig(config) || isChatGPT4oModelConfig(config) +); +export const useRegionalReferenceImageModels = buildModelsHook( (config) => isIPAdapterModelConfig(config) || isFluxReduxModelConfig(config) ); export const useLLaVAModels = buildModelsHook(isLLaVAModelConfig); diff --git a/invokeai/frontend/web/src/services/api/types.ts b/invokeai/frontend/web/src/services/api/types.ts index e4510db7ab7..4711beb36dd 100644 --- a/invokeai/frontend/web/src/services/api/types.ts +++ b/invokeai/frontend/web/src/services/api/types.ts @@ -65,7 +65,7 @@ export type CheckpointModelConfig = S['MainCheckpointConfig']; type CLIPVisionDiffusersConfig = S['CLIPVisionDiffusersConfig']; export type SigLipModelConfig = S['SigLIPConfig']; export type FLUXReduxModelConfig = S['FluxReduxConfig']; -type ApiModelConfig = S['ApiModelConfig']; +export type ApiModelConfig = S['ApiModelConfig']; export type MainModelConfig = DiffusersModelConfig | CheckpointModelConfig | ApiModelConfig; export type AnyModelConfig = | ControlLoRAModelConfig @@ -228,6 +228,10 @@ export const isFluxReduxModelConfig = (config: AnyModelConfig): config is FLUXRe return config.type === 'flux_redux'; }; +export const isChatGPT4oModelConfig = (config: AnyModelConfig): config is ApiModelConfig => { + return config.type === 'main' && config.base === 'chatgpt-4o'; +}; + export const isNonRefinerMainModelConfig = (config: AnyModelConfig): config is MainModelConfig => { return config.type === 'main' && config.base !== 'sdxl-refiner'; };