Stream API Vulkan Backend

filament/backend/CMakeLists.txt

@@ -199,6 +199,8 @@ if (FILAMENT_SUPPORTS_VULKAN)
             src/vulkan/VulkanDriverFactory.h
             src/vulkan/VulkanExternalImageManager.cpp
             src/vulkan/VulkanExternalImageManager.h
+            src/vulkan/VulkanStreamedImageManager.cpp
+            src/vulkan/VulkanStreamedImageManager.h
             src/vulkan/VulkanFboCache.cpp
             src/vulkan/VulkanFboCache.h
             src/vulkan/VulkanHandles.cpp

filament/backend/include/backend/Platform.h

@@ -66,6 +66,9 @@ class UTILS_PUBLIC Platform {
         ExternalImageHandle& operator=(ExternalImageHandle const& rhs) noexcept;
         ExternalImageHandle& operator=(ExternalImageHandle&& rhs) noexcept;
 
+        bool operator==(const ExternalImageHandle& rhs) const noexcept {
+            return mTarget == rhs.mTarget;
+        }
         explicit operator bool() const noexcept { return mTarget != nullptr; }
 
         ExternalImage* UTILS_NULLABLE get() noexcept { return mTarget; }

filament/backend/include/backend/platforms/VulkanPlatform.h

@@ -406,6 +406,9 @@ class VulkanPlatform : public Platform, utils::PrivateImplementation<VulkanPlatf
     virtual ImageData createVkImageFromExternal(ExternalImageHandleRef image) const {
         return {};
     }
+    virtual ExternalImageHandle createExternalImageFromRaw(void* image, bool sRGB) noexcept {
+        return {};
+    }
 
 protected:
     virtual ExtensionSet getSwapchainInstanceExtensions() const;

filament/backend/include/backend/platforms/VulkanPlatformAndroid.h

@@ -28,6 +28,8 @@ class VulkanPlatformAndroid : public VulkanPlatform {
 public:
     ExternalImageHandle UTILS_PUBLIC createExternalImage(AHardwareBuffer const* buffer,
             bool sRGB) noexcept;
+    virtual ExternalImageHandle createExternalImageFromRaw(void* image,
+            bool sRGB) noexcept override;
 
     struct UTILS_PUBLIC ExternalImageDescAndroid {
         uint32_t width;      // Texture width

filament/backend/src/vulkan/VulkanDriver.cpp

@@ -231,6 +231,7 @@ VulkanDriver::VulkanDriver(VulkanPlatform* platform, VulkanContext& context,
       mQueryManager(mPlatform->getDevice()),
       mExternalImageManager(platform, &mSamplerCache, &mYcbcrConversionCache, &mDescriptorSetCache,
               &mDescriptorSetLayoutCache),
+      mStreamedImageManager(&mExternalImageManager, &mDescriptorSetCache, &mSamplerCache),
       mIsSRGBSwapChainSupported(mPlatform->getCustomization().isSRGBSwapChainSupported),
       mStereoscopicType(driverConfig.stereoscopicType) {
 
@@ -455,13 +456,18 @@ void VulkanDriver::updateDescriptorSetTexture(
     if (UTILS_UNLIKELY(mExternalImageManager.isExternallySampledTexture(texture))) {
         mExternalImageManager.bindExternallySampledTexture(set, binding, texture, params);
         mAppState.hasBoundExternalImages = true;
-    } else {
+    } else if (bool(texture->getStream())) {
+        mStreamedImageManager.bindStreamedTexture(set, binding, texture, params);
+        mAppState.hasBoundExternalImages = true;
+    }
+    else {
         VulkanSamplerCache::Params cacheParams = {
             .sampler = params,
         };
         VkSampler const vksampler = mSamplerCache.getSampler(cacheParams);
         mDescriptorSetCache.updateSampler(set, binding, texture, vksampler);
         mExternalImageManager.clearTextureBinding(set, binding);
+        mStreamedImageManager.unbindStreamedTexture(set, binding);
     }
 }
 
@@ -965,6 +971,11 @@ void VulkanDriver::destroySwapChain(Handle<HwSwapChain> sch) {
 }
 
 void VulkanDriver::destroyStream(Handle<HwStream> sh) {
+    if (!sh) {
+        return;
+    }
+    auto stream = resource_ptr<VulkanStream>::cast(&mResourceManager, sh);
+    stream.dec();
 }
 
 void VulkanDriver::destroyTimerQuery(Handle<HwTimerQuery> tqh) {
@@ -991,25 +1002,99 @@ void VulkanDriver::destroyDescriptorSet(Handle<HwDescriptorSet> dsh) {
 }
 
 Handle<HwStream> VulkanDriver::createStreamNative(void* nativeStream) {
-    return {};
+    auto handle = mResourceManager.allocHandle<VulkanStream>();
+    auto stream = resource_ptr<VulkanStream>::make(&mResourceManager, handle);
+    stream.inc();
+    return handle;
 }
 
 Handle<HwStream> VulkanDriver::createStreamAcquired() {
-    return {};
+    auto handle = mResourceManager.allocHandle<VulkanStream>();
+    auto stream = resource_ptr<VulkanStream>::make(&mResourceManager, handle);
+    stream.inc();
+    return handle;
 }
 
 void VulkanDriver::setAcquiredImage(Handle<HwStream> sh, void* image, const math::mat3f& transform,
         CallbackHandler* handler, StreamCallback cb, void* userData) {
+    FVK_SYSTRACE_SCOPE();
+    auto stream = resource_ptr<VulkanStream>::cast(&mResourceManager, sh);
+    stream->acquire({ image, cb, userData, handler });
+    mStreamsWithPendingAcquiredImage.push_back(stream);
 }
 
 void VulkanDriver::setStreamDimensions(Handle<HwStream> sh, uint32_t width, uint32_t height) {
+    auto stream = resource_ptr<VulkanStream>::cast(&mResourceManager, sh);
+    stream->width = width;
+    stream->height = height;
 }
 
 int64_t VulkanDriver::getStreamTimestamp(Handle<HwStream> sh) {
     return 0;
 }
 
 void VulkanDriver::updateStreams(CommandStream* driver) {
+    if (UTILS_UNLIKELY(!mStreamsWithPendingAcquiredImage.empty())) {
+        for (auto& stream: mStreamsWithPendingAcquiredImage) {
+            if (stream->previousNeedsRelease()) {
+                scheduleRelease(stream->takePrevious());
+            }
+
+            auto texture = stream->getTexture(stream->getAcquired().image);
+            bool newImage = false;
+            if (!texture) {
+                auto externalImage =
+                        mPlatform->createExternalImageFromRaw(stream->getAcquired().image, false);
+                auto metadata = mPlatform->extractExternalImageMetadata(externalImage);
+                auto imgData = mPlatform->createVkImageFromExternal(externalImage);
+
+                assert_invariant(imgData.internal.valid() || imgData.external.valid());
+
+                VkFormat vkformat = metadata.format;
+                VkImage vkimage = VK_NULL_HANDLE;
+                VkDeviceMemory memory = VK_NULL_HANDLE;
+                if (imgData.internal.valid()) {
+                    metadata.externalFormat = 0;
+                    vkimage = imgData.internal.image;
+                    memory = imgData.internal.memory;
+                } else {
+                    vkformat = VK_FORMAT_UNDEFINED;
+                    vkimage = imgData.external.image;
+                    memory = imgData.external.memory;
+                }
+
+                VkSamplerYcbcrConversion const conversion =
+                        mExternalImageManager.getVkSamplerYcbcrConversion(metadata);
+
+                auto newTexture = resource_ptr<VulkanTexture>::construct(&mResourceManager, mContext,
+                        mPlatform->getDevice(), mAllocator, &mResourceManager, &mCommands, vkimage,
+                        memory, vkformat, conversion, metadata.samples, metadata.width,
+                        metadata.height, metadata.layers, metadata.filamentUsage, mStagePool);
+
+                // We shouldn't need to do this when we allocate the texture
+                if (true) {
+                    auto& commands = mCommands.get();
+                    // Unlike uploaded textures or swapchains, we need to explicit transition this
+                    // texture into the read layout.
+                    newTexture->transitionLayout(&commands, newTexture->getPrimaryViewRange(),
+                            VulkanLayout::FRAG_READ);
+                }
+
+                if (imgData.external.valid()) {
+                    mExternalImageManager.addExternallySampledTexture(newTexture, externalImage);
+                    // Cache the AHB backed image.
+                    stream->pushImage(stream->getAcquired().image, newTexture);
+                    newImage = true;
+                }
+
+                newTexture.inc();
+                texture = newTexture;
+            }
+
+            mStreamedImageManager.onStreamAcquireImage(texture, stream, newImage);
+        }
+        mStreamsWithPendingAcquiredImage.clear();
+    }
 }
 
 void VulkanDriver::destroyFence(Handle<HwFence> fh) {
@@ -1331,6 +1416,11 @@ TimerQueryResult VulkanDriver::getTimerQueryValue(Handle<HwTimerQuery> tqh, uint
 }
 
 void VulkanDriver::setExternalStream(Handle<HwTexture> th, Handle<HwStream> sh) {
+    auto t = resource_ptr<VulkanTexture>::cast(&mResourceManager, th);
+    assert_invariant(t);
+    auto s = resource_ptr<VulkanStream>::cast(&mResourceManager, sh);
+    assert_invariant(s);
+    t->setStream(s);
 }
 
 void VulkanDriver::generateMipmaps(Handle<HwTexture> th) {

filament/backend/src/vulkan/VulkanDriver.h

@@ -33,6 +33,7 @@
 #include "vulkan/VulkanDescriptorSetCache.h"
 #include "vulkan/VulkanDescriptorSetLayoutCache.h"
 #include "vulkan/VulkanExternalImageManager.h"
+#include "vulkan/VulkanStreamedImageManager.h"
 #include "vulkan/VulkanPipelineLayoutCache.h"
 #include "vulkan/memory/ResourceManager.h"
 #include "vulkan/memory/ResourcePointer.h"
@@ -153,6 +154,7 @@ class VulkanDriver final : public DriverBase {
     VulkanDescriptorSetCache mDescriptorSetCache;
     VulkanQueryManager mQueryManager;
     VulkanExternalImageManager mExternalImageManager;
+    VulkanStreamedImageManager mStreamedImageManager;
 
     // This is necessary for us to write to push constants after binding a pipeline.
     using DescriptorSetLayoutHandleList = std::array<resource_ptr<VulkanDescriptorSetLayout>,
@@ -188,6 +190,10 @@ class VulkanDriver final : public DriverBase {
 
     bool const mIsSRGBSwapChainSupported;
     backend::StereoscopicType const mStereoscopicType;
+
+    // setAcquiredImage is a DECL_DRIVER_API_SYNCHRONOUS_N which means we don't necessarily have the
+    // data to process it at call time. So we store it and process it during updateStreams.
+    std::vector<resource_ptr<VulkanStream>> mStreamsWithPendingAcquiredImage;
 };
 
 } // namespace filament::backend

filament/backend/src/vulkan/VulkanExternalImageManager.h

@@ -31,7 +31,7 @@ class VulkanSamplerCache;
 class VulkanDescriptorSetLayoutCache;
 class VulkanDescriptorSetCache;
 
-// Manages the logic of external images and their quirks wrt Vulikan.
+// Manages the logic of external images and their quirks wrt Vulkan.
 class VulkanExternalImageManager {
 public:
 

filament/backend/src/vulkan/VulkanStreamedImageManager.cpp

@@ -0,0 +1,83 @@
+/*
+ * Copyright (C) 2025 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include "VulkanStreamedImageManager.h"
+
+#include "VulkanDescriptorSetCache.h"
+#include "VulkanExternalImageManager.h"
+#include "VulkanDescriptorSetLayoutCache.h"
+#include "VulkanSamplerCache.h"
+
+namespace filament::backend {
+    VulkanStreamedImageManager::VulkanStreamedImageManager(
+        VulkanExternalImageManager* manager,
+        VulkanDescriptorSetCache* descriptorSet,
+        VulkanSamplerCache* samplerCache)
+    : mExternalImageManager(manager),
+      mDescriptorSetCache(descriptorSet),
+      mSamplerCache(samplerCache) {
+
+}
+
+VulkanStreamedImageManager::~VulkanStreamedImageManager() = default;
+
+void VulkanStreamedImageManager::terminate() {}
+
+void VulkanStreamedImageManager::bindStreamedTexture(
+        fvkmemory::resource_ptr<VulkanDescriptorSet> set,
+        uint8_t bindingPoint, fvkmemory::resource_ptr<VulkanTexture> image,
+        SamplerParams samplerParams) {
+    mStreamedTexturesBindings.push_back({ bindingPoint, image, set, samplerParams });
+}
+
+void VulkanStreamedImageManager::unbindStreamedTexture(
+        fvkmemory::resource_ptr<VulkanDescriptorSet> set,
+        uint8_t bindingPoint) {
+    auto iter = std::remove_if(mStreamedTexturesBindings.begin(), mStreamedTexturesBindings.end(),
+            [&](streamedTextureBinding& binding) {
+                return ((binding.set == set) && (binding.binding == bindingPoint));
+            });
+    mStreamedTexturesBindings.erase(iter, mStreamedTexturesBindings.end());
+}
+
+void VulkanStreamedImageManager::onStreamAcquireImage(fvkmemory::resource_ptr<VulkanTexture> image,
+        fvkmemory::resource_ptr<VulkanStream> stream, bool newImage) {
+    for (streamedTextureBinding const& data: mStreamedTexturesBindings) {
+        if (data.image->getStream() == stream) {
+            if (newImage) {
+                mExternalImageManager->bindExternallySampledTexture(data.set, data.binding, image,
+                        data.samplerParams);
+            } else {
+                // For some reason, some of the frames coming to us, are on streams where the
+                // descriptor set isn't external...
+                if (data.set->getExternalSamplerVkSet()) {
+                    mDescriptorSetCache->updateSamplerForExternalSamplerSet(data.set, data.binding,
+                            image);
+                } else {
+                    //... In this case we just default to using the normal path and update the
+                    //sampler.
+                    VulkanSamplerCache::Params cacheParams = {
+                        .sampler = data.samplerParams,
+                    };
+                    VkSampler const vksampler = mSamplerCache->getSampler(cacheParams);
+                    mDescriptorSetCache->updateSampler(data.set, data.binding, image, vksampler);
+                }
+            }
+        }
+    }
+}
+
+} // namespace filament::backend

filament/backend/src/vulkan/VulkanStreamedImageManager.h

@@ -0,0 +1,68 @@
+/*
+ * Copyright (C) 2025 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#ifndef TNT_FILAMENT_BACKEND_CACHING_VULKANSTREAMEDIMAGEMANAGER_H
+#define TNT_FILAMENT_BACKEND_CACHING_VULKANSTREAMEDIMAGEMANAGER_H
+
+#include "VulkanHandles.h"
+
+#include <backend/DriverEnums.h>
+
+#include <array>
+#include <vector>
+
+namespace filament::backend {
+
+class VulkanExternalImageManager;
+class VulkanDescriptorSetCache;
+class VulkanSamplerCache;
+
+// Manages the logic of streamed and streamed images.
+class VulkanStreamedImageManager {
+public:
+    VulkanStreamedImageManager(
+            VulkanExternalImageManager* manager,
+            VulkanDescriptorSetCache* descriptorSet,
+            VulkanSamplerCache* samplerCache);
+    ~VulkanStreamedImageManager();
+    void terminate();
+
+public:
+    void bindStreamedTexture(fvkmemory::resource_ptr<VulkanDescriptorSet> set, uint8_t bindingPoint,
+            fvkmemory::resource_ptr<VulkanTexture> image, SamplerParams samplerParams);
+    void unbindStreamedTexture(fvkmemory::resource_ptr<VulkanDescriptorSet> set,
+            uint8_t bindingPoint);
+    void onStreamAcquireImage(fvkmemory::resource_ptr<VulkanTexture> image,
+            fvkmemory::resource_ptr<VulkanStream> stream, bool newImage);
+
+private:
+    struct streamedTextureBinding {
+        uint8_t binding = 0;
+        fvkmemory::resource_ptr<VulkanTexture> image;
+        fvkmemory::resource_ptr<VulkanDescriptorSet> set;
+        SamplerParams samplerParams;
+    };
+    // keep track of all the stream bindings
+    std::vector<streamedTextureBinding> mStreamedTexturesBindings;
+
+    VulkanExternalImageManager* mExternalImageManager;
+    VulkanDescriptorSetCache*   mDescriptorSetCache;
+    VulkanSamplerCache*         mSamplerCache;
+};
+
+} // namespace filament::backend
+
+#endif // TNT_FILAMENT_BACKEND_CACHING_VULKANSTREAMEDIMAGEMANAGER_H