Stream API Vulkan Backend

filament/backend/CMakeLists.txt

@@ -199,6 +199,8 @@ if (FILAMENT_SUPPORTS_VULKAN)
             src/vulkan/VulkanDriverFactory.h
             src/vulkan/VulkanExternalImageManager.cpp
             src/vulkan/VulkanExternalImageManager.h
+            src/vulkan/VulkanStreamedImageManager.cpp
+            src/vulkan/VulkanStreamedImageManager.h
             src/vulkan/VulkanFboCache.cpp
             src/vulkan/VulkanFboCache.h
             src/vulkan/VulkanHandles.cpp

filament/backend/include/backend/Platform.h

@@ -66,6 +66,9 @@ class UTILS_PUBLIC Platform {
         ExternalImageHandle& operator=(ExternalImageHandle const& rhs) noexcept;
         ExternalImageHandle& operator=(ExternalImageHandle&& rhs) noexcept;
 
+        bool operator==(const ExternalImageHandle& rhs) const noexcept {
+            return mTarget == rhs.mTarget;
+        }
         explicit operator bool() const noexcept { return mTarget != nullptr; }
 
         ExternalImage* UTILS_NULLABLE get() noexcept { return mTarget; }

filament/backend/include/backend/platforms/VulkanPlatform.h

@@ -406,6 +406,9 @@ class VulkanPlatform : public Platform, utils::PrivateImplementation<VulkanPlatf
     virtual ImageData createVkImageFromExternal(ExternalImageHandleRef image) const {
         return {};
     }
+    virtual ExternalImageHandle createExternalImageFromRaw(void* image, bool sRGB) noexcept {
+        return {};
+    }
 
 protected:
     virtual ExtensionSet getSwapchainInstanceExtensions() const;

filament/backend/include/backend/platforms/VulkanPlatformAndroid.h

@@ -28,6 +28,8 @@ class VulkanPlatformAndroid : public VulkanPlatform {
 public:
     ExternalImageHandle UTILS_PUBLIC createExternalImage(AHardwareBuffer const* buffer,
             bool sRGB) noexcept;
+    virtual ExternalImageHandle createExternalImageFromRaw(void* image,
+            bool sRGB) noexcept override;
 
     struct UTILS_PUBLIC ExternalImageDescAndroid {
         uint32_t width;      // Texture width

filament/backend/src/vulkan/VulkanDriver.cpp

@@ -231,6 +231,7 @@ VulkanDriver::VulkanDriver(VulkanPlatform* platform, VulkanContext& context,
       mQueryManager(mPlatform->getDevice()),
       mExternalImageManager(platform, &mSamplerCache, &mYcbcrConversionCache, &mDescriptorSetCache,
               &mDescriptorSetLayoutCache),
+      mStreamedImageManager(&mExternalImageManager, &mDescriptorSetCache, &mSamplerCache),
       mIsSRGBSwapChainSupported(mPlatform->getCustomization().isSRGBSwapChainSupported),
       mStereoscopicType(driverConfig.stereoscopicType) {
 
@@ -455,13 +456,17 @@ void VulkanDriver::updateDescriptorSetTexture(
     if (UTILS_UNLIKELY(mExternalImageManager.isExternallySampledTexture(texture))) {
         mExternalImageManager.bindExternallySampledTexture(set, binding, texture, params);
         mAppState.hasBoundExternalImages = true;
+    } else if (bool(texture->getStream())) {
+        mStreamedImageManager.bindStreamedTexture(set, binding, texture, params);
+        mAppState.hasBoundExternalImages = true;
     } else {
         VulkanSamplerCache::Params cacheParams = {
             .sampler = params,
         };
         VkSampler const vksampler = mSamplerCache.getSampler(cacheParams);
         mDescriptorSetCache.updateSampler(set, binding, texture, vksampler);
         mExternalImageManager.clearTextureBinding(set, binding);
+        mStreamedImageManager.unbindStreamedTexture(set, binding);
     }
 }
 
@@ -988,6 +993,11 @@ void VulkanDriver::destroySwapChain(Handle<HwSwapChain> sch) {
 }
 
 void VulkanDriver::destroyStream(Handle<HwStream> sh) {
+    if (!sh) {
+        return;
+    }
+    auto stream = resource_ptr<VulkanStream>::cast(&mResourceManager, sh);
+    stream.dec();
 }
 
 void VulkanDriver::destroyTimerQuery(Handle<HwTimerQuery> tqh) {
@@ -1014,25 +1024,120 @@ void VulkanDriver::destroyDescriptorSet(Handle<HwDescriptorSet> dsh) {
 }
 
 Handle<HwStream> VulkanDriver::createStreamNative(void* nativeStream, utils::CString tag) {
+    FILAMENT_CHECK_PRECONDITION(false) << "createStreamNative not supported in Vulkan.";
     return {};
 }
 
 Handle<HwStream> VulkanDriver::createStreamAcquired(utils::CString tag) {
-    return {};
+    FVK_SYSTRACE_SCOPE();
+    auto handle = mResourceManager.allocHandle<VulkanStream>();
+    auto stream = resource_ptr<VulkanStream>::make(&mResourceManager, handle);
+    stream.inc();
+    return handle;
 }
 
 void VulkanDriver::setAcquiredImage(Handle<HwStream> sh, void* image, const math::mat3f& transform,
         CallbackHandler* handler, StreamCallback cb, void* userData) {
+    FVK_SYSTRACE_SCOPE();
+    auto stream = resource_ptr<VulkanStream>::cast(&mResourceManager, sh);
+    stream->acquire({ image, cb, userData, handler});
+    stream->setTransform(transform);
+    mStreamsWithPendingAcquiredImage.push_back(stream);
 }
 
 void VulkanDriver::setStreamDimensions(Handle<HwStream> sh, uint32_t width, uint32_t height) {
+    FVK_SYSTRACE_SCOPE();
+    auto stream = resource_ptr<VulkanStream>::cast(&mResourceManager, sh);
+    stream->width = width;
+    stream->height = height;
 }
 
 int64_t VulkanDriver::getStreamTimestamp(Handle<HwStream> sh) {
     return 0;
 }
 
+math::mat3f VulkanDriver::getStreamTransformMatrix(Handle<HwStream> sh) {
+    if (sh) {
+        auto stream = resource_ptr<VulkanStream>::cast(&mResourceManager, sh);
+        if (stream->streamType == StreamType::NATIVE) {
+            FILAMENT_CHECK_PRECONDITION(false) << "Native Stream not supported in Vulkan.";
+        } else {
+            return stream->getTransform();
+        }
+    }
+    return math::mat3f();
+}
+
 void VulkanDriver::updateStreams(CommandStream* driver) {
+    FVK_SYSTRACE_SCOPE();
+    if (UTILS_UNLIKELY(!mStreamsWithPendingAcquiredImage.empty())) {
+        for (auto& stream: mStreamsWithPendingAcquiredImage) {
+            if (stream->previousNeedsRelease()) {
+                scheduleRelease(stream->takePrevious());
+            }
+
+            // This executes on the backend thread (updateStreams is synchonous which means it
+            // executes on the user thread) Note: stream is captured by copy which is fine, this is
+            // a copy of a resource_ptr<VulkanStream>. We only need it find the associated stream
+            // inside the mStreamedImageManager texture bindings
+            driver->queueCommand([this, stream, s = stream.get(),
+                                         image = stream->getAcquired().image,
+                                         transform = stream->getTransform()]() {
+                auto texture = s->getTexture(image);
+                bool newImage = false;
+                if (!texture) {
+                    auto externalImage = mPlatform->createExternalImageFromRaw(image, false);
+                    auto metadata = mPlatform->extractExternalImageMetadata(externalImage);
+                    auto imgData = mPlatform->createVkImageFromExternal(externalImage);
+
+                    assert_invariant(imgData.internal.valid() || imgData.external.valid());
+
+                    VkFormat vkformat = metadata.format;
+                    VkImage vkimage = VK_NULL_HANDLE;
+                    VkDeviceMemory memory = VK_NULL_HANDLE;
+                    if (imgData.internal.valid()) {
+                        metadata.externalFormat = 0;
+                        vkimage = imgData.internal.image;
+                        memory = imgData.internal.memory;
+                    } else {
+                        vkformat = VK_FORMAT_UNDEFINED;
+                        vkimage = imgData.external.image;
+                        memory = imgData.external.memory;
+                    }
+
+                    VkSamplerYcbcrConversion const conversion =
+                            mExternalImageManager.getVkSamplerYcbcrConversion(metadata);
+
+                    auto newTexture = resource_ptr<VulkanTexture>::construct(&mResourceManager,
+                            mContext, mPlatform->getDevice(), mAllocator, &mResourceManager,
+                            &mCommands, vkimage, memory, vkformat, conversion, metadata.samples,
+                            metadata.width, metadata.height, metadata.layers,
+                            metadata.filamentUsage, mStagePool);
+
+                    auto& commands = mCommands.get();
+                    // Unlike uploaded textures or swapchains, we need to explicit transition this
+                    // texture into the read layout.
+                    newTexture->transitionLayout(&commands, newTexture->getPrimaryViewRange(),
+                            VulkanLayout::FRAG_READ);
+
+                    if (imgData.external.valid()) {
+                        mExternalImageManager.addExternallySampledTexture(newTexture,
+                                externalImage);
+                        // Cache the AHB backed image.
+                        s->pushImage(image, newTexture);
+                        newImage = true;
+                    }
+
+                    newTexture.inc();
+                    texture = newTexture;
+                }
+
+                // Note that we capture the
+                mStreamedImageManager.onStreamAcquireImage(texture, stream, newImage);
+            });
+        }
+        mStreamsWithPendingAcquiredImage.clear();
+    }
 }
 
 void VulkanDriver::destroyFence(Handle<HwFence> fh) {
@@ -1278,7 +1383,34 @@ void VulkanDriver::updateIndexBuffer(Handle<HwIndexBuffer> ibh, BufferDescriptor
 
 void VulkanDriver::registerBufferObjectStreams(Handle<HwBufferObject> boh,
         BufferObjectStreamDescriptor&& streams) {
-    // Noop
+
+    auto bo = resource_ptr<VulkanBufferObject>::cast(&mResourceManager, boh);
+    mStreamUniformDescriptors[bo->getVkBuffer()] = std::move(streams);
+}
+
+// This needs to be discussed because Vulkan has different default Matrix formats
+// right now this is following GL.
+static void copyMat3f(void* addr, size_t const offset, const math::mat3f& v) noexcept {
+    struct mat43 {
+        float v[3][4];
+    };
+
+    addr = static_cast<char*>(addr) + offset;
+    mat43& temp = *static_cast<mat43*>(addr);
+
+    temp.v[0][0] = v[0][0];
+    temp.v[0][1] = v[0][1];
+    temp.v[0][2] = v[0][2];
+
+    temp.v[1][0] = v[1][0];
+    temp.v[1][1] = v[1][1];
+    temp.v[1][2] = v[1][2];
+
+    temp.v[2][0] = v[2][0];
+    temp.v[2][1] = v[2][1];
+    temp.v[2][2] = v[2][2];
+
+    // don't store anything in temp.v[][3] because there could be uniforms packed there
 }
 
 void VulkanDriver::updateBufferObject(Handle<HwBufferObject> boh, BufferDescriptor&& bd,
@@ -1289,6 +1421,20 @@ void VulkanDriver::updateBufferObject(Handle<HwBufferObject> boh, BufferDescript
     commands.acquire(bo);
     bo->loadFromCpu(commands, bd.buffer, byteOffset, bd.size);
 
+    if (UTILS_UNLIKELY(!mStreamUniformDescriptors.empty())) {
+        auto streamDescriptors = mStreamUniformDescriptors.find(bo->getVkBuffer());
+        if (streamDescriptors != mStreamUniformDescriptors.end()) {
+            for (auto const& [offset, stream, associationType]:
+                    streamDescriptors->second.mStreams) {
+                if (associationType == BufferObjectStreamAssociationType::TRANSFORM_MATRIX) {
+                    auto transform = getStreamTransformMatrix(stream);
+                    copyMat3f(bd.buffer, offset, transform);
+                }
+            }
+            mStreamUniformDescriptors.erase(streamDescriptors);
+        }
+    }
+
     scheduleDestroy(std::move(bd));
 }
 
@@ -1354,6 +1500,12 @@ TimerQueryResult VulkanDriver::getTimerQueryValue(Handle<HwTimerQuery> tqh, uint
 }
 
 void VulkanDriver::setExternalStream(Handle<HwTexture> th, Handle<HwStream> sh) {
+    FVK_SYSTRACE_SCOPE();
+    auto t = resource_ptr<VulkanTexture>::cast(&mResourceManager, th);
+    assert_invariant(t);
+    auto s = resource_ptr<VulkanStream>::cast(&mResourceManager, sh);
+    assert_invariant(s);
+    t->setStream(s);
 }
 
 void VulkanDriver::generateMipmaps(Handle<HwTexture> th) {

filament/backend/src/vulkan/VulkanDriver.h

@@ -33,6 +33,7 @@
 #include "vulkan/VulkanDescriptorSetCache.h"
 #include "vulkan/VulkanDescriptorSetLayoutCache.h"
 #include "vulkan/VulkanExternalImageManager.h"
+#include "vulkan/VulkanStreamedImageManager.h"
 #include "vulkan/VulkanPipelineLayoutCache.h"
 #include "vulkan/memory/ResourceManager.h"
 #include "vulkan/memory/ResourcePointer.h"
@@ -153,6 +154,12 @@ class VulkanDriver final : public DriverBase {
     VulkanDescriptorSetCache mDescriptorSetCache;
     VulkanQueryManager mQueryManager;
     VulkanExternalImageManager mExternalImageManager;
+    VulkanStreamedImageManager mStreamedImageManager;
+
+    // Stream transforms
+    std::unordered_map<VkBuffer, BufferObjectStreamDescriptor> mStreamUniformDescriptors;
+    math::mat3f getStreamTransformMatrix(Handle<HwStream> sh);
+
 
     // This is necessary for us to write to push constants after binding a pipeline.
     using DescriptorSetLayoutHandleList = std::array<resource_ptr<VulkanDescriptorSetLayout>,
@@ -188,6 +195,10 @@ class VulkanDriver final : public DriverBase {
 
     bool const mIsSRGBSwapChainSupported;
     backend::StereoscopicType const mStereoscopicType;
+
+    // setAcquiredImage is a DECL_DRIVER_API_SYNCHRONOUS_N which means we don't necessarily have the
+    // data to process it at call time. So we store it and process it during updateStreams.
+    std::vector<resource_ptr<VulkanStream>> mStreamsWithPendingAcquiredImage;
 };
 
 } // namespace filament::backend

filament/backend/src/vulkan/VulkanExternalImageManager.h

@@ -31,7 +31,7 @@ class VulkanSamplerCache;
 class VulkanDescriptorSetLayoutCache;
 class VulkanDescriptorSetCache;
 
-// Manages the logic of external images and their quirks wrt Vulikan.
+// Manages the logic of external images and their quirks wrt Vulkan.
 class VulkanExternalImageManager {
 public:
 

filament/backend/src/vulkan/VulkanStreamedImageManager.cpp

@@ -0,0 +1,87 @@
+/*
+ * Copyright (C) 2025 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include "VulkanStreamedImageManager.h"
+
+#include "VulkanDescriptorSetCache.h"
+#include "VulkanExternalImageManager.h"
+#include "VulkanDescriptorSetLayoutCache.h"
+#include "VulkanSamplerCache.h"
+
+namespace filament::backend {
+    VulkanStreamedImageManager::VulkanStreamedImageManager(
+        VulkanExternalImageManager* manager,
+        VulkanDescriptorSetCache* descriptorSet,
+        VulkanSamplerCache* samplerCache)
+    : mExternalImageManager(manager),
+      mDescriptorSetCache(descriptorSet),
+      mSamplerCache(samplerCache) {
+
+}
+
+VulkanStreamedImageManager::~VulkanStreamedImageManager() = default;
+
+void VulkanStreamedImageManager::terminate() { mStreamedTexturesBindings.clear(); }
+
+void VulkanStreamedImageManager::bindStreamedTexture(
+        fvkmemory::resource_ptr<VulkanDescriptorSet> set,
+        uint8_t bindingPoint, fvkmemory::resource_ptr<VulkanTexture> image,
+        SamplerParams samplerParams) {
+    mStreamedTexturesBindings.push_back({ bindingPoint, image, set, samplerParams });
+}
+
+void VulkanStreamedImageManager::unbindStreamedTexture(
+        fvkmemory::resource_ptr<VulkanDescriptorSet> set,
+        uint8_t bindingPoint) {
+    auto iter = std::remove_if(mStreamedTexturesBindings.begin(), mStreamedTexturesBindings.end(),
+            [&](StreamedTextureBinding& binding) {
+                return ((binding.set == set) && (binding.binding == bindingPoint));
+            });
+    mStreamedTexturesBindings.erase(iter, mStreamedTexturesBindings.end());
+}
+
+void VulkanStreamedImageManager::onStreamAcquireImage(fvkmemory::resource_ptr<VulkanTexture> image,
+        fvkmemory::resource_ptr<VulkanStream> stream, bool newImage) {
+    for (StreamedTextureBinding const& data: mStreamedTexturesBindings) {
+        if (data.image->getStream() == stream) {
+            if (newImage) {
+                mExternalImageManager->bindExternallySampledTexture(data.set, data.binding, image,
+                        data.samplerParams);
+            } else {
+                // For some reason, some of the frames coming to us, are on streams where the
+                // descriptor set isn't external...
+                if (data.set->getExternalSamplerVkSet()) {
+                    // Eventually the updateSampler and updateSamplerForExternalSamplerSet 
+                    // will call to vkUpdateDescriptorSets with a VkWriteDescriptorSet 
+                    // type VK_DESCRIPTOR_TYPE_COMBINED_IMAGE_SAMPLER the 
+                    // VkDescriptorImageInfo will contain the view of the new image frame.
+                    mDescriptorSetCache->updateSamplerForExternalSamplerSet(data.set, data.binding,
+                            image);
+                } else {
+                    //... In this case we just default to using the normal path and update the
+                    //sampler.
+                    VulkanSamplerCache::Params cacheParams = {
+                        .sampler = data.samplerParams,
+                    };
+                    VkSampler const vksampler = mSamplerCache->getSampler(cacheParams);
+                    mDescriptorSetCache->updateSampler(data.set, data.binding, image, vksampler);
+                }
+            }
+        }
+    }
+}
+
+} // namespace filament::backend