Stream API Vulkan Backend

filament/backend/CMakeLists.txt

@@ -199,6 +199,8 @@ if (FILAMENT_SUPPORTS_VULKAN)
             src/vulkan/VulkanDriverFactory.h
             src/vulkan/VulkanExternalImageManager.cpp
             src/vulkan/VulkanExternalImageManager.h
+            src/vulkan/VulkanStreamedImageManager.cpp
+            src/vulkan/VulkanStreamedImageManager.h
             src/vulkan/VulkanFboCache.cpp
             src/vulkan/VulkanFboCache.h
             src/vulkan/VulkanHandles.cpp

filament/backend/include/backend/Platform.h

@@ -70,6 +70,9 @@ class UTILS_PUBLIC Platform {
         ExternalImageHandle& operator=(ExternalImageHandle const& rhs) noexcept;
         ExternalImageHandle& operator=(ExternalImageHandle&& rhs) noexcept;
 
+        bool operator==(const ExternalImageHandle& rhs) const noexcept {
+            return mTarget == rhs.mTarget;
+        }
         explicit operator bool() const noexcept { return mTarget != nullptr; }
 
         ExternalImage* UTILS_NULLABLE get() noexcept { return mTarget; }

filament/backend/src/vulkan/VulkanDriver.cpp

@@ -31,6 +31,7 @@
 #include "vulkan/memory/ResourcePointer.h"
 #include "vulkan/utils/Conversion.h"
 #include "vulkan/utils/Definitions.h"
+#include "vulkan/utils/Image.h"
 
 #include <backend/DriverEnums.h>
 #include <backend/platforms/VulkanPlatform.h>
@@ -145,6 +146,31 @@ static CallbackHandler::Callback syncCallbackWrapper = [](void* userData) {
     cbData->cb(cbData->sync, cbData->userData);
 };
 
+// This needs to be discussed because Vulkan has different default Matrix formats
+// right now this is following GL.
+static void copyMat3f(void* addr, size_t const offset, const math::mat3f& v) noexcept {
+    struct mat43 {
+        float v[3][4];
+    };
+
+    addr = static_cast<char*>(addr) + offset;
+    mat43& temp = *static_cast<mat43*>(addr);
+
+    temp.v[0][0] = v[0][0];
+    temp.v[0][1] = v[0][1];
+    temp.v[0][2] = v[0][2];
+
+    temp.v[1][0] = v[1][0];
+    temp.v[1][1] = v[1][1];
+    temp.v[1][2] = v[1][2];
+
+    temp.v[2][0] = v[2][0];
+    temp.v[2][1] = v[2][1];
+    temp.v[2][2] = v[2][2];
+
+    // don't store anything in temp.v[][3] because there could be uniforms packed there
+}
+
 }// anonymous namespace
 
 #if FVK_ENABLED(FVK_DEBUG_DEBUG_UTILS)
@@ -239,6 +265,7 @@ VulkanDriver::VulkanDriver(VulkanPlatform* platform, VulkanContext& context,
       mQueryManager(mPlatform->getDevice()),
       mExternalImageManager(platform, &mSamplerCache, &mYcbcrConversionCache, &mDescriptorSetCache,
               &mDescriptorSetLayoutCache),
+      mStreamedImageManager(&mExternalImageManager, &mDescriptorSetCache, &mSamplerCache),
       mIsSRGBSwapChainSupported(mPlatform->getCustomization().isSRGBSwapChainSupported),
       mIsMSAASwapChainSupported(false), // TODO: support MSAA swapchain
       mStereoscopicType(driverConfig.stereoscopicType) {
@@ -466,13 +493,17 @@ void VulkanDriver::updateDescriptorSetTexture(
     if (UTILS_UNLIKELY(mExternalImageManager.isExternallySampledTexture(texture))) {
         mExternalImageManager.bindExternallySampledTexture(set, binding, texture, params);
         mAppState.hasBoundExternalImages = true;
+    } else if (bool(texture->getStream())) {
+        mStreamedImageManager.bindStreamedTexture(set, binding, texture, params);
+        mAppState.hasBoundExternalImages = true;
     } else {
         VulkanSamplerCache::Params cacheParams = {
             .sampler = params,
         };
         VkSampler const vksampler = mSamplerCache.getSampler(cacheParams);
         mDescriptorSetCache.updateSampler(set, binding, texture, vksampler);
         mExternalImageManager.clearTextureBinding(set, binding);
+        mStreamedImageManager.unbindStreamedTexture(set, binding);
     }
 }
 
@@ -1050,6 +1081,12 @@ void VulkanDriver::destroySwapChain(Handle<HwSwapChain> sch) {
 }
 
 void VulkanDriver::destroyStream(Handle<HwStream> sh) {
+    FVK_SYSTRACE_SCOPE();
+    if (!sh) {
+        return;
+    }
+    auto stream = resource_ptr<VulkanStream>::cast(&mResourceManager, sh);
+    stream.dec();
 }
 
 void VulkanDriver::destroySync(Handle<HwSync> sh) {
@@ -1087,25 +1124,112 @@ void VulkanDriver::destroyDescriptorSet(Handle<HwDescriptorSet> dsh) {
 }
 
 Handle<HwStream> VulkanDriver::createStreamNative(void* nativeStream, utils::CString tag) {
+    FILAMENT_CHECK_PRECONDITION(false) << "createStreamNative not supported in Vulkan.";
     return {};
 }
 
 Handle<HwStream> VulkanDriver::createStreamAcquired(utils::CString tag) {
-    return {};
+    // @TODO This is still not thread-safe. We might have to revisit this question.
+    FVK_SYSTRACE_SCOPE();
+    auto handle = mResourceManager.allocHandle<VulkanStream>();
+    auto stream = resource_ptr<VulkanStream>::make(&mResourceManager, handle);
+    stream.inc();
+    return handle;
 }
 
 void VulkanDriver::setAcquiredImage(Handle<HwStream> sh, void* image, const math::mat3f& transform,
         CallbackHandler* handler, StreamCallback cb, void* userData) {
+    FVK_SYSTRACE_SCOPE();
+    auto stream = resource_ptr<VulkanStream>::cast(&mResourceManager, sh);
+    assert_invariant(stream->streamType == StreamType::ACQUIRED);
+    assert_invariant(image != nullptr);
+    // This covers the case where setAcquiredImage is called back to back (no updateStreams in between)
+    if (stream->previousNeedsRelease()) {
+        scheduleRelease(stream->takePrevious());
+    }
+    stream->acquire({ image, cb, userData, handler});
+    stream->setFrontEndTransform(transform);
+    mStreamsWithPendingAcquiredImage.push_back(stream);
 }
 
 void VulkanDriver::setStreamDimensions(Handle<HwStream> sh, uint32_t width, uint32_t height) {
+    FVK_SYSTRACE_SCOPE();
+    auto stream = resource_ptr<VulkanStream>::cast(&mResourceManager, sh);
+    stream->width = width;
+    stream->height = height;
 }
 
 int64_t VulkanDriver::getStreamTimestamp(Handle<HwStream> sh) {
     return 0;
 }
 
 void VulkanDriver::updateStreams(CommandStream* driver) {
+    FVK_SYSTRACE_SCOPE();
+    if (UTILS_UNLIKELY(!mStreamsWithPendingAcquiredImage.empty())) {
+        for (auto& stream: mStreamsWithPendingAcquiredImage) {
+            if (stream->previousNeedsRelease()) {
+                scheduleRelease(stream->takePrevious());
+            }
+
+            // This executes on the backend thread (updateStreams is synchonous which means it
+            // executes on the user thread) Note: stream is captured by copy which is fine, this is
+            // a copy of a resource_ptr<VulkanStream>. We only need it find the associated stream
+            // inside the mStreamedImageManager texture bindings
+            driver->queueCommand([this, stream, s = stream.get(),
+                                        image = stream->getAcquired().image,
+                                        transform = stream->getFrontEndTransform()]() {
+                auto texture = s->getTexture(image);
+                s->setBackendTransform(transform);
+                if (!texture) {
+                    auto externalImage = fvkutils::createExternalImageFromRaw(mPlatform, image, false);
+                    auto metadata = mPlatform->extractExternalImageMetadata(externalImage);
+                    auto imgData = mPlatform->createVkImageFromExternal(externalImage);
+
+                    assert_invariant(imgData.internal.valid() || imgData.external.valid());
+
+                    VkFormat vkformat = metadata.format;
+                    VkImage vkimage = VK_NULL_HANDLE;
+                    VkDeviceMemory memory = VK_NULL_HANDLE;
+                    if (imgData.internal.valid()) {
+                        metadata.externalFormat = 0;
+                        vkimage = imgData.internal.image;
+                        memory = imgData.internal.memory;
+                    } else {
+                        vkformat = VK_FORMAT_UNDEFINED;
+                        vkimage = imgData.external.image;
+                        memory = imgData.external.memory;
+                    }
+
+                    VkSamplerYcbcrConversion const conversion =
+                            mExternalImageManager.getVkSamplerYcbcrConversion(metadata);
+
+                    auto newTexture = resource_ptr<VulkanTexture>::construct(&mResourceManager,
+                            mContext, mPlatform->getDevice(), mAllocator, &mResourceManager,
+                            &mCommands, vkimage, memory, vkformat, conversion, metadata.samples,
+                            metadata.width, metadata.height, metadata.layers,
+                            metadata.filamentUsage, mStagePool);
+
+                    auto& commands = mCommands.get();
+                    // Unlike uploaded textures or swapchains, we need to explicit transition this
+                    // texture into the read layout.
+                    newTexture->transitionLayout(&commands, newTexture->getPrimaryViewRange(),
+                            VulkanLayout::FRAG_READ);
+
+                    if (imgData.external.valid()) {
+                        mExternalImageManager.addExternallySampledTexture(newTexture,
+                                externalImage);
+                        // Cache the AHB backed image. Acquires the image here.
+                        s->pushImage(image, newTexture);
+                    }
+
+                    texture = newTexture;
+                }
+
+                mStreamedImageManager.onStreamAcquireImage(texture, stream);
+            });
+        }
+        mStreamsWithPendingAcquiredImage.clear();
+    }
 }
 
 void VulkanDriver::destroyFence(Handle<HwFence> fh) {
@@ -1378,7 +1502,9 @@ void VulkanDriver::updateIndexBuffer(Handle<HwIndexBuffer> ibh, BufferDescriptor
 
 void VulkanDriver::registerBufferObjectStreams(Handle<HwBufferObject> boh,
         BufferObjectStreamDescriptor&& streams) {
-    // Noop
+
+    auto bo = resource_ptr<VulkanBufferObject>::cast(&mResourceManager, boh);
+    mStreamUniformDescriptors[bo.get()] = std::move(streams);
 }
 
 void VulkanDriver::updateBufferObject(Handle<HwBufferObject> boh, BufferDescriptor&& bd,
@@ -1389,6 +1515,33 @@ void VulkanDriver::updateBufferObject(Handle<HwBufferObject> boh, BufferDescript
     commands.acquire(bo);
     bo->loadFromCpu(commands, bd.buffer, byteOffset, bd.size);
 
+    if (UTILS_UNLIKELY(!mStreamUniformDescriptors.empty())) {
+        auto streamDescriptors = mStreamUniformDescriptors.find(bo.get());
+        if (streamDescriptors != mStreamUniformDescriptors.end()) {
+            for (auto const& [offset, streamHandle, associationType]:
+                    streamDescriptors->second.mStreams) {
+                if (associationType == BufferObjectStreamAssociationType::TRANSFORM_MATRIX) {
+                    math::mat3f transform;
+                    if (streamHandle) {
+                        auto stream =
+                                resource_ptr<VulkanStream>::cast(&mResourceManager, streamHandle);
+                        if (stream->streamType == StreamType::NATIVE) {
+                            FILAMENT_CHECK_PRECONDITION(false)
+                                    << "Native Stream not supported in Vulkan.";
+                        } else {
+                            // Backend call since getStreamTransformMatrix is called from async
+                            // updateBufferObject.
+                            transform = stream->getBackEndTransform();
+                        }
+                    }
+
+                    copyMat3f(bd.buffer, offset, transform);
+                }
+            }
+            mStreamUniformDescriptors.erase(streamDescriptors);
+        }
+    }
+
     scheduleDestroy(std::move(bd));
 }
 
@@ -1454,6 +1607,12 @@ TimerQueryResult VulkanDriver::getTimerQueryValue(Handle<HwTimerQuery> tqh, uint
 }
 
 void VulkanDriver::setExternalStream(Handle<HwTexture> th, Handle<HwStream> sh) {
+    FVK_SYSTRACE_SCOPE();
+    auto t = resource_ptr<VulkanTexture>::cast(&mResourceManager, th);
+    assert_invariant(t);
+    auto s = resource_ptr<VulkanStream>::cast(&mResourceManager, sh);
+    assert_invariant(s);
+    t->setStream(s);
 }
 
 void VulkanDriver::generateMipmaps(Handle<HwTexture> th) {

filament/backend/src/vulkan/VulkanDriver.h

@@ -33,6 +33,7 @@
 #include "vulkan/VulkanDescriptorSetCache.h"
 #include "vulkan/VulkanDescriptorSetLayoutCache.h"
 #include "vulkan/VulkanExternalImageManager.h"
+#include "vulkan/VulkanStreamedImageManager.h"
 #include "vulkan/VulkanPipelineLayoutCache.h"
 #include "vulkan/memory/ResourceManager.h"
 #include "vulkan/memory/ResourcePointer.h"
@@ -153,6 +154,12 @@ class VulkanDriver final : public DriverBase {
     VulkanDescriptorSetCache mDescriptorSetCache;
     VulkanQueryManager mQueryManager;
     VulkanExternalImageManager mExternalImageManager;
+    VulkanStreamedImageManager mStreamedImageManager;
+
+    // Stream transforms
+    std::unordered_map<VulkanBufferObject*, BufferObjectStreamDescriptor> mStreamUniformDescriptors;
+    math::mat3f getStreamTransformMatrix(Handle<HwStream> sh);
+
 
     // This is necessary for us to write to push constants after binding a pipeline.
     using DescriptorSetLayoutHandleList = std::array<resource_ptr<VulkanDescriptorSetLayout>,
@@ -189,6 +196,10 @@ class VulkanDriver final : public DriverBase {
     bool const mIsSRGBSwapChainSupported;
     bool const mIsMSAASwapChainSupported;
     backend::StereoscopicType const mStereoscopicType;
+
+    // setAcquiredImage is a DECL_DRIVER_API_SYNCHRONOUS_N which means we don't necessarily have the
+    // data to process it at call time. So we store it and process it during updateStreams.
+    std::vector<resource_ptr<VulkanStream>> mStreamsWithPendingAcquiredImage;
 };
 
 } // namespace filament::backend

filament/backend/src/vulkan/VulkanExternalImageManager.cpp

@@ -261,7 +261,17 @@ void VulkanExternalImageManager::bindExternallySampledTexture(
         fvkmemory::resource_ptr<VulkanTexture> image, SamplerParams samplerParams) {
     // Should we do duplicate validation here?
     auto& imageData = findImage(mImages, image);
-    mSetBindings.push_back({ bindingPoint, imageData.image, set, samplerParams });
+    auto itr = std::find_if(mSetBindings.begin(), mSetBindings.end(),
+            [&](SetBindingInfo const& binding) {
+                return (binding.set == set && binding.binding == bindingPoint);
+            });
+    if (itr == mSetBindings.end()) {
+        mSetBindings.push_back({ bindingPoint, imageData.image, set, samplerParams });
+    } else {
+        // override the image data in the binding point
+        itr->image = image;
+        itr->samplerParams = samplerParams;
+    }
 }
 
 void VulkanExternalImageManager::addExternallySampledTexture(

filament/backend/src/vulkan/VulkanExternalImageManager.h

@@ -31,7 +31,7 @@ class VulkanSamplerCache;
 class VulkanDescriptorSetLayoutCache;
 class VulkanDescriptorSetCache;
 
-// Manages the logic of external images and their quirks wrt Vulikan.
+// Manages the logic of external images and their quirks wrt Vulkan.
 class VulkanExternalImageManager {
 public: