.Net: Support Support BinaryContent in Gemini Connector

dotnet/samples/Concepts/ChatCompletion/Google_GeminiChatCompletionWithFile.cs

@@ -0,0 +1,145 @@
+// Copyright (c) Microsoft. All rights reserved.
+
+using Microsoft.SemanticKernel;
+using Microsoft.SemanticKernel.ChatCompletion;
+using Resources;
+
+namespace ChatCompletion;
+
+/// <summary>
+/// This sample shows how to use binary file and inline Base64 inputs, like PDFs, with Google Gemini's chat completion.
+/// </summary>
+public class Google_GeminiChatCompletionWithFile(ITestOutputHelper output) : BaseTest(output)
+{
+    [Fact]
+    public async Task GoogleAIChatCompletionWithLocalFile()
+    {
+        Console.WriteLine("============= Google AI - Gemini Chat Completion With Local File =============");
+
+        Assert.NotNull(TestConfiguration.GoogleAI.ApiKey);
+        Assert.NotNull(TestConfiguration.GoogleAI.Gemini.ModelId);
+
+        Kernel kernel = Kernel.CreateBuilder()
+            .AddGoogleAIGeminiChatCompletion(TestConfiguration.GoogleAI.Gemini.ModelId, TestConfiguration.GoogleAI.ApiKey)
+            .Build();
+
+        var fileBytes = await EmbeddedResource.ReadAllAsync("employees.pdf");
+
+        var chatHistory = new ChatHistory("You are a friendly assistant.");
+        chatHistory.AddUserMessage(
+        [
+            new TextContent("What's in this file?"),
+            new BinaryContent(fileBytes, "application/pdf")
+        ]);
+
+        var chatCompletionService = kernel.GetRequiredService<IChatCompletionService>();
+
+        var reply = await chatCompletionService.GetChatMessageContentAsync(chatHistory);
+
+        Console.WriteLine(reply.Content);
+    }
+
+    [Fact]
+    public async Task VertexAIChatCompletionWithLocalFile()
+    {
+        Console.WriteLine("============= Vertex AI - Gemini Chat Completion With Local File =============");
+
+        Assert.NotNull(TestConfiguration.VertexAI.BearerKey);
+        Assert.NotNull(TestConfiguration.VertexAI.Location);
+        Assert.NotNull(TestConfiguration.VertexAI.ProjectId);
+        Assert.NotNull(TestConfiguration.VertexAI.Gemini.ModelId);
+
+        Kernel kernel = Kernel.CreateBuilder()
+            .AddVertexAIGeminiChatCompletion(
+                modelId: TestConfiguration.VertexAI.Gemini.ModelId,
+                bearerKey: TestConfiguration.VertexAI.BearerKey,
+                location: TestConfiguration.VertexAI.Location,
+                projectId: TestConfiguration.VertexAI.ProjectId)
+            .Build();
+
+        var fileBytes = await EmbeddedResource.ReadAllAsync("employees.pdf");
+
+        var chatHistory = new ChatHistory("You are a friendly assistant.");
+        chatHistory.AddUserMessage(
+        [
+            new TextContent("What's in this file?"),
+            new BinaryContent(fileBytes, "application/pdf"),
+        ]);
+
+        var chatCompletionService = kernel.GetRequiredService<IChatCompletionService>();
+
+        var reply = await chatCompletionService.GetChatMessageContentAsync(chatHistory);
+
+        Console.WriteLine(reply.Content);
+    }
+
+    [Fact]
+    public async Task GoogleAIChatCompletionWithBase64DataUri()
+    {
+        Console.WriteLine("============= Google AI - Gemini Chat Completion With Base64 Data Uri =============");
+
+        Assert.NotNull(TestConfiguration.GoogleAI.ApiKey);
+        Assert.NotNull(TestConfiguration.GoogleAI.Gemini.ModelId);
+
+        Kernel kernel = Kernel.CreateBuilder()
+            .AddGoogleAIGeminiChatCompletion(TestConfiguration.GoogleAI.Gemini.ModelId, TestConfiguration.GoogleAI.ApiKey)
+            .Build();
+
+        var fileBytes = await EmbeddedResource.ReadAllAsync("employees.pdf");
+        var fileBase64 = Convert.ToBase64String(fileBytes.ToArray());
+        var dataUri = $"data:application/pdf;base64,{fileBase64}";
+
+        var chatHistory = new ChatHistory("You are a friendly assistant.");
+        chatHistory.AddUserMessage(
+        [
+            new TextContent("What's in this file?"),
+            new BinaryContent(dataUri)
+            // Google AI Gemini AI does not support arbitrary URIs but we can convert a Base64 URI into InlineData with the correct mimeType.
+        ]);
+
+        var chatCompletionService = kernel.GetRequiredService<IChatCompletionService>();
+
+        var reply = await chatCompletionService.GetChatMessageContentAsync(chatHistory);
+
+        Console.WriteLine(reply.Content);
+    }
+
+    [Fact]
+    public async Task VertexAIChatCompletionWithBase64DataUri()
+    {
+        Console.WriteLine("============= Vertex AI - Gemini Chat Completion With Base64 Data Uri =============");
+
+        Assert.NotNull(TestConfiguration.VertexAI.BearerKey);
+        Assert.NotNull(TestConfiguration.VertexAI.Location);
+        Assert.NotNull(TestConfiguration.VertexAI.ProjectId);
+        Assert.NotNull(TestConfiguration.VertexAI.Gemini.ModelId);
+
+        Kernel kernel = Kernel.CreateBuilder()
+            .AddVertexAIGeminiChatCompletion(
+                modelId: TestConfiguration.VertexAI.Gemini.ModelId,
+                bearerKey: TestConfiguration.VertexAI.BearerKey,
+                location: TestConfiguration.VertexAI.Location,
+                projectId: TestConfiguration.VertexAI.ProjectId)
+            .Build();
+
+        var fileBytes = await EmbeddedResource.ReadAllAsync("employees.pdf");
+        var fileBase64 = Convert.ToBase64String(fileBytes.ToArray());
+        var dataUri = $"data:application/pdf;base64,{fileBase64}";
+
+        var chatHistory = new ChatHistory("You are a friendly assistant.");
+        chatHistory.AddUserMessage(
+        [
+            new TextContent("What's in this file?"),
+            new BinaryContent(dataUri)
+            // Vertex AI API does not support URIs outside of inline Base64 or GCS buckets within the same project. The bucket that stores the file must be in the same Google Cloud project that's sending the request. You must always provide the mimeType via the metadata property.
+            // var content = new BinaryContent(gs://generativeai-downloads/files/employees.pdf);
+            // content.Metadata = new Dictionary<string, object?> { { "mimeType", "application/pdf" } };
+        ]);
+
+        var chatCompletionService = kernel.GetRequiredService<IChatCompletionService>();
+
+        var reply = await chatCompletionService.GetChatMessageContentAsync(chatHistory);
+
+        Console.WriteLine(reply.Content);
+    }
+}

dotnet/samples/Concepts/README.md

@@ -64,6 +64,7 @@ dotnet test -l "console;verbosity=detailed" --filter "FullyQualifiedName=ChatCom
 - [Google_GeminiChatCompletion](https://github.com/microsoft/semantic-kernel/blob/main/dotnet/samples/Concepts/ChatCompletion/Google_GeminiChatCompletion.cs)
 - [Google_GeminiChatCompletionStreaming](https://github.com/microsoft/semantic-kernel/blob/main/dotnet/samples/Concepts/ChatCompletion/Google_GeminiChatCompletionStreaming.cs)
 - [Google_GeminiChatCompletionWithThinkingBudget](https://github.com/microsoft/semantic-kernel/blob/main/dotnet/samples/Concepts/ChatCompletion/Google_GeminiChatCompletionWithThinkingBudget.cs)
+- [Google_GeminiChatCompletionWithFile.cs](https://github.com/microsoft/semantic-kernel/blob/main/dotnet/samples/Concepts/ChatCompletion/Google_GeminiChatCompletionWithFile.cs)
 - [Google_GeminiGetModelResult](https://github.com/microsoft/semantic-kernel/blob/main/dotnet/samples/Concepts/ChatCompletion/Google_GeminiGetModelResult.cs)
 - [Google_GeminiStructuredOutputs](https://github.com/microsoft/semantic-kernel/blob/main/dotnet/samples/Concepts/ChatCompletion/Google_GeminiStructuredOutputs.cs)
 - [Google_GeminiVision](https://github.com/microsoft/semantic-kernel/blob/main/dotnet/samples/Concepts/ChatCompletion/Google_GeminiVision.cs)

dotnet/src/Connectors/Connectors.Google.UnitTests/Core/Gemini/GeminiRequestTests.cs

@@ -371,6 +371,33 @@ public void FromChatHistoryAudioAsAudioContentItReturnsWithChatHistory()
                 .SequenceEqual(Convert.FromBase64String(c.Parts![0].InlineData!.InlineData))));
     }
 
+    [Fact]
+    public void FromChatHistoryPdfAsBinaryContentItReturnsWithChatHistory()
+    {
+        // Arrange
+        ReadOnlyMemory<byte> pdfAsBytes = new byte[] { 0x00, 0x01, 0x02, 0x03 };
+        ChatHistory chatHistory = [];
+        chatHistory.AddUserMessage("user-message");
+        chatHistory.AddAssistantMessage("assist-message");
+        chatHistory.AddUserMessage(contentItems:
+            [new BinaryContent(new Uri("https://example-file.com/file.pdf")) { MimeType = "application/pdf" }]);
+        chatHistory.AddUserMessage(contentItems:
+            [new BinaryContent(pdfAsBytes, "application/pdf")]);
+        var executionSettings = new GeminiPromptExecutionSettings();
+
+        // Act
+        var request = GeminiRequest.FromChatHistoryAndExecutionSettings(chatHistory, executionSettings);
+
+        // Assert
+        Assert.Collection(request.Contents,
+            c => Assert.Equal(chatHistory[0].Content, c.Parts![0].Text),
+            c => Assert.Equal(chatHistory[1].Content, c.Parts![0].Text),
+            c => Assert.Equal(chatHistory[2].Items.Cast<BinaryContent>().Single().Uri,
+                c.Parts![0].FileData!.FileUri),
+            c => Assert.True(pdfAsBytes.ToArray()
+                .SequenceEqual(Convert.FromBase64String(c.Parts![0].InlineData!.InlineData))));
+    }
+
     [Fact]
     public void FromChatHistoryUnsupportedContentItThrowsNotSupportedException()
     {

dotnet/src/Connectors/Connectors.Google.UnitTests/Services/GoogleAIGeminiChatCompletionServiceTests.cs

@@ -5,7 +5,9 @@
 using System.IO;
 using System.Net.Http;
 using System.Text;
+using System.Text.Json;
 using System.Threading.Tasks;
+using Microsoft.SemanticKernel;
 using Microsoft.SemanticKernel.ChatCompletion;
 using Microsoft.SemanticKernel.Connectors.Google;
 using Microsoft.SemanticKernel.Services;
@@ -144,6 +146,78 @@ public async Task RequestBodyIncludesThinkingConfigWhenSetAsync(int? thinkingBud
         }
     }
 
+    [Fact]
+    public async Task GetChatMessageContentsAsyncThrowsExceptionWithEmptyBinaryContentAsync()
+    {
+        // Arrange
+        var sut = new GoogleAIGeminiChatCompletionService("gemini-2.5-pro", "key");
+
+        var chatHistory = new ChatHistory();
+        chatHistory.AddUserMessage([new BinaryContent()]);
+
+        // Act & Assert
+        await Assert.ThrowsAsync<InvalidOperationException>(() => sut.GetChatMessageContentsAsync(chatHistory));
+    }
+
+    [Fact]
+    public async Task GetChatMessageContentsThrowsExceptionUriOnlyReferenceBinaryContentAsync()
+    {
+        // Arrange
+        var sut = new GoogleAIGeminiChatCompletionService("gemini-2.5-pro", "key");
+
+        var chatHistory = new ChatHistory();
+        chatHistory.AddUserMessage([new BinaryContent(new Uri("file://testfile.pdf"))]);
+
+        // Act & Assert
+        await Assert.ThrowsAsync<InvalidOperationException>(() => sut.GetChatMessageContentsAsync(chatHistory));
+    }
+
+    [Theory]
+    [InlineData(true)]
+    [InlineData(false)]
+    public async Task ItSendsBinaryContentCorrectlyAsync(bool useUriData)
+    {
+        // Arrange
+        var sut = new GoogleAIGeminiChatCompletionService("gemini-2.5-pro", "key", httpClient: this._httpClient);
+
+        var mimeType = "application/pdf";
+        var chatHistory = new ChatHistory();
+        chatHistory.AddUserMessage([
+            new TextContent("What's in this file?"),
+            useUriData
+                ? new BinaryContent($"data:{mimeType};base64,{PdfBase64Data}")
+                : new BinaryContent(Convert.FromBase64String(PdfBase64Data), mimeType)
+        ]);
+
+        // Act
+        await sut.GetChatMessageContentsAsync(chatHistory);
+
+        // Assert
+        var actualRequestContent = Encoding.UTF8.GetString(this._messageHandlerStub.RequestContent!);
+        Assert.NotNull(actualRequestContent);
+        var optionsJson = JsonSerializer.Deserialize<JsonElement>(actualRequestContent);
+
+        var contents = optionsJson.GetProperty("contents");
+        Assert.Equal(1, contents.GetArrayLength());
+
+        var parts = contents[0].GetProperty("parts");
+        Assert.Equal(2, parts.GetArrayLength());
+
+        Assert.True(parts[0].TryGetProperty("text", out var prompt));
+        Assert.Equal("What's in this file?", prompt.ToString());
+
+        // Check for the file data
+        Assert.True(parts[1].TryGetProperty("inlineData", out var inlineData));
+        Assert.Equal(JsonValueKind.Object, inlineData.ValueKind);
+        Assert.Equal(mimeType, inlineData.GetProperty("mimeType").GetString());
+        Assert.Equal(PdfBase64Data, inlineData.GetProperty("data").ToString());
+    }
+
+    /// <summary>
+    /// Sample PDF data URI for testing.
+    /// </summary>
+    private const string PdfBase64Data = "JVBERi0xLjQKMSAwIG9iago8PC9UeXBlIC9DYXRhbG9nCi9QYWdlcyAyIDAgUgo+PgplbmRvYmoKMiAwIG9iago8PC9UeXBlIC9QYWdlcwovS2lkcyBbMyAwIFJdCi9Db3VudCAxCj4+CmVuZG9iagozIDAgb2JqCjw8L1R5cGUgL1BhZ2UKL1BhcmVudCAyIDAgUgovTWVkaWFCb3ggWzAgMCA1OTUgODQyXQovQ29udGVudHMgNSAwIFIKL1Jlc291cmNlcyA8PC9Qcm9jU2V0IFsvUERGIC9UZXh0XQovRm9udCA8PC9GMSA0IDAgUj4+Cj4+Cj4+CmVuZG9iago0IDAgb2JqCjw8L1R5cGUgL0ZvbnQKL1N1YnR5cGUgL1R5cGUxCi9OYW1lIC9GMQovQmFzZUZvbnQgL0hlbHZldGljYQovRW5jb2RpbmcgL01hY1JvbWFuRW5jb2RpbmcKPj4KZW5kb2JqCjUgMCBvYmoKPDwvTGVuZ3RoIDUzCj4+CnN0cmVhbQpCVAovRjEgMjAgVGYKMjIwIDQwMCBUZAooRHVtbXkgUERGKSBUagpFVAplbmRzdHJlYW0KZW5kb2JqCnhyZWYKMCA2CjAwMDAwMDAwMDAgNjU1MzUgZgowMDAwMDAwMDA5IDAwMDAwIG4KMDAwMDAwMDA2MyAwMDAwMCBuCjAwMDAwMDAxMjQgMDAwMDAgbgowMDAwMDAwMjc3IDAwMDAwIG4KMDAwMDAwMDM5MiAwMDAwMCBuCnRyYWlsZXIKPDwvU2l6ZSA2Ci9Sb290IDEgMCBSCj4+CnN0YXJ0eHJlZgo0OTUKJSVFT0YK";
+
     public void Dispose()
     {
         this._httpClient.Dispose();

dotnet/src/Connectors/Connectors.Google.UnitTests/Services/VertexAIGeminiChatCompletionServiceTests.cs

@@ -5,7 +5,9 @@
 using System.IO;
 using System.Net.Http;
 using System.Text;
+using System.Text.Json;
 using System.Threading.Tasks;
+using Microsoft.SemanticKernel;
 using Microsoft.SemanticKernel.ChatCompletion;
 using Microsoft.SemanticKernel.Connectors.Google;
 using Microsoft.SemanticKernel.Services;
@@ -156,6 +158,78 @@ public async Task RequestBodyIncludesThinkingConfigWhenSetAsync(int? thinkingBud
         }
     }
 
+    [Fact]
+    public async Task GetChatMessageContentsAsyncThrowsExceptionWithEmptyBinaryContentAsync()
+    {
+        // Arrange
+        var sut = new VertexAIGeminiChatCompletionService("gemini-2.5-pro", "key", "location", "project");
+
+        var chatHistory = new ChatHistory();
+        chatHistory.AddUserMessage([new BinaryContent()]);
+
+        // Act & Assert
+        await Assert.ThrowsAsync<InvalidOperationException>(() => sut.GetChatMessageContentsAsync(chatHistory));
+    }
+
+    [Fact]
+    public async Task GetChatMessageContentsThrowsExceptionUriOnlyReferenceBinaryContentAsync()
+    {
+        // Arrange
+        var sut = new VertexAIGeminiChatCompletionService("gemini-2.5-pro", "key", "location", "project");
+
+        var chatHistory = new ChatHistory();
+        chatHistory.AddUserMessage([new BinaryContent(new Uri("file://testfile.pdf"))]);
+
+        // Act & Assert
+        await Assert.ThrowsAsync<InvalidOperationException>(() => sut.GetChatMessageContentsAsync(chatHistory));
+    }
+
+    [Theory]
+    [InlineData(true)]
+    [InlineData(false)]
+    public async Task ItSendsBinaryContentCorrectlyAsync(bool useUriData)
+    {
+        // Arrange
+        var sut = new VertexAIGeminiChatCompletionService("gemini-2.5-pro", "key", "location", "project", httpClient: this._httpClient);
+
+        var mimeType = "application/pdf";
+        var chatHistory = new ChatHistory();
+        chatHistory.AddUserMessage([
+            new TextContent("What's in this file?"),
+            useUriData
+                ? new BinaryContent($"data:{mimeType};base64,{PdfBase64Data}")
+                : new BinaryContent(Convert.FromBase64String(PdfBase64Data), mimeType)
+        ]);
+
+        // Act
+        await sut.GetChatMessageContentsAsync(chatHistory);
+
+        // Assert
+        var actualRequestContent = Encoding.UTF8.GetString(this._messageHandlerStub.RequestContent!);
+        Assert.NotNull(actualRequestContent);
+        var optionsJson = JsonSerializer.Deserialize<JsonElement>(actualRequestContent);
+
+        var contents = optionsJson.GetProperty("contents");
+        Assert.Equal(1, contents.GetArrayLength());
+
+        var parts = contents[0].GetProperty("parts");
+        Assert.Equal(2, parts.GetArrayLength());
+
+        Assert.True(parts[0].TryGetProperty("text", out var prompt));
+        Assert.Equal("What's in this file?", prompt.ToString());
+
+        // Check for the file data
+        Assert.True(parts[1].TryGetProperty("inlineData", out var inlineData));
+        Assert.Equal(JsonValueKind.Object, inlineData.ValueKind);
+        Assert.Equal(mimeType, inlineData.GetProperty("mimeType").GetString());
+        Assert.Equal(PdfBase64Data, inlineData.GetProperty("data").ToString());
+    }
+
+    /// <summary>
+    /// Sample PDF data URI for testing.
+    /// </summary>
+    private const string PdfBase64Data = "JVBERi0xLjQKMSAwIG9iago8PC9UeXBlIC9DYXRhbG9nCi9QYWdlcyAyIDAgUgo+PgplbmRvYmoKMiAwIG9iago8PC9UeXBlIC9QYWdlcwovS2lkcyBbMyAwIFJdCi9Db3VudCAxCj4+CmVuZG9iagozIDAgb2JqCjw8L1R5cGUgL1BhZ2UKL1BhcmVudCAyIDAgUgovTWVkaWFCb3ggWzAgMCA1OTUgODQyXQovQ29udGVudHMgNSAwIFIKL1Jlc291cmNlcyA8PC9Qcm9jU2V0IFsvUERGIC9UZXh0XQovRm9udCA8PC9GMSA0IDAgUj4+Cj4+Cj4+CmVuZG9iago0IDAgb2JqCjw8L1R5cGUgL0ZvbnQKL1N1YnR5cGUgL1R5cGUxCi9OYW1lIC9GMQovQmFzZUZvbnQgL0hlbHZldGljYQovRW5jb2RpbmcgL01hY1JvbWFuRW5jb2RpbmcKPj4KZW5kb2JqCjUgMCBvYmoKPDwvTGVuZ3RoIDUzCj4+CnN0cmVhbQpCVAovRjEgMjAgVGYKMjIwIDQwMCBUZAooRHVtbXkgUERGKSBUagpFVAplbmRzdHJlYW0KZW5kb2JqCnhyZWYKMCA2CjAwMDAwMDAwMDAgNjU1MzUgZgowMDAwMDAwMDA5IDAwMDAwIG4KMDAwMDAwMDA2MyAwMDAwMCBuCjAwMDAwMDAxMjQgMDAwMDAgbgowMDAwMDAwMjc3IDAwMDAwIG4KMDAwMDAwMDM5MiAwMDAwMCBuCnRyYWlsZXIKPDwvU2l6ZSA2Ci9Sb290IDEgMCBSCj4+CnN0YXJ0eHJlZgo0OTUKJSVFT0YK";
+
     public void Dispose()
     {
         this._httpClient.Dispose();