add: appending to FileDataSink

Author: stefanedwards

Interfaces/Cosmos.DataTransfer.Common.UnitTests/FileDataSinkTests.cs

@@ -1,19 +1,19 @@
 using System.IO.Compression;
 using Microsoft.Extensions.Logging.Abstractions;
+using Microsoft.Extensions.Configuration;
 using System.Text;
 using Microsoft.VisualStudio.TestTools.UnitTesting;
 using Cosmos.DataTransfer.JsonExtension;
 using Cosmos.DataTransfer.JsonExtension.UnitTests;
 using Cosmos.DataTransfer.Interfaces;
 using Moq;
 
-namespace Cosmos.DataTransfer.Common.UnitTest;
+namespace Cosmos.DataTransfer.Common.UnitTests;
 
 [TestClass]
 public class FileSinkDataSinkTests {
 
     public static IEnumerable<object[]> Test_WriteToTargetAsyncData { get {
-
         yield return new object[] { CompressionEnum.None, "", "" };
         yield return new object[] { CompressionEnum.None, ".txt", ".txt" };
         yield return new object[] { CompressionEnum.Brotli, "", ".br" };
@@ -26,15 +26,15 @@ public static IEnumerable<object[]> Test_WriteToTargetAsyncData { get {
     } }
 
     [TestMethod]
-    [DoNotParallelize]
     [DynamicData(nameof(Test_WriteToTargetAsyncData))]
     public async Task Test_WriteToTargetAsync(CompressionEnum compression, string suffix, string expected_ext) {
         var source = new Mock<IDataSourceExtension>();
         FileDataSink sink = new();
         var filePath = Path.GetTempFileName();
         var config = TestHelpers.CreateConfig(new Dictionary<string,string>() {
             { "FilePath", filePath + suffix },
-            { "Compression", compression.ToString() }
+            { "Compression", compression.ToString() }, 
+            { "Append", "false" }
         });
 
         var str = Encoding.UTF8.GetBytes("Hello world!");
@@ -57,4 +57,44 @@ public void Test_GetSettings() {
         Assert.IsInstanceOfType(response[0], typeof(FileSinkSettings));
     }
 
+    public static IEnumerable<object[]> Test_WriteToTargetAsyncAppendData { get {
+        yield return new object[] { "foobar.txt", CompressionEnum.None };
+        yield return new object[] { "foobar.txt.gz", CompressionEnum.Gzip };
+        yield return new object[] { "foobar.txt.br", CompressionEnum.Brotli };
+        yield return new object[] { "foobar.txt.zz", CompressionEnum.Deflate };
+    } }
+
+    [TestMethod]
+    [DynamicData(nameof(Test_WriteToTargetAsyncAppendData))]
+    public async Task Test_WriteToTargetAsyncAppend(string filePath, CompressionEnum compression) {
+        var source = new Mock<IDataSourceExtension>();
+        FileDataSink sink = new();
+        var tmpdir = FileSinkDataSourceTests.GetTemporaryDirectory();
+        var destfile = Path.Combine(tmpdir, filePath);
+        File.Copy(Path.Combine("Data", filePath), destfile);
+        var config = TestHelpers.CreateConfig(new Dictionary<string,string>() {
+            { "FilePath", destfile },
+            { "Compression", compression.ToString() }, 
+            { "Append", "true" }
+        });
+
+        if (compression != CompressionEnum.None) {
+            var e = Assert.ThrowsException<AggregateException>(() => {
+                var settings = config.Get<FileSinkSettings>();
+                settings.Validate();
+            });
+            return;
+        }
+        var str = Encoding.UTF8.GetBytes("\nIt's Valentines day! Lovely!");
+        await sink.WriteToTargetAsync(writer => writer.WriteAsync(str).AsTask(),
+            config, source.Object, NullLogger.Instance);
+
+        var dataSource = new FileDataSource();
+        var stream = dataSource.ReadFile(destfile, CompressionEnum.None, NullLogger.Instance)!;
+        var buffer = new byte[100];
+        await stream.ReadAsync(buffer.AsMemory(0, buffer.Length));
+        var result = Encoding.UTF8.GetString(buffer);
+        Assert.AreEqual("Hello world!\nIt's Valentines day! Lovely!", result.TrimEnd('\0'));
+    }
+
 }

Interfaces/Cosmos.DataTransfer.Common.UnitTests/FileDataSourceTests.cs

@@ -1,10 +1,9 @@
 using System.Text;
 using Microsoft.VisualStudio.TestTools.UnitTesting;
-using Microsoft.Extensions.Logging;
 using Microsoft.Extensions.Logging.Abstractions;
 using Cosmos.DataTransfer.JsonExtension.UnitTests;
 
-namespace Cosmos.DataTransfer.Common.UnitTest;
+namespace Cosmos.DataTransfer.Common.UnitTests;
 
 [TestClass]
 public class FileSinkDataSourceTests {
@@ -50,7 +49,7 @@ public async Task Test_ReadSourceAsync_SingleFile(string filePath, CompressionEn
         Assert.AreEqual(expected, result.TrimEnd('\0'), $"file: {filePath}, compression: {compression}.");
     }
 
-    private string GetTemporaryDirectory() {
+    internal static string GetTemporaryDirectory() {
         string tempDirectory = Path.Combine(Path.GetTempPath(), Path.GetRandomFileName());
         if (File.Exists(tempDirectory)) 
         {

Interfaces/Cosmos.DataTransfer.Common.UnitTests/FileSinkSettingsTests.cs

@@ -0,0 +1,49 @@
+using Microsoft.VisualStudio.TestTools.UnitTesting;
+using Cosmos.DataTransfer.Interfaces;
+
+namespace Cosmos.DataTransfer.Common.UnitTests;
+
+[TestClass]
+public class FileSinkSettingsTests {
+
+    [TestMethod]
+    public void TestValidate_PassMinimum() {
+        var settings = new FileSinkSettings() {
+            FilePath = "A file",
+        };
+        settings.Validate();
+    }
+
+    [TestMethod]
+    public void TestValidate_Fails() {
+        var settings = new FileSinkSettings() {
+            FilePath = " ",
+        };
+        var e = Assert.ThrowsException<AggregateException>(() => settings.Validate());
+        Assert.AreEqual(e.InnerException!.Message, "The FilePath field is required.");
+    }
+
+    [TestMethod]
+    [DataRow(false, "None", true)]
+    [DataRow(false, "Gzip", true)]
+    [DataRow(false, "Brotli", true)]
+    [DataRow(false, "Deflate", true)]
+    [DataRow(true, "None", true)]
+    [DataRow(true, "Gzip", false)]
+    [DataRow(true, "Brotli", false)]
+    [DataRow(true, "Deflate", false)]
+    public void TestValidate_TestCombinations(bool append, string compression, bool pass) {
+        var settings = new FileSinkSettings() {
+            FilePath = "A file",
+            Append = append,
+            Compression = Enum.Parse<CompressionEnum>(compression),
+        };
+        if (!pass) {
+            Assert.ThrowsException<AggregateException>(() => settings.Validate());
+        } else {
+            settings.Validate();
+        }
+    }
+
+
+}

Interfaces/Cosmos.DataTransfer.Common/FileDataSink.cs

@@ -15,6 +15,7 @@ public async Task WriteToTargetAsync(Func<Stream, Task> writeToStream, IConfigur
         {
             using var writer = GetCompressor(settings.Compression, settings.FilePath, settings.Append);
             await writeToStream(writer);
+            writer.Close();
         }
     }
 

Interfaces/Cosmos.DataTransfer.Common/FileSinkSettings.cs

@@ -3,10 +3,22 @@
 
 namespace Cosmos.DataTransfer.Common;
 
-public class FileSinkSettings : IDataExtensionSettings
+public class FileSinkSettings : IDataExtensionSettings, IValidatableObject
 {
     [Required]
     public string? FilePath { get; set; }
     public bool Append { get; set; } = false;
     public CompressionEnum Compression { get; set; } = CompressionEnum.None;
+
+    public IEnumerable<ValidationResult> Validate(ValidationContext validationContext)
+    {
+        if (this.Append && Compression != CompressionEnum.None) {
+            // Technically we can, but the .NET methods here
+            // cannot read the concatenated, compressed files.
+            yield return new ValidationResult(
+                "Cannot Append to any compressed files.",
+                new string[] { "Append", "Compression" }
+            );
+        }
+    }
 }

Interfaces/Cosmos.DataTransfer.Common/README.md

@@ -26,9 +26,14 @@ Gzip, bzip2, and zlib compressed files are automatically detected by file extens
 Use parameter `Compression` to enable compression of output file. 
 The relevant extension is automatically added to the file name, if missing.
 
+It is possible to build further upon a file by setting the `Append` parameter
+to `true`. Note however that it is not compatible with any compression methods,
+as the result a concatenation of multiple compressed files.
+
 ```json
 {
     "FilePath": "",
-    "Compression": "None" | "Gzip" | "Brotli" | "Deflate"
+    "Compression": "None" | "Gzip" | "Brotli" | "Deflate",
+    "Append": false
 }
 ```