feat: SqlSeverSource can point to an SQL file

Adds `FilePath` argument to point to a text file with SQL statement,
instead of including the entire SQL statement in the migrationsettings-file.

add: Coverage to SqlServer extension

Author: stefanedwards

Extensions/SqlServer/Cosmos.DataTransfer.SqlServerExtension.UnitTests/Cosmos.DataTransfer.SqlServerExtension.UnitTests.csproj

@@ -6,6 +6,9 @@
     <Nullable>enable</Nullable>
 
     <IsPackable>false</IsPackable>
+    <CollectCoverage>true</CollectCoverage>
+    <CoverletOutputFormat>cobertura</CoverletOutputFormat>
+    <CoverletOutput>../../../Extensions/SqlServer/Cosmos.DataTransfer.SqlServerExtension/coverage.cobertura.xml</CoverletOutput>
   </PropertyGroup>
 
   <ItemGroup>
@@ -15,14 +18,20 @@
     <PackageReference Include="MSTest.TestFramework" Version="2.2.10" />
     <PackageReference Include="coverlet.collector" Version="3.1.2" />
     <PackageReference Include="System.Linq.Async" Version="6.0.1" />
+    <PackageReference Include="coverlet.msbuild" Version="2.8.0">
+      <IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>
+      <PrivateAssets>all</PrivateAssets>
+    </PackageReference>
   </ItemGroup>
 
   <ItemGroup>
+    <ProjectReference Include="..\..\..\Interfaces\Cosmos.DataTransfer.Common.UnitTests\Cosmos.DataTransfer.Common.UnitTests.csproj" />
     <ProjectReference Include="..\Cosmos.DataTransfer.SqlServerExtension\Cosmos.DataTransfer.SqlServerExtension.csproj" />
   </ItemGroup>
 
+
   <ItemGroup>
-    <None Update="Data\FlatFile.json">
+    <None Update="Data\SqlStatement.sql">
       <CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
     </None>
   </ItemGroup>

Extensions/SqlServer/Cosmos.DataTransfer.SqlServerExtension.UnitTests/Data/FlatFile.json

@@ -0,0 +1 @@
+SELECT * FROM foobar;

Extensions/SqlServer/Cosmos.DataTransfer.SqlServerExtension.UnitTests/Data/SqlStatement.sql

@@ -0,0 +1,71 @@
+using Cosmos.DataTransfer.Interfaces;
+using System.ComponentModel.DataAnnotations;
+
+namespace Cosmos.DataTransfer.SqlServerExtension.UnitTests;
+
+[TestClass]
+public class SqlServerSourceSettingsTests 
+{
+
+    [TestMethod]
+    public void TestSourceSettings_ValidationFails1()
+    {
+        var settings = new SqlServerSourceSettings {
+        };
+
+        var validationResults = settings.Validate(new ValidationContext(settings)).ToList();
+        var expected = new List<ValidationResult>() {
+            new ValidationResult("The `ConnectionString` field is required.",
+            new string[] { "ConnectionString" }),
+            new ValidationResult("Either `QueryText` or `FilePath` are required!",
+                new string[] { "QueryText", "FilePath"})
+        };
+        CollectionAssert.AreEquivalent(expected.Select(x => x.ErrorMessage).ToList(),
+            validationResults.Select(x => x.ErrorMessage).ToList());
+                CollectionAssert.AreEquivalent(expected.SelectMany(x => x.MemberNames).ToList(),
+            validationResults.SelectMany(x => x.MemberNames).ToList());
+
+        Assert.ThrowsException<AggregateException>(() => settings.Validate());
+    }
+
+    [TestMethod]
+    public void TestSourceSettings_ValidationFails2()
+    {
+        var settings = new SqlServerSourceSettings {
+            QueryText = "SELECT 1;",
+            FilePath = "dmt-query.sql"
+        };
+
+        var validationResults = settings.Validate(new ValidationContext(settings)).ToList();
+        var expected = new List<ValidationResult>() {
+            new ValidationResult("The `ConnectionString` field is required.",
+            new string[] { "ConnectionString" }),
+            new ValidationResult("Both `QueryText` and `FilePath` are not allowed.",
+                new string[] { "QueryText", "FilePath"})
+        };
+        CollectionAssert.AreEquivalent(expected.Select(x => x.ErrorMessage).ToList(),
+            validationResults.Select(x => x.ErrorMessage).ToList());
+                CollectionAssert.AreEquivalent(expected.SelectMany(x => x.MemberNames).ToList(),
+            validationResults.SelectMany(x => x.MemberNames).ToList());
+
+        Assert.ThrowsException<AggregateException>(() => settings.Validate());
+    }
+
+    [TestMethod]
+    [DataRow("SELECT 1", null)]
+    [DataRow("SELECT 1", " ")]
+    [DataRow(null, "filename")]
+    [DataRow("  ", "filename")]
+    public void TestSourceSettings_ValidationSuccess(string queryText, string filePath) {
+        var settings = new SqlServerSourceSettings {
+            ConnectionString = "Server",
+            QueryText = queryText,
+            FilePath = filePath
+        };
+
+        var validationResults = settings.Validate(new ValidationContext(settings));
+        Assert.AreEqual(0, validationResults.Count());
+
+        settings.Validate();
+    }
+}

Extensions/SqlServer/Cosmos.DataTransfer.SqlServerExtension.UnitTests/SqlServerExtensionTests.cs

@@ -1,4 +1,5 @@
 using System.ComponentModel.Composition;
+using System.IO;
 using System.Runtime.CompilerServices;
 using Cosmos.DataTransfer.Interfaces;
 using Microsoft.Data.SqlClient;
@@ -17,9 +18,14 @@ public async IAsyncEnumerable<IDataItem> ReadAsync(IConfiguration config, ILogge
             var settings = config.Get<SqlServerSourceSettings>();
             settings.Validate();
 
+            string queryText = settings.QueryText!;
+            if (settings.FilePath != null) {
+                queryText = File.ReadAllText(queryText);
+            }
+            
             await using var connection = new SqlConnection(settings.ConnectionString);
             await connection.OpenAsync(cancellationToken);
-            await using SqlCommand command = new SqlCommand(settings.QueryText, connection);
+            await using SqlCommand command = new SqlCommand(queryText, connection);
             await using SqlDataReader reader = await command.ExecuteReaderAsync(cancellationToken);
             while (await reader.ReadAsync(cancellationToken))
             {

Extensions/SqlServer/Cosmos.DataTransfer.SqlServerExtension.UnitTests/SqlServerSourceSettingsTests.cs

@@ -4,14 +4,32 @@
 
 namespace Cosmos.DataTransfer.SqlServerExtension
 {
-    public class SqlServerSourceSettings : IDataExtensionSettings
+    public class SqlServerSourceSettings : IDataExtensionSettings, IValidatableObject
     {
-        [Required]
         [SensitiveValue]
         public string? ConnectionString { get; set; }
 
-        [Required]
         public string? QueryText { get; set; }
 
+        public string? FilePath { get; set; }
+
+        public IEnumerable<ValidationResult> Validate(ValidationContext validationContext)
+        {
+            if (String.IsNullOrWhiteSpace(this.ConnectionString)) {
+                yield return new ValidationResult("The `ConnectionString` field is required.",
+                    new string[] { "ConnectionString" });
+            } 
+            if (String.IsNullOrWhiteSpace(this.QueryText) &&
+                String.IsNullOrWhiteSpace(this.FilePath)) {
+                    yield return new ValidationResult(
+                        "Either `QueryText` or `FilePath` are required!",
+                        new string[] { "QueryText", "FilePath"});
+            } else if (String.IsNullOrWhiteSpace(this.QueryText) == false &&
+                String.IsNullOrWhiteSpace(this.FilePath) == false) {
+                    yield return new ValidationResult(
+                        "Both `QueryText` and `FilePath` are not allowed.",
+                        new string[] { "QueryText", "FilePath"});
+            }
+        }
     }
 }

Extensions/SqlServer/Cosmos.DataTransfer.SqlServerExtension/SqlServerDataSourceExtension.cs

@@ -8,14 +8,18 @@ The SQL Server data transfer extension provides source and sink capabilities for
 
 Source and sink settings both require a `ConnectionString` parameter.
 
-Source settings also require a `QueryText` parameter to define the data to select from SQL. This can combine data from multiple tables, views, etc. but should produce a single result set.
+Source settings also require either a `QueryText` or `FilePath` parameter with the SQL statement that defines the data to select.
+`QueryText` can be the SQL statement, while `FilePath` can point to a file with the SQL statement. 
+In both cases, the SQL statement can combine data from multiple tables, views, etc., 
+but should produce a single result set.
 
 ### Source
 
 ```json
 {
     "ConnectionString": "",
-    "QueryText": ""
+    "QueryText": "", // required if FilePath not set.
+    "FilePath": ""   // required if QueryText not set.
 }
 ```