Add utility for config loading (#521)

Just a simple utility class containing code for fetching configs from CDF. It's maybe a bit early for this, since I still don't have a perfectly clear idea of how it's going to be used, but most of the logic here makes sense in general, and is a foundation for future work.

Author: einarmo

Cognite.Config/Configuration.cs

@@ -119,8 +119,11 @@ public static int GetVersionFromString(string yaml)
         /// in case of yaml parsing errors.</exception>
         public static T TryReadConfigFromString<T>(string yaml, params int[]? acceptedConfigVersions) where T : VersionedConfig
         {
-            int configVersion = GetVersionFromString(yaml);
-            CheckVersion(configVersion, acceptedConfigVersions);
+            if ((acceptedConfigVersions?.Length ?? 0) > 0)
+            {
+                int configVersion = GetVersionFromString(yaml);
+                CheckVersion(configVersion, acceptedConfigVersions);
+            }
 
             var config = ReadString<T>(yaml);
             config.GenerateDefaults();
@@ -142,8 +145,11 @@ public static T TryReadConfigFromString<T>(string yaml, params int[]? acceptedCo
         /// the yaml file is not found or in case of yaml parsing error.</exception>
         public static T TryReadConfigFromFile<T>(string path, params int[]? acceptedConfigVersions) where T : VersionedConfig
         {
-            int configVersion = GetVersionFromFile(path);
-            CheckVersion(configVersion, acceptedConfigVersions);
+            if ((acceptedConfigVersions?.Length ?? 0) > 0)
+            {
+                int configVersion = GetVersionFromFile(path);
+                CheckVersion(configVersion, acceptedConfigVersions);
+            }
 
             var config = Read<T>(path);
             config.GenerateDefaults();

ExtractorUtils.Test/unit/Unstable/CheckInWorkerTests.cs

@@ -22,6 +22,7 @@
 using Cognite.ExtractorUtils.Unstable.Tasks;
 using Microsoft.Extensions.Logging;
 using CogniteSdk.Alpha;
+using CogniteSdk;
 
 
 namespace ExtractorUtils.Test.Unit.Unstable
@@ -64,11 +65,9 @@ private ConnectionConfig GetConfig()
         private (ServiceProvider, CheckInWorker) GetCheckInWorker()
         {
             var config = GetConfig();
-            var baseCogniteConfig = new BaseCogniteConfig();
 
             var services = new ServiceCollection();
             services.AddConfig(config, typeof(ConnectionConfig));
-            services.AddConfig(baseCogniteConfig, typeof(BaseCogniteConfig));
             var mocks = TestUtilities.GetMockedHttpClientFactory(mockCheckInAsync);
             var mockHttpMessageHandler = mocks.handler;
             var mockFactory = mocks.factory;
@@ -77,9 +76,9 @@ private ConnectionConfig GetConfig()
             DestinationUtilsUnstable.AddCogniteClient(services, "myApp", null, setLogger: true, setMetrics: true, setHttpClient: true);
             var provider = services.BuildServiceProvider();
 
-            var dest = provider.GetRequiredService<CogniteDestination>();
+            var client = provider.GetRequiredService<Client>();
 
-            return (provider, new CheckInWorker(config.Integration, provider.GetRequiredService<ILogger<CheckInWorker>>(), dest.CogniteClient));
+            return (provider, new CheckInWorker(config.Integration, provider.GetRequiredService<ILogger<CheckInWorker>>(), client));
         }
 
         [Fact]

ExtractorUtils.Test/unit/Unstable/ConnectionConfigTests.cs

@@ -14,26 +14,37 @@
 using Cognite.Extensions;
 using Cognite.Extractor.Utils;
 using Cognite.ExtractorUtils.Unstable.Configuration;
+using CogniteSdk.Alpha;
+using System.Net.Http.Headers;
+using CogniteSdk;
+using Microsoft.Extensions.Logging;
+using System.IO;
+using Cognite.ExtractorUtils.Unstable.Tasks;
+using ExtractorUtils.Test.unit.Unstable;
+using YamlDotNet.Core;
+using Cognite.Extractor.Common;
+using System.Dynamic;
+using Newtonsoft.Json;
 
 namespace ExtractorUtils.Test.Unit.Unstable
 {
     public class ConnectionConfigTests
     {
-        private static int _tokenCounter;
+        private int _tokenCounter;
 
         private readonly ITestOutputHelper _output;
         public ConnectionConfigTests(ITestOutputHelper output)
         {
             _output = output;
         }
 
-        [Fact]
-        public async Task TestGetClient()
+        private ConnectionConfig GetConfig()
         {
-            var config = new ConnectionConfig
+            return new ConnectionConfig
             {
                 Project = "project",
                 BaseUrl = "https://greenfield.cognitedata.com",
+                Integration = "test-integration",
                 Authentication = new ClientCredentialsConfig
                 {
                     ClientId = "someId",
@@ -45,6 +56,13 @@ public async Task TestGetClient()
                     TokenUrl = "http://example.url/token",
                 }
             };
+        }
+
+        [Fact]
+        public async Task TestGetClient()
+        {
+            var config = GetConfig();
+            _tokenCounter = 0;
 
             var baseCogniteConfig = new BaseCogniteConfig();
 
@@ -57,6 +75,7 @@ public async Task TestGetClient()
             services.AddSingleton(mockFactory.Object);
             services.AddTestLogging(_output);
             DestinationUtilsUnstable.AddCogniteClient(services, "myApp", null, setLogger: true, setMetrics: true, setHttpClient: true);
+            services.AddCogniteDestination();
             using var provider = services.BuildServiceProvider();
 
             var auth = provider.GetRequiredService<IAuthenticator>();
@@ -68,7 +87,7 @@ public async Task TestGetClient()
         }
 
 
-        private static Task<HttpResponseMessage> mockAuthSendAsync(HttpRequestMessage message, CancellationToken token)
+        private Task<HttpResponseMessage> mockAuthSendAsync(HttpRequestMessage message, CancellationToken token)
         {
             // Verify endpoint and method
             Assert.Equal($@"http://example.url/token", message.RequestUri.ToString());
@@ -86,10 +105,205 @@ private static Task<HttpResponseMessage> mockAuthSendAsync(HttpRequestMessage me
             {
                 StatusCode = HttpStatusCode.OK,
                 Content = new StringContent(reply)
-
             };
 
             return Task.FromResult(response);
         }
+
+        class MyFancyConfig : VersionedConfig
+        {
+            public int Foo { get; set; }
+            public string Bar { get; set; }
+
+            public override void GenerateDefaults()
+            {
+            }
+        }
+
+        private (ConfigSource<MyFancyConfig>, DummySink) GetConfigSource(string configPath)
+        {
+            var config = GetConfig();
+            _tokenCounter = 0;
+
+            var services = new ServiceCollection();
+            services.AddConfig(config, typeof(ConnectionConfig));
+            var mocks = TestUtilities.GetMockedHttpClientFactory(mockGetConfig);
+            var mockHttpMessageHandler = mocks.handler;
+            var mockFactory = mocks.factory;
+            services.AddSingleton(mockFactory.Object);
+            services.AddTestLogging(_output);
+            DestinationUtilsUnstable.AddCogniteClient(services, "myApp", null, setLogger: true, setMetrics: true, setHttpClient: true);
+            var provider = services.BuildServiceProvider();
+
+            Directory.CreateDirectory(configPath);
+
+            var configFile = configPath + "/config.yml";
+
+            var source = new ConfigSource<MyFancyConfig>(
+                provider.GetRequiredService<Client>(),
+                provider.GetRequiredService<ILogger<ConfigSource<MyFancyConfig>>>(),
+                "test-integration",
+                configFile,
+                true);
+
+            var reporter = new DummySink();
+
+            return (source, reporter);
+        }
+
+        [Fact]
+        public async Task TestConfigSource()
+        {
+            var config = GetConfig();
+
+            var configPath = TestUtils.AlphaNumericPrefix("dotnet_extractor_test") + "_config";
+            var (source, reporter) = GetConfigSource(configPath);
+            var configFile = source.ConfigFilePath;
+
+            // Try to load a new config when one doesn't exist.
+            await Assert.ThrowsAnyAsync<Exception>(async () => await source.ResolveLocalConfig(reporter, CancellationToken.None));
+
+            // Write an invalid local file.
+            System.IO.File.WriteAllText(configFile, @"
+foo: 123
+baz: test
+            ");
+            await Assert.ThrowsAsync<ConfigurationException>(async () => await source.ResolveLocalConfig(reporter, CancellationToken.None));
+
+            // 2 start, 2 end.
+            Assert.Equal(4, reporter.Errors.Count);
+
+            await Assert.ThrowsAsync<ConfigurationException>(async () => await source.ResolveLocalConfig(reporter, CancellationToken.None));
+            // Nothing has changed, no new errors.
+            Assert.Equal(4, reporter.Errors.Count);
+
+            // Write a valid local file.
+            System.IO.File.WriteAllText(configFile, @"
+foo: 123
+bar: test
+            ");
+            var isNew = await source.ResolveLocalConfig(reporter, CancellationToken.None);
+            Assert.True(isNew);
+
+            isNew = await source.ResolveLocalConfig(reporter, CancellationToken.None);
+            Assert.False(isNew);
+            Assert.Equal(123, source.Config.Foo);
+            Assert.Equal("test", source.Config.Bar);
+
+            // Fail to fetch remote config
+            await Assert.ThrowsAnyAsync<Exception>(async () => await source.ResolveRemoteConfig(null, reporter, CancellationToken.None));
+            // Another 2 error reports.
+            Assert.Equal(6, reporter.Errors.Count);
+
+            // Fail to fetch again with the same error.
+            await Assert.ThrowsAnyAsync<Exception>(async () => await source.ResolveRemoteConfig(null, reporter, CancellationToken.None));
+            // No new reports.
+            Assert.Equal(6, reporter.Errors.Count);
+
+            _responseRevision = new ConfigRevision
+            {
+                ExternalId = "test-integration",
+                Config = @"
+foo: 321
+bar: test
+",
+                Revision = 1,
+            };
+
+            Assert.Equal(2, _getConfigCount);
+
+            isNew = await source.ResolveRemoteConfig(null, reporter, CancellationToken.None);
+            Assert.True(isNew);
+            Assert.Equal(321, source.Config.Foo);
+            Assert.Equal("test", source.Config.Bar);
+
+            isNew = await source.ResolveRemoteConfig(1, reporter, CancellationToken.None);
+            Assert.False(isNew);
+            // Only one new request
+            Assert.Equal(3, _getConfigCount);
+
+            Assert.True(System.IO.File.Exists(configPath + "/_temp_config.yml"));
+
+            Directory.Delete(configPath, true);
+        }
+
+        [Fact]
+        public async Task TestBufferConfigFile()
+        {
+            var config = GetConfig();
+            var configPath = TestUtils.AlphaNumericPrefix("dotnet_extractor_test") + "_config";
+            var (source, reporter) = GetConfigSource(configPath);
+            var configFile = source.ConfigFilePath;
+            var bufferFile = configPath + "/_temp_config.yml";
+
+            var okRevision = new ConfigRevision
+            {
+                ExternalId = "test-integration",
+                Config = @"
+foo: 321
+bar: test
+",
+                Revision = 1,
+            };
+            _responseRevision = okRevision;
+
+            var isNew = await source.ResolveRemoteConfig(null, reporter, CancellationToken.None);
+            Assert.True(isNew);
+            Assert.True(System.IO.File.Exists(bufferFile));
+
+            // We can load the config from the buffer file.
+            _responseRevision = null;
+            isNew = await source.ResolveRemoteConfig(null, reporter, CancellationToken.None);
+            Assert.True(isNew);
+
+            // Make the file write protected. Now loading remote config should fail until we delete it.
+            System.IO.File.SetAttributes(bufferFile, FileAttributes.ReadOnly);
+            _responseRevision = okRevision;
+            await Assert.ThrowsAnyAsync<Exception>(async () => await source.ResolveRemoteConfig(null, reporter, CancellationToken.None));
+
+            // Delete the buffer file, loading remote config should now fail.
+            System.IO.File.SetAttributes(bufferFile, FileAttributes.Normal);
+            System.IO.File.Delete(bufferFile);
+            _responseRevision = null;
+            await Assert.ThrowsAnyAsync<Exception>(async () => await source.ResolveRemoteConfig(null, reporter, CancellationToken.None));
+
+            Directory.Delete(configPath, true);
+        }
+
+        private ConfigRevision _responseRevision;
+        private int _getConfigCount;
+
+        private async Task<HttpResponseMessage> mockGetConfig(HttpRequestMessage message, CancellationToken token)
+        {
+            var uri = message.RequestUri.ToString();
+            if (uri == "http://example.url/token") return await mockAuthSendAsync(message, token);
+
+            Assert.Contains("/integrations/config", uri);
+            _getConfigCount++;
+
+            if (_responseRevision == null)
+            {
+                dynamic res = new ExpandoObject();
+                res.error = new ExpandoObject();
+                res.error.code = 400;
+                res.error.message = "Something went wrong";
+                return new HttpResponseMessage
+                {
+                    StatusCode = HttpStatusCode.BadRequest,
+                    Content = new StringContent(JsonConvert.SerializeObject(res)),
+                };
+            }
+
+            var resBody = System.Text.Json.JsonSerializer.Serialize(_responseRevision, Oryx.Cognite.Common.jsonOptions);
+            var fresponse = new HttpResponseMessage
+            {
+                StatusCode = HttpStatusCode.OK,
+                Content = new StringContent(resBody)
+            };
+            fresponse.Content.Headers.ContentType = new MediaTypeHeaderValue("application/json");
+            fresponse.Headers.Add("x-request-id", "1");
+
+            return fresponse;
+        }
     }
 }

ExtractorUtils.Test/unit/Unstable/TaskSchedulerTest.cs

@@ -10,12 +10,16 @@
 
 namespace ExtractorUtils.Test.unit.Unstable
 {
-    class DummySink : IIntegrationSink
+    class DummySink : BaseErrorReporter, IIntegrationSink
     {
         public List<ExtractorError> Errors { get; } = new();
         public List<(string, DateTime)> TaskStart { get; } = new();
         public List<(string, DateTime)> TaskEnd { get; } = new();
 
+        public override ExtractorError NewError(ErrorLevel level, string description, string details = null, DateTime? now = null)
+        {
+            return new ExtractorError(level, description, this, details, null, now);
+        }
 
         public void ReportError(ExtractorError error)
         {