Add option for changing ignoreBadDataPoints in GetExtractedRanges (#646)

* Add option for changing ignoreBadDataPoints in GetExtractedRanges

* remove false change

Author: ozangoktan

Cognite.Extensions/TimeSeries/DataPointExtensions.cs

@@ -466,13 +466,15 @@ private static async Task<HashSet<Identity>> DeleteDataPointsIgnoreErrorsChunk(
         /// <param name="chunkSize">Number of timeseries per request</param>
         /// <param name="throttleSize">Maximum number of parallel requests</param>
         /// <param name="token">Cancellation token</param>
+        /// <param name="ignoreBadDataPoints">Ignore bad status datapoints, true by default</param>
         /// <returns>Dictionary from externalId to last timestamp, only contains existing timeseries</returns>
         public static async Task<IDictionary<Identity, DateTime>> GetLatestTimestamps(
             this DataPointsResource dataPoints,
             IEnumerable<(Identity id, DateTime before)> ids,
             int chunkSize,
             int throttleSize,
-            CancellationToken token)
+            CancellationToken token,
+            bool ignoreBadDataPoints = true)
         {
             var ret = new ConcurrentDictionary<Identity, DateTime>();
             var idSet = new HashSet<Identity>(ids.Select(id => id.id));
@@ -497,6 +499,7 @@ public static async Task<IDictionary<Identity, DateTime>> GetLatestTimestamps(
                     {
                         idt.Before = pair.before.ToUnixTimeMilliseconds().ToString();
                     }
+                    idt.IgnoreBadDataPoints = ignoreBadDataPoints;
                     return idt;
                 })
                 .ChunkBy(chunkSize)
@@ -558,21 +561,23 @@ await generators
         /// <param name="chunkSize">Number of timeseries per request</param>
         /// <param name="throttleSize">Maximum number of parallel requests</param>
         /// <param name="token">Cancellation token</param>
+        /// <param name="ignoreBadDataPoints">Ignore bad status datapoints, true by default</param>
         /// <returns>Dictionary from externalId to first timestamp, only contains existing timeseries</returns>
         public static async Task<IDictionary<Identity, DateTime>> GetEarliestTimestamps(
             this DataPointsResource dataPoints,
             IEnumerable<(Identity id, DateTime after)> ids,
             int chunkSize,
             int throttleSize,
-            CancellationToken token)
+            CancellationToken token,
+            bool ignoreBadDataPoints = true)
         {
             var ret = new ConcurrentDictionary<Identity, DateTime>();
             var idSet = new HashSet<Identity>(ids.Select(id => id.id));
 
             var chunks = ids
                 .Select((pair) =>
                 {
-                    var query = new DataPointsQueryItem();
+                    var query = new DataPointsQueryItem() { IgnoreBadDataPoints = ignoreBadDataPoints };
                     if (pair.id.Id.HasValue)
                     {
                         query.Id = pair.id.Id.Value;
@@ -660,6 +665,7 @@ await generators
         /// <param name="latest">If true, fetch latest timestamps</param>
         /// <param name="earliest">If true, fetch earliest timestamps</param>
         /// <param name="token">Cancellation token</param>
+        /// <param name="ignoreBadDataPoints">Ignore bad status datapoints, true by default</param>
         /// <returns></returns>
         public static async Task<IDictionary<Identity, TimeRange>> GetExtractedRanges(
             this DataPointsResource dataPoints,
@@ -669,7 +675,8 @@ public static async Task<IDictionary<Identity, TimeRange>> GetExtractedRanges(
             int throttleSize,
             bool latest,
             bool earliest,
-            CancellationToken token)
+            CancellationToken token,
+            bool ignoreBadDataPoints = true)
         {
             if (ids == null)
             {
@@ -684,12 +691,12 @@ public static async Task<IDictionary<Identity, TimeRange>> GetExtractedRanges(
             if (latest)
             {
                 tasks.Add(dataPoints.GetLatestTimestamps(ids.Select(pair => (pair.id, pair.limit?.Last ?? DateTime.MaxValue)),
-                    chunkSizeLatest, throttleSize, token));
+                    chunkSizeLatest, throttleSize, token, ignoreBadDataPoints));
             }
             if (earliest)
             {
                 tasks.Add(dataPoints.GetEarliestTimestamps(ids.Select(pair => (pair.id, pair.limit?.First ?? CogniteTime.DateTimeEpoch)),
-                    chunkSizeEarliest, throttleSize, token));
+                    chunkSizeEarliest, throttleSize, token, ignoreBadDataPoints));
             }
             var results = await Task.WhenAll(tasks).ConfigureAwait(false);
             if (latest)

ExtractorUtils/Cognite/CogniteDestination.cs

@@ -719,12 +719,14 @@ public Task DeleteRowsAsync(string dbName, string tableName, IEnumerable<string>
         /// <param name="token">Cancellation token</param>
         /// <param name="earliest">If true, fetch earliest timestamps, default true</param>
         /// <param name="latest">If true, fetch latest timestamps, default true</param>
+        /// <param name="ignoreBadDataPoints">Ignore bad status datapoints, true by default</param>
         /// <returns></returns>
         public Task<IDictionary<Identity, TimeRange>> GetExtractedRanges(
             IEnumerable<Identity> ids,
             CancellationToken token,
             bool earliest = true,
-            bool latest = true
+            bool latest = true,
+            bool ignoreBadDataPoints = true
             )
         {
             return _client.DataPoints.GetExtractedRanges(
@@ -734,7 +736,8 @@ public Task<IDictionary<Identity, TimeRange>> GetExtractedRanges(
                 _throttling.Ranges,
                 latest,
                 earliest,
-                token);
+                token,
+                ignoreBadDataPoints);
         }
 
 
@@ -747,12 +750,14 @@ public Task<IDictionary<Identity, TimeRange>> GetExtractedRanges(
         /// <param name="token">Cancellation token</param>
         /// <param name="earliest">If true, fetch earliest timestamps, default true</param>
         /// <param name="latest">If true, fetch latest timestamps, default true</param>
+        /// <param name="ignoreBadDataPoints">Ignore bad status datapoints, true by default</param>
         /// <returns></returns>
         public Task<IDictionary<Identity, TimeRange>> GetExtractedRanges(
             IEnumerable<(Identity id, TimeRange limit)> ids,
             CancellationToken token,
             bool earliest = true,
-            bool latest = true
+            bool latest = true,
+            bool ignoreBadDataPoints = true
             )
         {
             return _client.DataPoints.GetExtractedRanges(
@@ -762,7 +767,8 @@ public Task<IDictionary<Identity, TimeRange>> GetExtractedRanges(
                 _throttling.Ranges,
                 latest,
                 earliest,
-                token);
+                token,
+                ignoreBadDataPoints);
         }
 
 

version

@@ -1 +1 @@
-1.39.1
+1.40.0