Skip to content

Commit 76e10bd

Browse files
einarmoeinarmo
committed
Schemas for the unstable module, and tests for schemas
1 parent db85f0e commit 76e10bd

File tree

8 files changed

+464
-16
lines changed

8 files changed

+464
-16
lines changed

‎.github/workflows/build-and-test.yml‎

Lines changed: 21 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -54,3 +54,24 @@ jobs:
5454
uses: codecov/codecov-action@v5
5555
with:
5656
token: ${{ secrets.CODECOV_TOKEN }}
57+
58+
test_schemas:
59+
runs-on: ubuntu-latest
60+
61+
steps:
62+
- uses: actions/checkout@v6
63+
- name: Install python
64+
uses: actions/setup-python@v6
65+
with:
66+
python-version: '3.12'
67+
68+
- name: Install extractor uploader
69+
env:
70+
PYPI_ARTIFACTORY_USERNAME: ${{ secrets.ARTIFACTORY_READONLY_TOKEN_USER_PUBLIC_REPOS }}
71+
PYPI_ARTIFACTORY_PASSWORD: ${{ secrets.ARTIFACTORY_READONLY_TOKEN_PUBLIC_REPOS }}
72+
run: |
73+
python -m pip install --upgrade pip
74+
pip install cognite-extractor-publisher --extra-index-url "https://${PYPI_ARTIFACTORY_USERNAME}:${PYPI_ARTIFACTORY_PASSWORD}@cognite.jfrog.io/cognite/api/pypi/snakepit/simple"
75+
76+
- name: Test schemas
77+
run: ./ExtractorUtils.Test/test_schemas.sh

‎ExtractorUtils.Test/test_schemas.sh‎

Lines changed: 17 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,17 @@
1+
#!/bin/bash
2+
3+
set -e -x
4+
5+
target_schemas=(
6+
"schema/base_config.schema.json"
7+
"schema/unstable/connection_config.schema.json"
8+
"schema/unstable/base_cognite_config.schema.json"
9+
)
10+
11+
for schema in "${target_schemas[@]}"; do
12+
echo "Processing $schema"
13+
publish-extractor schema --schema "$schema" --output bundled.schema.json
14+
echo "Generating docs for $schema"
15+
publish-extractor docs --schema bundled.schema.json
16+
rm bundled.schema.json
17+
done

‎ExtractorUtils/Unstable/Configuration/Connection.cs‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -37,9 +37,9 @@ public class ConnectionConfig : VersionedConfig
3737
/// <summary>
3838
/// Configuration for the connection to CDF.
3939
/// </summary>
40-
public CdfConnectionConfig CdfConnection { get => _cdfConnection; set { _cdfConnection = value ?? _cdfConnection; } }
40+
public CdfConnectionConfig Connection { get => _connection; set { _connection = value ?? _connection; } }
4141

42-
private CdfConnectionConfig _cdfConnection = new CdfConnectionConfig();
42+
private CdfConnectionConfig _connection = new CdfConnectionConfig();
4343

4444
/// <summary>
4545
/// Register any necessary yaml converters.
@@ -157,4 +157,4 @@ public class SdkLoggingConfig
157157

158158
}
159159

160-
}
160+
}

‎ExtractorUtils/Unstable/DestinationUtilsUnstable.cs‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -110,7 +110,7 @@ public static IHttpClientBuilder ConfigureCogniteHttpClientHandlers(this IHttpCl
110110
{
111111
try
112112
{
113-
var retryConfig = provider.GetService<ConnectionConfig>()?.CdfConnection.Retries ?? new Configuration.RetryConfig();
113+
var retryConfig = provider.GetService<ConnectionConfig>()?.Connection.Retries ?? new Configuration.RetryConfig();
114114
return CogniteExtensions.GetRetryPolicy(provider.GetService<ILogger<Client>>(),
115115
retryConfig.MaxRetries, retryConfig.MaxBackoffValue.Value);
116116
}
@@ -123,7 +123,7 @@ public static IHttpClientBuilder ConfigureCogniteHttpClientHandlers(this IHttpCl
123123
{
124124
try
125125
{
126-
var retryConfig = provider.GetService<ConnectionConfig>()?.CdfConnection.Retries ?? new Configuration.RetryConfig();
126+
var retryConfig = provider.GetService<ConnectionConfig>()?.Connection.Retries ?? new Configuration.RetryConfig();
127127
return CogniteExtensions.GetTimeoutPolicy(retryConfig.TimeoutValue.Value);
128128
}
129129
catch (ObjectDisposedException)
@@ -136,7 +136,7 @@ public static IHttpClientBuilder ConfigureCogniteHttpClientHandlers(this IHttpCl
136136
{
137137
try
138138
{
139-
var certConfig = provider.GetService<ConnectionConfig>()?.CdfConnection?.SslCertificates;
139+
var certConfig = provider.GetService<ConnectionConfig>()?.Connection?.SslCertificates;
140140
return GetClientHandler(certConfig);
141141
}
142142
catch (ObjectDisposedException)
@@ -295,4 +295,4 @@ public static Client.Builder Configure(
295295
return builder;
296296
}
297297
}
298-
}
298+
}

‎schema/cognite_config.schema.json‎

Lines changed: 22 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -250,6 +250,13 @@
250250
"description": "Maximum number of data modeling instances per get/create instance request",
251251
"max": 1000,
252252
"min": 1
253+
},
254+
"stream-records": {
255+
"type": "integer",
256+
"default": 1000,
257+
"description": "Maximum number of stream records per get/create stream records request",
258+
"max": 1000,
259+
"min": 1
253260
}
254261
},
255262
"unevaluatedProperties": false
@@ -260,51 +267,57 @@
260267
"properties": {
261268
"time-series": {
262269
"type": "integer",
263-
"default": 20,
270+
"default": 5,
264271
"description": "Maximum number of parallel requests per timeseries operation",
265272
"min": 1
266273
},
267274
"assets": {
268275
"type": "integer",
269-
"default": 20,
276+
"default": 5,
270277
"description": "Maximum number of parallel requests per assets operation",
271278
"min": 1
272279
},
273280
"data-points": {
274281
"type": "integer",
275-
"default": 10,
282+
"default": 5,
276283
"description": "Maximum number of parallel requests per datapoints operation",
277284
"min": 1
278285
},
279286
"raw": {
280287
"type": "integer",
281-
"default": 10,
288+
"default": 5,
282289
"description": "Maximum number of parallel requests per raw operation",
283290
"min": 1
284291
},
285292
"ranges": {
286293
"type": "integer",
287-
"default": 20,
294+
"default": 5,
288295
"description": "Maximum number of parallel requests per get first/last datapoint operation",
289296
"min": 1
290297
},
291298
"events": {
292299
"type": "integer",
293-
"default": 20,
300+
"default": 5,
294301
"description": "Maximum number of parallel requests per events operation",
295302
"min": 1
296303
},
297304
"sequences": {
298305
"type": "integer",
299-
"default": 10,
306+
"default": 5,
300307
"description": "Maximum number of parallel requests per sequences operation",
301308
"min": 1
302309
},
303310
"instances": {
304311
"type": "integer",
305-
"default": 4,
312+
"default": 2,
306313
"description": "Maximum number of parallel requests per data modeling instances operation",
307314
"min": 1
315+
},
316+
"stream-records": {
317+
"type": "integer",
318+
"default": 4,
319+
"description": "Maximum number of parallel requests per stream records operation",
320+
"min": 1
308321
}
309322
},
310323
"unevaluatedProperties": false
@@ -382,4 +395,4 @@
382395
"unevaluatedProperties": false
383396
}
384397
}
385-
}
398+
}

‎schema/unstable/base_cognite_config.schema.json‎

Lines changed: 192 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,192 @@
1+
{
2+
"$id": "base_cognite_config.schema.json",
3+
"$schema": "https://json-schema.org/draft/2020-12/schema",
4+
"type": "object",
5+
"unevaluatedProperties": false,
6+
"properties": {
7+
"cdf-chunking": {
8+
"type": "object",
9+
"description": "Configure chunking of data on requests to CDF. Note that increasing these may cause requests to fail due to limits in the API itself",
10+
"properties": {
11+
"time-series": {
12+
"type": "integer",
13+
"default": 1000,
14+
"description": "Maximum number of timeseries per get/create timeseries request",
15+
"max": 1000,
16+
"min": 1
17+
},
18+
"assets": {
19+
"type": "integer",
20+
"default": 1000,
21+
"description": "Maximum number of assets per get/create assets request",
22+
"max": 1000,
23+
"min": 1
24+
},
25+
"data-point-time-series": {
26+
"type": "integer",
27+
"default": 10000,
28+
"description": "Maximum number of timeseries per datapoint create request",
29+
"max": 10000,
30+
"min": 1
31+
},
32+
"data-point-delete": {
33+
"type": "integer",
34+
"default": 10000,
35+
"description": "Maximum number of ranges per delete datapoints request",
36+
"max": 10000,
37+
"min": 1
38+
},
39+
"data-point-list": {
40+
"type": "integer",
41+
"default": 100,
42+
"description": "Maximum number of timeseries per datapoint read request. Used when getting the first point in a timeseries.",
43+
"max": 100,
44+
"min": 1
45+
},
46+
"data-points": {
47+
"type": "integer",
48+
"default": 100000,
49+
"description": "Maximum number of datapoints per datapoints create request",
50+
"max": 100000,
51+
"min": 1
52+
},
53+
"data-points-gzip-limit": {
54+
"type": "integer",
55+
"default": 5000,
56+
"description": "Minimum number of datapoints in request to switch to using gzip. Set to -1 to disable, and 0 to always enable (not recommended). The minimum HTTP packet size is generally 1500 bytes, so this should never be set below 100 for numeric datapoints. Even for larger packages gzip is efficient enough that packages are compressed below 1500 bytes. At 5000 it is always a performance gain. It can be set lower if bandwidth is a major issue",
57+
"max": 100000,
58+
"min": 1
59+
},
60+
"raw-rows": {
61+
"type": "integer",
62+
"default": 10000,
63+
"description": "Maximum number of rows per request to cdf raw",
64+
"max": 10000,
65+
"min": 1
66+
},
67+
"raw-rows-delete": {
68+
"type": "integer",
69+
"default": 1000,
70+
"description": "Maximum number of row keys per delete request to raw",
71+
"max": 1000,
72+
"min": 1
73+
},
74+
"data-point-latest": {
75+
"type": "integer",
76+
"default": 100,
77+
"description": "Maximum number of timeseries per datapoint read latest request",
78+
"max": 100,
79+
"min": 1
80+
},
81+
"events": {
82+
"type": "integer",
83+
"default": 1000,
84+
"description": "Maximum number of events per get/create events request",
85+
"max": 1000,
86+
"min": 1
87+
},
88+
"sequences": {
89+
"type": "integer",
90+
"default": 1000,
91+
"description": "Maximum number of sequences per get/create sequences request",
92+
"max": 1000,
93+
"min": 1
94+
},
95+
"sequence-row-sequences": {
96+
"type": "integer",
97+
"default": 1000,
98+
"description": "Maximum number of sequences per create sequence rows request",
99+
"max": 1000,
100+
"min": 1
101+
},
102+
"sequence-rows": {
103+
"type": "integer",
104+
"default": 10000,
105+
"description": "Maximum number of sequence rows per sequence when creating rows",
106+
"max": 10000,
107+
"min": 1
108+
},
109+
"instances": {
110+
"type": "integer",
111+
"default": 1000,
112+
"description": "Maximum number of data modeling instances per get/create instance request",
113+
"max": 1000,
114+
"min": 1
115+
},
116+
"stream-records": {
117+
"type": "integer",
118+
"default": 1000,
119+
"description": "Maximum number of stream records per get/create stream records request",
120+
"max": 1000,
121+
"min": 1
122+
}
123+
},
124+
"unevaluatedProperties": false
125+
},
126+
"cdf-throttling": {
127+
"type": "object",
128+
"description": "Configure the maximum number of parallel requests for different CDF resources.",
129+
"properties": {
130+
"time-series": {
131+
"type": "integer",
132+
"default": 5,
133+
"description": "Maximum number of parallel requests per timeseries operation",
134+
"min": 1
135+
},
136+
"assets": {
137+
"type": "integer",
138+
"default": 5,
139+
"description": "Maximum number of parallel requests per assets operation",
140+
"min": 1
141+
},
142+
"data-points": {
143+
"type": "integer",
144+
"default": 5,
145+
"description": "Maximum number of parallel requests per datapoints operation",
146+
"min": 1
147+
},
148+
"raw": {
149+
"type": "integer",
150+
"default": 5,
151+
"description": "Maximum number of parallel requests per raw operation",
152+
"min": 1
153+
},
154+
"ranges": {
155+
"type": "integer",
156+
"default": 5,
157+
"description": "Maximum number of parallel requests per get first/last datapoint operation",
158+
"min": 1
159+
},
160+
"events": {
161+
"type": "integer",
162+
"default": 5,
163+
"description": "Maximum number of parallel requests per events operation",
164+
"min": 1
165+
},
166+
"sequences": {
167+
"type": "integer",
168+
"default": 5,
169+
"description": "Maximum number of parallel requests per sequences operation",
170+
"min": 1
171+
},
172+
"instances": {
173+
"type": "integer",
174+
"default": 2,
175+
"description": "Maximum number of parallel requests per data modeling instances operation",
176+
"min": 1
177+
},
178+
"stream-records": {
179+
"type": "integer",
180+
"default": 4,
181+
"description": "Maximum number of parallel requests per stream records operation",
182+
"min": 1
183+
}
184+
},
185+
"unevaluatedProperties": false
186+
},
187+
"nan-replacement": {
188+
"type": "number",
189+
"description": "Replacement for NaN values when writing to CDF. If left out, NaN values are skipped."
190+
}
191+
}
192+
}

0 commit comments

Comments
 (0)
⚡