Skip to content

Commit 41e17cd

Browse files
janvanmansumjanvanmansum
authored
DD-2110 Custom properties support. (#17)
Fixes DD-2110 # Description of changes This PR adds support for custom properties in version properties files. The main change is updating the system to allow properties prefixed with "custom." to be passed through as OCFL object version properties, alongside the required version info fields (user name, email, and message). **Changes:** - Renamed `VersionInfoReader` to `VersionPropertiesReader` and refactored it to handle both standard version info and custom properties - Updated `OcflRepositoryProvider` to read and apply custom properties to OCFL object versions - Added comprehensive test coverage for custom properties functionality # Related PRs * DANS-KNAW/dd-transfer-to-vault#65 # Notify @DANS-KNAW/core-systems
1 parent 49de639 commit 41e17cd

8 files changed

Lines changed: 261 additions & 51 deletions

File tree

‎docs/index.md‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -48,6 +48,10 @@ batch-dir
4848
* `message` - the commit message for this version
4949
If no properties file is present, the service will use default values for these properties. These default values can be configured in the configuration
5050
file, under `dataVault.defaultVersionInfo`. If no default values are configured, an error will be raised.
51+
* `vN.properties` can optionally have custom properties. These are properties prefixed with the string `custom.`. The service will add these as object version
52+
properties using the mechanism defined by the [Object Version Properties]{:target=_blank} extension.
53+
54+
[Object Version Properties]: {{ object_version_properties_ext }}
5155

5256
Processing
5357
----------

‎mkdocs.yml‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -33,6 +33,9 @@ nav:
3333
- Database for testing: test-db.md
3434
- Testing with dmftar: test-dmftar.md
3535

36+
extra:
37+
object_version_properties_ext: https://dans-knaw.github.io/dans-ocfl-extensions/object-version-properties/object-version-properties/
38+
3639
plugins:
3740
- markdownextradata
3841
- search

‎pom.xml‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -34,6 +34,8 @@
3434
<inceptionYear>2024</inceptionYear>
3535

3636
<properties>
37+
<!-- TODO: move to dd-parent -->
38+
<commons-validator.version>1.7</commons-validator.version>
3739
<main-class>nl.knaw.dans.datavault.DdDataVaultApplication</main-class>
3840
</properties>
3941

@@ -83,6 +85,11 @@
8385
<groupId>nl.knaw.dans</groupId>
8486
<artifactId>dans-java-utils</artifactId>
8587
</dependency>
88+
<dependency>
89+
<groupId>commons-validator</groupId>
90+
<artifactId>commons-validator</artifactId>
91+
<version>${commons-validator.version}</version>
92+
</dependency>
8693
<dependency>
8794
<groupId>nl.knaw.dans</groupId>
8895
<artifactId>dans-validation-lib</artifactId>

‎src/main/assembly/dist/cfg/ocfl-root-extensions/property-registry/config.json‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,11 @@
55
"description": "The packaging format of the current object version, as defined in the packaging-format-registry extension.",
66
"type": "string",
77
"required": true
8+
},
9+
"dataset-version": {
10+
"description": "The version of the dataset that this object version is an export of, as defined by the depositor",
11+
"type": "string",
12+
"required": false
813
}
914
}
1015
}

‎src/main/java/nl/knaw/dans/datavault/core/OcflRepositoryProvider.java‎

Lines changed: 12 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -19,7 +19,6 @@
1919
import io.ocfl.api.OcflRepository;
2020
import io.ocfl.api.exception.NotFoundException;
2121
import io.ocfl.api.model.ObjectVersionId;
22-
import io.ocfl.api.model.User;
2322
import io.ocfl.api.model.VersionInfo;
2423
import io.ocfl.api.model.VersionNum;
2524
import io.ocfl.core.OcflRepositoryBuilder;
@@ -57,9 +56,7 @@
5756
@Slf4j
5857
@RequiredArgsConstructor(access = AccessLevel.PRIVATE) // Builder should be used to create instances
5958
public class OcflRepositoryProvider implements RepositoryProvider, Managed {
60-
public static final String PACKAGING_FORMAT_KEY = "packaging-format";
61-
public static final String DANS_RDA_BAG_PACK_PROFILE_0_1_0 = "DANS RDA BagPack Profile/0.1.0";
62-
private final VersionInfoReader versionInfoReader;
59+
private final DefaultVersionInfoConfig defaultVersionInfoConfig;
6360

6461
@NonNull
6562
private final LayeredItemStore layeredItemStore;
@@ -78,7 +75,7 @@ public class OcflRepositoryProvider implements RepositoryProvider, Managed {
7875
@Builder
7976
public static OcflRepositoryProvider create(LayeredItemStore itemStore, Path workDir, DefaultVersionInfoConfig defaultVersionInfoConfig, LayerConsistencyChecker layerConsistencyChecker,
8077
Path rootExtensionsSourcePath) {
81-
return new OcflRepositoryProvider(new VersionInfoReader(defaultVersionInfoConfig), itemStore, workDir, layerConsistencyChecker, rootExtensionsSourcePath);
78+
return new OcflRepositoryProvider(defaultVersionInfoConfig, itemStore, workDir, layerConsistencyChecker, rootExtensionsSourcePath);
8279
}
8380

8481
@Override
@@ -88,9 +85,11 @@ public void addVersion(String objectId, int version, Path objectVersionDirectory
8885
throw new IllegalStateException("OCFL repository is not yet started");
8986
}
9087
// putObject wants the version number of HEAD, so we need to subtract 1 from the version number
91-
ocflRepository.putObject(ObjectVersionId.version(objectId, version - 1), objectVersionDirectory, createVersionInfoFor(objectVersionDirectory));
88+
var versionInfoFile = objectVersionDirectory.resolveSibling(objectVersionDirectory.getFileName().toString() + ".properties");
89+
var reader = createVersionPropertiesReader(versionInfoFile);
90+
ocflRepository.putObject(ObjectVersionId.version(objectId, version - 1), objectVersionDirectory, reader.getVersionInfo());
9291

93-
updateObjectVersionProperties(objectId, version, PACKAGING_FORMAT_KEY, DANS_RDA_BAG_PACK_PROFILE_0_1_0);
92+
reader.getCustomProperties().forEach((key, value) -> updateObjectVersionProperties(objectId, version, key, value));
9493
}
9594

9695
@Override
@@ -99,10 +98,11 @@ public void addHeadVersion(String objectId, Path objectVersionDirectory) {
9998
if (ocflRepository == null) {
10099
throw new IllegalStateException("OCFL repository is not yet started");
101100
}
102-
ocflRepository.putObject(ObjectVersionId.head(objectId), objectVersionDirectory, createVersionInfoFor(objectVersionDirectory));
101+
var reader = createVersionPropertiesReader(objectVersionDirectory.resolveSibling(objectVersionDirectory.getFileName().toString() + ".properties"));
102+
ocflRepository.putObject(ObjectVersionId.head(objectId), objectVersionDirectory, reader.getVersionInfo());
103103
long headVersion = Optional.ofNullable(ObjectVersionId.head(objectId).getVersionNum()).map(VersionNum::getVersionNum).orElse(1L);
104104

105-
updateObjectVersionProperties(objectId, headVersion, PACKAGING_FORMAT_KEY, DANS_RDA_BAG_PACK_PROFILE_0_1_0);
105+
reader.getCustomProperties().forEach((key, value) -> updateObjectVersionProperties(objectId, headVersion, key, value));
106106
}
107107

108108
private void updateObjectVersionProperties(String objectId, long version, String key, Object value) {
@@ -130,12 +130,12 @@ public Optional<OcflObjectVersionDto> getOcflObjectVersion(String objectId, int
130130
}
131131
}
132132

133-
private VersionInfo createVersionInfoFor(Path objectVersionDirectory) {
133+
private VersionPropertiesReader createVersionPropertiesReader(Path versionPropertiesFile) {
134134
try {
135-
return versionInfoReader.read(objectVersionDirectory.resolveSibling(objectVersionDirectory.getFileName().toString() + ".properties"));
135+
return new VersionPropertiesReader(versionPropertiesFile, defaultVersionInfoConfig);
136136
}
137137
catch (IOException e) {
138-
throw new RuntimeException("Failed to read version info from " + objectVersionDirectory, e);
138+
throw new RuntimeException("Failed to read version info from " + versionPropertiesFile, e);
139139
}
140140
}
141141

src/main/java/nl/knaw/dans/datavault/core/VersionInfoReader.java renamed to src/main/java/nl/knaw/dans/datavault/core/VersionPropertiesReader.java

Lines changed: 44 additions & 21 deletions
Original file line numberDiff line numberDiff line change
@@ -17,59 +17,82 @@
1717

1818
import io.ocfl.api.model.User;
1919
import io.ocfl.api.model.VersionInfo;
20-
import lombok.RequiredArgsConstructor;
2120
import nl.knaw.dans.datavault.config.DefaultVersionInfoConfig;
21+
import org.apache.commons.validator.routines.EmailValidator;
2222

2323
import java.io.IOException;
2424
import java.nio.file.Files;
2525
import java.nio.file.Path;
26+
import java.util.Map;
2627
import java.util.Properties;
2728
import java.util.Set;
29+
import java.util.stream.Collectors;
2830
import java.util.regex.Pattern;
2931

30-
@RequiredArgsConstructor
31-
public class VersionInfoReader {
32-
private static final Pattern validEmailPattern = Pattern.compile("^mailto:[A-Za-z0-9._%+-]+@[A-Za-z0-9-]+\\.[A-Za-z]{2,}$");
33-
private static final Set<String> ALLOWED_KEYS = Set.of("user.name", "user.email", "message");
32+
public class VersionPropertiesReader {
33+
private static final String MAILTO_PREFIX = "mailto:";
34+
private static final Set<String> VERSION_INFO_KEYS = Set.of("user.name", "user.email", "message");
3435
private final DefaultVersionInfoConfig defaultConfig;
36+
private final Properties props;
37+
38+
public VersionPropertiesReader(Path file, DefaultVersionInfoConfig defaultConfig) throws IOException {
39+
this.defaultConfig = defaultConfig;
3540

36-
public VersionInfo read(Path file) throws IOException {
3741
if (file == null || !Files.exists(file)) {
3842
if (defaultConfig == null) {
3943
throw new IllegalArgumentException("No version info file " + file + " provided and no default configuration available");
4044
}
45+
this.props = null;
46+
}
47+
else {
48+
this.props = new Properties();
49+
try (var in = Files.newInputStream(file)) {
50+
this.props.load(in);
51+
}
4152

42-
return createDefaultVersionInfo();
53+
for (var key : this.props.stringPropertyNames()) {
54+
if (!VERSION_INFO_KEYS.contains(key) && !key.startsWith("custom.")) {
55+
throw new IllegalArgumentException("Unknown property in version info file: " + key);
56+
}
57+
}
4358
}
59+
}
4460

45-
var props = new Properties();
46-
try (var in = Files.newInputStream(file)) {
47-
props.load(in);
61+
public Map<String, String> getCustomProperties() {
62+
if (this.props == null) {
63+
return Map.of();
4864
}
4965

50-
for (var key : props.stringPropertyNames()) {
51-
if (!ALLOWED_KEYS.contains(key)) {
52-
throw new IllegalArgumentException("Unknown property in version info file: " + key);
53-
}
66+
return this.props.stringPropertyNames().stream()
67+
.filter(key -> key.startsWith("custom."))
68+
.collect(Collectors.toMap(
69+
key -> key.substring("custom.".length()),
70+
this.props::getProperty
71+
));
72+
}
73+
74+
public VersionInfo getVersionInfo() {
75+
if (this.props == null) {
76+
return createDefaultVersionInfo();
5477
}
5578

5679
var info = new VersionInfo();
5780
var user = new User();
5881
user.setName(getOrThrow(props, "user.name"));
59-
// Add mailto: if not present yet
60-
var mail = getOrThrow(props, "user.email");
61-
if (!mail.startsWith("mailto:")) {
62-
mail = "mailto:" + mail;
82+
var email = getOrThrow(props, "user.email");
83+
if (!email.startsWith(MAILTO_PREFIX)) {
84+
email = MAILTO_PREFIX + email;
6385
}
64-
validateEmail(mail);
65-
user.setAddress(mail);
86+
validateEmail(email);
87+
user.setAddress(email);
6688
info.setUser(user);
6789
info.setMessage(getOrThrow(props, "message"));
6890
return info;
6991
}
7092

7193
private void validateEmail(String email) {
72-
if (!validEmailPattern.matcher(email).matches()) {
94+
var mailWithoutMailTo = email.startsWith(MAILTO_PREFIX) ? email.substring(MAILTO_PREFIX.length()) : email;
95+
if (!EmailValidator.getInstance().isValid(mailWithoutMailTo)) {
7396
throw new IllegalArgumentException("Invalid email address: " + email);
7497
}
7598
}

‎src/test/java/nl/knaw/dans/datavault/core/OcflRepositoryProviderTest.java‎

Lines changed: 95 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -36,6 +36,7 @@
3636
import org.junit.jupiter.api.extension.ExtendWith;
3737

3838
import java.net.URI;
39+
import java.nio.file.Files;
3940
import java.nio.file.Path;
4041
import java.util.Map;
4142

@@ -100,6 +101,13 @@ public void setUp() throws Exception {
100101
public void addHeadVersion_should_create_new_object() throws Exception {
101102
// Given
102103
copyToTestDir("simple-object/v1", TEST_INPUT);
104+
var properties = """
105+
user.name=Test User
106+
user.email=test.user@mail.com
107+
message=Initial version
108+
custom.packaging-format=DANS RDA BagPack Profile/0.1.0
109+
""";
110+
Files.writeString(testDir.resolve(TEST_INPUT + "/v1.properties"), properties);
103111

104112
// When
105113
ocflRepositoryProvider.addHeadVersion("urn:nbn:o1", testDir.resolve(TEST_INPUT + "/v1"));
@@ -132,18 +140,30 @@ public void addHeadVersion_should_create_new_object() throws Exception {
132140
assertThat(objectVersionProperties.keySet()).hasSize(1);
133141
assertThat(objectVersionProperties).containsKey("v1");
134142
assertThat(objectVersionProperties.get("v1").keySet()).hasSize(1);
135-
assertThat(objectVersionProperties.get("v1")).containsEntry(OcflRepositoryProvider.PACKAGING_FORMAT_KEY,
136-
OcflRepositoryProvider.DANS_RDA_BAG_PACK_PROFILE_0_1_0);
137-
143+
assertThat(objectVersionProperties.get("v1")).containsEntry("packaging-format", "DANS RDA BagPack Profile/0.1.0");
138144
}
139145

140146

141147
@Test
142148
public void addVersion_should_add_version_to_existing_object() throws Exception {
143149
// Given
144150
copyToTestDir("simple-object/v1", TEST_INPUT);
151+
var propertiesV1 = """
152+
user.name=Test User
153+
user.email=test.user@mail.com
154+
message=Initial version
155+
custom.packaging-format=DANS RDA BagPack Profile/0.1.0
156+
""";
157+
Files.writeString(testDir.resolve(TEST_INPUT + "/v1.properties"), propertiesV1);
145158
ocflRepositoryProvider.addHeadVersion("urn:nbn:o1", testDir.resolve(TEST_INPUT + "/v1"));
146159
copyToTestDir("simple-object/v2", TEST_INPUT);
160+
var propertiesV2 = """
161+
user.name=Test User
162+
user.email=test.user@mail.com
163+
message=Version 2
164+
custom.packaging-format=DANS RDA BagPack Profile/0.1.0
165+
""";
166+
Files.writeString(testDir.resolve(TEST_INPUT + "/v2.properties"), propertiesV2);
147167

148168
// When
149169
ocflRepositoryProvider.addVersion("urn:nbn:o1", 2, testDir.resolve(TEST_INPUT + "/v2"));
@@ -174,14 +194,82 @@ public void addVersion_should_add_version_to_existing_object() throws Exception
174194
assertThat(objectVersionProperties.keySet()).hasSize(2);
175195
assertThat(objectVersionProperties).containsKey("v1");
176196
assertThat(objectVersionProperties.get("v1").keySet()).hasSize(1);
177-
assertThat(objectVersionProperties.get("v1")).containsEntry(OcflRepositoryProvider.PACKAGING_FORMAT_KEY,
178-
OcflRepositoryProvider.DANS_RDA_BAG_PACK_PROFILE_0_1_0);
197+
assertThat(objectVersionProperties.get("v1")).containsEntry("packaging-format", "DANS RDA BagPack Profile/0.1.0");
179198
assertThat(objectVersionProperties).containsKey("v2");
180199
assertThat(objectVersionProperties.get("v2").keySet()).hasSize(1);
181-
assertThat(objectVersionProperties.get("v2")).containsEntry(OcflRepositoryProvider.PACKAGING_FORMAT_KEY,
182-
OcflRepositoryProvider.DANS_RDA_BAG_PACK_PROFILE_0_1_0);
200+
assertThat(objectVersionProperties.get("v2")).containsEntry("packaging-format", "DANS RDA BagPack Profile/0.1.0");
201+
}
202+
203+
@Test
204+
public void addHeadVersion_should_add_custom_properties() throws Exception {
205+
// Given
206+
copyToTestDir("simple-object/v1", TEST_INPUT);
207+
var properties = """
208+
user.name=Test User
209+
user.email=test.user@mail.com
210+
message=Initial version
211+
custom.key1=Value 1
212+
custom.key2=Value 2
213+
custom.packaging-format=DANS RDA BagPack Profile/0.1.0
214+
""";
215+
Files.writeString(testDir.resolve(TEST_INPUT + "/v1.properties"), properties);
216+
217+
// When
218+
ocflRepositoryProvider.addHeadVersion("urn:nbn:o1", testDir.resolve(TEST_INPUT + "/v1"));
219+
220+
// Then
221+
long layerId = layerManager.getTopLayer().getId();
222+
var objectRoot = testDir.resolve(LAYER_STAGING_ROOT).resolve(Long.toString(layerId)).resolve("000/000/0o1/o1");
223+
224+
// Verify that the object version properties are set correctly
225+
Map<String, Map<String, Object>> objectVersionProperties = mapper.readValue(
226+
objectRoot.resolve("extensions/object-version-properties/object_version_properties.json").toFile(),
227+
mapper.getTypeFactory().constructMapType(Map.class, String.class, Map.class)
228+
);
229+
230+
assertThat(objectVersionProperties.get("v1")).containsEntry("key1", "Value 1");
231+
assertThat(objectVersionProperties.get("v1")).containsEntry("key2", "Value 2");
232+
assertThat(objectVersionProperties.get("v1")).containsEntry("packaging-format", "DANS RDA BagPack Profile/0.1.0");
183233
}
184234

235+
@Test
236+
public void addVersion_should_add_custom_properties() throws Exception {
237+
// Given
238+
copyToTestDir("simple-object/v1", TEST_INPUT);
239+
var propertiesV1 = """
240+
user.name=Test User
241+
user.email=test.user@mail.com
242+
message=Initial version
243+
custom.packaging-format=DANS RDA BagPack Profile/0.1.0
244+
""";
245+
Files.writeString(testDir.resolve(TEST_INPUT + "/v1.properties"), propertiesV1);
246+
ocflRepositoryProvider.addHeadVersion("urn:nbn:o1", testDir.resolve(TEST_INPUT + "/v1"));
247+
copyToTestDir("simple-object/v2", TEST_INPUT);
248+
var propertiesV2 = """
249+
user.name=Test User
250+
user.email=test.user@mail.com
251+
message=Second version
252+
custom.key3=Value 3
253+
custom.packaging-format=DANS RDA BagPack Profile/0.1.0
254+
""";
255+
Files.writeString(testDir.resolve(TEST_INPUT + "/v2.properties"), propertiesV2);
256+
257+
// When
258+
ocflRepositoryProvider.addVersion("urn:nbn:o1", 2, testDir.resolve(TEST_INPUT + "/v2"));
259+
260+
// Then
261+
long layerId = layerManager.getTopLayer().getId();
262+
var objectRoot = testDir.resolve(LAYER_STAGING_ROOT).resolve(Long.toString(layerId)).resolve("000/000/0o1/o1");
263+
264+
// Verify that the object version properties are set correctly
265+
Map<String, Map<String, Object>> objectVersionProperties = mapper.readValue(
266+
objectRoot.resolve("extensions/object-version-properties/object_version_properties.json").toFile(),
267+
mapper.getTypeFactory().constructMapType(Map.class, String.class, Map.class)
268+
);
269+
270+
assertThat(objectVersionProperties.get("v2")).containsEntry("key3", "Value 3");
271+
assertThat(objectVersionProperties.get("v2")).containsEntry("packaging-format", "DANS RDA BagPack Profile/0.1.0");
272+
}
185273

186274

187275
// TODO: sidecar file must have the algorithm as inventory sidecar file (this must then first be made configurable in OcflRepositoryProvider)

0 commit comments

Comments
 (0)
⚡