diff --git a/doc/release-notes/11217-source-name-harvesting-client.md b/doc/release-notes/11217-source-name-harvesting-client.md new file mode 100644 index 00000000000..53347de694c --- /dev/null +++ b/doc/release-notes/11217-source-name-harvesting-client.md @@ -0,0 +1,13 @@ +### Metadata Source Facet Can Now Differentiate Between Harvested Sources + +The behavior of the feature flag `index-harvested-metadata-source` and the "Metadata Source" facet, which were added and updated, respectively, in [Dataverse 6.3](https://github.com/IQSS/dataverse/releases/tag/v6.3) (through pull requests #10464 and #10651), have been updated. A new field called "Source Name" has been added to harvesting clients. + +Before Dataverse 6.3, all harvested content (datasets and files) appeared together under "Harvested" under the "Metadata Source" facet. This is still the behavior of Dataverse out of the box. Since Dataverse 6.3, enabling the `index-harvested-metadata-source` feature flag (and reindexing) resulted in harvested content appearing under the nickname for whatever harvesting client was used to bring in the content. This meant that instead of having all harvested content lumped together under "Harvested", content would appear under "client1", "client2", etc. + +Now, as this release, enabling the `index-harvested-metadata-source` feature flag, populating a new field for harvesting clients called "Source Name" ("sourceName" in the [API](https://dataverse-guide--11217.org.readthedocs.build/en/11217/api/native-api.html#create-a-harvesting-client)), and reindexing (see upgrade instructions below), results in the source name appearing under the "Metadata Source" facet rather than the harvesting client nickname. This gives you more control over the name that appears under the "Metadata Source" facet and allows you to group harvested content from various harvesting clients under the same name if you wish (by reusing the same source name). + +Previously, `index-harvested-metadata-source` was not documented in the guides, but now you can find information about it under [Feature Flags](https://dataverse-guide--11217.org.readthedocs.build/en/11217/installation/config.html#feature-flags). See also #10217 and #11217. + +## Upgrade instructions + +If you have enabled the `dataverse.feature.index-harvested-metadata-source` feature flag and given some of your harvesting clients a source name, you should reindex to have those source names appear under the "Metadata Source" facet. diff --git a/doc/sphinx-guides/source/_static/api/harvesting-client.json b/doc/sphinx-guides/source/_static/api/harvesting-client.json new file mode 100644 index 00000000000..82a817fc38f --- /dev/null +++ b/doc/sphinx-guides/source/_static/api/harvesting-client.json @@ -0,0 +1,11 @@ +{ + "nickName": "zenodo", + "dataverseAlias": "zenodoHarvested", + "harvestUrl": "https://zenodo.org/oai2d", + "archiveUrl": "https://zenodo.org", + "archiveDescription": "Moissonné depuis la collection LMOPS de l'entrepôt Zenodo. En cliquant sur ce jeu de données, vous serez redirigé vers Zenodo.", + "metadataFormat": "oai_dc", + "customHeaders": "x-oai-api-key: xxxyyyzzz", + "set": "user-lmops", + "allowHarvestingMissingCVV":true +} diff --git a/doc/sphinx-guides/source/api/native-api.rst b/doc/sphinx-guides/source/api/native-api.rst index 7927d063dd6..350cd2020ca 100644 --- a/doc/sphinx-guides/source/api/native-api.rst +++ b/doc/sphinx-guides/source/api/native-api.rst @@ -5559,7 +5559,7 @@ Create a Harvesting Set To create a harvesting set you must supply a JSON file that contains the following fields: -- Name: Alpha-numeric may also contain -, _, or %, but no spaces. Must also be unique in the installation. +- Name: Alpha-numeric may also contain -, _, or %, but no spaces. It must also be unique in the installation. - Definition: A search query to select the datasets to be harvested. For example, a query containing authorName:YYY would include all datasets where ‘YYY’ is the authorName. - Description: Text that describes the harvesting set. The description appears in the Manage Harvesting Sets dashboard and in API responses. This field is optional. @@ -5655,20 +5655,43 @@ The following API can be used to create and manage "Harvesting Clients". A Harve List All Configured Harvesting Clients ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -Shows all the Harvesting Clients configured:: +Shows all the harvesting clients configured. - GET http://$SERVER/api/harvest/clients/ +.. note:: See :ref:`curl-examples-and-environment-variables` if you are unfamiliar with the use of export below. + +.. code-block:: bash + + export SERVER_URL=https://demo.dataverse.org + + curl "$SERVER_URL/api/harvest/clients" + +The fully expanded example above (without the environment variables) looks like this: + +.. code-block:: bash + + curl "https://demo.dataverse.org/api/harvest/clients" Show a Specific Harvesting Client ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -Shows a Harvesting Client with a defined nickname:: +Shows a harvesting client by nickname. - GET http://$SERVER/api/harvest/clients/$nickname +.. code-block:: bash + + export SERVER_URL=https://demo.dataverse.org + export NICKNAME=myclient + + curl "$SERVER_URL/api/harvest/clients/$NICKNAME" + +The fully expanded example above (without the environment variables) looks like this: .. code-block:: bash - curl "http://localhost:8080/api/harvest/clients/myclient" + curl "https://demo.dataverse.org/api/harvest/clients/myclient" + +The output will look something like the following. + +.. code-block:: bash { "status":"OK", @@ -5684,6 +5707,7 @@ Shows a Harvesting Client with a defined nickname:: "type": "oai", "dataverseAlias": "fooData", "nickName": "myClient", + "sourceName": "", "set": "fooSet", "useOaiIdentifiersAsPids": false "schedule": "none", @@ -5697,16 +5721,12 @@ Shows a Harvesting Client with a defined nickname:: } +.. _create-a-harvesting-client: + Create a Harvesting Client ~~~~~~~~~~~~~~~~~~~~~~~~~~ - -To create a new harvesting client:: - - POST http://$SERVER/api/harvest/clients/$nickname - -``nickName`` is the name identifying the new client. It should be alpha-numeric and may also contain -, _, or %, but no spaces. Must also be unique in the installation. -You must supply a JSON file that describes the configuration, similarly to the output of the GET API above. The following fields are mandatory: +To create a harvesting client you must supply a JSON file that describes the configuration, similarly to the output of the GET API above. The following fields are mandatory: - dataverseAlias: The alias of an existing collection where harvested datasets will be deposited - harvestUrl: The URL of the remote OAI archive @@ -5715,6 +5735,7 @@ You must supply a JSON file that describes the configuration, similarly to the o The following optional fields are supported: +- sourceName: When ``index-harvested-metadata-source`` is enabled (see :ref:`feature-flags`), sourceName will override the nickname in the Metadata Source facet. It can be used to group the content from many harvesting clients under the same name. - archiveDescription: What the name suggests. If not supplied, will default to "This Dataset is harvested from our partners. Clicking the link will take you directly to the archival source of the data." - set: The OAI set on the remote server. If not supplied, will default to none, i.e., "harvest everything". - style: Defaults to "default" - a generic OAI archive. (Make sure to use "dataverse" when configuring harvesting from another Dataverse installation). @@ -5723,38 +5744,35 @@ The following optional fields are supported: - useOaiIdentifiersAsPids: Defaults to false; if set to true, the harvester will attempt to use the identifier from the OAI-PMH record header as the **first choice** for the persistent id of the harvested dataset. When set to false, Dataverse will still attempt to use this identifier, but only if none of the `` entries in the OAI_DC record contain a valid persistent id (this is new as of v6.5). Generally, the API will accept the output of the GET version of the API for an existing client as valid input, but some fields will be ignored. For example, as of writing this there is no way to configure a harvesting schedule via this API. - -An example JSON file would look like this:: - { - "nickName": "zenodo", - "dataverseAlias": "zenodoHarvested", - "harvestUrl": "https://zenodo.org/oai2d", - "archiveUrl": "https://zenodo.org", - "archiveDescription": "Moissonné depuis la collection LMOPS de l'entrepôt Zenodo. En cliquant sur ce jeu de données, vous serez redirigé vers Zenodo.", - "metadataFormat": "oai_dc", - "customHeaders": "x-oai-api-key: xxxyyyzzz", - "set": "user-lmops", - "allowHarvestingMissingCVV":true - } +You can download this :download:`harvesting-client.json <../_static/api/harvesting-client.json>` file to use as a starting point. -Something important to keep in mind about this API is that, unlike the harvesting clients GUI, it will create a client with the values supplied without making any attempts to validate them in real time. In other words, for the `harvestUrl` it will accept anything that looks like a well-formed url, without making any OAI calls to verify that the name of the set and/or the metadata format entered are supported by it. This is by design, to give an admin an option to still be able to create a client, in a rare case when it cannot be done via the GUI because of some real time failures in an exchange with an otherwise valid OAI server. This however puts the responsibility on the admin to supply the values already confirmed to be valid. +.. literalinclude:: ../_static/api/harvesting-client.json +Something important to keep in mind about this API is that, unlike the harvesting clients GUI, it will create a client with the values supplied without making any attempts to validate them in real time. In other words, for the `harvestUrl` it will accept anything that looks like a well-formed url, without making any OAI calls to verify that the name of the set and/or the metadata format entered are supported by it. This is by design, to give an admin an option to still be able to create a client, in a rare case when it cannot be done via the GUI because of some real time failures in an exchange with an otherwise valid OAI server. This however puts the responsibility on the admin to supply the values already confirmed to be valid. .. note:: See :ref:`curl-examples-and-environment-variables` if you are unfamiliar with the use of export below. + +``nickName`` in the JSON file and ``$NICKNAME`` in the URL path below is the name identifying the new client. It should be alpha-numeric and may also contain -, _, or %, but no spaces. It must be unique in the installation. + .. code-block:: bash export API_TOKEN=xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx export SERVER_URL=http://localhost:8080 + export NICKNAME=zenodo - curl -H "X-Dataverse-key:$API_TOKEN" -X POST -H "Content-Type: application/json" "$SERVER_URL/api/harvest/clients/zenodo" --upload-file client.json + curl -H "X-Dataverse-key:$API_TOKEN" -X POST -H "Content-Type: application/json" "$SERVER_URL/api/harvest/clients/$NICKNAME" --upload-file harvesting-client.json The fully expanded example above (without the environment variables) looks like this: .. code-block:: bash - curl -H "X-Dataverse-key:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" -X POST -H "Content-Type: application/json" "http://localhost:8080/api/harvest/clients/zenodo" --upload-file "client.json" + curl -H "X-Dataverse-key:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" -X POST -H "Content-Type: application/json" "http://localhost:8080/api/harvest/clients/zenodo" --upload-file "harvesting-client.json" + +The output will look something like the following. + +.. code-block:: bash { "status": "OK", @@ -5788,15 +5806,21 @@ Similar to the API above, using the same JSON format, but run on an existing cli Delete a Harvesting Client ~~~~~~~~~~~~~~~~~~~~~~~~~~ -Self-explanatory: - .. code-block:: bash - curl -H "X-Dataverse-key:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" -X DELETE "http://localhost:8080/api/harvest/clients/$nickName" + export API_TOKEN=xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx + export SERVER_URL=http://localhost:8080 + export NICKNAME=zenodo -Only users with superuser permissions may delete harvesting clients. + curl -H "X-Dataverse-key:$API_TOKEN" -X DELETE "$SERVER_URL/api/harvest/clients/$NICKNAME" +The fully expanded example above (without the environment variables) looks like this: + +.. code-block:: bash + curl -H "X-Dataverse-key:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" -X DELETE "http://localhost:8080/api/harvest/clients/zenodo" + +Only users with superuser permissions may delete harvesting clients. .. _pids-api: diff --git a/doc/sphinx-guides/source/installation/config.rst b/doc/sphinx-guides/source/installation/config.rst index 0f5aec8c942..6c683a82ba0 100644 --- a/doc/sphinx-guides/source/installation/config.rst +++ b/doc/sphinx-guides/source/installation/config.rst @@ -3493,6 +3493,9 @@ please find all known feature flags below. Any of these flags can be activated u * - globus-use-experimental-async-framework - Activates a new experimental implementation of Globus polling of ongoing remote data transfers that does not rely on the instance staying up continuously for the duration of the transfers and saves the state information about Globus upload requests in the database. Added in v6.4. Affects :ref:`:GlobusPollingInterval`. Note that the JVM option :ref:`dataverse.files.globus-monitoring-server` described above must also be enabled on one (and only one, in a multi-node installation) Dataverse instance. - ``Off`` + * - index-harvested-metadata-source + - Index the nickname or the source name (See the optional ``sourceName`` field in :ref:`create-a-harvesting-client`) of the harvesting client as the "metadata source" of harvested datasets and files. If enabled, the Metadata Source facet will show separate groupings of the content harvested from different sources (by harvesting client nickname or source name) instead of the default behavior where there is one "Harvested" grouping for all harvested content. + - ``Off`` **Note:** Feature flags can be set via any `supported MicroProfile Config API source`_, e.g. the environment variable ``DATAVERSE_FEATURE_XXX`` (e.g. ``DATAVERSE_FEATURE_API_SESSION_AUTH=1``). These environment variables can be set in your shell before starting Payara. If you are using :doc:`Docker for development `, you can set them in the `docker compose `_ file. diff --git a/docker-compose-dev.yml b/docker-compose-dev.yml index fdde14cdee5..0de90f7ec2a 100644 --- a/docker-compose-dev.yml +++ b/docker-compose-dev.yml @@ -17,6 +17,7 @@ services: SKIP_DEPLOY: "${SKIP_DEPLOY}" DATAVERSE_JSF_REFRESH_PERIOD: "1" DATAVERSE_FEATURE_API_BEARER_AUTH: "1" + DATAVERSE_FEATURE_INDEX_HARVESTED_METADATA_SOURCE: "1" DATAVERSE_FEATURE_API_BEARER_AUTH_PROVIDE_MISSING_CLAIMS: "1" DATAVERSE_MAIL_SYSTEM_EMAIL: "dataverse@localhost" DATAVERSE_MAIL_MTA_HOST: "smtp" diff --git a/src/main/java/edu/harvard/iq/dataverse/HarvestingClientsPage.java b/src/main/java/edu/harvard/iq/dataverse/HarvestingClientsPage.java index f008db1403f..1effd137e0e 100644 --- a/src/main/java/edu/harvard/iq/dataverse/HarvestingClientsPage.java +++ b/src/main/java/edu/harvard/iq/dataverse/HarvestingClientsPage.java @@ -78,7 +78,7 @@ public class HarvestingClientsPage implements java.io.Serializable { private Long dataverseId = null; private HarvestingClient selectedClient; private boolean setListTruncated = false; - + //private static final String solrDocIdentifierDataset = "dataset_"; public enum PageMode { @@ -242,6 +242,7 @@ public void editClient(HarvestingClient harvestingClient) { setSelectedClient(harvestingClient); this.newNickname = harvestingClient.getName(); + this.sourceName = harvestingClient.getSourceName(); this.newHarvestingUrl = harvestingClient.getHarvestingUrl(); this.customHeader = harvestingClient.getCustomHttpHeaders(); this.initialSettingsValidated = false; @@ -323,10 +324,9 @@ public void deleteClient() { } public void createClient(ActionEvent ae) { - - HarvestingClient newHarvestingClient = new HarvestingClient(); // will be set as type OAI by default - - newHarvestingClient.setName(newNickname); + + // will be set as type OAI by default + HarvestingClient newHarvestingClient = fillHarvestingClient(new HarvestingClient()); if (getSelectedDestinationDataverse() == null) { JsfHelper.JH.addMessage(FacesMessage.SEVERITY_ERROR,BundleUtil.getStringFromBundle("harvest.create.error")); @@ -338,35 +338,6 @@ public void createClient(ActionEvent ae) { } getSelectedDestinationDataverse().getHarvestingClientConfigs().add(newHarvestingClient); - newHarvestingClient.setHarvestingUrl(newHarvestingUrl); - newHarvestingClient.setCustomHttpHeaders(customHeader); - if (!StringUtils.isEmpty(newOaiSet)) { - newHarvestingClient.setHarvestingSet(newOaiSet); - } - newHarvestingClient.setMetadataPrefix(newMetadataFormat); - newHarvestingClient.setHarvestStyle(newHarvestingStyle); - - if (isNewHarvestingScheduled()) { - newHarvestingClient.setScheduled(true); - - if (isNewHarvestingScheduledWeekly()) { - newHarvestingClient.setSchedulePeriod(HarvestingClient.SCHEDULE_PERIOD_WEEKLY); - if (getWeekDayNumber() == null) { - // create a "week day is required..." error message, etc. - // but we may be better off not even giving them an opportunity - // to leave the field blank - ? - } - newHarvestingClient.setScheduleDayOfWeek(getWeekDayNumber()); - } else { - newHarvestingClient.setSchedulePeriod(HarvestingClient.SCHEDULE_PERIOD_DAILY); - } - - if (getHourOfDay() == null) { - // see the comment above, about the day of week. same here. - } - newHarvestingClient.setScheduleHourOfDay(getHourOfDay()); - } - // make default archive url (used to generate links pointing back to the // archival sources, when harvested datasets are displayed in search results), // from the harvesting url: @@ -412,51 +383,9 @@ public void createClient(ActionEvent ae) { // this saves an existing client that the user has edited: public void saveClient(ActionEvent ae) { - - HarvestingClient harvestingClient = getSelectedClient(); - - if (harvestingClient == null) { - // TODO: - // tell the user somehow that the client cannot be saved, and advise - // them to save the settings they have entered. - // as of now - we will show an error message, but only after the - // edit form has been closed. - } - - // nickname is not editable for existing clients: - //harvestingClient.setName(newNickname); - harvestingClient.setHarvestingUrl(newHarvestingUrl); - harvestingClient.setCustomHttpHeaders(customHeader); - harvestingClient.setHarvestingSet(newOaiSet); - harvestingClient.setMetadataPrefix(newMetadataFormat); - harvestingClient.setHarvestStyle(newHarvestingStyle); - - if (isNewHarvestingScheduled()) { - harvestingClient.setScheduled(true); - - if (isNewHarvestingScheduledWeekly()) { - harvestingClient.setSchedulePeriod(HarvestingClient.SCHEDULE_PERIOD_WEEKLY); - if (getWeekDayNumber() == null) { - // create a "week day is required..." error message, etc. - // but we may be better off not even giving them an opportunity - // to leave the field blank - ? - } - harvestingClient.setScheduleDayOfWeek(getWeekDayNumber()); - } else { - harvestingClient.setSchedulePeriod(HarvestingClient.SCHEDULE_PERIOD_DAILY); - } - - if (getHourOfDay() == null) { - // see the comment above, about the day of week. same here. - } - harvestingClient.setScheduleHourOfDay(getHourOfDay()); - } else { - harvestingClient.setScheduled(false); - } - - // will try to save it now: - try { + HarvestingClient harvestingClient = fillHarvestingClient(getSelectedClient()); + harvestingClient = engineService.submit( new UpdateHarvestingClientCommand(dvRequestService.getDataverseRequest(), harvestingClient)); configuredHarvestingClients = harvestingClientService.getAllHarvestingClients(); @@ -477,9 +406,50 @@ public void saveClient(ActionEvent ae) { } setPageMode(PageMode.VIEW); - + } - + + /** + * Based on a new harvestingClient instance or an existing one, it will update basics fields with new UI fields values + * @param harvestingClient new or existing harvestingClient to update + * @return harvestingClient with updated values + */ + private HarvestingClient fillHarvestingClient(HarvestingClient harvestingClient) { + // update nickname if it's a new object otherwise is not editable for existing clients + if(harvestingClient.getId() == null) { + harvestingClient.setName(newNickname); + } + harvestingClient.setSourceName(sourceName); + harvestingClient.setHarvestingUrl(newHarvestingUrl); + harvestingClient.setCustomHttpHeaders(customHeader); + if (!StringUtils.isEmpty(newOaiSet)) { + harvestingClient.setHarvestingSet(newOaiSet); + } + harvestingClient.setMetadataPrefix(newMetadataFormat); + harvestingClient.setHarvestStyle(newHarvestingStyle); + + harvestingClient.setScheduled(isNewHarvestingScheduled()); + if (isNewHarvestingScheduled()) { + if (isNewHarvestingScheduledWeekly()) { + harvestingClient.setSchedulePeriod(HarvestingClient.SCHEDULE_PERIOD_WEEKLY); + if (getWeekDayNumber() == null) { + // create a "week day is required..." error message, etc. + // but we may be better off not even giving them an opportunity + // to leave the field blank - ? + } + harvestingClient.setScheduleDayOfWeek(getWeekDayNumber()); + } else { + harvestingClient.setSchedulePeriod(HarvestingClient.SCHEDULE_PERIOD_DAILY); + } + + if (getHourOfDay() == null) { + // see the comment above, about the day of week. same here. + } + harvestingClient.setScheduleHourOfDay(getHourOfDay()); + } + return harvestingClient; + } + public void validateMetadataFormat(FacesContext context, UIComponent toValidate, Object rawValue) { String value = (String) rawValue; UIInput input = (UIInput) toValidate; @@ -717,6 +687,7 @@ public void backToStepThree() { UIInput selectedDataverseMenu; private String newNickname = ""; + private String sourceName = ""; private String newHarvestingUrl = ""; private String customHeader = null; private boolean initialSettingsValidated = false; @@ -741,6 +712,7 @@ public void backToStepThree() { public void initNewClient(ActionEvent ae) { //this.selectedClient = new HarvestingClient(); this.newNickname = ""; + this.sourceName = ""; this.newHarvestingUrl = ""; this.customHeader = null; this.initialSettingsValidated = false; @@ -842,6 +814,14 @@ public int getHarvestingScheduleRadio() { public void setHarvestingScheduleRadio(int harvestingScheduleRadio) { this.harvestingScheduleRadio = harvestingScheduleRadio; } + + public String getSourceName() { + return sourceName; + } + + public void setSourceName(String sourceName) { + this.sourceName = sourceName; + } public boolean isNewHarvestingScheduled() { return this.harvestingScheduleRadio != harvestingScheduleRadioNone; diff --git a/src/main/java/edu/harvard/iq/dataverse/api/HarvestingClients.java b/src/main/java/edu/harvard/iq/dataverse/api/HarvestingClients.java index dfc9f48dd1a..e4300099244 100644 --- a/src/main/java/edu/harvard/iq/dataverse/api/HarvestingClients.java +++ b/src/main/java/edu/harvard/iq/dataverse/api/HarvestingClients.java @@ -278,7 +278,10 @@ public Response modifyHarvestingClient(@Context ContainerRequestContext crc, Str // Go through the supported editable fields and update the client accordingly: // TODO: We may want to reevaluate whether we really want/need *all* // of these fields to be editable. - + + if (newHarvestingClient.getSourceName() != null) { + harvestingClient.setSourceName(newHarvestingClient.getSourceName()); + } if (newHarvestingClient.getHarvestingUrl() != null) { harvestingClient.setHarvestingUrl(newHarvestingClient.getHarvestingUrl()); } diff --git a/src/main/java/edu/harvard/iq/dataverse/harvest/client/HarvestingClient.java b/src/main/java/edu/harvard/iq/dataverse/harvest/client/HarvestingClient.java index e73310650b4..9d949b6a0dd 100644 --- a/src/main/java/edu/harvard/iq/dataverse/harvest/client/HarvestingClient.java +++ b/src/main/java/edu/harvard/iq/dataverse/harvest/client/HarvestingClient.java @@ -29,13 +29,11 @@ import jakarta.persistence.NamedQueries; import jakarta.persistence.NamedQuery; import jakarta.persistence.OneToMany; -import jakarta.persistence.OneToOne; import jakarta.persistence.OrderBy; import jakarta.persistence.Table; -import jakarta.persistence.Temporal; -import jakarta.persistence.TemporalType; import jakarta.validation.constraints.Pattern; import jakarta.validation.constraints.Size; +import org.apache.commons.lang3.StringUtils; import org.hibernate.validator.constraints.NotBlank; /** @@ -192,6 +190,20 @@ public void setHarvestingUrl(String harvestingUrl) { this.harvestingUrl = harvestingUrl.trim(); } } + + private String sourceName; + + public String getSourceName() { + return sourceName; + } + + public void setSourceName(String sourceName) { + this.sourceName = sourceName; + } + + public String getMetadataSource() { + return StringUtils.isNotBlank(this.sourceName) ? this.sourceName : this.name; + } private String archiveUrl; @@ -476,5 +488,4 @@ public boolean equals(Object object) { public String toString() { return "edu.harvard.iq.dataverse.harvest.client.HarvestingClient[ id=" + id + " ]"; } - } diff --git a/src/main/java/edu/harvard/iq/dataverse/search/IndexServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/search/IndexServiceBean.java index a2149b44c41..5fbcd6ea520 100644 --- a/src/main/java/edu/harvard/iq/dataverse/search/IndexServiceBean.java +++ b/src/main/java/edu/harvard/iq/dataverse/search/IndexServiceBean.java @@ -1005,7 +1005,7 @@ public SolrInputDocuments toSolrDocs(IndexableDataset indexableDataset, Set#{bundle['harvestclients.newClientDialog.nickname.helptext']}

+ +
+ +
+ + +

#{bundle['harvestclients.newClientDialog.sourcename.helptext']}

+
+
diff --git a/src/test/java/edu/harvard/iq/dataverse/api/HarvestingClientsIT.java b/src/test/java/edu/harvard/iq/dataverse/api/HarvestingClientsIT.java index f84c5ad1a20..7e06d6a9181 100644 --- a/src/test/java/edu/harvard/iq/dataverse/api/HarvestingClientsIT.java +++ b/src/test/java/edu/harvard/iq/dataverse/api/HarvestingClientsIT.java @@ -184,14 +184,19 @@ public void testCreateEditDeleteClient() throws InterruptedException { @Test public void testHarvestingClientRun_AllowHarvestingMissingCVV_False() throws InterruptedException { - harvestingClientRun(false); + harvestingClientRun(false, false); } @Test public void testHarvestingClientRun_AllowHarvestingMissingCVV_True() throws InterruptedException { - harvestingClientRun(true); + harvestingClientRun(true, false); } - private void harvestingClientRun(boolean allowHarvestingMissingCVV) throws InterruptedException { + @Test + public void testHarvestingClientRun_AllowHarvestingMissingCVV_True_WithSourceName() throws InterruptedException { + harvestingClientRun(true, true); + } + + private void harvestingClientRun(boolean allowHarvestingMissingCVV, boolean testingSourceName) throws InterruptedException { int expectedNumberOfSetsHarvested = allowHarvestingMissingCVV ? DATASETS_IN_CONTROL_SET : DATASETS_IN_CONTROL_SET - 1; // This test will create a client and attempt to perform an actual @@ -203,16 +208,18 @@ private void harvestingClientRun(boolean allowHarvestingMissingCVV) throws Inte // from confirming the expected HTTP status code. String nickName = "h" + UtilIT.getRandomString(6); + String sourceName = testingSourceName ? "AnotherSourceName" : ""; clientApiPath = String.format(HARVEST_CLIENTS_API+"%s", nickName); String clientJson = String.format("{\"dataverseAlias\":\"%s\"," + "\"type\":\"oai\"," + + "\"sourceName\":\"%s\"," + "\"harvestUrl\":\"%s\"," + "\"archiveUrl\":\"%s\"," + "\"set\":\"%s\"," + "\"allowHarvestingMissingCVV\":%s," + "\"metadataFormat\":\"%s\"}", - harvestCollectionAlias, HARVEST_URL, ARCHIVE_URL, CONTROL_OAI_SET, allowHarvestingMissingCVV, HARVEST_METADATA_FORMAT); + harvestCollectionAlias, sourceName, HARVEST_URL, ARCHIVE_URL, CONTROL_OAI_SET, allowHarvestingMissingCVV, HARVEST_METADATA_FORMAT); Response createResponse = given() .header(UtilIT.API_TOKEN_HTTP_HEADER, adminUserAPIKey) @@ -290,7 +297,7 @@ private void harvestingClientRun(boolean allowHarvestingMissingCVV) throws Inte Thread.sleep(1000L); // Requires the index-harvested-metadata-source Flag feature to be enabled to search on the nickName // Otherwise, the search must be performed with metadataSource:Harvested - Response searchHarvestedDatasets = UtilIT.search("metadataSource:" + nickName, normalUserAPIKey); + Response searchHarvestedDatasets = UtilIT.search("metadataSource:" + (testingSourceName ? sourceName : nickName), normalUserAPIKey); searchHarvestedDatasets.then().assertThat().statusCode(OK.getStatusCode()); searchHarvestedDatasets.prettyPrint(); // Get all global ids for cleanup