Merge branch 'develop' into 9570-untranslated-error-ingest-message #9570

Author: pdurbin

.github/workflows/container_app_pr.yml

@@ -0,0 +1,83 @@
+---
+name: Preview Application Container Image
+
+on:
+    # We only run the push commands if we are asked to by an issue comment with the correct command.
+    # This workflow is always taken from the default branch and runs in repo context with access to secrets.
+    repository_dispatch:
+        types: [ push-image-command ]
+
+env:
+    IMAGE_TAG: unstable
+    BASE_IMAGE_TAG: unstable
+    PLATFORMS: "linux/amd64,linux/arm64"
+
+jobs:
+    deploy:
+        name: "Package & Push"
+        runs-on: ubuntu-latest
+        # Only run in upstream repo - avoid unnecessary runs in forks
+        if: ${{ github.repository_owner == 'IQSS' }}
+        steps:
+            # Checkout the pull request code as when merged
+            - uses: actions/checkout@v3
+              with:
+                  ref: 'refs/pull/${{ github.event.client_payload.pull_request.number }}/merge'
+            - uses: actions/setup-java@v3
+              with:
+                  java-version: "11"
+                  distribution: 'adopt'
+            - uses: actions/cache@v3
+              with:
+                  path: ~/.m2
+                  key: ${{ runner.os }}-m2-${{ hashFiles('**/pom.xml') }}
+                  restore-keys: ${{ runner.os }}-m2
+
+            # Note: Accessing, pushing tags etc. to GHCR will only succeed in upstream because secrets.
+            - name: Login to Github Container Registry
+              uses: docker/login-action@v2
+              with:
+                  registry: ghcr.io
+                  username: ${{ secrets.GHCR_USERNAME }}
+                  password: ${{ secrets.GHCR_TOKEN }}
+
+            - name: Set up QEMU for multi-arch builds
+              uses: docker/setup-qemu-action@v2
+
+            # Get the image tag from either the command or default to branch name (Not used for now)
+            #-   name: Get the target tag name
+            #    id: vars
+            #    run: |
+            #        tag=${{ github.event.client_payload.slash_command.args.named.tag }}
+            #        if [[ -z "$tag" ]]; then tag=$(echo "${{ github.event.client_payload.pull_request.head.ref }}" | tr '\\/_:&+,;#*' '-'); fi
+            #        echo "IMAGE_TAG=$tag" >> $GITHUB_ENV
+
+            # Set image tag to branch name of the PR
+            - name: Set image tag to branch name
+              run: |
+                  echo "IMAGE_TAG=$(echo "${{ github.event.client_payload.pull_request.head.ref }}" | tr '\\/_:&+,;#*' '-')" >> $GITHUB_ENV
+
+            - name: Deploy multi-arch application container image
+              run: mvn -Pct deploy -Dapp.image.tag=${{ env.IMAGE_TAG }} -Dbase.image.tag=${{ env.BASE_IMAGE_TAG }} -Ddocker.registry=ghcr.io -Ddocker.platforms=${{ env.PLATFORMS }}
+
+            - uses: marocchino/sticky-pull-request-comment@v2
+              with:
+                  header: app-registry-push
+                  hide_and_recreate: true
+                  hide_classify: "OUTDATED"
+                  number: ${{ github.event.client_payload.pull_request.number }}
+                  message: |
+                      :package: Pushed preview application image as
+                      ```
+                      ghcr.io/gdcc/dataverse:${{ env.IMAGE_TAG }}
+                      ```
+                      :ship: [See on GHCR](https://github.com/orgs/gdcc/packages/container/package/dataverse). Use by referencing with full name as printed above, mind the registry name.
+
+            # Leave a note when things have gone sideways
+            - uses: peter-evans/create-or-update-comment@v3
+              if: ${{ failure() }}
+              with:
+                  issue-number: ${{ github.event.client_payload.pull_request.number }}
+                  body: >
+                      :package: Could not push preview image :disappointed:.
+                      See [log](https://github.com/IQSS/dataverse/actions/runs/${{ github.run_id }}) for details.

.github/workflows/container_app_push.yml

@@ -18,10 +18,11 @@ env:
     BASE_IMAGE_TAG: unstable
     REGISTRY: "" # Empty means default to Docker Hub
     PLATFORMS: "linux/amd64,linux/arm64"
+    MASTER_BRANCH_TAG: alpha
 
 jobs:
     build:
-        name: Build & deploy
+        name: "Build & Test"
         runs-on: ubuntu-latest
         permissions:
             contents: read
@@ -49,19 +50,64 @@ jobs:
             - name: Build app container image with local architecture
               run: mvn -Pct package
 
-            # Note: Accessing, pushing tags etc. to DockerHub or GHCR will only succeed in upstream because secrets.
+            # TODO: add smoke / integration testing here
 
-            # Run this when triggered via push or schedule as reused workflow from base / maven unit tests
-            - if: ${{ github.event_name != 'pull_request' && github.ref_name == 'develop' }}
-              name: Push description to DockerHub
-              uses: peter-evans/dockerhub-description@v3
+    hub-description:
+        needs: build
+        name: Push image description to Docker Hub
+        # Run this when triggered via push or schedule as reused workflow from base / maven unit tests.
+        # Excluding PRs here means we will have no trouble with secrets access. Also avoid runs in forks.
+        if: ${{ github.event_name != 'pull_request' && github.ref_name == 'develop' && github.repository_owner == 'IQSS' }}
+        runs-on: ubuntu-latest
+        steps:
+            - uses: actions/checkout@v3
+            - uses: peter-evans/dockerhub-description@v3
               with:
                   username: ${{ secrets.DOCKERHUB_USERNAME }}
                   password: ${{ secrets.DOCKERHUB_TOKEN }}
                   repository: gdcc/dataverse
                   short-description: "Dataverse Application Container Image providing the executable"
                   readme-filepath: ./src/main/docker/README.md
 
+    # Note: Accessing, pushing tags etc. to DockerHub or GHCR will only succeed in upstream because secrets.
+    # We check for them here and subsequent jobs can rely on this to decide if they shall run.
+    check-secrets:
+        needs: build
+        name: Check for Secrets Availability
+        runs-on: ubuntu-latest
+        outputs:
+            available: ${{ steps.secret-check.outputs.available }}
+        steps:
+            - id: secret-check
+              # perform secret check & put boolean result as an output
+              shell: bash
+              run: |
+                  if [ "${{ secrets.DOCKERHUB_TOKEN }}" != '' ]; then
+                      echo "available=true" >> $GITHUB_OUTPUT;
+                  else
+                      echo "available=false" >> $GITHUB_OUTPUT;
+                  fi
+
+    deploy:
+        needs: check-secrets
+        name: "Package & Publish"
+        runs-on: ubuntu-latest
+        # Only run this job if we have access to secrets. This is true for events like push/schedule which run in
+        # context of main repo, but for PRs only true if coming from the main repo! Forks have no secret access.
+        if: needs.check-secrets.outputs.available == 'true'
+        steps:
+            - uses: actions/checkout@v3
+            - uses: actions/setup-java@v3
+              with:
+                  java-version: "11"
+                  distribution: 'adopt'
+            - uses: actions/cache@v3
+              with:
+                  path: ~/.m2
+                  key: ${{ runner.os }}-m2-${{ hashFiles('**/pom.xml') }}
+                  restore-keys: ${{ runner.os }}-m2
+
+            # Depending on context, we push to different targets. Login accordingly.
             - if: ${{ github.event_name != 'pull_request' }}
               name: Log in to Docker Hub registry
               uses: docker/login-action@v2
@@ -82,8 +128,8 @@ jobs:
             - name: Re-set image tag based on branch (if master)
               if: ${{ github.ref_name == 'master' }}
               run: |
-                  echo "IMAGE_TAG=alpha" >> $GITHUB_ENV
-                  echo "BASE_IMAGE_TAG=alpha" >> $GITHUB_ENV
+                  echo "IMAGE_TAG=${{ env.MASTER_BRANCH_TAG }}" >> $GITHUB_ENV
+                  echo "BASE_IMAGE_TAG=${{ env.MASTER_BRANCH_TAG }}" >> $GITHUB_ENV
             - name: Re-set image tag and container registry when on PR
               if: ${{ github.event_name == 'pull_request' }}
               run: |
@@ -97,6 +143,11 @@ jobs:
               if: ${{ github.event_name == 'pull_request' }}
               with:
                   header: app-registry-push
+                  hide_and_recreate: true
+                  hide_classify: "OUTDATED"
                   message: |
-                      Pushed preview application image as [`ghcr.io/gdcc/dataverse:${{ env.IMAGE_TAG }}`](https://github.com/orgs/gdcc/packages/container/package/dataverse).
-                      Use it by referencing it with its full name as printed above.
+                      :package: Pushed preview application image as
+                      ```
+                      ghcr.io/gdcc/dataverse:${{ env.IMAGE_TAG }}
+                      ```
+                      :ship: [See on GHCR](https://github.com/orgs/gdcc/packages/container/package/dataverse). Use by referencing with full name as printed above, mind the registry name.

.github/workflows/pr_comment_commands.yml

@@ -0,0 +1,20 @@
+name: PR Comment Commands
+on:
+    issue_comment:
+        types: [created]
+jobs:
+    dispatch:
+        # Avoid being triggered by forks in upstream
+        if: ${{ github.repository_owner == 'IQSS' }}
+        runs-on: ubuntu-latest
+        steps:
+            - name: Dispatch
+              uses: peter-evans/slash-command-dispatch@v3
+              with:
+                  # This token belongs to @dataversebot and has sufficient scope.
+                  token: ${{ secrets.GHCR_TOKEN }}
+                  commands: |
+                      push-image
+                  repository: IQSS/dataverse
+                  # Commenter must have at least write permission to repo to trigger dispatch
+                  permission: write

.readthedocs.yml

@@ -0,0 +1,21 @@
+version: 2
+
+# HTML is always built, these are additional formats only
+formats:
+    - pdf
+
+build:
+    os: ubuntu-22.04
+    tools:
+        python: "3.10"
+    apt_packages:
+        - graphviz
+
+python:
+    install:
+        - requirements: doc/sphinx-guides/requirements.txt
+
+
+sphinx:
+    configuration: doc/sphinx-guides/source/conf.py
+    fail_on_warning: true

conf/keycloak/oidc-keycloak-auth-provider.json

@@ -3,6 +3,6 @@
   "factoryAlias": "oidc",
   "title": "OIDC-Keycloak",
   "subtitle": "OIDC-Keycloak",
-  "factoryData": "type: oidc | issuer: http://localhost:8090/auth/realms/oidc-realm | clientId: oidc-client | clientSecret: ss6gE8mODCDfqesQaSG3gwUwZqZt547E",
+  "factoryData": "type: oidc | issuer: http://keycloak.mydomain.com:8090/realms/oidc-realm | clientId: oidc-client | clientSecret: ss6gE8mODCDfqesQaSG3gwUwZqZt547E",
   "enabled": true
 }

conf/solr/8.11.1/schema.xml

@@ -233,6 +233,9 @@
     <field name="geolocation" type="location_rpt" multiValued="true" stored="true" indexed="true"/>
     <!-- https://solr.apache.org/guide/8_11/spatial-search.html#bboxfield -->
     <field name="boundingBox" type="bbox" multiValued="true" stored="true" indexed="true"/>
+
+    <!-- incomplete datasets issue 8822 -->
+    <field name="datasetValid" type="boolean" stored="true" indexed="true" multiValued="false"/>
 
     <!--
         METADATA SCHEMA FIELDS
@@ -470,6 +473,8 @@
     <!-- <copyField source="*_ss" dest="_text_" maxChars="3000"/> -->
     <!-- <copyField source="*_i" dest="_text_" maxChars="3000"/> -->
 
+    <copyField source="datasetValid" dest="_text_" maxChars="3000"/>
+    
     <!--
         METADATA SCHEMA FIELDS
         Now following: copyFields to copy the contents of the metadata fields above to a

doc/release-notes/8822-incomplete-datasets-via-api.md

@@ -0,0 +1,14 @@
+### Creating datasets with incomplete metadata through API
+
+The create dataset API call (POST to /api/dataverses/#dataverseId/datasets) is extended with the "doNotValidate" parameter. However, in order to be able to create a dataset with incomplete metadata, the solr configuration must be updated first with the new "schema.xml" file (do not forget to run the metadata fields update script when you use custom metadata). Reindexing is optional, but recommended. Also, even when this feature is not used, it is recommended to update the solar configuration and reindex the metadata. Finally, this new feature can be activated with the "dataverse.api.allow-incomplete-metadata" JVM option.
+
+You can also enable a valid/incomplete metadata filter in the "My Data" page using the "dataverse.ui.show-validity-filter" JVM option. By default, this filter is not shown. When you wish to use this filter, you must reindex the datasets first, otherwise datasets with valid metadata will not be shown in the results.
+
+It is not possible to publish datasets with incomplete or incomplete metadata. By default, you also cannot send such datasets for review. If you wish to enable sending for review of datasets with incomplete metadata, turn on the "dataverse.ui.allow-review-for-incomplete" JVM option.
+
+In order to customize the wording and add translations to the UI sections extended by this feature, you can edit the "Bundle.properties" file and the localized versions of that file. The property keys used by this feature are:
+- incomplete
+- valid
+- dataset.message.incomplete.warning
+- mydataFragment.validity
+- dataverses.api.create.dataset.error.mustIncludeAuthorName

doc/release-notes/9100-schema.org-updates.md

@@ -0,0 +1,3 @@
+Changes made in v5.13 and v5.14 in multiple PRs to improve the embedded Schema.org metadata in dataset pages will only be propagated to the Schema.Org JSON-LD metadata export if a reExportAll() is done.
+
+The 5.14 release notes should include the standard instructions for doing a reExportAll after updating the code.

doc/release-notes/9148-license-via-api.md

@@ -0,0 +1,28 @@
+# License management via API
+
+See https://github.com/IQSS/dataverse/issues/9148. 
+
+When publishing a dataset via API, it now requires the dataset to either have a standard license configured, or have valid Custom Terms of Use (if allowed by the instance). Attempting to publish a dataset without such **will fail with an error message**. This introduces a backward incompatibility, and if you have scripts that automatically create, update and publish datasets, this last step may start failing. Because, unfortunately, there were some problems with the datasets APIs that made it difficult to manage licenses, so an API user was likely to end up with a dataset missing either of the above. In this release we have addressed it by making the following fixes:
+
+We fixed the incompatibility between the format in which license information was *exported* in json, and the format the create and update APIs were expecting it for *import* (https://github.com/IQSS/dataverse/issues/9155). This means that the following json format can now be imported:
+```
+"license": {
+   "name": "CC0 1.0",
+   "uri": "http://creativecommons.org/publicdomain/zero/1.0"
+}
+```
+However, for the sake of backward compatibility the old format
+```
+"license" : "CC0 1.0"
+```
+will be accepted as well.
+
+We have added the default license (CC0) to the model json file that we provide and recommend to use as the model in the Native API Guide (https://github.com/IQSS/dataverse/issues/9364).
+
+And we have corrected the misleading language in the same guide where we used to recommend to users that they select, edit and re-import only the `.metadataBlocks` fragment of the json metadata representing the latest version. There are in fact other useful pieces of information that need to be preserved in the update (such as the `"license"` section above). So the recommended way of creating base json for updates via the API is to select *everything but* the `"files"` section, with (for example) the following `jq` command:
+
+```
+jq '.data | del(.files)'
+```
+
+Please see the [Update Metadata For a Dataset](https://guides.dataverse.org/en/latest/api/native-api.html#update-metadata-for-a-dataset) section of our Native Api guide for more information. 

doc/release-notes/9185-contact-email-updates.md

@@ -0,0 +1,12 @@
+## Contact Email Improvements
+
+Email sent from the contact forms to the contact(s) for a collection, dataset, or datafile can now optionally be cc'd to a support email address. The support email address can be changed from the default :SystemEmail address to a separate :SupportEmail address. When multiple contacts are listed, the system will now send one email to all contacts (with the optional cc if configured) instead of separate emails to each contact. Contact names with a comma that refer to Organizations will no longer have the name parts reversed in the email greeting. A new protected feedback API has been added.
+
+## Backward Incompatibilities
+
+When there are multiple contacts, the system will now send one email with all of the contacts in the To: header instead of sending one email to each contact (with no indication that others have been notified).
+
+## New JVM/MicroProfile Settings
+
+dataverse.mail.support-email - allows a separate email, distinct from the :SystemEmail to be used as the to address in emails from the contact form/ feedback api.
+dataverse.mail.cc-support-on-contact-emails - include the support email address as a CC: entry when contact/feedback emails are sent to the contacts for a collection, dataset, or datafile.