DD-2118, DD-2119 Applying bag exporter and archivalStatus api fixes and improvements to v6.7.1-DANS-DataStation

doc/release-notes/12063-ORE-and-Bag-updates.md

@@ -0,0 +1,14 @@
+This release contains multiple updates to the OAI-ORE metadata export and archival Bag output:
+
+OAI-ORE
+- now uses URI for checksum algorithms
+- a bug causing failures with deaccessioned versions when the deaccession note ("Deaccession Reason" in the UI) was null (which has been allowed via the API). 
+- the "https://schema.org/additionalType" is updated to "Dataverse OREMap Format v1.0.2" to indicate that the out has changed 
+
+Archival Bag 
+- for dataset versions with no files, the (empty) manifest-<alg>.txt file created will now use the default algorithm defined by the "FileFixityChecksumAlgorithm" setting rather than always defaulting to "md5"
+- a bug causing the bag-info.txt to not have information on contacts when the dataset version has more than one contact has been fixed
+- values used in the bag-info.txt file that may be multi-line (with embedded CR or LF characters) are now properly indented/formatted per the BagIt specification (i.e. Internal-Sender-Identifier, External-Description, Source-Organization, Organization-Address).
+- the name of the dataset is no longer used as a subdirectory under the data directory (dataset names can be long enough to cause failures when unzipping)
+- a new key, "Dataverse-Bag-Version" has been added to bag-info.txt with a value "1.0", allowing tracking of changes to Dataverse's arhival bag generation
+- improvements to file retrieval w.r.t. retries on errors or throttling

doc/release-notes/12065-allow-crud-on-archival-status-for-deaccessioned.md

@@ -0,0 +1 @@
+This release removes an undocumented restriction on the API calls to get, set, and delete archival status. They did not work on deaccessioned dataset versions and now do. (See https://guides.dataverse.org/en/latest/api/native-api.html#get-the-archival-status-of-a-dataset-by-version )

src/main/java/edu/harvard/iq/dataverse/DataFile.java

@@ -109,18 +109,22 @@ public class DataFile extends DvObject implements Comparable {
      * The list of types should be limited to the list above in the technote
      * because the string gets passed into MessageDigest.getInstance() and you
      * can't just pass in any old string.
+     * 
+     * The URIs are used in the OAI_ORE export. They are taken from the associated XML Digital Signature standards.
      */
     public enum ChecksumType {
 
-        MD5("MD5"),
-        SHA1("SHA-1"),
-        SHA256("SHA-256"),
-        SHA512("SHA-512");
+        MD5("MD5", "http://www.w3.org/2001/04/xmldsig-more#md5"),
+        SHA1("SHA-1", "http://www.w3.org/2000/09/xmldsig#sha1"),
+        SHA256("SHA-256", "http://www.w3.org/2001/04/xmlenc#sha256"),
+        SHA512("SHA-512", "http://www.w3.org/2001/04/xmlenc#sha512");
 
         private final String text;
+        private final String uri;
 
-        private ChecksumType(final String text) {
+        private ChecksumType(final String text, final String uri) {
             this.text = text;
+            this.uri = uri;
         }
 
         public static ChecksumType fromString(String text) {
@@ -131,13 +135,30 @@ public static ChecksumType fromString(String text) {
                     }
                 }
             }
-            throw new IllegalArgumentException("ChecksumType must be one of these values: " + Arrays.asList(ChecksumType.values()) + ".");
+            throw new IllegalArgumentException(
+                    "ChecksumType must be one of these values: " + Arrays.asList(ChecksumType.values()) + ".");
+        }
+
+        public static ChecksumType fromUri(String uri) {
+            if (uri != null) {
+                for (ChecksumType checksumType : ChecksumType.values()) {
+                    if (uri.equals(checksumType.uri)) {
+                        return checksumType;
+                    }
+                }
+            }
+            throw new IllegalArgumentException(
+                    "ChecksumType must be one of these values: " + Arrays.asList(ChecksumType.values()) + ".");
         }
 
         @Override
         public String toString() {
             return text;
         }
+
+        public String toUri() {
+            return uri;
+        }
     }
 
     //@Expose

src/main/java/edu/harvard/iq/dataverse/api/Datasets.java

@@ -5011,7 +5011,7 @@ public Response getDatasetVersionArchivalStatus(@Context ContainerRequestContext
             }
             DataverseRequest req = createDataverseRequest(au);
             DatasetVersion dsv = getDatasetVersionOrDie(req, versionNumber, findDatasetOrDie(datasetId), uriInfo,
-                    headers);
+                    headers, true);
 
             if (dsv.getArchivalCopyLocation() == null) {
                 return error(Status.NOT_FOUND, "This dataset version has not been archived");
@@ -5053,7 +5053,7 @@ public Response setDatasetVersionArchivalStatus(@Context ContainerRequestContext
 
                     DataverseRequest req = createDataverseRequest(au);
                     DatasetVersion dsv = getDatasetVersionOrDie(req, versionNumber, findDatasetOrDie(datasetId),
-                            uriInfo, headers);
+                            uriInfo, headers, true);
 
                     if (dsv == null) {
                         return error(Status.NOT_FOUND, "Dataset version not found");
@@ -5100,7 +5100,7 @@ public Response deleteDatasetVersionArchivalStatus(@Context ContainerRequestCont
 
             DataverseRequest req = createDataverseRequest(au);
             DatasetVersion dsv = getDatasetVersionOrDie(req, versionNumber, findDatasetOrDie(datasetId), uriInfo,
-                    headers);
+                    headers, true);
             if (dsv == null) {
                 return error(Status.NOT_FOUND, "Dataset version not found");
             }

src/main/java/edu/harvard/iq/dataverse/settings/SettingsServiceBean.java

@@ -692,6 +692,7 @@ Whether Harvesting (OAI) service is enabled
         PostExternalSearchUrl,
         //Experimental setting to provide a display name for the POST external search service
         PostExternalSearchName,
+        BagGeneratorThreads,
         ;
 
         @Override