Fix URL signing api for URLs containing special characters

doc/release-notes/fix-url-signing-special-characters.md

@@ -0,0 +1,45 @@
+### Signed URLs work again for URLs with special characters
+
+Requesting a signed URL (e.g. via `/api/admin/requestSignedUrl`, used by external tools, the Globus
+integration and third-party integrations such as the `rdm-integration` connector) was broken in 6.10
+for URLs whose query contained special characters — most notably persistent IDs such as
+`doi:10.5072/FK2/ABC` (which contain `:` and `/`), as well as spaces, percent-encoded values and
+non-ASCII characters. In 6.10 the signing step began re-encoding/normalizing the URL (for example
+percent-encoding `:` and `/`) before computing the signature, while the request is validated against
+the URL the caller actually presents back. The re-encoded signature no longer matched, so validation
+failed with authentication / "signature does not match" errors.
+
+Signing no longer re-encodes the URL: it is signed exactly as provided, with only the reserved
+signing parameters (`until`, `user`, `method`, `token`, `key`, `signed`) stripped out; the rest of
+the URL is left untouched, character for character.
+
+**This restores the URL-signing behavior used before 6.10, so it is compatible with older versions
+and with existing integrations.** Clients and connectors that build or consume signed URLs the way
+they did before 6.10 keep working unchanged, signatures are computed the same way as before the
+regression, and URLs containing special characters validate again. No client-side changes are
+required.
+
+### A signing secret is now required for signed URLs
+
+Separately from the fix above, Dataverse no longer falls back to a weak signing key when
+`dataverse.api.signing-secret` is unset. Previously, with no secret configured, signed URLs were
+signed using only the user's API token (or, for a guest, a value derived from the public URL), which
+is too weak to be a signing key. A non-empty `dataverse.api.signing-secret` is now required wherever
+URLs are signed with a key based on a user's API token:
+
+- The endpoints that issue a signed URL on request - `/api/admin/requestSignedUrl` and the
+  guestbook-response file download (`POST /api/access/datafile/{id}`) - return an error instead of
+  issuing a weakly-signed URL.
+- Signed callbacks and links built internally (external tool launches, Globus transfers, the
+  permission-history CSV links) are sent unsigned, with a warning logged, rather than weakly signed.
+
+Remote and Globus overlay stores are unaffected: they sign with their own per-store secret key, not
+`dataverse.api.signing-secret`.
+
+**Upgrade note:** installations that rely on signed URLs - including the `rdm-integration` connector,
+signed guestbook-response downloads, and external tools or Globus transfers that use signed callbacks -
+must set `dataverse.api.signing-secret`. See the
+[Configuration Guide](https://guides.dataverse.org/en/latest/installation/config.html#dataverse-api-signing-secret).
+Treat the value like a password. Because the signing secret is part of the signing key, setting (or
+later changing) it invalidates previously issued signed URLs: any existing signed URLs that have not
+yet expired will stop working, and clients/integrations will need to request new ones.

doc/sphinx-guides/source/installation/config.rst

@@ -3349,9 +3349,10 @@ are time limited and only allow the action of the API call in the URL. See :ref:
 :ref:`api-native-signed-url` for more details. 
 
 The key used to sign a URL is created from the API token of the creating user plus a signing-secret provided by an administrator.
-**Using a signing-secret is highly recommended.** This setting defaults to an empty string. Using a non-empty 
-signing-secret makes it impossible for someone who knows an API token from forging signed URLs and provides extra security by 
-making the overall signing key longer.
+**A non-empty signing-secret is required to request signed URLs through the API.** If it is not configured, the
+``/api/admin/requestSignedUrl`` endpoint (see :ref:`api-native-signed-url`) returns an error instead of issuing a
+weakly-signed URL. (The setting otherwise defaults to an empty string.) A non-empty signing-secret makes it impossible for
+someone who only knows an API token to forge signed URLs, and provides extra security by making the overall signing key longer.
 
 **WARNING**:
 *Since the signing-secret is sensitive, you should treat it like a password.*

docker-compose-dev.yml

@@ -60,6 +60,7 @@ services:
         -Ddataverse.pid.fake.label=FakeDOIProvider
         -Ddataverse.pid.fake.authority=10.5072
         -Ddataverse.pid.fake.shoulder=FK2/
+        -Ddataverse.api.signing-secret=dev-only-signing-secret-change-me
         -Ddataverse.cors.origin=* \
         -Ddataverse.cors.methods=GET,POST,PUT,DELETE,OPTIONS \
         -Ddataverse.cors.headers.allow=range,content-type,x-dataverse-key,accept \

src/main/java/edu/harvard/iq/dataverse/ManageFilePermissionsPage.java

@@ -22,7 +22,6 @@
 import edu.harvard.iq.dataverse.engine.command.exception.PermissionException;
 import edu.harvard.iq.dataverse.engine.command.impl.AssignRoleCommand;
 import edu.harvard.iq.dataverse.engine.command.impl.RevokeRoleCommand;
-import edu.harvard.iq.dataverse.settings.JvmSettings;
 import edu.harvard.iq.dataverse.util.BundleUtil;
 import edu.harvard.iq.dataverse.util.DateUtil;
 import edu.harvard.iq.dataverse.util.JsfHelper;
@@ -640,9 +639,8 @@ public String getSignedUrlForRAHistoryCsv() {
                     key = apiToken.getTokenString();
                 }
             }
-            key = JvmSettings.API_SIGNING_SECRET.lookupOptional().orElse("") + key;
-            if(key.length() >= 36) {
-                return UrlSignerUtil.signUrl(fullApiPath, 10, userId, "GET", key);
+            if (key != null && UrlSignerUtil.isSigningSecretConfigured()) {
+                return UrlSignerUtil.signUrlWithApiKey(fullApiPath, 10, userId, "GET", key);
             }
         } catch (Exception e) {
             logger.log(Level.SEVERE, "Error generating signed URL for permissions history CSV: " + e.getMessage(), e);

src/main/java/edu/harvard/iq/dataverse/ManagePermissionsPage.java

@@ -20,7 +20,6 @@
 import edu.harvard.iq.dataverse.engine.command.impl.CreateRoleCommand;
 import edu.harvard.iq.dataverse.engine.command.impl.RevokeRoleCommand;
 import edu.harvard.iq.dataverse.engine.command.impl.UpdateDataverseDefaultContributorRoleCommand;
-import edu.harvard.iq.dataverse.settings.JvmSettings;
 import edu.harvard.iq.dataverse.util.BundleUtil;
 import edu.harvard.iq.dataverse.util.JsfHelper;
 import static edu.harvard.iq.dataverse.util.JsfHelper.JH;
@@ -736,9 +735,8 @@ public String getSignedUrlForRAHistoryCsv() {
                     key = apiToken.getTokenString();
                 }
             }
-            key = JvmSettings.API_SIGNING_SECRET.lookupOptional().orElse("") + key;
-            if(key.length() >= 36) {
-                return UrlSignerUtil.signUrl(fullApiPath, 10, userId, "GET", key);
+            if (key != null && UrlSignerUtil.isSigningSecretConfigured()) {
+                return UrlSignerUtil.signUrlWithApiKey(fullApiPath, 10, userId, "GET", key);
             }
         } catch (Exception e) {
             logger.log(Level.SEVERE, "Error generating signed URL for permissions history CSV: " + e.getMessage(), e);

src/main/java/edu/harvard/iq/dataverse/api/Access.java

@@ -29,7 +29,6 @@
 import edu.harvard.iq.dataverse.makedatacount.MakeDataCountLoggingServiceBean;
 import edu.harvard.iq.dataverse.makedatacount.MakeDataCountLoggingServiceBean.MakeDataCountEntry;
 import edu.harvard.iq.dataverse.mydata.Pager;
-import edu.harvard.iq.dataverse.settings.JvmSettings;
 import edu.harvard.iq.dataverse.settings.SettingsServiceBean;
 import edu.harvard.iq.dataverse.util.*;
 import edu.harvard.iq.dataverse.util.json.JsonParseException;
@@ -529,6 +528,12 @@ private Map<Long, DataFile> getDatafilesMap(DataverseRequest req, String fileIds
     }
 
     private Response returnSignedUrl(ContainerRequestContext crc, UriInfo uriInfo, User user, String id, String gbrids) {
+        // Require a signing secret: without it the key is only the user's API token (or, for a guest,
+        // a guessable value derived from the URL), which is too weak. Mirrors Admin.getSignedUrl.
+        if (!UrlSignerUtil.isSigningSecretConfigured()) {
+            return error(INTERNAL_SERVER_ERROR,
+                    "Requesting signed URLs requires a signing secret to be configured. Please set the dataverse.api.signing-secret JVM option.");
+        }
         // Create the signed URL
         String userIdentifier = null;
         String key = null;
@@ -564,8 +569,7 @@ private Response returnSignedUrl(ContainerRequestContext crc, UriInfo uriInfo, U
         String baseUrlEncoded = builder.build().toString();
         String baseUrl = URLDecoder.decode(baseUrlEncoded, StandardCharsets.UTF_8);
         baseUrl = baseUrl.replace(":persistentId", id);
-        key = JvmSettings.API_SIGNING_SECRET.lookupOptional().orElse("") + key;
-        String signedUrl = UrlSignerUtil.signUrl(baseUrl, GUESTBOOK_RESPONSE_SIGNEDURL_TIMEOUT_MINUTES, userIdentifier, "GET", key);
+        String signedUrl = UrlSignerUtil.signUrlWithApiKey(baseUrl, GUESTBOOK_RESPONSE_SIGNEDURL_TIMEOUT_MINUTES, userIdentifier, "GET", key);
         return ok(Json.createObjectBuilder().add(URLTokenUtil.SIGNED_URL, signedUrl));
     }
 

src/main/java/edu/harvard/iq/dataverse/api/Admin.java

@@ -17,7 +17,6 @@
 import edu.harvard.iq.dataverse.DvObjectServiceBean;
 import edu.harvard.iq.dataverse.FileMetadata;
 import edu.harvard.iq.dataverse.api.auth.AuthRequired;
-import edu.harvard.iq.dataverse.settings.JvmSettings;
 import edu.harvard.iq.dataverse.settings.SettingsValidationException;
 import edu.harvard.iq.dataverse.util.StringUtil;
 import edu.harvard.iq.dataverse.util.cache.CacheFactoryBean;
@@ -2453,7 +2452,13 @@ public Response getSignedUrl(@Context ContainerRequestContext crc, JsonObject ur
         if (superuser == null || !superuser.isSuperuser()) {
             return error(Response.Status.FORBIDDEN, "Requesting signed URLs is restricted to superusers.");
         }
-
+
+        // Require a signing secret: without it the key is only the user's API token, which is too weak.
+        if (!UrlSignerUtil.isSigningSecretConfigured()) {
+            return error(Response.Status.INTERNAL_SERVER_ERROR,
+                    "Requesting signed URLs requires a signing secret to be configured. Please set the dataverse.api.signing-secret JVM option.");
+        }
+
         String userId = urlInfo.getString("user");
         String key=null;
         if (userId != null) {
@@ -2475,14 +2480,13 @@ public Response getSignedUrl(@Context ContainerRequestContext crc, JsonObject ur
             if (key == null) {
                 return error(Response.Status.CONFLICT, "Do not have a valid user with apiToken");
             }
-            key = JvmSettings.API_SIGNING_SECRET.lookupOptional().orElse("") + key;
         }
-        
+
         String baseUrl = urlInfo.getString("url");
         int timeout = urlInfo.getInt(URLTokenUtil.TIMEOUT, 10);
         String method = urlInfo.getString(URLTokenUtil.HTTP_METHOD, "GET");
-        
-        String signedUrl = UrlSignerUtil.signUrl(baseUrl, timeout, userId, method, key); 
+
+        String signedUrl = UrlSignerUtil.signUrlWithApiKey(baseUrl, timeout, userId, method, key);
 
         return ok(Json.createObjectBuilder().add(URLTokenUtil.SIGNED_URL, signedUrl));
     }

src/main/java/edu/harvard/iq/dataverse/externaltools/ExternalToolHandler.java

@@ -4,7 +4,6 @@
 import edu.harvard.iq.dataverse.Dataset;
 import edu.harvard.iq.dataverse.FileMetadata;
 import edu.harvard.iq.dataverse.authorization.users.ApiToken;
-import edu.harvard.iq.dataverse.settings.JvmSettings;
 import edu.harvard.iq.dataverse.util.SystemConfig;
 import edu.harvard.iq.dataverse.util.URLTokenUtil;
 
@@ -111,8 +110,12 @@ public String handleRequest(boolean preview) {
                             + externalTool.getId();
                 }
                 if (apiToken != null) {
-                    callback = UrlSignerUtil.signUrl(callback, 5, apiToken.getAuthenticatedUser().getUserIdentifier(), HttpMethod.GET,
-                        JvmSettings.API_SIGNING_SECRET.lookupOptional().orElse("") + apiToken.getTokenString());
+                    if (UrlSignerUtil.isSigningSecretConfigured()) {
+                        callback = UrlSignerUtil.signUrlWithApiKey(callback, 5, apiToken.getAuthenticatedUser().getUserIdentifier(),
+                                HttpMethod.GET, apiToken.getTokenString());
+                    } else {
+                        logger.warning("Cannot sign external tool callback: no signing secret configured (dataverse.api.signing-secret). Sending an unsigned callback.");
+                    }
                 }
                 paramsString= "?callback=" + Base64.getEncoder().encodeToString(StringUtils.getBytesUtf8(callback));
                 if (getLocaleCode() != null) {

src/main/java/edu/harvard/iq/dataverse/globus/GlobusServiceBean.java

@@ -776,13 +776,14 @@ public String getGlobusAppUrlForDataset(Dataset d, boolean upload, List<DataFile
                     + "/globusDownloadParameters?locale=" + localeCode + "&downloadId=" + downloadId;
 
         }
-        if (apiToken != null) {
-            callback = UrlSignerUtil.signUrl(callback, 5, apiToken.getAuthenticatedUser().getUserIdentifier(),
-                    HttpMethod.GET,
-                    JvmSettings.API_SIGNING_SECRET.lookupOptional().orElse("") + apiToken.getTokenString());
-        } else {
+        if (apiToken == null) {
             // Shouldn't happen
             logger.warning("Unable to get api token for user: " + user.getIdentifier());
+        } else if (UrlSignerUtil.isSigningSecretConfigured()) {
+            callback = UrlSignerUtil.signUrlWithApiKey(callback, 5, apiToken.getAuthenticatedUser().getUserIdentifier(),
+                    HttpMethod.GET, apiToken.getTokenString());
+        } else {
+            logger.warning("Cannot sign Globus callback: no signing secret configured (dataverse.api.signing-secret). Sending an unsigned callback.");
         }
         appUrl = appUrl + "&callback=" + Base64.getEncoder().encodeToString(StringUtils.getBytesUtf8(callback));
 

src/main/java/edu/harvard/iq/dataverse/util/URLTokenUtil.java

@@ -18,7 +18,6 @@
 import edu.harvard.iq.dataverse.FileMetadata;
 import edu.harvard.iq.dataverse.GlobalId;
 import edu.harvard.iq.dataverse.authorization.users.ApiToken;
-import edu.harvard.iq.dataverse.settings.JvmSettings;
 import edu.harvard.iq.dataverse.util.json.JsonUtil;
 
 import static edu.harvard.iq.dataverse.api.ApiConstants.DS_VERSION_DRAFT;
@@ -227,9 +226,12 @@ public JsonObjectBuilder createPostBody(JsonObject params, JsonArray allowedApiC
                 // Sign if apiToken exists, otherwise send unsigned URL (i.e. for guest users)
                 ApiToken apiToken = getApiToken();
                 if (apiToken != null) {
-                    url = UrlSignerUtil.signUrl(apiPath, timeout, apiToken.getAuthenticatedUser().getUserIdentifier(),
-                            httpmethod, JvmSettings.API_SIGNING_SECRET.lookupOptional().orElse("")
-                                    + getApiToken().getTokenString());
+                    if (UrlSignerUtil.isSigningSecretConfigured()) {
+                        url = UrlSignerUtil.signUrlWithApiKey(apiPath, timeout, apiToken.getAuthenticatedUser().getUserIdentifier(),
+                                httpmethod, apiToken.getTokenString());
+                    } else {
+                        logger.warning("Cannot sign URL: no signing secret configured (dataverse.api.signing-secret). Sending an unsigned URL.");
+                    }
                 }
                 logger.fine("Signed URL: " + url);
                 apisBuilder.add(Json.createObjectBuilder().add(NAME, name).add(HTTP_METHOD, httpmethod)