Skip to content

HDDS-14847. [STS] Expose ExpiredToken Error #9935

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
sodonnel merged 4 commits into from
Apr 14, 2026
Merged

HDDS-14847. [STS] Expose ExpiredToken Error #9935

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
0d4cd7c
expose expiredToken error
Mar 17, 2026
fa6d824
fix additional location for token expiration
Mar 30, 2026
0b7fd74
handle latent internal 500 error generated via RuntimeException
Mar 30, 2026
f00f62a
address PR review comment for Russole
Apr 7, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
13 changes: 9 additions & 4 deletions ...p-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/security/STSSecurityUtil.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,7 @@
package org.apache.hadoop.ozone.security;

import static org.apache.hadoop.ozone.om.exceptions.OMException.ResultCodes.INVALID_TOKEN;
import static org.apache.hadoop.ozone.om.exceptions.OMException.ResultCodes.TOKEN_EXPIRED;

import com.google.common.annotations.VisibleForTesting;
import com.google.protobuf.InvalidProtocolBufferException;
Expand Down Expand Up @@ -71,7 +72,7 @@ public static STSTokenIdentifier constructValidateAndDecryptSTSToken(String sess
* @throws SecretManager.InvalidToken if the token is invalid
*/
private static STSTokenIdentifier verifyAndDecryptToken(Token<STSTokenIdentifier> token,
SecretKeyClient secretKeyClient, Clock clock) throws SecretManager.InvalidToken {
SecretKeyClient secretKeyClient, Clock clock) throws SecretManager.InvalidToken, OMException {
if (!STSTokenIdentifier.KIND_NAME.equals(token.getKind())) {
throw new SecretManager.InvalidToken("Invalid STS token - kind is incorrect: " + token.getKind());
}
Expand Down Expand Up @@ -100,6 +101,8 @@ private static STSTokenIdentifier verifyAndDecryptToken(Token<STSTokenIdentifier
secretKey = getValidatedSecretKey(secretKeyId, secretKeyClient);
tokenId.setEncryptionKey(secretKey.getSecretKey().getEncoded());
tokenId.readFromByteArray(tokenBytes);
} catch (OMException e) {
throw e;
} catch (IOException e) {
throw new SecretManager.InvalidToken("Invalid STS token - could not readFromByteArray: " + e.getMessage());
}
Expand All @@ -109,7 +112,7 @@ private static STSTokenIdentifier verifyAndDecryptToken(Token<STSTokenIdentifier

// Check expiration
if (tokenId.isExpired(clock.instant())) {
throw new SecretManager.InvalidToken("Invalid STS token - token expired at " + tokenId.getExpiry());
throw new OMException("Invalid STS token - token expired at " + tokenId.getExpiry(), TOKEN_EXPIRED);
}

// Verify token signature against the original identifier bytes
Expand All @@ -121,7 +124,7 @@ private static STSTokenIdentifier verifyAndDecryptToken(Token<STSTokenIdentifier
}

private static ManagedSecretKey getValidatedSecretKey(UUID secretKeyId, SecretKeyClient secretKeyClient)
throws SecretManager.InvalidToken {
throws SecretManager.InvalidToken, OMException {
if (secretKeyId == null) {
throw new SecretManager.InvalidToken("STS token missing secret key ID");
}
Expand All @@ -138,7 +141,9 @@ private static ManagedSecretKey getValidatedSecretKey(UUID secretKeyId, SecretKe
}

if (secretKey.isExpired()) {
throw new SecretManager.InvalidToken("Token cannot be verified due to expired secret key " + secretKeyId);
throw new OMException(
"Token cannot be verified due to expired secret key: " + secretKeyId + " Token expired at " +
secretKey.getExpiryTime(), TOKEN_EXPIRED);
}

return secretKey;
Expand Down
11 changes: 7 additions & 4 deletions ...one/ozone-manager/src/test/java/org/apache/hadoop/ozone/security/TestSTSSecurityUtil.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,7 @@

package org.apache.hadoop.ozone.security;

import static org.apache.hadoop.ozone.om.exceptions.OMException.ResultCodes.TOKEN_EXPIRED;
import static org.assertj.core.api.Assertions.assertThat;
import static org.assertj.core.api.Assertions.assertThatThrownBy;
import static org.mockito.ArgumentMatchers.any;
Expand Down Expand Up @@ -179,7 +180,8 @@ public void testConstructValidateAndDecryptSTSTokenExpired() throws Exception {
assertThatThrownBy(() ->
STSSecurityUtil.constructValidateAndDecryptSTSToken(tokenString, secretKeyClient, clock))
.isInstanceOf(OMException.class)
.hasMessageContaining("Invalid STS token format: Invalid STS token - token expired at");
.satisfies(exception -> assertThat(((OMException) exception).getResult()).isEqualTo(TOKEN_EXPIRED))
.hasMessageContaining("Invalid STS token - token expired at");
}

@Test
Expand Down Expand Up @@ -236,6 +238,8 @@ public void testConstructValidateAndDecryptSTSTokenExpiredSecretKey() throws Exc
// Create a mock secret key that is expired
final ManagedSecretKey expiredSecretKey = mock(ManagedSecretKey.class);
when(expiredSecretKey.isExpired()).thenReturn(true);
final Instant now = Instant.now();
when(expiredSecretKey.getExpiryTime()).thenReturn(now);

final SecretKeyClient mockKeyClient = mock(SecretKeyClient.class);
when(mockKeyClient.getSecretKey(any())).thenReturn(expiredSecretKey);
Expand All @@ -244,9 +248,8 @@ public void testConstructValidateAndDecryptSTSTokenExpiredSecretKey() throws Exc
assertThatThrownBy(() ->
STSSecurityUtil.constructValidateAndDecryptSTSToken(validTokenString, mockKeyClient, clock))
.isInstanceOf(OMException.class)
.hasMessage(
"Invalid STS token format: Invalid STS token - could not readFromByteArray: Token cannot be " +
"verified due to expired secret key " + secretKeyId);
.satisfies(exception -> assertThat(((OMException) exception).getResult()).isEqualTo(TOKEN_EXPIRED))
.hasMessage("Token cannot be verified due to expired secret key: " + secretKeyId + " Token expired at " + now);
}

@Test
Expand Down
4 changes: 4 additions & 0 deletions hadoop-ozone/s3gateway/pom.xml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -30,6 +30,10 @@
</properties>

<dependencies>
<dependency>
<groupId>com.fasterxml.jackson.core</groupId>
<artifactId>jackson-annotations</artifactId>
</dependency>
<dependency>
<groupId>com.fasterxml.jackson.core</groupId>
<artifactId>jackson-databind</artifactId>
Expand Down
4 changes: 4 additions & 0 deletions ...p-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/BucketAclHandler.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -113,6 +113,8 @@ Response handleGetRequest(S3RequestContext context, String bucketName)
auditReadFailure(context.getAction(), ex);
if (ex.getResult() == ResultCodes.BUCKET_NOT_FOUND) {
throw newError(S3ErrorTable.NO_SUCH_BUCKET, bucketName, ex);
} else if (isExpiredToken(ex)) {
throw newError(S3ErrorTable.EXPIRED_TOKEN, bucketName, ex);
} else if (isAccessDenied(ex)) {
throw newError(S3ErrorTable.ACCESS_DENIED, bucketName, ex);
} else {
Expand Down Expand Up @@ -232,6 +234,8 @@ Response handlePutRequest(S3RequestContext context, String bucketName, InputStre
auditWriteFailure(context.getAction(), exception);
if (exception.getResult() == ResultCodes.BUCKET_NOT_FOUND) {
throw newError(S3ErrorTable.NO_SUCH_BUCKET, bucketName, exception);
} else if (isExpiredToken(exception)) {
throw newError(S3ErrorTable.EXPIRED_TOKEN, bucketName, exception);
} else if (isAccessDenied(exception)) {
throw newError(S3ErrorTable.ACCESS_DENIED, bucketName, exception);
}
Expand Down
2 changes: 2 additions & 0 deletions ...-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/BucketCrudHandler.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -111,6 +111,8 @@ Response handleDeleteRequest(S3RequestContext context, String bucketName)
throw newError(S3ErrorTable.BUCKET_NOT_EMPTY, bucketName, ex);
} else if (ex.getResult() == OMException.ResultCodes.BUCKET_NOT_FOUND) {
throw newError(S3ErrorTable.NO_SUCH_BUCKET, bucketName, ex);
} else if (isExpiredToken(ex)) {
throw newError(S3ErrorTable.EXPIRED_TOKEN, bucketName, ex);
} else if (isAccessDenied(ex)) {
throw newError(S3ErrorTable.ACCESS_DENIED, bucketName, ex);
} else {
Expand Down
120 changes: 71 additions & 49 deletions hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/BucketEndpoint.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -163,14 +163,7 @@ public Response get(
} catch (OMException ex) {
auditReadFailure(s3GAction, ex);
getMetrics().updateGetBucketFailureStats(startNanos);
if (isAccessDenied(ex)) {
throw newError(S3ErrorTable.ACCESS_DENIED, bucketName, ex);
} else if (ex.getResult() == ResultCodes.FILE_NOT_FOUND) {
// File not found, continue and send normal response with 0 keyCount
LOG.debug("Key Not found prefix: {}", prefix);
} else {
throw ex;
}
handleOMException(ex, bucketName, prefix);
} catch (Exception ex) {
getMetrics().updateGetBucketFailureStats(startNanos);
auditReadFailure(s3GAction, ex);
Expand Down Expand Up @@ -210,53 +203,67 @@ public Response get(
String lastKey = null;
int count = 0;
if (maxKeys > 0) {
while (ozoneKeyIterator != null && ozoneKeyIterator.hasNext()) {
OzoneKey next = ozoneKeyIterator.next();
if (bucket != null && bucket.getBucketLayout().isFileSystemOptimized() &&
StringUtils.isNotEmpty(prefix) &&
!next.getName().startsWith(prefix)) {
// prefix has delimiter but key don't have
// example prefix: dir1/ key: dir123
continue;
}
if (startAfter != null && count == 0 && Objects.equals(startAfter, next.getName())) {
continue;
}
String relativeKeyName = next.getName().substring(prefix.length());

int depth = StringUtils.countMatches(relativeKeyName, delimiter);
if (!StringUtils.isEmpty(delimiter)) {
if (depth > 0) {
// means key has multiple delimiters in its value.
// ex: dir/dir1/dir2, where delimiter is "/" and prefix is dir/
String dirName = relativeKeyName.substring(0, relativeKeyName
.indexOf(delimiter));
if (!dirName.equals(prevDir)) {
response.addPrefix(EncodingTypeObject.createNullable(
prefix + dirName + delimiter, encodingType));
prevDir = dirName;
try {
while (ozoneKeyIterator != null && ozoneKeyIterator.hasNext()) {
OzoneKey next = ozoneKeyIterator.next();
if (bucket != null && bucket.getBucketLayout().isFileSystemOptimized() &&
StringUtils.isNotEmpty(prefix) &&
!next.getName().startsWith(prefix)) {
// prefix has delimiter but key don't have
// example prefix: dir1/ key: dir123
continue;
}
if (startAfter != null && count == 0 && Objects.equals(startAfter, next.getName())) {
continue;
}
String relativeKeyName = next.getName().substring(prefix.length());

int depth = StringUtils.countMatches(relativeKeyName, delimiter);
if (!StringUtils.isEmpty(delimiter)) {
if (depth > 0) {
// means key has multiple delimiters in its value.
// ex: dir/dir1/dir2, where delimiter is "/" and prefix is dir/
String dirName = relativeKeyName.substring(0, relativeKeyName
.indexOf(delimiter));
if (!dirName.equals(prevDir)) {
response.addPrefix(EncodingTypeObject.createNullable(
prefix + dirName + delimiter, encodingType));
prevDir = dirName;
count++;
}
} else if (relativeKeyName.endsWith(delimiter)) {
// means or key is same as prefix with delimiter at end and ends with
// delimiter. ex: dir/, where prefix is dir and delimiter is /
response.addPrefix(
EncodingTypeObject.createNullable(relativeKeyName, encodingType));
count++;
} else {
// means our key is matched with prefix if prefix is given and it
// does not have any common prefix.
addKey(response, next);
count++;
}
} else if (relativeKeyName.endsWith(delimiter)) {
// means or key is same as prefix with delimiter at end and ends with
// delimiter. ex: dir/, where prefix is dir and delimiter is /
response.addPrefix(
EncodingTypeObject.createNullable(relativeKeyName, encodingType));
count++;
} else {
// means our key is matched with prefix if prefix is given and it
// does not have any common prefix.
addKey(response, next);
count++;
}
} else {
addKey(response, next);
count++;
}

if (count == maxKeys) {
lastKey = next.getName();
break;
if (count == maxKeys) {
lastKey = next.getName();
break;
}
}
} catch (RuntimeException ex) {
getMetrics().updateGetBucketFailureStats(startNanos);
auditReadFailure(s3GAction, ex);
if (ex.getCause() instanceof OMException) {
final OMException omException = (OMException) ex.getCause();
if (omException.getResult() == ResultCodes.FILE_NOT_FOUND) {
throw ex;
}
handleOMException(omException, bucketName, prefix);
} else {
Comment on lines +259 to +265
Copy link
Copy Markdown
Contributor

@Russole Russole Apr 4, 2026

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

The comment for ResultCodes.FILE_NOT_FOUND says "continue and send normal response with 0 keyCount", which makes sense for the outer case (before iteration starts).
However, in the new iterator-side runtime path, FILE_NOT_FOUND will stop the loop and return the current response.
So the semantics seem different. Should iterator-side FILE_NOT_FOUND really be treated the same way?

Copy link
Copy Markdown
Contributor Author

@fmorg-git fmorg-git Apr 7, 2026

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

good catch - updated

Copy link
Copy Markdown
Contributor Author

@fmorg-git fmorg-git Apr 9, 2026

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

hi @Russole - I think your comment might have been reversed. Before this PR, the inner loop would have thrown on FILE_NOT_FOUND but my initial commit would have swallowed it. So I excluded the FILE_NOT_FOUND case so it will behave the same as before and still handle the AccessDenied and ExpiredToken cases.

throw ex;
}
}
}
Expand Down Expand Up @@ -362,7 +369,9 @@ public Response listMultipartUploads(
} catch (OMException exception) {
auditReadFailure(s3GAction, exception);
getMetrics().updateListMultipartUploadsFailureStats(startNanos);
if (isAccessDenied(exception)) {
if (isExpiredToken(exception)) {
throw newError(S3ErrorTable.EXPIRED_TOKEN, prefix, exception);
} else if (isAccessDenied(exception)) {
throw newError(S3ErrorTable.ACCESS_DENIED, prefix, exception);
}
throw exception;
Expand Down Expand Up @@ -517,4 +526,17 @@ private void addHandler(BucketOperationHandler handler) {
copyDependenciesTo(handler);
handlers.add(handler);
}

private void handleOMException(OMException ex, String bucketName, String prefix) throws OMException {
if (isExpiredToken(ex)) {
throw newError(S3ErrorTable.EXPIRED_TOKEN, bucketName, ex);
} else if (isAccessDenied(ex)) {
throw newError(S3ErrorTable.ACCESS_DENIED, bucketName, ex);
} else if (ex.getResult() == ResultCodes.FILE_NOT_FOUND) {
// File not found, continue and send normal response with 0 keyCount
LOG.debug("Key Not found prefix: {}", prefix);
} else {
throw ex;
}
}
}
14 changes: 14 additions & 0 deletions hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/EndpointBase.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -194,6 +194,8 @@ protected OzoneBucket getBucket(OzoneVolume volume, String bucketName)
} catch (OMException ex) {
if (ex.getResult() == ResultCodes.BUCKET_NOT_FOUND) {
throw newError(S3ErrorTable.NO_SUCH_BUCKET, bucketName, ex);
} else if (isExpiredToken(ex)) {
throw newError(S3ErrorTable.EXPIRED_TOKEN, s3Auth.getAccessID(), ex);
} else if (ex.getResult() == ResultCodes.INVALID_TOKEN) {
throw newError(S3ErrorTable.ACCESS_DENIED,
s3Auth.getAccessID(), ex);
Expand Down Expand Up @@ -259,6 +261,8 @@ protected OzoneBucket getBucket(String bucketName)
if (ex.getResult() == ResultCodes.BUCKET_NOT_FOUND
|| ex.getResult() == ResultCodes.VOLUME_NOT_FOUND) {
throw newError(S3ErrorTable.NO_SUCH_BUCKET, bucketName, ex);
} else if (isExpiredToken(ex)) {
throw newError(S3ErrorTable.EXPIRED_TOKEN, s3Auth.getAccessID(), ex);
} else if (ex.getResult() == ResultCodes.INVALID_TOKEN) {
throw newError(S3ErrorTable.ACCESS_DENIED,
s3Auth.getAccessID(), ex);
Expand Down Expand Up @@ -294,6 +298,8 @@ protected String createS3Bucket(String bucketName) throws
getMetrics().updateCreateBucketFailureStats(startNanos);
if (ex.getResult() == ResultCodes.PERMISSION_DENIED) {
throw newError(S3ErrorTable.ACCESS_DENIED, bucketName, ex);
} else if (isExpiredToken(ex)) {
throw newError(S3ErrorTable.EXPIRED_TOKEN, s3Auth.getAccessID(), ex);
} else if (ex.getResult() == ResultCodes.INVALID_TOKEN) {
throw newError(S3ErrorTable.ACCESS_DENIED,
s3Auth.getAccessID(), ex);
Expand Down Expand Up @@ -322,6 +328,8 @@ protected void deleteS3Bucket(String s3BucketName)
if (ex.getResult() == ResultCodes.PERMISSION_DENIED) {
throw newError(S3ErrorTable.ACCESS_DENIED,
s3BucketName, ex);
} else if (isExpiredToken(ex)) {
throw newError(S3ErrorTable.EXPIRED_TOKEN, s3Auth.getAccessID(), ex);
} else if (ex.getResult() == ResultCodes.INVALID_TOKEN) {
throw newError(S3ErrorTable.ACCESS_DENIED,
s3Auth.getAccessID(), ex);
Expand Down Expand Up @@ -377,6 +385,8 @@ private Iterator<? extends OzoneBucket> iterateBuckets(
} else if (e.getResult() == ResultCodes.PERMISSION_DENIED) {
throw newError(S3ErrorTable.ACCESS_DENIED,
"listBuckets", e);
} else if (isExpiredToken(e)) {
throw newError(S3ErrorTable.EXPIRED_TOKEN, s3Auth.getAccessID(), e);
} else if (e.getResult() == ResultCodes.INVALID_TOKEN) {
throw newError(S3ErrorTable.ACCESS_DENIED,
s3Auth.getAccessID(), e);
Expand Down Expand Up @@ -685,6 +695,10 @@ protected boolean isAccessDenied(OMException ex) {
|| result == ResultCodes.REVOKED_TOKEN;
}

protected boolean isExpiredToken(OMException ex) {
return ex.getResult() == ResultCodes.TOKEN_EXPIRED;
}

protected ReplicationConfig getReplicationConfig(OzoneBucket ozoneBucket) throws OS3Exception {
String storageType = getHeaders().getHeaderString(STORAGE_CLASS_HEADER);
String storageConfig = getHeaders().getHeaderString(CUSTOM_METADATA_HEADER_PREFIX + STORAGE_CONFIG_HEADER);
Expand Down
2 changes: 2 additions & 0 deletions ...zone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/MultipartKeyHandler.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -164,6 +164,8 @@ private Response listParts(OzoneBucket ozoneBucket, String key, String uploadId,
} catch (OMException ex) {
if (ex.getResult() == ResultCodes.NO_SUCH_MULTIPART_UPLOAD_ERROR) {
throw newError(NO_SUCH_UPLOAD, uploadId, ex);
} else if (isExpiredToken(ex)) {
throw newError(S3ErrorTable.EXPIRED_TOKEN, bucketName + "/" + key + "/" + uploadId, ex);
} else if (isAccessDenied(ex)) {
throw newError(S3ErrorTable.ACCESS_DENIED,
bucketName + "/" + key + "/" + uploadId, ex);
Expand Down
Loading
Loading