Java 类com.amazonaws.services.s3.model.AmazonS3Exception 实例源码

@Override
public InputStream readBlob(String blobName) throws IOException {
    int retry = 0;
    while (retry <= blobStore.numberOfRetries()) {
        try {
            S3Object s3Object = SocketAccess.doPrivileged(() -> blobStore.client().getObject(blobStore.bucket(), buildKey(blobName)));
            return s3Object.getObjectContent();
        } catch (AmazonClientException e) {
            if (blobStore.shouldRetry(e) && (retry < blobStore.numberOfRetries())) {
                retry++;
            } else {
                if (e instanceof AmazonS3Exception) {
                    if (404 == ((AmazonS3Exception) e).getStatusCode()) {
                        throw new NoSuchFileException("Blob object [" + blobName + "] not found: " + e.getMessage());
                    }
                }
                throw e;
            }
        }
    }
    throw new BlobStoreException("retries exhausted while attempting to access blob object [name:" + blobName + ", bucket:" + blobStore.bucket() + "]");
}

@Override
public void move(String sourceBlobName, String targetBlobName) throws IOException {
    try {
        CopyObjectRequest request = new CopyObjectRequest(blobStore.bucket(), buildKey(sourceBlobName),
            blobStore.bucket(), buildKey(targetBlobName));

        if (blobStore.serverSideEncryption()) {
            ObjectMetadata objectMetadata = new ObjectMetadata();
            objectMetadata.setSSEAlgorithm(ObjectMetadata.AES_256_SERVER_SIDE_ENCRYPTION);
            request.setNewObjectMetadata(objectMetadata);
        }

        SocketAccess.doPrivilegedVoid(() -> {
            blobStore.client().copyObject(request);
            blobStore.client().deleteObject(blobStore.bucket(), buildKey(sourceBlobName));
        });

    } catch (AmazonS3Exception e) {
        throw new IOException(e);
    }
}

@Override
public PutObjectResult putObject(PutObjectRequest putObjectRequest)
        throws AmazonClientException, AmazonServiceException {
    String blobName = putObjectRequest.getKey();
    DigestInputStream stream = (DigestInputStream) putObjectRequest.getInputStream();

    if (blobs.containsKey(blobName)) {
        throw new AmazonS3Exception("[" + blobName + "] already exists.");
    }

    blobs.put(blobName, stream);

    // input and output md5 hashes need to match to avoid an exception
    String md5 = Base64.encodeAsString(stream.getMessageDigest().digest());
    PutObjectResult result = new PutObjectResult();
    result.setContentMd5(md5);

    return result;
}

@Override
public S3Object getObject(GetObjectRequest getObjectRequest)
        throws AmazonClientException, AmazonServiceException {
    // in ESBlobStoreContainerTestCase.java, the prefix is empty,
    // so the key and blobName are equivalent to each other
    String blobName = getObjectRequest.getKey();

    if (!blobs.containsKey(blobName)) {
        throw new AmazonS3Exception("[" + blobName + "] does not exist.");
    }

    // the HTTP request attribute is irrelevant for reading
    S3ObjectInputStream stream = new S3ObjectInputStream(
            blobs.get(blobName), null, false);
    S3Object s3Object = new S3Object();
    s3Object.setObjectContent(stream);
    return s3Object;
}

@Override
public CopyObjectResult copyObject(CopyObjectRequest copyObjectRequest)
        throws AmazonClientException, AmazonServiceException {
    String sourceBlobName = copyObjectRequest.getSourceKey();
    String targetBlobName = copyObjectRequest.getDestinationKey();

    if (!blobs.containsKey(sourceBlobName)) {
        throw new AmazonS3Exception("Source blob [" +
                sourceBlobName + "] does not exist.");
    }

    if (blobs.containsKey(targetBlobName)) {
        throw new AmazonS3Exception("Target blob [" +
                targetBlobName + "] already exists.");
    }

    blobs.put(targetBlobName, blobs.get(sourceBlobName));
    return new CopyObjectResult(); // nothing is done with it
}

/**
 * This method deletes the url for code
 * 
 * @param code
 */
public void deleteShortUrl(String code) {
    try {
        // get the object
        ObjectMetadata metaData = this.s3Client.getObjectMetadata(this.bucket, code);
        String url = metaData.getUserMetaDataOf("url");
        logger.info("The url to be deleted {}", url);
        this.s3Client.deleteObject(this.bucket, code);
        this.s3Client.deleteObject(this.bucket + "-dummy", code + Base64.encodeBase64String(url.getBytes()));

    } catch (AmazonS3Exception ex) {
        if (ex.getStatusCode() == HttpStatus.SC_NOT_FOUND) {
            return;
        }
        logger.warn("Unable to get object status", ex);
        throw ex;
    }

}

/**
 * Attempt to fetch a secret from S3.
 *
 * @param s3path where to fetch it from
 * @return the content of the file found on S3
 * @throws IOException on problems streaming the content of the file
 * @throws AmazonS3Exception on problems communicating with amazon
 */
private String getS3Value(final SecretPath s3path) throws IOException, AmazonS3Exception {
    LOG.info("Fetching secret from s3://" + s3path.bucket + "/" + s3path.key);
    if (s3Client == null) {
        if (awsCredentialsProvider != null) {
            s3Client = AmazonS3ClientBuilder.standard().withCredentials(awsCredentialsProvider)
                    .build();
        } else {
            s3Client = AmazonS3ClientBuilder.standard().build();
        }
    }
    final S3Object s3object
            = s3Client.getObject(new GetObjectRequest(s3path.bucket, s3path.key));
    final BufferedReader reader
            = new BufferedReader(new InputStreamReader(s3object.getObjectContent()));
    final StringBuilder b = new StringBuilder();
    String line;
    while((line = reader.readLine()) != null) {
        b.append(line);
    }
    LOG.info("Found secret");
    reader.close();
    return b.toString();
}

private boolean shouldRetryCompleteMultipartUpload(AmazonWebServiceRequest originalRequest,
                                                   AmazonS3Exception exception,
                                                   int retriesAttempted) {

    final RetryPolicy retryPolicy = clientConfiguration.getRetryPolicy();

    if (retryPolicy == null || retryPolicy.getRetryCondition() == null) {
        return false;
    }

    if (retryPolicy == PredefinedRetryPolicies.NO_RETRY_POLICY) {
        return false;
    }

    return completeMultipartUploadRetryCondition.shouldRetry
            (originalRequest, exception, retriesAttempted);
}

/**
 * Retrieves the region of the bucket by making a HeadBucket request to us-west-1 region.
 *
 * Currently S3 doesn't return region in a HEAD Bucket request if the bucket
 * owner has enabled bucket to accept only SigV4 requests via bucket
 * policies.
 */
private String getBucketRegionViaHeadRequest(String bucketName) {
    String bucketRegion = null;

    try {
        Request<HeadBucketRequest> request = createRequest(bucketName, null,
                new HeadBucketRequest(bucketName), HttpMethodName.HEAD);

        HeadBucketResult result = invoke(request, new HeadBucketResultHandler(), bucketName, null, true);
        bucketRegion = result.getBucketRegion();
    } catch (AmazonS3Exception exception) {
        if (exception.getAdditionalDetails() != null) {
            bucketRegion = exception.getAdditionalDetails().get(
                Headers.S3_BUCKET_REGION);
        }
    }

    if (bucketRegion == null && log.isDebugEnabled()) {
        log.debug("Not able to derive region of the " + bucketName + " from the HEAD Bucket requests.");
    }

    return bucketRegion;
}

private AmazonS3Exception createExceptionFromHeaders(
        HttpResponse errorResponse, String errorResponseXml) {
    final Map<String, String> headers = errorResponse.getHeaders();
    final int statusCode = errorResponse.getStatusCode();
    final AmazonS3ExceptionBuilder exceptionBuilder = new AmazonS3ExceptionBuilder();
    exceptionBuilder.setErrorMessage(errorResponse.getStatusText());
    exceptionBuilder.setErrorResponseXml(errorResponseXml);
    exceptionBuilder.setStatusCode(statusCode);
    exceptionBuilder
            .setExtendedRequestId(headers.get(Headers.EXTENDED_REQUEST_ID));
    exceptionBuilder.setRequestId(headers.get(Headers.REQUEST_ID));
    exceptionBuilder.setCloudFrontId(headers.get(Headers.CLOUD_FRONT_ID));
    exceptionBuilder
            .setErrorCode(statusCode + " " + errorResponse.getStatusText());
    exceptionBuilder.addAdditionalDetail(Headers.S3_BUCKET_REGION,
            errorResponse.getHeaders().get(Headers.S3_BUCKET_REGION));
    return exceptionBuilder.build();
}

/**
 * Tests if Object can be uploaded with wrong KMS Key
 */
@Test
public void shouldNotUploadStreamingWithWrongEncryptionKey() {
  final byte[] bytes = UPLOAD_FILE_NAME.getBytes();
  final InputStream stream = new ByteArrayInputStream(bytes);
  final String objectKey = UUID.randomUUID().toString();
  s3Client.createBucket(BUCKET_NAME);
  final ObjectMetadata metadata = new ObjectMetadata();
  metadata.setContentLength(bytes.length);
  final PutObjectRequest putObjectRequest =
      new PutObjectRequest(BUCKET_NAME, objectKey, stream, metadata);
  putObjectRequest.setSSEAwsKeyManagementParams(new SSEAwsKeyManagementParams(TEST_WRONG_KEYREF));

  thrown.expect(AmazonS3Exception.class);
  thrown.expectMessage(containsString("Status Code: 400; Error Code: KMS.NotFoundException"));
  s3Client.putObject(putObjectRequest);
}

/**
 * Tests that an object wont be copied with wrong encryption Key
 *
 * @throws Exception if an Exception occurs
 */
@Test
public void shouldNotObjectCopyWithWrongEncryptionKey() {
  final File uploadFile = new File(UPLOAD_FILE_NAME);
  final String sourceKey = UPLOAD_FILE_NAME;
  final String destinationBucketName = "destinationBucket";
  final String destinationKey = "copyOf" + sourceKey;

  s3Client.putObject(new PutObjectRequest(BUCKET_NAME, sourceKey, uploadFile));

  final CopyObjectRequest copyObjectRequest =
      new CopyObjectRequest(BUCKET_NAME, sourceKey, destinationBucketName, destinationKey);
  copyObjectRequest
      .setSSEAwsKeyManagementParams(new SSEAwsKeyManagementParams(TEST_WRONG_KEYREF));

  thrown.expect(AmazonS3Exception.class);
  thrown.expectMessage(containsString("Status Code: 400; Error Code: KMS.NotFoundException"));
  s3Client.copyObject(copyObjectRequest);
}

/**
 * Tests if the Metadata of an existing file can be retrieved.
 */
@Test
public void shouldGetObjectMetadata() {
  final String nonExistingFileName = "nonExistingFileName";
  final File uploadFile = new File(UPLOAD_FILE_NAME);
  s3Client.createBucket(BUCKET_NAME);

  final PutObjectResult putObjectResult =
      s3Client.putObject(new PutObjectRequest(BUCKET_NAME, UPLOAD_FILE_NAME, uploadFile));
  final ObjectMetadata metadataExisting =
      s3Client.getObjectMetadata(BUCKET_NAME, UPLOAD_FILE_NAME);

  assertThat("The ETags should be identically!", metadataExisting.getETag(),
      is(putObjectResult.getETag()));

  thrown.expect(AmazonS3Exception.class);
  thrown.expectMessage(containsString("Status Code: 404"));
  s3Client.getObjectMetadata(BUCKET_NAME, nonExistingFileName);
}

/**
 * A method that returns true if a correct s3 URI was provided and false otherwise.
 *
 * @param uri The provided URI for the file.
 * @return a boolean value that shows whether the correct URI was provided
 */
boolean isFileExisting(AmazonS3URI uri) {

    boolean exist = true;

    try {
        aws.getObjectMetadata(uri.getBucket(), uri.getKey());
    } catch (AmazonS3Exception e) {
        if (e.getStatusCode() == HttpStatus.SC_FORBIDDEN
                || e.getStatusCode() == HttpStatus.SC_NOT_FOUND) {
            exist = false;
        } else {
            throw e;
        }
    }
    return exist;
}

private FileStatus getFileStatusKeyBased(String key, Path path) throws AmazonS3Exception {
  LOG.trace("internal method - get file status by key {}, path {}", key, path);
  FileStatus cachedFS = memoryCache.getFileStatus(path.toString());
  if (cachedFS != null) {
    return cachedFS;
  }
  ObjectMetadata meta = mClient.getObjectMetadata(mBucket, key);
  String sparkOrigin = meta.getUserMetaDataOf("data-origin");
  boolean stocatorCreated = false;
  if (sparkOrigin != null) {
    String tmp = (String) sparkOrigin;
    if (tmp.equals("stocator")) {
      stocatorCreated = true;
    }
  }
  mCachedSparkOriginated.put(key, Boolean.valueOf(stocatorCreated));
  FileStatus fs = createFileStatus(meta.getContentLength(), key, meta.getLastModified(), path);
  memoryCache.putFileStatus(path.toString(), fs);
  return fs;
}

@Override
public void retrieve( char[] path )
        throws IOException
{
    String pathValue = String.valueOf( path );
    try
    {
        LOG.log( Level.FINE, () -> "Retrieving " + getBucketName() + ":" + pathValue );
        S3Object obj = getS3().getObject( new GetObjectRequest( getBucketName(), pathValue ) );
        FileSystemUtils.copyFromRemote( () -> obj.getObjectContent(), getDelegate(), path );
        LOG.log( Level.FINE, () -> "Retrieved " + getBucketName() + ":" + pathValue );
    } catch( AmazonS3Exception ex )
    {
        LOG.log( Level.FINE, () -> "Error " + ex.getStatusCode() + " " + getBucketName() + ":" + pathValue );
        if( ex.getStatusCode() == 404 )
        {
            throw new FileNotFoundException( pathValue );
        }
        throw new IOException( "Cannot access " + pathValue, ex );
    }
}

@Test(expected = RuntimeException.class)
public void testThatHandlerErrorsWhenWeCantFindTheConfigFile() {
    String arn = "arn:aws:lambda:us-west-2:1111111:function:dev-gateway-fas342452-6d86-LambdaWAFBlacklistingFun-1LSORI5GUP95H";
    String bucketName = "dev-cerberusconfigbucket";

    List<Bucket> bucketList = Lists.newLinkedList();
    bucketList.add(new Bucket(bucketName));

    AmazonS3Exception e = new AmazonS3Exception("foo");
    e.setErrorCode("NoSuchKey");

    when(amazonS3Client.getObject(any())).thenThrow(e);

    when(amazonS3Client.listBuckets()).thenReturn(bucketList);

    handler.getConfiguration(arn);
}

public String getRegionForBucket(String bucket) {
    // Just querying for the location for a bucket can be done with the local client
    AmazonS3 client = getLocalS3Client();
    try {
        String region = client.getBucketLocation(bucket);
        if ("US".equals(region)) {
            // GetBucketLocation requests return null for us-east-1 which the SDK then replaces with "US".
            // So change it to the actual region.
            region = "us-east-1";
        }
        return region;
    } catch (AmazonS3Exception e) {
        if (e.getStatusCode() == Response.Status.NOT_FOUND.getStatusCode()) {
            // If the bucket doesn't exist then return null
            return null;
        }
        throw e;
    }
}

@Override
protected boolean writeScanCompleteFile(URI fileUri, byte[] contents)
        throws IOException {
    String bucket = fileUri.getHost();
    String key = getKeyFromPath(fileUri);

    try {
        // The following will throw an exception unless the file already exists
        _amazonS3.getObjectMetadata(bucket, key);
        return false;
    } catch (AmazonS3Exception e) {
        if (e.getStatusCode() != Response.Status.NOT_FOUND.getStatusCode()) {
            // Expected case is not found, meaning the file does not exist
            // All other cases are some unexpected error
            throw new IOException(e);
        }
    }

    uploadContents(bucket, key, contents);
    return true;
}

public static List<S3ObjectSummary> listObjects(String bucket, String prefix, AmazonS3 s3) {
  List<S3ObjectSummary> objects = new ArrayList<>();
  ObjectListing listing;

  try {
    if (prefix == null) {
      listing = s3.listObjects(bucket);
    } else {
      listing = s3.listObjects(bucket, prefix);
    }

    objects.addAll(listing.getObjectSummaries());
    while (listing.isTruncated()) {
      listing = s3.listNextBatchOfObjects(listing);
      objects.addAll(listing.getObjectSummaries());
    }
  } catch (AmazonS3Exception e) {
   log.warn("listObjects for bucket '{}' prefix '{}' returned error code: {}", bucket, prefix, e.getStatusCode());
  }

  return objects;
}

/**
 * {@inheritDoc}
 */
@Override
public Map<String, AlexaStateObject> readValues(final Map<String, AlexaScope> idsInScope) throws AlexaStateException {
    final Map<String, AlexaStateObject> stateObjectMap = new HashMap<>();
    // first read all the session-scoped items and put to result map
    stateObjectMap.putAll(super.readValues(idsInScope));

    idsInScope.forEach((id, scope) -> {
        if (scope.isIn(AlexaScope.USER, AlexaScope.APPLICATION)) {
            final String filePath = AlexaScope.USER.includes(scope) ?
                    getUserScopedFilePath(id) : getAppScopedFilePath(id);
            try {
                // get S3 file
                getS3FileContentsAsString(filePath)
                        // wrap its contents in state object
                        .map(fileContents -> new AlexaStateObject(id, fileContents, scope))
                        // add to result map
                        .ifPresent(stateObject -> stateObjectMap.putIfAbsent(id, stateObject));
            } catch (final AlexaStateException | AmazonS3Exception e) {
                // we are fine with an exception likely caused by file (state) not exists
                log.warn("Could not read from '" + filePath + "'.", e);
            }
        }
    });
    return stateObjectMap;
}

public void testFetchOffsetNewTopic() throws Exception {
  AmazonS3 s3Mock = mock(AmazonS3.class);
  S3Writer s3Writer = new S3Writer(testBucket, "pfx", s3Mock);

  // Non existing topic should return 0 offset
  // Since the file won't exist. code will expect the initial fetch to 404
  AmazonS3Exception ase = new AmazonS3Exception("The specified key does not exist.");
  ase.setStatusCode(404);
  when(s3Mock.getObject(eq(testBucket), eq("pfx/last_chunk_index.new_topic-00000.txt")))
    .thenThrow(ase)
    .thenReturn(null);

  TopicPartition tp = new TopicPartition("new_topic", 0);
  long offset = s3Writer.fetchOffset(tp);
  assertEquals(0, offset);
  verify(s3Mock).getObject(eq(testBucket), eq("pfx/last_chunk_index.new_topic-00000.txt"));
}

private ObjectMetadata getConfigMetadata() throws Exception
{
    try
    {
        ObjectMetadata metadata = s3Client.getObjectMetadata(arguments.getBucket(), arguments.getKey());
        if ( metadata.getContentLength() > 0 )
        {
            return metadata;
        }
    }
    catch ( AmazonS3Exception e )
    {
        if ( !isNotFoundError(e) )
        {
            throw e;
        }
    }
    return null;
}

private S3Object getConfigObject() throws Exception
{
    try
    {
        S3Object object = s3Client.getObject(arguments.getBucket(), arguments.getKey());
        if ( object.getObjectMetadata().getContentLength() > 0 )
        {
            return object;
        }
    }
    catch ( AmazonS3Exception e )
    {
        if ( !isNotFoundError(e) )
        {
            throw e;
        }
    }
    return null;
}

@Override
public boolean hasFullControlPermission(final String bucketName)
        throws AmazonClientException, AmazonServiceException,
        AmazonS3Exception {
    LOGGER.info("Checking full controll permission on bucket..");
    boolean hasFullControl = false;
    final AccessControlList acl =  getBucketAccessControlList(bucketName);
    final List<Grant> grantList = acl.getGrantsAsList();
    for (final Grant grant : grantList) {
        if(Permission.FullControl.equals(grant.getPermission())){
            hasFullControl = true;
            LOGGER.info("Permissions validated, hasFullControl: {}", hasFullControl);
            break;
        }
    }
    return hasFullControl;
}

@Override
public boolean checkBucketPermission(final String bucketName, final Permission permission)
        throws AmazonClientException, AmazonServiceException, AmazonS3Exception {
    LOGGER.info("Checking bucket permission..");
    boolean hasPermission = false;
    final AccessControlList acl =  getBucketAccessControlList(bucketName);
    final List<Grant> grantList = acl.getGrantsAsList();
    for (final Grant grant : grantList) {
        if(permission.equals(grant.getPermission())){
            hasPermission = true;
            LOGGER.info("Permissions validated,hasPermission: {}",hasPermission);
            break;
        }
    }
    return hasPermission;
}

@Override
public boolean checkObjectPermission(final String bucketName, final String key, final Permission permission)
        throws AmazonClientException, AmazonServiceException, AmazonS3Exception {
    LOGGER.info("Checking object permission..");
    boolean hasPermission = false;
    final AccessControlList objectAcl = s3client.getObjectAcl(bucketName, key);
    final List<Grant> grantList = objectAcl.getGrantsAsList();
    for (final Grant grant : grantList) {
        if(permission.equals(grant.getPermission())){
            hasPermission = true;
            LOGGER.info("Permissions validated,hasPermission: {}",hasPermission);
            break;
        }
    }
    return hasPermission;
}

@SuppressWarnings({"ResultOfMethodCallIgnored", "OverlyStrongTypeCast", "ConstantConditions"})
@Test
public void testReadRetryCounters()
        throws Exception
{
    try (PrestoS3FileSystem fs = new PrestoS3FileSystem()) {
        int maxRetries = 2;
        MockAmazonS3 s3 = new MockAmazonS3();
        s3.setGetObjectHttpErrorCode(SC_INTERNAL_SERVER_ERROR);
        Configuration configuration = new Configuration();
        configuration.set(S3_MAX_BACKOFF_TIME, "1ms");
        configuration.set(S3_MAX_RETRY_TIME, "5s");
        configuration.setInt(S3_MAX_CLIENT_RETRIES, maxRetries);
        fs.initialize(new URI("s3n://test-bucket/"), configuration);
        fs.setS3Client(s3);
        try (FSDataInputStream inputStream = fs.open(new Path("s3n://test-bucket/test"))) {
            inputStream.read();
        }
        catch (Throwable expected) {
            assertInstanceOf(expected, AmazonS3Exception.class);
            assertEquals(((AmazonS3Exception) expected).getStatusCode(), SC_INTERNAL_SERVER_ERROR);
            assertEquals(PrestoS3FileSystem.getFileSystemStats().getReadRetries().getTotalCount(), maxRetries);
            assertEquals(PrestoS3FileSystem.getFileSystemStats().getGetObjectRetries().getTotalCount(), (maxRetries + 1L) * maxRetries);
        }
    }
}

@SuppressWarnings({"OverlyStrongTypeCast", "ConstantConditions"})
@Test
public void testGetMetadataRetryCounter()
{
    int maxRetries = 2;
    try (PrestoS3FileSystem fs = new PrestoS3FileSystem()) {
        MockAmazonS3 s3 = new MockAmazonS3();
        s3.setGetObjectMetadataHttpCode(SC_INTERNAL_SERVER_ERROR);
        Configuration configuration = new Configuration();
        configuration.set(S3_MAX_BACKOFF_TIME, "1ms");
        configuration.set(S3_MAX_RETRY_TIME, "5s");
        configuration.setInt(S3_MAX_CLIENT_RETRIES, maxRetries);
        fs.initialize(new URI("s3n://test-bucket/"), configuration);
        fs.setS3Client(s3);
        fs.getS3ObjectMetadata(new Path("s3n://test-bucket/test"));
    }
    catch (Throwable expected) {
        assertInstanceOf(expected, AmazonS3Exception.class);
        assertEquals(((AmazonS3Exception) expected).getStatusCode(), SC_INTERNAL_SERVER_ERROR);
        assertEquals(PrestoS3FileSystem.getFileSystemStats().getGetMetadataRetries().getTotalCount(), maxRetries);
    }
}

@Override
protected InputStream getInputStream(int x, int y) throws IOException {
    try {
        GetObjectRequest req = new GetObjectRequest(bucketName, String.format("%d/%d.pbf.gz", x, y));
        // the LODES bucket is requester-pays.
        req.setRequesterPays(true);
        return s3.getObject(req).getObjectContent();
    } catch (AmazonS3Exception e) {
        // there is no data in this tile
        if ("NoSuchKey".equals(e.getErrorCode()))
            return null;
        else
            // re-throw, something else is amiss
            throw e;
    }
}

private void validateConnection(
    Stage.Context context,
    String configPrefix,
    List<Stage.ConfigIssue> issues
) {
  try {
    //check if the credentials are right by trying to list an object in the common prefix
    getS3Client().listObjects(new ListObjectsRequest(bucket, commonPrefix, null, delimiter, 1).withEncodingType("url"));
  } catch (AmazonS3Exception e) {
    LOG.debug(Errors.S3_SPOOLDIR_20.getMessage(), e.toString(), e);
    issues.add(
        context.createConfigIssue(
            Groups.S3.name(),
            configPrefix + S3ConnectionBaseConfig.AWS_CONFIG_PREFIX + "awsAccessKeyId",
            Errors.S3_SPOOLDIR_20,
            e.toString()
        )
    );
  }
}

private boolean etagMatches(String bucket, String key, String etag) {
    try {
        S3Object object = s3.getObject(bucket, key);
        if (object != null) {
            String existingETag = object.getObjectMetadata().getETag();
            if (etag.equals(existingETag)) {
                logger.info("etag matches, skip uploading, bucket={}, key={}", bucket, key);
                return true;
            }
        }
    } catch (AmazonS3Exception e) {
        if (!"NoSuchKey".equals(e.getErrorCode())) {
            throw e;
        }
    }
    return false;
}

@Test
public void testAwsV4SignatureBadIdentity() throws Exception {
    client = AmazonS3ClientBuilder.standard()
            .withCredentials(new AWSStaticCredentialsProvider(
                    new BasicAWSCredentials(
                            "bad-access-key", awsCreds.getAWSSecretKey())))
            .withEndpointConfiguration(s3EndpointConfig)
            .build();

    ObjectMetadata metadata = new ObjectMetadata();
    metadata.setContentLength(BYTE_SOURCE.size());

    try {
        client.putObject(containerName, "foo",
                BYTE_SOURCE.openStream(), metadata);
        Fail.failBecauseExceptionWasNotThrown(AmazonS3Exception.class);
    } catch (AmazonS3Exception e) {
        assertThat(e.getErrorCode()).isEqualTo("InvalidAccessKeyId");
    }
}

@Test
public void testAwsV4SignatureBadCredential() throws Exception {
    client = AmazonS3ClientBuilder.standard()
            .withCredentials(new AWSStaticCredentialsProvider(
                    new BasicAWSCredentials(
                            awsCreds.getAWSAccessKeyId(),
                            "bad-secret-key")))
            .withEndpointConfiguration(s3EndpointConfig)
            .build();

    ObjectMetadata metadata = new ObjectMetadata();
    metadata.setContentLength(BYTE_SOURCE.size());

    try {
        client.putObject(containerName, "foo",
                BYTE_SOURCE.openStream(), metadata);
        Fail.failBecauseExceptionWasNotThrown(AmazonS3Exception.class);
    } catch (AmazonS3Exception e) {
        assertThat(e.getErrorCode()).isEqualTo("SignatureDoesNotMatch");
    }
}

@Test
public void testUpdateBlobXmlAcls() throws Exception {
    assumeTrue(!Quirks.NO_BLOB_ACCESS_CONTROL.contains(blobStoreType));
    String blobName = "testUpdateBlobXmlAcls-blob";
    ObjectMetadata metadata = new ObjectMetadata();
    metadata.setContentLength(BYTE_SOURCE.size());
    client.putObject(containerName, blobName, BYTE_SOURCE.openStream(),
            metadata);
    AccessControlList acl = client.getObjectAcl(containerName, blobName);

    acl.grantPermission(GroupGrantee.AllUsers, Permission.Read);
    client.setObjectAcl(containerName, blobName, acl);
    assertThat(client.getObjectAcl(containerName, blobName)).isEqualTo(acl);

    acl.revokeAllPermissions(GroupGrantee.AllUsers);
    client.setObjectAcl(containerName, blobName, acl);
    assertThat(client.getObjectAcl(containerName, blobName)).isEqualTo(acl);

    acl.grantPermission(GroupGrantee.AllUsers, Permission.Write);
    try {
        client.setObjectAcl(containerName, blobName, acl);
        Fail.failBecauseExceptionWasNotThrown(AmazonS3Exception.class);
    } catch (AmazonS3Exception e) {
        assertThat(e.getErrorCode()).isEqualTo("NotImplemented");
    }
}

@Test
public void testPartNumberMarker() throws Exception {
    String blobName = "foo";
    InitiateMultipartUploadResult result = client.initiateMultipartUpload(
            new InitiateMultipartUploadRequest(containerName, blobName));
    ListPartsRequest request = new ListPartsRequest(containerName,
            blobName, result.getUploadId());

    client.listParts(request.withPartNumberMarker(0));

    try {
        client.listParts(request.withPartNumberMarker(1));
        Fail.failBecauseExceptionWasNotThrown(AmazonS3Exception.class);
    } catch (AmazonS3Exception e) {
        assertThat(e.getErrorCode()).isEqualTo("NotImplemented");
    }
}

@Test
public void testBlobRemove() throws Exception {
    String blobName = "blob";
    ObjectMetadata metadata = new ObjectMetadata();
    metadata.setContentLength(BYTE_SOURCE.size());
    client.putObject(containerName, blobName, BYTE_SOURCE.openStream(),
            metadata);
    assertThat(client.getObjectMetadata(containerName, blobName))
            .isNotNull();

    client.deleteObject(containerName, blobName);
    try {
        client.getObjectMetadata(containerName, blobName);
        Fail.failBecauseExceptionWasNotThrown(AmazonS3Exception.class);
    } catch (AmazonS3Exception e) {
        assertThat(e.getErrorCode()).isEqualTo("404 Not Found");
    }

    client.deleteObject(containerName, blobName);
}

@Override
public boolean mapShortCode(String code, String paramString) throws IOException {
    if(!enabled) {
        throw new IllegalStateException("Shortlink feature disabled");
    }

    try {
        // does object exist?
        client.getObjectMetadata(bucket, OBJECT_PREFIX + code);
    } catch (AmazonS3Exception e) {
        if (e.getStatusCode() != 404) {
            log.warn(e);
            return false;
        }
    }

    byte[] paramStringBytes = paramString.getBytes(StringUtils.UTF8);
    InputStream is = new ByteArrayInputStream(paramStringBytes);
    ObjectMetadata metadata = new ObjectMetadata();
    metadata.setContentType("text/plain");
    metadata.setContentLength(paramStringBytes.length);

    client.putObject(new PutObjectRequest(bucket, OBJECT_PREFIX + code, is, metadata));
    return true;
}

private void findAllResourcesThatMatches(String bucketName, Set<Resource> resources, String prefix, String keyPattern) {
    ListObjectsRequest listObjectsRequest = new ListObjectsRequest().withBucketName(bucketName).withPrefix(prefix);
    ObjectListing objectListing = null;

    do {
        try {
            if (objectListing == null) {
                objectListing = this.amazonS3.listObjects(listObjectsRequest);
            } else {
                objectListing = this.amazonS3.listNextBatchOfObjects(objectListing);
            }
            Set<Resource> newResources = getResourcesFromObjectSummaries(bucketName, keyPattern, objectListing.getObjectSummaries());
            if (!newResources.isEmpty()) {
                resources.addAll(newResources);
            }
        } catch (AmazonS3Exception e) {
            if (301 != e.getStatusCode()) {
                throw e;
            }
        }
    } while (objectListing != null && objectListing.isTruncated());
}

private ObjectMetadata getObjectMetadata() {
    if (this.objectMetadata == null) {
        try {
            GetObjectMetadataRequest metadataRequest = new GetObjectMetadataRequest(this.bucketName, this.objectName);
            if (this.versionId != null) {
                metadataRequest.setVersionId(this.versionId);
            }
            this.objectMetadata = this.amazonS3.getObjectMetadata(metadataRequest);
        } catch (AmazonS3Exception e) {
            // Catch 404 (object not found) and 301 (bucket not found, moved permanently)
            if (e.getStatusCode() == 404 || e.getStatusCode() == 301) {
                this.objectMetadata = null;
            } else {
                throw e;
            }
        }
    }
    return this.objectMetadata;
}