chore: add client specification and Duvet annotations (#481)

* use implication/implementation/test annotation types instead of just comments for everything

Author: kessplas

.gitmodules

@@ -0,0 +1,4 @@
+[submodule "specification"]
+	path = specification
+	url = [email protected]:awslabs/aws-encryption-sdk-specification.git
+	branch = kessplas/s3-ec-v3

Makefile

@@ -0,0 +1,14 @@
+# Used for misc supporting functions like Duvet and prettier. Builds, tests, etc. should use the usual Java/Maven tooling.
+
+duvet: | duvet_extract duvet_report
+
+duvet_extract:
+	rm -rf compliance
+	$(foreach file, $(shell find specification/s3-encryption -name '*.md'), duvet extract -o compliance -f MARKDOWN $(file);)
+
+duvet_report:
+	duvet \
+		report \
+		--spec-pattern "compliance/**/*.toml" \
+		--source-pattern "src/**/*.java" \
+		--html specification_compliance_report.html

specification

@@ -75,7 +75,13 @@ private DecryptionMaterials prepareMaterialsFromRequest(final GetObjectRequest g
         }
 
         AlgorithmSuite algorithmSuite = contentMetadata.algorithmSuite();
+        //= specification/s3-encryption/client.md#enable-legacy-unauthenticated-modes
+        //= type=implication
+        //# When enabled, the S3EC MUST be able to decrypt objects encrypted with all content encryption algorithms (both legacy and fully supported).
         if (!_enableLegacyUnauthenticatedModes && algorithmSuite.isLegacy()) {
+            //= specification/s3-encryption/client.md#enable-legacy-unauthenticated-modes
+            //= type=exception
+            //# When disabled, the S3EC MUST NOT decrypt objects encrypted using legacy content encryption algorithms; it MUST throw an exception when attempting to decrypt an object encrypted with a legacy content encryption algorithm.
             throw new S3EncryptionClientException("Enable legacy unauthenticated modes to use legacy content decryption: " + algorithmSuite.cipherName());
         }
 
@@ -146,11 +152,17 @@ public void onStream(SdkPublisher<ByteBuffer> ciphertextPublisher) {
             if (algorithmSuite.equals(AlgorithmSuite.ALG_AES_256_CBC_IV16_NO_KDF)
                     || algorithmSuite.equals(AlgorithmSuite.ALG_AES_256_CTR_IV16_TAG16_NO_KDF)
                     || _enableDelayedAuthentication) {
+                //= specification/s3-encryption/client.md#enable-delayed-authentication
+                //= type=implication
+                //# When enabled, the S3EC MAY release plaintext from a stream which has not been authenticated.
                 // CBC and GCM with delayed auth enabled use a standard publisher
                 CipherPublisher plaintextPublisher = new CipherPublisher(ciphertextPublisher,
                         getObjectResponse.contentLength(), desiredRange, contentMetadata.contentRange(), algorithmSuite.cipherTagLengthBits(), materials, iv);
                 wrappedAsyncResponseTransformer.onStream(plaintextPublisher);
             } else {
+                //= specification/s3-encryption/client.md#enable-delayed-authentication
+                //= type=implication
+                //# When disabled the S3EC MUST NOT release plaintext from a stream which has not been authenticated.
                 // Use buffered publisher for GCM when delayed auth is not enabled
                 BufferedCipherPublisher plaintextPublisher = new BufferedCipherPublisher(ciphertextPublisher,
                         getObjectResponse.contentLength(), materials, iv, _bufferSize);

src/main/java/software/amazon/encryption/s3/S3EncryptionClient.java

@@ -74,6 +74,9 @@ public CreateMultipartUploadResponse createMultipartUpload(CreateMultipartUpload
                 .overrideConfiguration(API_NAME_INTERCEPTOR)
                 .build();
 
+        //= specification/s3-encryption/client.md#api-operations
+        //= type=implication
+        //# If implemented, CreateMultipartUpload MUST initiate a multipart upload.
         CreateMultipartUploadResponse response = _s3AsyncClient.createMultipartUpload(request).join();
 
         MultipartUploadMaterials mpuMaterials = MultipartUploadMaterials.builder()
@@ -133,13 +136,21 @@ public UploadPartResponse uploadPart(UploadPartRequest request, RequestBody requ
             throw new S3EncryptionClientException("No client-side information available on upload ID " + uploadId);
         }
         final UploadPartResponse response;
-        // Checks the parts are uploaded in series
+        //= specification/s3-encryption/client.md#api-operations
+        //= type=implication
+        //# Each part MUST be encrypted in sequence.
         materials.beginPartUpload(actualRequest.partNumber(), partContentLength);
+        //= specification/s3-encryption/client.md#api-operations
+        //= type=implication
+        //# Each part MUST be encrypted using the same cipher instance for each part.
         Cipher cipher = materials.getCipher(materials.getIv());
 
         ExecutorService singleThreadExecutor = Executors.newSingleThreadExecutor();
 
         try {
+            //= specification/s3-encryption/client.md#api-operations
+            //= type=implication
+            //# UploadPart MUST encrypt each part.
             final AsyncRequestBody cipherAsyncRequestBody = new CipherAsyncRequestBody(
                 AsyncRequestBody.fromInputStream(
                     requestBody.contentStreamProvider().newStream(),
@@ -159,6 +170,8 @@ public UploadPartResponse uploadPart(UploadPartRequest request, RequestBody requ
             }
             // Ensures parts are not retried to avoid corrupting ciphertext
             AsyncRequestBody noRetryBody = new NoRetriesAsyncRequestBody(cipherAsyncRequestBody);
+            //= specification/s3-encryption/client.md#api-operations
+            //= type=implication
             response =  _s3AsyncClient.uploadPart(actualRequest, noRetryBody).join();
         } finally {
             materials.endPartUpload();
@@ -187,6 +200,9 @@ public CompleteMultipartUploadResponse completeMultipartUpload(CompleteMultipart
                 .overrideConfiguration(API_NAME_INTERCEPTOR)
                 .build();
 
+        //= specification/s3-encryption/client.md#api-operations
+        //= type=implication
+        //# CompleteMultipartUpload MUST complete the multipart upload.
         CompleteMultipartUploadResponse response = _s3AsyncClient.completeMultipartUpload(actualRequest).join();
 
         _multipartUploadMaterials.remove(uploadId);
@@ -198,6 +214,9 @@ public AbortMultipartUploadResponse abortMultipartUpload(AbortMultipartUploadReq
         AbortMultipartUploadRequest actualRequest = request.toBuilder()
                 .overrideConfiguration(API_NAME_INTERCEPTOR)
                 .build();
+        //= specification/s3-encryption/client.md#api-operations
+        //= type=implication
+        //# AbortMultipartUpload MUST abort the multipart upload.
         return _s3AsyncClient.abortMultipartUpload(actualRequest).join();
     }
 

src/main/java/software/amazon/encryption/s3/internal/GetEncryptedObjectPipeline.java

@@ -5,13 +5,13 @@
 import edu.umd.cs.findbugs.annotations.SuppressFBWarnings;
 import software.amazon.encryption.s3.S3EncryptionClientException;
 
+import javax.crypto.SecretKey;
 import java.nio.charset.StandardCharsets;
 import java.security.GeneralSecurityException;
 import java.security.SecureRandom;
 import java.util.ArrayList;
 import java.util.List;
 import java.util.Map;
-import javax.crypto.SecretKey;
 
 /**
  * This serves as the base class for all the keyrings in the S3 encryption client.
@@ -115,7 +115,13 @@ public DecryptionMaterials onDecrypt(final DecryptionMaterials materials, List<E
             throw new S3EncryptionClientException("The keyring does not support the object's key wrapping algorithm: " + keyProviderInfo);
         }
 
+        //= specification/s3-encryption/client.md#enable-legacy-wrapping-algorithms
+        //= type=implication
+        //# When enabled, the S3EC MUST be able to decrypt objects encrypted with all supported wrapping algorithms (both legacy and fully supported).
         if (decryptStrategy.isLegacy() && !_enableLegacyWrappingAlgorithms) {
+            //= specification/s3-encryption/client.md#enable-legacy-wrapping-algorithms
+            //= type=exception
+            //# When disabled, the S3EC MUST NOT decrypt objects encrypted using legacy wrapping algorithms; it MUST throw an exception when attempting to decrypt an object encrypted with a legacy wrapping algorithm.
             throw new S3EncryptionClientException("Enable legacy wrapping algorithms to use legacy key wrapping algorithm: " + keyProviderInfo);
         }
 

src/main/java/software/amazon/encryption/s3/internal/MultipartUploadObjectPipeline.java

@@ -836,6 +836,9 @@ public void AesCbcV1toV3FailsWhenUnauthencticateModeDisabled() {
         final String input = "AesCbcV1toV3";
         v1Client.putObject(BUCKET, objectKey, input);
 
+        //= specification/s3-encryption/client.md#enable-legacy-wrapping-algorithms
+        //= type=test
+        //# When disabled, the S3EC MUST NOT decrypt objects encrypted using legacy wrapping algorithms; it MUST throw an exception when attempting to decrypt an object encrypted with a legacy wrapping algorithm.
         assertThrows(S3EncryptionClientException.class, () -> v3Client.getObjectAsBytes(builder -> builder
                 .bucket(BUCKET)
                 .key(objectKey)));
@@ -870,6 +873,9 @@ public void AesCbcV1toV3FailsWhenLegacyKeyringDisabled() {
         final String input = "AesCbcV1toV3";
         v1Client.putObject(BUCKET, objectKey, input);
 
+        //= specification/s3-encryption/client.md#enable-legacy-wrapping-algorithms
+        //= type=test
+        //# When disabled, the S3EC MUST NOT decrypt objects encrypted using legacy wrapping algorithms; it MUST throw an exception when attempting to decrypt an object encrypted with a legacy wrapping algorithm.
         assertThrows(S3EncryptionClientException.class, () -> v3Client.getObjectAsBytes(builder -> builder
                 .bucket(BUCKET)
                 .key(objectKey)));

src/main/java/software/amazon/encryption/s3/materials/S3Keyring.java

@@ -41,6 +41,8 @@
 import software.amazon.encryption.s3.materials.CryptographicMaterialsManager;
 import software.amazon.encryption.s3.materials.DefaultCryptoMaterialsManager;
 import software.amazon.encryption.s3.materials.KmsKeyring;
+import software.amazon.encryption.s3.materials.PartialRsaKeyPair;
+import software.amazon.encryption.s3.materials.RsaKeyring;
 import software.amazon.encryption.s3.utils.BoundedInputStream;
 import software.amazon.encryption.s3.utils.S3EncryptionClientTestResources;
 
@@ -98,6 +100,9 @@ public static void setUp() throws NoSuchAlgorithmException {
         RSA_KEY_PAIR = keyPairGen.generateKeyPair();
     }
 
+    //= specification/s3-encryption/client.md#aws-sdk-compatibility
+    //= type=test
+    //# The S3EC SHOULD support invoking operations unrelated to client-side encryption e.g. CopyObject as the conventional AWS SDK S3 client would.
     @Test
     public void copyObjectTransparently() {
         final String objectKey = appendTestSuffix("copy-object-from-here");
@@ -161,10 +166,15 @@ public void deleteObjectWithInstructionFileSuccess() {
         v3Client.deleteObject(builder -> builder.bucket(BUCKET).key(objectKey));
 
         S3Client s3Client = S3Client.builder().build();
-        // Assert throw NoSuchKeyException when getObject for objectKey
+        //= specification/s3-encryption/client.md#api-operations
+        //= type=test
+        //# DeleteObject MUST delete the given object key.
         assertThrows(S3Exception.class, () -> s3Client.getObject(builder -> builder
                 .bucket(BUCKET)
                 .key(objectKey)));
+        //= specification/s3-encryption/client.md#api-operations
+        //= type=test
+        //# DeleteObject MUST delete the associated instruction file using the default instruction file suffix.
         assertThrows(S3Exception.class, () -> s3Client.getObject(builder -> builder
                 .bucket(BUCKET)
                 .key(objectKey + ".instruction")));
@@ -208,10 +218,15 @@ public void deleteObjectsWithInstructionFilesSuccess() {
                 .delete(builder1 -> builder1.objects(objects)));
 
         S3Client s3Client = S3Client.builder().build();
-        // Assert throw NoSuchKeyException when getObject for any of objectKeys
+        //= specification/s3-encryption/client.md#api-operations
+        //= type=test
+        //# DeleteObjects MUST delete each of the given objects.
         assertThrows(S3Exception.class, () -> s3Client.getObject(builder -> builder
                 .bucket(BUCKET)
                 .key(objectKeys[0])));
+        //= specification/s3-encryption/client.md#api-operations
+        //= type=test
+        //# DeleteObjects MUST delete each of the corresponding instruction files using the default instruction file suffix.
         assertThrows(S3Exception.class, () -> s3Client.getObject(builder -> builder
                 .bucket(BUCKET)
                 .key(objectKeys[0] + ".instruction")));
@@ -292,6 +307,9 @@ public void getNonExistentObject() {
         v3Client.close();
     }
 
+    //= specification/s3-encryption/client.md#cryptographic-materials
+    //= type=test
+    //# The S3EC MUST accept either one CMM or one Keyring instance upon initialization.
     @Test
     public void s3EncryptionClientWithMultipleKeyringsFails() {
         assertThrows(S3EncryptionClientException.class, () -> S3EncryptionClient.builder()
@@ -300,6 +318,22 @@ public void s3EncryptionClientWithMultipleKeyringsFails() {
                 .build());
     }
 
+    //= specification/s3-encryption/client.md#cryptographic-materials
+    //= type=test
+    //# If both a CMM and a Keyring are provided, the S3EC MUST throw an exception.
+    @Test
+    public void s3EncryptionClientWithCMMAndKeyringFails() {
+        CryptographicMaterialsManager defaultCMM = DefaultCryptoMaterialsManager.builder()
+                        .keyring(RsaKeyring.builder()
+                                .wrappingKeyPair(new PartialRsaKeyPair(RSA_KEY_PAIR))
+                                .build())
+                                .build();
+        assertThrows(S3EncryptionClientException.class, () -> S3EncryptionClient.builder()
+                .aesKey(AES_KEY)
+                .cryptoMaterialsManager(defaultCMM)
+                .build());
+    }
+
     @Test
     public void s3EncryptionClientWithNoKeyringsFails() {
         assertThrows(S3EncryptionClientException.class, () -> S3EncryptionClient.builder()
@@ -440,6 +474,9 @@ public void s3EncryptionClientWithCmmFromKmsKeyIdSucceeds() {
         v3Client.close();
     }
 
+    //= specification/s3-encryption/client.md#wrapped-s3-client-s
+    //= type=test
+    //# The S3EC MUST support the option to provide an SDK S3 client instance during its initialization.
     @Test
     public void s3EncryptionClientWithWrappedS3ClientSucceeds() {
         final String objectKey = appendTestSuffix("wrapped-s3-client-with-kms-key-id");
@@ -462,6 +499,9 @@ public void s3EncryptionClientWithWrappedS3ClientSucceeds() {
         wrappingClient.close();
     }
 
+    //= specification/s3-encryption/client.md#wrapped-s3-client-s
+    //= type=test
+    //# The S3EC MUST NOT support use of S3EC as the provided S3 client during its initialization; it MUST throw an exception in this case.
     /**
      * S3EncryptionClient implements S3Client, so it can be passed into the builder as a wrappedClient.
      * However, is not a supported use case, and the builder should throw an exception if this happens.
@@ -854,6 +894,9 @@ public void s3EncryptionClientTopLevelCredentialsNullCreds() {
         }
     }
 
+    //= specification/s3-encryption/client.md#inherited-sdk-configuration
+    //= type=test
+    //# If the S3EC accepts SDK client configuration, the configuration MUST be applied to all wrapped SDK clients including the KMS client.
     @Test
     public void s3EncryptionClientTopLevelAlternateCredentials() {
         final String objectKey = appendTestSuffix("wrapped-s3-client-with-top-level-credentials");