Version 1.0.17

Fixes an issue where BatchGrantPermissions was not present, causing all back-permission grants from mesh to producer to fail and resulting in a None table reference.

Author: IanMeyers

setup.cfg

@@ -1,6 +1,6 @@
 [metadata]
 name = aws-data-mesh-utils
-version = 1.0.16
+version = 1.0.17
 author = Ian Meyers
 author_email = [email protected]
 license = Apache 2.0

setup.py

@@ -3,10 +3,10 @@
 from setuptools import find_packages, setup
 
 setup(
-    install_requires=['boto3~=1.20.51',
+    install_requires=['boto3~=1.26.121',
                       'pystache~=0.6.0',
-                      'setuptools~=60.8.1',
-                      'shortuuid~=1.0.8',
-                      'botocore~=1.23.51'],
+                      'setuptools~=67.7.2',
+                      'shortuuid~=1.0.11',
+                      'botocore~=1.29.121'],
     include_package_data=True
 )

src/data_mesh_util/DataMeshProducer.py

@@ -89,7 +89,7 @@ def _create_mesh_table(self, table_def: dict, data_mesh_glue_client, source_data
                            data_mesh_database_name: str,
                            producer_account_id: str,
                            data_mesh_account_id: str, create_public_metadata: bool = True,
-                           expose_table_references_with_suffix: str = "_link", use_original_table_name: bool = False):
+                           expose_table_references_with_suffix: str = "_link", use_original_table_name: bool = False) -> tuple:
         '''
         API to create a table as a data product in the data mesh
         :param table_def:
@@ -162,28 +162,27 @@ def _create_mesh_table(self, table_def: dict, data_mesh_glue_client, source_data
             self._logger.info(f"Granted Describe on {table_name} to {DATA_MESH_READONLY_ROLENAME}")
 
         # in the producer account, accept the RAM share after 1 second - seems to be an async delay
-        if permissions_granted > 0:
-            time.sleep(1)
-            self._producer_automator.accept_pending_lf_resource_shares(
-                sender_account=data_mesh_account_id
-            )
+        time.sleep(1)
+        self._producer_automator.accept_pending_lf_resource_shares(
+            sender_account=data_mesh_account_id
+        )
 
-            # create a resource link for the data mesh table in producer account
-            if use_original_table_name is True:
-                link_table_name = table_name
-            else:
-                link_table_name = f"{table_name}_link"
-                if expose_table_references_with_suffix is not None:
-                    link_table_name = f"{table_name}{expose_table_references_with_suffix}"
+        # create a resource link for the data mesh table in producer account
+        if use_original_table_name is True:
+            link_table_name = table_name
+        else:
+            link_table_name = f"{table_name}_link"
+            if expose_table_references_with_suffix is not None:
+                link_table_name = f"{table_name}{expose_table_references_with_suffix}"
 
-            self._producer_automator.create_remote_table(
-                data_mesh_account_id=self._data_mesh_account_id,
-                database_name=data_mesh_database_name,
-                local_table_name=link_table_name,
-                remote_table_name=table_name
-            )
+        self._producer_automator.create_remote_table(
+            data_mesh_account_id=self._data_mesh_account_id,
+            database_name=data_mesh_database_name,
+            local_table_name=link_table_name,
+            remote_table_name=table_name
+        )
 
-            return table_name, link_table_name
+        return table_name, link_table_name
 
     def _make_database_name(self, database_name: str):
         return "%s-%s" % (database_name, self._data_producer_identity.get('Account'))

src/data_mesh_util/lib/ApiAutomator.py

@@ -927,7 +927,11 @@ def lf_batch_grant_permissions(self,
         if 'Failures' in response:
             perms_added -= len(response.get('Failures'))
 
-        return perms_added
+        if perms_added == 0:
+            self._logger.error(response.get('Failures'))
+            raise Exception(f"Failed to grant permissions on Account {data_mesh_account_id}")
+        else:
+            return perms_added
 
     def lf_grant_permissions(self, data_mesh_account_id: str, principal: str, database_name: str,
                              table_name: str = None,

src/data_mesh_util/resource/producer_account_policy.pystache

@@ -41,6 +41,7 @@
             "Sid": "ProducerAccess4",
             "Effect": "Allow",
             "Action": [
+                "lakeformation:BatchGrantPermissions",
                 "lakeformation:GrantPermissions",
                 "lakeformation:GetResourceLFTags",
                 "lakeformation:GetLFTag"