git rebase and setup.sh changes to pass securityEnabled parameters to bicep

Author: sjyang18

deploy/README.md

@@ -48,6 +48,12 @@ Steps 2 through 4 can instead be deployed using a single script below:
 ./deploy/setup.sh <environmentCode> <location> <pipelineName> <envTag>
 
 ```
+To enabled security features like Synapse managed VNET, managed private endpoints, and privates enpoints to 3 synapse endpoints, set 'SECURITY_ENABLED=true' when running setup.sh:
+```
+SECURITY_ENABLED=true ./deploy/setup.sh <environmentCode> <location> <pipelineName> <envTag>
+```
+**Note that if you turn on SECURITY_ENABLED during setup, Synapse endpoints are restricted to the custom VNET in the deployment environment. Thus, you need to create a Windows jumpbox inside the VNET to connect to Synapse Studio.**
+
 If you like to package other pipelines or re-package an updated/modified pipeline, follow the instructions under `Packaging the Synapse pipeline` section. The script mentioned in that section can be rerun multiple times.
 
 Arguments | Required | Sample value
@@ -83,6 +89,12 @@ To install infrastructure execute install.sh script as follows
 
 ```
 
+To enabled security features like Synapse managed VNET, managed private endpoints, and privates enpoints to 3 synapse endpoints, set 'SECURITY_ENABLED=true' when running install.sh:
+```
+SECURITY_ENABLED=true ./deploy/install.sh <environmentCode> <location> <envTag>
+```
+**Note that if you turn on SECURITY_ENABLED during setup, Synapse endpoints are restricted to the custom VNET in the deployment environment. Thus, you need to create a Windows jumpbox inside the VNET to connect to Synapse Studio.**
+
 Default values for the parameters are provided in the script itself.
 
 Arguments | Required | Sample value
@@ -113,6 +125,11 @@ For eg.
 az deployment sub create -l <region> -n aoi -f main.bicep -p location=<region> environmentCode=aoi environment=synapse-aoi
 ```
 
+To enabled security features like Synapse managed VNET, managed private endpoints, and privates enpoints to 3 Synapse endpoints, pass parameter 'securityEnabled=true' when running bicep:
+```
+bash
+az deployment sub create -l <region_name> -n <deployment_name> -f main.bicep -p location=<region_name> environmentCode=<environment_name_prefix> environment=<tag_value> securityEnabled=true
+```
 
 ## Configuring the Resources
 
@@ -142,7 +159,10 @@ Once the above step completes, a zip file is generated. Upload the generated zip
 4. When prompted to select a file, pick the zip file generated in the previous step
 5. Pipelines and its dependencies are imported to the Synapse Studio. Validate the components being imported for any errors
 6. Click "Publish all" and wait for the imported components to be published
-NOTE: You may run into this error during import "Invalid template, please check the template file". It is a known issue that we are working on with the product team. In the interim, we suggest importing from Git Repository as described below.  
+NOTE: You may run into this error during import "Invalid template, please check the template file". It is a known issue that we are working on with the product team. In the interim, we suggest importing from Git Repository as described below.
+
+**Note that if you turn on SECURITY_ENABLED during setup, Synapse endpoints are restricted to the custom VNET in the deployment environment. Thus, you need to create a Windows jumpbox inside the VNET to connect to Synapse Studio and import the package.**
+
 ## Importing from Git Repository
 
 Another way to get import pipeline into the Synape Studio is through Source Control repository like GitHub or Azure DevOps repository. Refer to the document on [Source Control](https://docs.microsoft.com/azure/synapse-analytics/cicd/source-control) to learn about Git Integration for Azure Synapse Analytics and how to setup.

deploy/addManagedPE.sh

@@ -10,6 +10,25 @@ if [[ -z "$1" ]]
     exit 1
 fi
 ENVCODE=$1
+PE_APPROVAL_DESCRIPTION="Approved by script"
+
+approved_managed_private_endpoint_request_exists() {
+    local groupId=$1
+    local resourceName=$2
+    local resourceType=$3
+
+    local peList=$(az network private-endpoint-connection list \
+        -g $groupId -n $resourceName --type $resourceType -ojson 2>/dev/null || echo '')
+    local result=''
+    
+    if [[ -n peList ]];
+    then
+        result=$( echo $peList \
+            | jq -r ".[] | select (.properties.privateLinkServiceConnectionState.description == \"${PE_APPROVAL_DESCRIPTION}\").id" )
+        echo $result
+    fi
+    echo $result     
+}
 
 create_synapase_managed_private_endpoint() {
     local tmpfile=$(mktemp)
@@ -63,7 +82,7 @@ approve_synapase_managed_private_endpoint() {
         if [[ $PE_CONNECTION_APPROVAL_STATUS != "Approved" ]];
         then
             az network private-endpoint-connection approve \
-                --id $PE_CONNECTION_ID --description "Approved by script"
+                --id $PE_CONNECTION_ID --description "$PE_APPROVAL_DESCRIPTION"
             echo "$PE_CONNECTION_ID got approved"
         fi
     fi
@@ -76,7 +95,11 @@ do
    sleep 30
    SYNAPSE_STORAGE_ACCT=$(az storage account list --query "[?tags.store && tags.store == 'synapse'].name" -o tsv -g $ENVCODE-pipeline-rg)
 done
-approve_synapase_managed_private_endpoint $ENVCODE-pipeline-rg $SYNAPSE_STORAGE_ACCT "Microsoft.Storage/storageAccounts"
+result=$(approved_managed_private_endpoint_request_exists $ENVCODE-pipeline-rg $SYNAPSE_STORAGE_ACCT "Microsoft.Storage/storageAccounts")
+if [[ -z $result ]];
+then
+    approve_synapase_managed_private_endpoint $ENVCODE-pipeline-rg $SYNAPSE_STORAGE_ACCT "Microsoft.Storage/storageAccounts"
+fi
 
 # Create Managed Private Endpoints (PE) if not exist
 PIPELINE_KV=$(az keyvault list --query "[?tags.usage && tags.usage == 'linkedService']" -ojson -g $ENVCODE-pipeline-rg)
@@ -87,7 +110,12 @@ do
 done
 PIPELINE_KV_NAME=$(echo $PIPELINE_KV | jq -r '.[0].name')
 PIPELINE_KV_ID=$(echo $PIPELINE_KV | jq -r '.[0].id')
-create_synapase_managed_private_endpoint "$ENVCODE-pipeline-syn-ws" "$ENVCODE-mpe-pipeline-kv" "vault" "$PIPELINE_KV_ID"
+result=$(approved_managed_private_endpoint_request_exists $ENVCODE-pipeline-rg $PIPELINE_KV_NAME "Microsoft.Keyvault/vaults")
+if [[ -z $result ]]
+then
+    create_synapase_managed_private_endpoint "$ENVCODE-pipeline-syn-ws" "$ENVCODE-mpe-pipeline-kv" "vault" "$PIPELINE_KV_ID"
+    approve_synapase_managed_private_endpoint $ENVCODE-pipeline-rg $PIPELINE_KV_NAME "Microsoft.Keyvault/vaults"
+fi 
 
 DATA_STORAGE_ACCT=$(az storage account list --query "[?tags.store && tags.store == 'raw']" -ojson -g $ENVCODE-data-rg)
 while [[ $DATA_STORAGE_ACCT == '[]' ]]
@@ -97,7 +125,12 @@ do
 done
 DATA_STORAGE_ACCT_NAME=$(echo $DATA_STORAGE_ACCT | jq -r '.[0].name')
 DATA_STORAGE_ACCT_ID=$(echo $DATA_STORAGE_ACCT | jq -r '.[0].id')
-create_synapase_managed_private_endpoint "$ENVCODE-pipeline-syn-ws" "$ENVCODE-mpe-data-raw" "dfs" "$DATA_STORAGE_ACCT_ID"
+result=$(approved_managed_private_endpoint_request_exists $ENVCODE-data-rg $DATA_STORAGE_ACCT_NAME "Microsoft.Storage/storageAccounts")
+if [[ -z $result ]]
+then
+    create_synapase_managed_private_endpoint "$ENVCODE-pipeline-syn-ws" "$ENVCODE-mpe-data-raw" "dfs" "$DATA_STORAGE_ACCT_ID"
+    approve_synapase_managed_private_endpoint $ENVCODE-data-rg $DATA_STORAGE_ACCT_NAME "Microsoft.Storage/storageAccounts"
+fi
 
 DATA_KV=$(az keyvault list --query "[?tags.usage && tags.usage == 'general']" -ojson -g $ENVCODE-data-rg)
 while [[ $DATA_KV == '[]' ]];
@@ -107,10 +140,9 @@ do
 done
 DATA_KV_NAME=$(echo $DATA_KV | jq -r '.[0].name')
 DATA_KV_ID=$(echo $DATA_KV | jq -r '.[0].id')
-create_synapase_managed_private_endpoint "$ENVCODE-pipeline-syn-ws" "$ENVCODE-mpe-data-kv" "vault" "$DATA_KV_ID"
-
-
-# Approve remaining Managed Private Endpoints (PE)
-approve_synapase_managed_private_endpoint $ENVCODE-pipeline-rg $PIPELINE_KV_NAME "Microsoft.Keyvault/vaults"
-approve_synapase_managed_private_endpoint $ENVCODE-data-rg $DATA_STORAGE_ACCT_NAME "Microsoft.Storage/storageAccounts"
-approve_synapase_managed_private_endpoint $ENVCODE-data-rg $DATA_KV_NAME "Microsoft.Keyvault/vaults"
+result=$(approved_managed_private_endpoint_request_exists $ENVCODE-data-rg $DATA_KV_NAME "Microsoft.Keyvault/vaults")
+if [[ -z $result ]]
+then
+    create_synapase_managed_private_endpoint "$ENVCODE-pipeline-syn-ws" "$ENVCODE-mpe-data-kv" "vault" "$DATA_KV_ID"
+    approve_synapase_managed_private_endpoint $ENVCODE-data-rg $DATA_KV_NAME "Microsoft.Keyvault/vaults"
+fi

deploy/infra/security-addons.bicep

@@ -91,11 +91,6 @@ module addSynapseSqlOnDemandPrivateEndpoint 'modules/privateendpoints.bicep' = {
   ]
 }
 
-output customVnetId string = customVnet.id
-output customVnetName string = customVnet.name
-output pipelineSubnetId string = pipelineSubnet.id
-output pipelineSubnetName string = pipelineSubnet.name
-output synapseWorkspaceProperties object = synapseWorkspace.properties
 
 
 

deploy/install.sh

@@ -22,17 +22,17 @@ envCode=${envCode:-"${1}"}
 location=${location:-"${2}"}
 envTag=${envTag:-"synapse-${envCode}"}
 deploymentName=${3:-"${envTag}-deploy"}
-securityEnabled=${securityEnabled:-false}
-preventDataExfiltration=${preventDataExfiltration:-false}
+SECURITY_ENABLED=${SECURITY_ENABLED:-false}
+PREVENT_DATA_EXFILTRATION=${PREVENT_DATA_EXFILTRATION:-false}
 
 DEPLOYMENT_SCRIPT="az deployment sub create -l $location -n $deploymentName \
     -f ./deploy/infra/main.bicep \
     -p \
     location=$location \
     environmentCode=$envCode \
     environment=$envTag \
-    securityEnabled=$securityEnabled \
-    preventDataExfiltration=$preventDataExfiltration"
+    securityEnabled=$SECURITY_ENABLED \
+    preventDataExfiltration=$PREVENT_DATA_EXFILTRATION"
 $DEPLOYMENT_SCRIPT
 
 if [[ $securityEnabled ]]

deploy/setup.sh

@@ -8,6 +8,8 @@ LOCATION=$2
 PIPELINE_NAME=$3
 ENVTAG=$4
 
+export SECURITY_ENABLED=${SECURITY_ENABLED:-false}
+export PREVENT_DATA_EXFILTRATION=${PREVENT_DATA_EXFILTRATION:-false}
 
 set -x