From e0335ed2cf356cc31eae3e903ebda99ac634a4dd Mon Sep 17 00:00:00 2001 From: Seokwon Yang Date: Wed, 20 Apr 2022 14:53:53 -0700 Subject: [PATCH 1/6] adding synapse managed vnet feature --- deploy/addManagedPE.sh | 116 +++++++++++++++++++ deploy/infra/groups/pipeline.bicep | 6 +- deploy/infra/main.bicep | 8 ++ deploy/infra/modules/privateendpoints.bicep | 60 ++++++++++ deploy/infra/modules/privatelink.bicep | 23 ++++ deploy/infra/modules/synapse.workspace.bicep | 61 ++++++---- deploy/infra/security-addons.bicep | 106 +++++++++++++++++ deploy/install.sh | 27 ++++- 8 files changed, 382 insertions(+), 25 deletions(-) create mode 100755 deploy/addManagedPE.sh create mode 100644 deploy/infra/modules/privateendpoints.bicep create mode 100644 deploy/infra/modules/privatelink.bicep create mode 100644 deploy/infra/security-addons.bicep diff --git a/deploy/addManagedPE.sh b/deploy/addManagedPE.sh new file mode 100755 index 0000000..dc49732 --- /dev/null +++ b/deploy/addManagedPE.sh @@ -0,0 +1,116 @@ +#!/usr/bin/env bash + +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +set +x +if [[ -z "$1" ]] + then + echo "Environment Code value not supplied" + exit 1 +fi +ENVCODE=$1 + +create_synapase_managed_private_endpoint() { + local tmpfile=$(mktemp) + local synpaseWorkspace=$1 + local peName=$2 + local groupId=$3 + local privateLinkResourceId=$4 + + echo "creating MPE if not exist for $peName" + # check if peName exists + local checkPeExists=$(az synapse managed-private-endpoints show \ + --pe-name $peName -otsv --query "id" --workspace-name $synpaseWorkspace 2>/dev/null || echo '') + + if [[ -z $checkPeExists ]]; + then + jq -n -r \ + --arg groupId "$groupId" \ + --arg privateLinkResourceId "$privateLinkResourceId" \ + '{groupId:$groupId, privateLinkResourceId:$privateLinkResourceId}' > $tmpfile + + az synapse managed-private-endpoints create \ + --file @$tmpfile \ + --pe-name $peName \ + --workspace-name $1 + + sleep 60 + local provisioningState=$(az synapse managed-private-endpoints show --pe-name $peName \ + --workspace-name $synpaseWorkspace -o tsv --query "properties.provisioningState") + while [[ $provisioningState != "Succeeded" ]]; + do + sleep 10 + provisioningState=$(az synapse managed-private-endpoints show --pe-name $peName \ + --workspace-name $synpaseWorkspace -o tsv --query "properties.provisioningState") + echo "provisioningState of $peName: $provisioningState" + done + fi +} + +approve_synapase_managed_private_endpoint() { + local resourceGroup=$1 + local resourceName=$2 + local resourceType=$3 + + local PE_CONNECTION=$(az network private-endpoint-connection list -g $resourceGroup -n $resourceName \ + --type $resourceType --query "[0]" -ojson 2>/dev/null || echo '') + if [[ -n $PE_CONNECTION ]]; + then + local PE_CONNECTION_ID=$(echo $PE_CONNECTION | jq -r '.id') + local PE_CONNECTION_APPROVAL_STATUS=$(echo $PE_CONNECTION | jq -r '.properties.privateLinkServiceConnectionState.status') + + if [[ $PE_CONNECTION_APPROVAL_STATUS != "Approved" ]]; + then + az network private-endpoint-connection approve \ + --id $PE_CONNECTION_ID --description "Approved by script" + echo "$PE_CONNECTION_ID got approved" + fi + fi +} + +# wait for SYNAPSE_STORAGE_ACCT showing up in azcli and approve its managed private endpoint first. +SYNAPSE_STORAGE_ACCT=$(az storage account list --query "[?tags.store && tags.store == 'synapse'].name" -o tsv -g $ENVCODE-pipeline-rg) +while [[ -z $SYNAPSE_STORAGE_ACCT ]]; +do + sleep 30 + SYNAPSE_STORAGE_ACCT=$(az storage account list --query "[?tags.store && tags.store == 'synapse'].name" -o tsv -g $ENVCODE-pipeline-rg) +done +approve_synapase_managed_private_endpoint $ENVCODE-pipeline-rg $SYNAPSE_STORAGE_ACCT "Microsoft.Storage/storageAccounts" + +# Create Managed Private Endpoints (PE) if not exist +PIPELINE_KV=$(az keyvault list --query "[?tags.usage && tags.usage == 'linkedService']" -ojson -g $ENVCODE-pipeline-rg) +while [[ $PIPELINE_KV == '[]' ]]; +do + sleep 30 + PIPELINE_KV=$(az keyvault list --query "[?tags.usage && tags.usage == 'linkedService']" -ojson -g $ENVCODE-pipeline-rg) +done +PIPELINE_KV_NAME=$(echo $PIPELINE_KV | jq -r '.[0].name') +PIPELINE_KV_ID=$(echo $PIPELINE_KV | jq -r '.[0].id') +create_synapase_managed_private_endpoint "$ENVCODE-pipeline-syn-ws" "$ENVCODE-mpe-pipeline-kv" "vault" "$PIPELINE_KV_ID" + +DATA_STORAGE_ACCT=$(az storage account list --query "[?tags.store && tags.store == 'raw']" -ojson -g $ENVCODE-data-rg) +while [[ $DATA_STORAGE_ACCT == '[]' ]] +do + sleep 30 + DATA_STORAGE_ACCT=$(az storage account list --query "[?tags.store && tags.store == 'raw']" -ojson -g $ENVCODE-data-rg) +done +DATA_STORAGE_ACCT_NAME=$(echo $DATA_STORAGE_ACCT | jq -r '.[0].name') +DATA_STORAGE_ACCT_ID=$(echo $DATA_STORAGE_ACCT | jq -r '.[0].id') +create_synapase_managed_private_endpoint "$ENVCODE-pipeline-syn-ws" "$ENVCODE-mpe-data-raw" "dfs" "$DATA_STORAGE_ACCT_ID" + +DATA_KV=$(az keyvault list --query "[?tags.usage && tags.usage == 'general']" -ojson -g $ENVCODE-data-rg) +while [[ $DATA_KV == '[]' ]]; +do + sleep 30 + DATA_KV=$(az keyvault list --query "[?tags.usage && tags.usage == 'general']" -ojson -g $ENVCODE-data-rg) +done +DATA_KV_NAME=$(echo $DATA_KV | jq -r '.[0].name') +DATA_KV_ID=$(echo $DATA_KV | jq -r '.[0].id') +create_synapase_managed_private_endpoint "$ENVCODE-pipeline-syn-ws" "$ENVCODE-mpe-data-kv" "vault" "$DATA_KV_ID" + + +# Approve remaining Managed Private Endpoints (PE) +approve_synapase_managed_private_endpoint $ENVCODE-pipeline-rg $PIPELINE_KV_NAME "Microsoft.Keyvault/vaults" +approve_synapase_managed_private_endpoint $ENVCODE-data-rg $DATA_STORAGE_ACCT_NAME "Microsoft.Storage/storageAccounts" +approve_synapase_managed_private_endpoint $ENVCODE-data-rg $DATA_KV_NAME "Microsoft.Keyvault/vaults" diff --git a/deploy/infra/groups/pipeline.bicep b/deploy/infra/groups/pipeline.bicep index 3bec0ec..8b13c96 100644 --- a/deploy/infra/groups/pipeline.bicep +++ b/deploy/infra/groups/pipeline.bicep @@ -84,6 +84,8 @@ param synapseMIStorageAccountRoles array = [ ] param logAnalyticsWorkspaceId string +param securityEnabled bool = false +param preventDataExfiltration bool = false var namingPrefix = '${environmentCode}-${projectName}' var synapseResourceGroupNameVar = empty(synapseResourceGroupName) ? '${namingPrefix}-rg' : synapseResourceGroupName @@ -129,8 +131,6 @@ module synapseHnsStorageAccount '../modules/storage.hns.bicep' = { } } - - module synapseWorkspace '../modules/synapse.workspace.bicep' = { name: '${namingPrefix}-workspace' params:{ @@ -153,6 +153,8 @@ module synapseWorkspace '../modules/synapse.workspace.bicep' = { gitRepoRootFolder: synapseGitRepoRootFolder gitRepoVstsTenantId: synapseGitRepoVstsTenantId gitRepoType: synapseGitRepoType + createManagedVnet: securityEnabled + preventDataExfiltration: preventDataExfiltration } dependsOn: [ synapseHnsStorageAccount diff --git a/deploy/infra/main.bicep b/deploy/infra/main.bicep index 5534073..68e57b8 100644 --- a/deploy/infra/main.bicep +++ b/deploy/infra/main.bicep @@ -14,6 +14,12 @@ param environmentCode string @description('Environment will be used as Tag on the resource group') param environment string +@description('Flag to set whether security resources such as Synapse managed vnet, NSG, etc are created or not') +param securityEnabled bool = false + +@description('preventDataExfiltration for Synapse managed vnet') +param preventDataExfiltration bool = false + @description('Used for naming of the network resource group and its resources') param networkModulePrefix string = 'network' @@ -105,6 +111,8 @@ module pipelineModule 'groups/pipeline.bicep' = { environmentCode: environmentCode environmentTag: environment logAnalyticsWorkspaceId: monitorModule.outputs.workspaceId + securityEnabled: securityEnabled + preventDataExfiltration: preventDataExfiltration } dependsOn: [ networkModule diff --git a/deploy/infra/modules/privateendpoints.bicep b/deploy/infra/modules/privateendpoints.bicep new file mode 100644 index 0000000..49c6441 --- /dev/null +++ b/deploy/infra/modules/privateendpoints.bicep @@ -0,0 +1,60 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT license. + +param environmentCode string +param location string +param subnetId string +param privateLinkServiceId string +param privateDnsZoneName string +param groupIds array + +resource privateDnsZone 'Microsoft.Network/privateDnsZones@2020-06-01' existing = { + name: privateDnsZoneName +} + +resource privateDnsZoneLink 'Microsoft.Network/privateDnsZones/virtualNetworkLinks@2020-06-01' existing = { + parent: privateDnsZone + name: privateDnsZoneName +} + +resource privateEndpoints 'Microsoft.Network/privateEndpoints@2021-05-01' = { + name: guid(environmentCode, privateLinkServiceId, groupIds[0]) + location: location + dependsOn: [ + privateDnsZoneLink + ] + properties: { + subnet: { + id: subnetId + } + privateLinkServiceConnections: [ + { + name: guid(environmentCode, privateLinkServiceId, groupIds[0]) + properties: { + privateLinkServiceId: privateLinkServiceId + groupIds: groupIds + privateLinkServiceConnectionState: { + status: 'Approved' + description: 'Auto-Approved' + actionsRequired: 'None' + } + } + } + ] + } +} + +resource privateDnsZoneGroup 'Microsoft.Network/privateEndpoints/privateDnsZoneGroups@2021-05-01' = { + parent: privateEndpoints + name: guid(environmentCode, privateLinkServiceId, groupIds[0]) + properties: { + privateDnsZoneConfigs: [ + { + name: privateDnsZone.name + properties: { + privateDnsZoneId: privateDnsZone.id + } + } + ] + } +} diff --git a/deploy/infra/modules/privatelink.bicep b/deploy/infra/modules/privatelink.bicep new file mode 100644 index 0000000..b6eb65c --- /dev/null +++ b/deploy/infra/modules/privatelink.bicep @@ -0,0 +1,23 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT license. + +param privateDnsZoneName string +param customVnetId string + +resource privateDnsZone 'Microsoft.Network/privateDnsZones@2020-06-01' = { + name: privateDnsZoneName + location: 'global' +} + +resource privateDnsZoneLink 'Microsoft.Network/privateDnsZones/virtualNetworkLinks@2020-06-01' = { + parent: privateDnsZone + name: privateDnsZoneName + location: 'global' + properties : { + registrationEnabled: false + virtualNetwork: { + id: customVnetId + } + } +} + diff --git a/deploy/infra/modules/synapse.workspace.bicep b/deploy/infra/modules/synapse.workspace.bicep index 4174dc0..7675ea2 100644 --- a/deploy/infra/modules/synapse.workspace.bicep +++ b/deploy/infra/modules/synapse.workspace.bicep @@ -26,6 +26,46 @@ param synapseSqlAdminPasswordSecretName string = 'synapse-sqladmin-password' param utcValue string = utcNow() param workspaceId string = 'default' +param createManagedVnet bool = false +@allowed([ + 'default' + '' +]) +param managedVirtualNetwork string = 'default' +param preventDataExfiltration bool = false +param managedVirtualNetworkSettings object = { + managedVirtualNetworkSettings : { + allowedAadTenantIdsForLinking: [] + preventDataExfiltration: preventDataExfiltration + } + managedVirtualNetwork : managedVirtualNetwork +} + +var defaultDataLakeStorageSettings = { + resourceId: hnsStorage.id + accountUrl: hnsStorage.properties.primaryEndpoints.dfs + filesystem: hnsStorageFileSystem + createManagedPrivateEndpoint: createManagedVnet +} + +var synapseCommonProperties = { + defaultDataLakeStorage: defaultDataLakeStorageSettings + sqlAdministratorLogin: sqlAdminLogin + sqlAdministratorLoginPassword: sqlAdminLoginPassword + workspaceRepositoryConfiguration:(empty(gitRepoType))? {}: { + accountName: gitRepoAccountName + collaborationBranch: gitRepoCollaborationBranch + hostName: gitRepoHostName + lastCommitId: gitRepoLastCommitId + projectName: gitRepoVstsProjectName + repositoryName: gitRepoRepositoryName + rootFolder: gitRepoRootFolder + tenantId: gitRepoVstsTenantId + type: gitRepoType + } +} +var selectedSynapseProperties = createManagedVnet ? union(synapseCommonProperties, managedVirtualNetworkSettings) : synapseCommonProperties + resource hnsStorage 'Microsoft.Storage/storageAccounts@2021-08-01' existing = { name: hnsStorageAccountName } @@ -40,26 +80,7 @@ resource synapseWorspace 'Microsoft.Synapse/workspaces@2021-06-01' = { identity: { type: 'SystemAssigned' } - properties: { - defaultDataLakeStorage: { - resourceId: hnsStorage.id - accountUrl: hnsStorage.properties.primaryEndpoints.dfs - filesystem: hnsStorageFileSystem - } - sqlAdministratorLogin: sqlAdminLogin - sqlAdministratorLoginPassword: sqlAdminLoginPassword - workspaceRepositoryConfiguration:(empty(gitRepoType))? {}: { - accountName: gitRepoAccountName - collaborationBranch: gitRepoCollaborationBranch - hostName: gitRepoHostName - lastCommitId: gitRepoLastCommitId - projectName: gitRepoVstsProjectName - repositoryName: gitRepoRepositoryName - rootFolder: gitRepoRootFolder - tenantId: gitRepoVstsTenantId - type: gitRepoType - } - } + properties: selectedSynapseProperties } resource synapseWorkspaceFwRules 'Microsoft.Synapse/workspaces/firewallRules@2021-06-01' = { diff --git a/deploy/infra/security-addons.bicep b/deploy/infra/security-addons.bicep new file mode 100644 index 0000000..cc74d1c --- /dev/null +++ b/deploy/infra/security-addons.bicep @@ -0,0 +1,106 @@ +targetScope='subscription' + +param environmentCode string +param location string +param synapseSuffix string = 'azuresynapse.net' +var pipelineRgName = '${environmentCode}-pipeline-rg' +var networkRgName = '${environmentCode}-network-rg' + +resource customVnet 'Microsoft.Network/virtualNetworks@2021-05-01' existing = { + name: '${environmentCode}-vnet' + scope: resourceGroup(networkRgName) +} +resource pipelineSubnet 'Microsoft.Network/virtualNetworks/subnets@2020-06-01' existing = { + name: '${environmentCode}-vnet/pipeline-subnet' + scope: resourceGroup(networkRgName) +} + +resource synapseWorkspace 'Microsoft.Synapse/workspaces@2021-06-01' existing = { + name: '${environmentCode}-pipeline-syn-ws' + scope: resourceGroup(pipelineRgName) +} + +module addSynapaseDevPrivateLink 'modules/privatelink.bicep' = { + name: '${environmentCode}-synpasews-dev-plink' + scope: resourceGroup(networkRgName) + params: { + customVnetId: customVnet.id + privateDnsZoneName: 'privatelink.dev.${synapseSuffix}' + } +} +module addSynapseSqlPrivateLink 'modules/privatelink.bicep' = { + name: '${environmentCode}-sql-plink' + scope: resourceGroup(networkRgName) + params: { + customVnetId: customVnet.id + privateDnsZoneName: 'privatelink.sql.${synapseSuffix}' + } +} + +module addSynapseDevPrivateEndpoint 'modules/privateendpoints.bicep' = { + name: '${environmentCode}-synpasews-dev-pe' + scope: resourceGroup(networkRgName) + params: { + environmentCode: environmentCode + location:location + subnetId: pipelineSubnet.id + privateLinkServiceId: synapseWorkspace.id + privateDnsZoneName: 'privatelink.dev.${synapseSuffix}' + groupIds: [ + 'Dev' + ] + } + dependsOn: [ + addSynapaseDevPrivateLink + ] +} + +module addSynapseSqlPrivateEndpoint 'modules/privateendpoints.bicep' = { + name: '${environmentCode}-synpasews-sql-pe' + scope: resourceGroup(networkRgName) + params: { + environmentCode: environmentCode + location:location + subnetId: pipelineSubnet.id + privateLinkServiceId: synapseWorkspace.id + privateDnsZoneName: 'privatelink.sql.${synapseSuffix}' + groupIds: [ + 'Sql' + ] + } + dependsOn: [ + addSynapseSqlPrivateLink + ] +} + +module addSynapseSqlOnDemandPrivateEndpoint 'modules/privateendpoints.bicep' = { + name: '${environmentCode}-synpasews-sql-ondemand-pe' + scope: resourceGroup(networkRgName) + params: { + environmentCode: environmentCode + location:location + subnetId: pipelineSubnet.id + privateLinkServiceId: synapseWorkspace.id + privateDnsZoneName: 'privatelink.sql.${synapseSuffix}' + groupIds: [ + 'SqlOnDemand' + ] + } + dependsOn: [ + addSynapseSqlPrivateLink + ] +} + +output customVnetId string = customVnet.id +output customVnetName string = customVnet.name +output pipelineSubnetId string = pipelineSubnet.id +output pipelineSubnetName string = pipelineSubnet.name +output synapseWorkspaceProperties object = synapseWorkspace.properties + + + + + + + + diff --git a/deploy/install.sh b/deploy/install.sh index 292f91c..f7ac008 100755 --- a/deploy/install.sh +++ b/deploy/install.sh @@ -4,7 +4,7 @@ # Licensed under the MIT license. set -x - +PRJ_ROOT="$(cd `dirname "${BASH_SOURCE}"`/..; pwd)" if [[ -z "$1" ]] then echo "Environment Code value not supplied" @@ -22,13 +22,34 @@ envCode=${envCode:-"${1}"} location=${location:-"${2}"} envTag=${envTag:-"synapse-${envCode}"} deploymentName=${3:-"${envTag}-deploy"} +securityEnabled=${securityEnabled:-false} +preventDataExfiltration=${preventDataExfiltration:-false} DEPLOYMENT_SCRIPT="az deployment sub create -l $location -n $deploymentName \ -f ./deploy/infra/main.bicep \ -p \ location=$location \ environmentCode=$envCode \ - environment=$envTag" + environment=$envTag \ + securityEnabled=$securityEnabled \ + preventDataExfiltration=$preventDataExfiltration" $DEPLOYMENT_SCRIPT -set +x +if [[ $securityEnabled ]] +then + # make sure to add synapse managed private endpoints before adding + # private endpoints for synapse to custom vnet + ${PRJ_ROOT}/deploy/addManagedPE.sh $envCode + + # after adding managed vnet, now we are adding private endpoints to + # custom vnet using bicep + DEPLOYMENT_SCRIPT="az deployment sub create -l $location \ + -n $deploymentName-security-addon \ + -f ${PRJ_ROOT}/deploy/infra/security-addons.bicep \ + -p \ + environmentCode=$envCode \ + location=$location" + $DEPLOYMENT_SCRIPT +fi + +set +x \ No newline at end of file From 2dd4746b7a6ef2925fa70c54150e49e8f7204e9d Mon Sep 17 00:00:00 2001 From: Seokwon Yang Date: Mon, 2 May 2022 10:49:35 -0700 Subject: [PATCH 2/6] git rebase and setup.sh changes to pass securityEnabled parameters to bicep --- deploy/README.md | 22 +++++++++++- deploy/addManagedPE.sh | 54 ++++++++++++++++++++++++------ deploy/infra/security-addons.bicep | 5 --- deploy/install.sh | 8 ++--- deploy/setup.sh | 2 ++ 5 files changed, 70 insertions(+), 21 deletions(-) diff --git a/deploy/README.md b/deploy/README.md index 61c8d4c..891a4d7 100644 --- a/deploy/README.md +++ b/deploy/README.md @@ -48,6 +48,12 @@ Steps 2 through 4 can instead be deployed using a single script below: ./deploy/setup.sh ``` +To enabled security features like Synapse managed VNET, managed private endpoints, and privates enpoints to 3 synapse endpoints, set 'SECURITY_ENABLED=true' when running setup.sh: +``` +SECURITY_ENABLED=true ./deploy/setup.sh +``` +**Note that if you turn on SECURITY_ENABLED during setup, Synapse endpoints are restricted to the custom VNET in the deployment environment. Thus, you need to create a Windows jumpbox inside the VNET to connect to Synapse Studio.** + If you like to package other pipelines or re-package an updated/modified pipeline, follow the instructions under `Packaging the Synapse pipeline` section. The script mentioned in that section can be rerun multiple times. Arguments | Required | Sample value @@ -83,6 +89,12 @@ To install infrastructure execute install.sh script as follows ``` +To enabled security features like Synapse managed VNET, managed private endpoints, and privates enpoints to 3 synapse endpoints, set 'SECURITY_ENABLED=true' when running install.sh: +``` +SECURITY_ENABLED=true ./deploy/install.sh +``` +**Note that if you turn on SECURITY_ENABLED during setup, Synapse endpoints are restricted to the custom VNET in the deployment environment. Thus, you need to create a Windows jumpbox inside the VNET to connect to Synapse Studio.** + Default values for the parameters are provided in the script itself. Arguments | Required | Sample value @@ -113,6 +125,11 @@ For eg. az deployment sub create -l -n aoi -f main.bicep -p location= environmentCode=aoi environment=synapse-aoi ``` +To enabled security features like Synapse managed VNET, managed private endpoints, and privates enpoints to 3 Synapse endpoints, pass parameter 'securityEnabled=true' when running bicep: +``` +bash +az deployment sub create -l -n -f main.bicep -p location= environmentCode= environment= securityEnabled=true +``` ## Configuring the Resources @@ -142,7 +159,10 @@ Once the above step completes, a zip file is generated. Upload the generated zip 4. When prompted to select a file, pick the zip file generated in the previous step 5. Pipelines and its dependencies are imported to the Synapse Studio. Validate the components being imported for any errors 6. Click "Publish all" and wait for the imported components to be published -NOTE: You may run into this error during import "Invalid template, please check the template file". It is a known issue that we are working on with the product team. In the interim, we suggest importing from Git Repository as described below. +NOTE: You may run into this error during import "Invalid template, please check the template file". It is a known issue that we are working on with the product team. In the interim, we suggest importing from Git Repository as described below. + +**Note that if you turn on SECURITY_ENABLED during setup, Synapse endpoints are restricted to the custom VNET in the deployment environment. Thus, you need to create a Windows jumpbox inside the VNET to connect to Synapse Studio and import the package.** + ## Importing from Git Repository Another way to get import pipeline into the Synape Studio is through Source Control repository like GitHub or Azure DevOps repository. Refer to the document on [Source Control](https://docs.microsoft.com/azure/synapse-analytics/cicd/source-control) to learn about Git Integration for Azure Synapse Analytics and how to setup. diff --git a/deploy/addManagedPE.sh b/deploy/addManagedPE.sh index dc49732..1dce9e9 100755 --- a/deploy/addManagedPE.sh +++ b/deploy/addManagedPE.sh @@ -10,6 +10,25 @@ if [[ -z "$1" ]] exit 1 fi ENVCODE=$1 +PE_APPROVAL_DESCRIPTION="Approved by script" + +approved_managed_private_endpoint_request_exists() { + local groupId=$1 + local resourceName=$2 + local resourceType=$3 + + local peList=$(az network private-endpoint-connection list \ + -g $groupId -n $resourceName --type $resourceType -ojson 2>/dev/null || echo '') + local result='' + + if [[ -n peList ]]; + then + result=$( echo $peList \ + | jq -r ".[] | select (.properties.privateLinkServiceConnectionState.description == \"${PE_APPROVAL_DESCRIPTION}\").id" ) + echo $result + fi + echo $result +} create_synapase_managed_private_endpoint() { local tmpfile=$(mktemp) @@ -63,7 +82,7 @@ approve_synapase_managed_private_endpoint() { if [[ $PE_CONNECTION_APPROVAL_STATUS != "Approved" ]]; then az network private-endpoint-connection approve \ - --id $PE_CONNECTION_ID --description "Approved by script" + --id $PE_CONNECTION_ID --description "$PE_APPROVAL_DESCRIPTION" echo "$PE_CONNECTION_ID got approved" fi fi @@ -76,7 +95,11 @@ do sleep 30 SYNAPSE_STORAGE_ACCT=$(az storage account list --query "[?tags.store && tags.store == 'synapse'].name" -o tsv -g $ENVCODE-pipeline-rg) done -approve_synapase_managed_private_endpoint $ENVCODE-pipeline-rg $SYNAPSE_STORAGE_ACCT "Microsoft.Storage/storageAccounts" +result=$(approved_managed_private_endpoint_request_exists $ENVCODE-pipeline-rg $SYNAPSE_STORAGE_ACCT "Microsoft.Storage/storageAccounts") +if [[ -z $result ]]; +then + approve_synapase_managed_private_endpoint $ENVCODE-pipeline-rg $SYNAPSE_STORAGE_ACCT "Microsoft.Storage/storageAccounts" +fi # Create Managed Private Endpoints (PE) if not exist PIPELINE_KV=$(az keyvault list --query "[?tags.usage && tags.usage == 'linkedService']" -ojson -g $ENVCODE-pipeline-rg) @@ -87,7 +110,12 @@ do done PIPELINE_KV_NAME=$(echo $PIPELINE_KV | jq -r '.[0].name') PIPELINE_KV_ID=$(echo $PIPELINE_KV | jq -r '.[0].id') -create_synapase_managed_private_endpoint "$ENVCODE-pipeline-syn-ws" "$ENVCODE-mpe-pipeline-kv" "vault" "$PIPELINE_KV_ID" +result=$(approved_managed_private_endpoint_request_exists $ENVCODE-pipeline-rg $PIPELINE_KV_NAME "Microsoft.Keyvault/vaults") +if [[ -z $result ]] +then + create_synapase_managed_private_endpoint "$ENVCODE-pipeline-syn-ws" "$ENVCODE-mpe-pipeline-kv" "vault" "$PIPELINE_KV_ID" + approve_synapase_managed_private_endpoint $ENVCODE-pipeline-rg $PIPELINE_KV_NAME "Microsoft.Keyvault/vaults" +fi DATA_STORAGE_ACCT=$(az storage account list --query "[?tags.store && tags.store == 'raw']" -ojson -g $ENVCODE-data-rg) while [[ $DATA_STORAGE_ACCT == '[]' ]] @@ -97,7 +125,12 @@ do done DATA_STORAGE_ACCT_NAME=$(echo $DATA_STORAGE_ACCT | jq -r '.[0].name') DATA_STORAGE_ACCT_ID=$(echo $DATA_STORAGE_ACCT | jq -r '.[0].id') -create_synapase_managed_private_endpoint "$ENVCODE-pipeline-syn-ws" "$ENVCODE-mpe-data-raw" "dfs" "$DATA_STORAGE_ACCT_ID" +result=$(approved_managed_private_endpoint_request_exists $ENVCODE-data-rg $DATA_STORAGE_ACCT_NAME "Microsoft.Storage/storageAccounts") +if [[ -z $result ]] +then + create_synapase_managed_private_endpoint "$ENVCODE-pipeline-syn-ws" "$ENVCODE-mpe-data-raw" "dfs" "$DATA_STORAGE_ACCT_ID" + approve_synapase_managed_private_endpoint $ENVCODE-data-rg $DATA_STORAGE_ACCT_NAME "Microsoft.Storage/storageAccounts" +fi DATA_KV=$(az keyvault list --query "[?tags.usage && tags.usage == 'general']" -ojson -g $ENVCODE-data-rg) while [[ $DATA_KV == '[]' ]]; @@ -107,10 +140,9 @@ do done DATA_KV_NAME=$(echo $DATA_KV | jq -r '.[0].name') DATA_KV_ID=$(echo $DATA_KV | jq -r '.[0].id') -create_synapase_managed_private_endpoint "$ENVCODE-pipeline-syn-ws" "$ENVCODE-mpe-data-kv" "vault" "$DATA_KV_ID" - - -# Approve remaining Managed Private Endpoints (PE) -approve_synapase_managed_private_endpoint $ENVCODE-pipeline-rg $PIPELINE_KV_NAME "Microsoft.Keyvault/vaults" -approve_synapase_managed_private_endpoint $ENVCODE-data-rg $DATA_STORAGE_ACCT_NAME "Microsoft.Storage/storageAccounts" -approve_synapase_managed_private_endpoint $ENVCODE-data-rg $DATA_KV_NAME "Microsoft.Keyvault/vaults" +result=$(approved_managed_private_endpoint_request_exists $ENVCODE-data-rg $DATA_KV_NAME "Microsoft.Keyvault/vaults") +if [[ -z $result ]] +then + create_synapase_managed_private_endpoint "$ENVCODE-pipeline-syn-ws" "$ENVCODE-mpe-data-kv" "vault" "$DATA_KV_ID" + approve_synapase_managed_private_endpoint $ENVCODE-data-rg $DATA_KV_NAME "Microsoft.Keyvault/vaults" +fi \ No newline at end of file diff --git a/deploy/infra/security-addons.bicep b/deploy/infra/security-addons.bicep index cc74d1c..7581166 100644 --- a/deploy/infra/security-addons.bicep +++ b/deploy/infra/security-addons.bicep @@ -91,11 +91,6 @@ module addSynapseSqlOnDemandPrivateEndpoint 'modules/privateendpoints.bicep' = { ] } -output customVnetId string = customVnet.id -output customVnetName string = customVnet.name -output pipelineSubnetId string = pipelineSubnet.id -output pipelineSubnetName string = pipelineSubnet.name -output synapseWorkspaceProperties object = synapseWorkspace.properties diff --git a/deploy/install.sh b/deploy/install.sh index f7ac008..435e18b 100755 --- a/deploy/install.sh +++ b/deploy/install.sh @@ -22,8 +22,8 @@ envCode=${envCode:-"${1}"} location=${location:-"${2}"} envTag=${envTag:-"synapse-${envCode}"} deploymentName=${3:-"${envTag}-deploy"} -securityEnabled=${securityEnabled:-false} -preventDataExfiltration=${preventDataExfiltration:-false} +SECURITY_ENABLED=${SECURITY_ENABLED:-false} +PREVENT_DATA_EXFILTRATION=${PREVENT_DATA_EXFILTRATION:-false} DEPLOYMENT_SCRIPT="az deployment sub create -l $location -n $deploymentName \ -f ./deploy/infra/main.bicep \ @@ -31,8 +31,8 @@ DEPLOYMENT_SCRIPT="az deployment sub create -l $location -n $deploymentName \ location=$location \ environmentCode=$envCode \ environment=$envTag \ - securityEnabled=$securityEnabled \ - preventDataExfiltration=$preventDataExfiltration" + securityEnabled=$SECURITY_ENABLED \ + preventDataExfiltration=$PREVENT_DATA_EXFILTRATION" $DEPLOYMENT_SCRIPT if [[ $securityEnabled ]] diff --git a/deploy/setup.sh b/deploy/setup.sh index 0bad4b5..b619941 100755 --- a/deploy/setup.sh +++ b/deploy/setup.sh @@ -8,6 +8,8 @@ LOCATION=$2 PIPELINE_NAME=$3 ENVTAG=$4 +export SECURITY_ENABLED=${SECURITY_ENABLED:-false} +export PREVENT_DATA_EXFILTRATION=${PREVENT_DATA_EXFILTRATION:-false} set -x From 03c9124249e0f7187fd99ab9ed311ce439f3a3d7 Mon Sep 17 00:00:00 2001 From: Seokwon Yang Date: Tue, 3 May 2022 18:24:13 -0700 Subject: [PATCH 3/6] idempotency fix + updated with review comments --- deploy/README.md | 18 ++++++++---------- deploy/infra/main.bicep | 5 ++++- deploy/install.sh | 13 +++++++++++++ 3 files changed, 25 insertions(+), 11 deletions(-) diff --git a/deploy/README.md b/deploy/README.md index 891a4d7..aea7364 100644 --- a/deploy/README.md +++ b/deploy/README.md @@ -52,7 +52,7 @@ To enabled security features like Synapse managed VNET, managed private endpoint ``` SECURITY_ENABLED=true ./deploy/setup.sh ``` -**Note that if you turn on SECURITY_ENABLED during setup, Synapse endpoints are restricted to the custom VNET in the deployment environment. Thus, you need to create a Windows jumpbox inside the VNET to connect to Synapse Studio.** +Before opting in security features, we advise you to get familiar with restrictions we listed below [restrictions-with-security-features](#restrictions-with-security-features) If you like to package other pipelines or re-package an updated/modified pipeline, follow the instructions under `Packaging the Synapse pipeline` section. The script mentioned in that section can be rerun multiple times. @@ -93,7 +93,6 @@ To enabled security features like Synapse managed VNET, managed private endpoint ``` SECURITY_ENABLED=true ./deploy/install.sh ``` -**Note that if you turn on SECURITY_ENABLED during setup, Synapse endpoints are restricted to the custom VNET in the deployment environment. Thus, you need to create a Windows jumpbox inside the VNET to connect to Synapse Studio.** Default values for the parameters are provided in the script itself. @@ -125,12 +124,6 @@ For eg. az deployment sub create -l -n aoi -f main.bicep -p location= environmentCode=aoi environment=synapse-aoi ``` -To enabled security features like Synapse managed VNET, managed private endpoints, and privates enpoints to 3 Synapse endpoints, pass parameter 'securityEnabled=true' when running bicep: -``` -bash -az deployment sub create -l -n -f main.bicep -p location= environmentCode= environment= securityEnabled=true -``` - ## Configuring the Resources If you have deployed the solution using `setup.sh` script, you should skip this step. However, if you have not run the `setup.sh` script, the steps outlined in this section are required. @@ -161,8 +154,6 @@ Once the above step completes, a zip file is generated. Upload the generated zip 6. Click "Publish all" and wait for the imported components to be published NOTE: You may run into this error during import "Invalid template, please check the template file". It is a known issue that we are working on with the product team. In the interim, we suggest importing from Git Repository as described below. -**Note that if you turn on SECURITY_ENABLED during setup, Synapse endpoints are restricted to the custom VNET in the deployment environment. Thus, you need to create a Windows jumpbox inside the VNET to connect to Synapse Studio and import the package.** - ## Importing from Git Repository Another way to get import pipeline into the Synape Studio is through Source Control repository like GitHub or Azure DevOps repository. Refer to the document on [Source Control](https://docs.microsoft.com/azure/synapse-analytics/cicd/source-control) to learn about Git Integration for Azure Synapse Analytics and how to setup. @@ -324,6 +315,13 @@ To run the pipeline, open the Synapse Studio for the Synapse workspace that you - Wait for the pipeline to complete. + +# Restrictions with security features +- Once SECURITY_ENABLED is turned on during setup/install, Synapse endpoints are restricted to the custom VNET in the deployment environment. Thus, you need to create a Windows jumpbox inside the VNET to connect to Synapse Studio. For more details, refer to [synapse-security-diagram](https://docs.microsoft.com/en-us/azure/synapse-analytics/guidance/media/security-white-paper-network-security/private-endpoints.png) and [synapse-network-securiy-section](https://docs.microsoft.com/en-us/azure/synapse-analytics/guidance/security-white-paper-network-security). +- Existing Azure Synapse workspace can not convert to security-enabled one. And, the reverse (i.e. secured one to public one) is not supported either. You cannot change Synapse workspace configuration after the workspace is created. Please refer to [Azure-Synapse-Documentation](https://docs.microsoft.com/en-us/azure/synapse-analytics/security/synapse-workspace-managed-vnet#managed-workspace-virtual-network). +- Once security feature is enabled, azure-cli that goes to Azure Synapse Studio like `az synapse managed-private-endpoints` may only work inside the custom VNET and Azure VM attached to it. + + # Cleanup Script We have a cleanup script to cleanup the resource groups and thus the resources provisioned using the `environmentCode`. diff --git a/deploy/infra/main.bicep b/deploy/infra/main.bicep index 68e57b8..334aec7 100644 --- a/deploy/infra/main.bicep +++ b/deploy/infra/main.bicep @@ -20,6 +20,9 @@ param securityEnabled bool = false @description('preventDataExfiltration for Synapse managed vnet') param preventDataExfiltration bool = false +@description('Flag to determine whether VNET and Subnet will be created or re-created') +param createVnetSubnets bool = true + @description('Used for naming of the network resource group and its resources') param networkModulePrefix string = 'network' @@ -51,7 +54,7 @@ module networkResourceGroup 'modules/resourcegroup.bicep' = { } } -module networkModule 'groups/networking.bicep' = { +module networkModule 'groups/networking.bicep' = if (createVnetSubnets) { name: '${networkModulePrefix}-module' scope: resourceGroup(networkResourceGroup.name) params: { diff --git a/deploy/install.sh b/deploy/install.sh index 435e18b..8706d47 100755 --- a/deploy/install.sh +++ b/deploy/install.sh @@ -24,6 +24,18 @@ envTag=${envTag:-"synapse-${envCode}"} deploymentName=${3:-"${envTag}-deploy"} SECURITY_ENABLED=${SECURITY_ENABLED:-false} PREVENT_DATA_EXFILTRATION=${PREVENT_DATA_EXFILTRATION:-false} +# Check if vnet exists. +# This is needed since vnet template tries to delete existing private enpoints +# from the security enabled vnet when re-running its template. +VNET_ID=$(az network vnet show \ + -n "${envCode}-vnet" -g "${envCode}-network-rg" --query id -otsv \ + 2>/dev/null || echo '') +if [[ -z "${VNET_ID}" ]] +then + CREATE_VNET_SUBNETS=true +else + CREATE_VNET_SUBNETS=false +fi DEPLOYMENT_SCRIPT="az deployment sub create -l $location -n $deploymentName \ -f ./deploy/infra/main.bicep \ @@ -31,6 +43,7 @@ DEPLOYMENT_SCRIPT="az deployment sub create -l $location -n $deploymentName \ location=$location \ environmentCode=$envCode \ environment=$envTag \ + createVnetSubnets=$CREATE_VNET_SUBNETS \ securityEnabled=$SECURITY_ENABLED \ preventDataExfiltration=$PREVENT_DATA_EXFILTRATION" $DEPLOYMENT_SCRIPT From 1c94d6d6a96f77ce1d1d88d99e67b1453289dd52 Mon Sep 17 00:00:00 2001 From: Seokwon Yang Date: Wed, 4 May 2022 14:45:57 -0700 Subject: [PATCH 4/6] updated with reviews(2) --- deploy/addManagedPE.sh | 31 +++++++++---------------------- deploy/install.sh | 2 +- 2 files changed, 10 insertions(+), 23 deletions(-) diff --git a/deploy/addManagedPE.sh b/deploy/addManagedPE.sh index 1dce9e9..f2499e3 100755 --- a/deploy/addManagedPE.sh +++ b/deploy/addManagedPE.sh @@ -10,7 +10,7 @@ if [[ -z "$1" ]] exit 1 fi ENVCODE=$1 -PE_APPROVAL_DESCRIPTION="Approved by script" +PE_APPROVAL_DESCRIPTION="Approved by addManagedPE.sh script" approved_managed_private_endpoint_request_exists() { local groupId=$1 @@ -33,14 +33,14 @@ approved_managed_private_endpoint_request_exists() { create_synapase_managed_private_endpoint() { local tmpfile=$(mktemp) local synpaseWorkspace=$1 - local peName=$2 + local synapseManagedPrivateEndpointName=$2 local groupId=$3 local privateLinkResourceId=$4 - echo "creating MPE if not exist for $peName" - # check if peName exists + echo "creating MPE if not exist for $synapseManagedPrivateEndpointName" + # check if synapseManagedPrivateEndpointName exists local checkPeExists=$(az synapse managed-private-endpoints show \ - --pe-name $peName -otsv --query "id" --workspace-name $synpaseWorkspace 2>/dev/null || echo '') + --pe-name $synapseManagedPrivateEndpointName -otsv --query "id" --workspace-name $synpaseWorkspace 2>/dev/null || echo '') if [[ -z $checkPeExists ]]; then @@ -51,18 +51,18 @@ create_synapase_managed_private_endpoint() { az synapse managed-private-endpoints create \ --file @$tmpfile \ - --pe-name $peName \ + --pe-name $synapseManagedPrivateEndpointName \ --workspace-name $1 sleep 60 - local provisioningState=$(az synapse managed-private-endpoints show --pe-name $peName \ + local provisioningState=$(az synapse managed-private-endpoints show --pe-name $synapseManagedPrivateEndpointName \ --workspace-name $synpaseWorkspace -o tsv --query "properties.provisioningState") while [[ $provisioningState != "Succeeded" ]]; do sleep 10 - provisioningState=$(az synapse managed-private-endpoints show --pe-name $peName \ + provisioningState=$(az synapse managed-private-endpoints show --pe-name $synapseManagedPrivateEndpointName \ --workspace-name $synpaseWorkspace -o tsv --query "properties.provisioningState") - echo "provisioningState of $peName: $provisioningState" + echo "provisioningState of $synapseManagedPrivateEndpointName: $provisioningState" done fi } @@ -88,19 +88,6 @@ approve_synapase_managed_private_endpoint() { fi } -# wait for SYNAPSE_STORAGE_ACCT showing up in azcli and approve its managed private endpoint first. -SYNAPSE_STORAGE_ACCT=$(az storage account list --query "[?tags.store && tags.store == 'synapse'].name" -o tsv -g $ENVCODE-pipeline-rg) -while [[ -z $SYNAPSE_STORAGE_ACCT ]]; -do - sleep 30 - SYNAPSE_STORAGE_ACCT=$(az storage account list --query "[?tags.store && tags.store == 'synapse'].name" -o tsv -g $ENVCODE-pipeline-rg) -done -result=$(approved_managed_private_endpoint_request_exists $ENVCODE-pipeline-rg $SYNAPSE_STORAGE_ACCT "Microsoft.Storage/storageAccounts") -if [[ -z $result ]]; -then - approve_synapase_managed_private_endpoint $ENVCODE-pipeline-rg $SYNAPSE_STORAGE_ACCT "Microsoft.Storage/storageAccounts" -fi - # Create Managed Private Endpoints (PE) if not exist PIPELINE_KV=$(az keyvault list --query "[?tags.usage && tags.usage == 'linkedService']" -ojson -g $ENVCODE-pipeline-rg) while [[ $PIPELINE_KV == '[]' ]]; diff --git a/deploy/install.sh b/deploy/install.sh index 8706d47..7e974dd 100755 --- a/deploy/install.sh +++ b/deploy/install.sh @@ -48,7 +48,7 @@ DEPLOYMENT_SCRIPT="az deployment sub create -l $location -n $deploymentName \ preventDataExfiltration=$PREVENT_DATA_EXFILTRATION" $DEPLOYMENT_SCRIPT -if [[ $securityEnabled ]] +if [[ $SECURITY_ENABLED ]] then # make sure to add synapse managed private endpoints before adding # private endpoints for synapse to custom vnet From f8090c8b0b22ced0b114a56125a4c75b794955cf Mon Sep 17 00:00:00 2001 From: Seokwon Yang Date: Wed, 4 May 2022 15:07:24 -0700 Subject: [PATCH 5/6] adding back synapse storage account managed private approval function call --- deploy/addManagedPE.sh | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/deploy/addManagedPE.sh b/deploy/addManagedPE.sh index f2499e3..a7dab36 100755 --- a/deploy/addManagedPE.sh +++ b/deploy/addManagedPE.sh @@ -87,6 +87,18 @@ approve_synapase_managed_private_endpoint() { fi fi } +# wait for SYNAPSE_STORAGE_ACCT showing up in azcli and approve its managed private endpoint first. +SYNAPSE_STORAGE_ACCT=$(az storage account list --query "[?tags.store && tags.store == 'synapse'].name" -o tsv -g $ENVCODE-pipeline-rg) +while [[ -z $SYNAPSE_STORAGE_ACCT ]]; +do + sleep 30 + SYNAPSE_STORAGE_ACCT=$(az storage account list --query "[?tags.store && tags.store == 'synapse'].name" -o tsv -g $ENVCODE-pipeline-rg) +done +result=$(approved_managed_private_endpoint_request_exists $ENVCODE-pipeline-rg $SYNAPSE_STORAGE_ACCT "Microsoft.Storage/storageAccounts") +if [[ -z $result ]]; +then + approve_synapase_managed_private_endpoint $ENVCODE-pipeline-rg $SYNAPSE_STORAGE_ACCT "Microsoft.Storage/storageAccounts" +fi # Create Managed Private Endpoints (PE) if not exist PIPELINE_KV=$(az keyvault list --query "[?tags.usage && tags.usage == 'linkedService']" -ojson -g $ENVCODE-pipeline-rg) From 2ec48ad6c689bd756a0712b41a63d3884a6cb9f3 Mon Sep 17 00:00:00 2001 From: Seokwon Yang Date: Fri, 6 May 2022 17:06:47 -0700 Subject: [PATCH 6/6] updated README.md with restrictions with a minor bug fix --- deploy/README.md | 5 +++-- deploy/install.sh | 2 +- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/deploy/README.md b/deploy/README.md index aea7364..d98c3d0 100644 --- a/deploy/README.md +++ b/deploy/README.md @@ -317,9 +317,10 @@ To run the pipeline, open the Synapse Studio for the Synapse workspace that you # Restrictions with security features -- Once SECURITY_ENABLED is turned on during setup/install, Synapse endpoints are restricted to the custom VNET in the deployment environment. Thus, you need to create a Windows jumpbox inside the VNET to connect to Synapse Studio. For more details, refer to [synapse-security-diagram](https://docs.microsoft.com/en-us/azure/synapse-analytics/guidance/media/security-white-paper-network-security/private-endpoints.png) and [synapse-network-securiy-section](https://docs.microsoft.com/en-us/azure/synapse-analytics/guidance/security-white-paper-network-security). - Existing Azure Synapse workspace can not convert to security-enabled one. And, the reverse (i.e. secured one to public one) is not supported either. You cannot change Synapse workspace configuration after the workspace is created. Please refer to [Azure-Synapse-Documentation](https://docs.microsoft.com/en-us/azure/synapse-analytics/security/synapse-workspace-managed-vnet#managed-workspace-virtual-network). -- Once security feature is enabled, azure-cli that goes to Azure Synapse Studio like `az synapse managed-private-endpoints` may only work inside the custom VNET and Azure VM attached to it. +- Once SECURITY_ENABLED is turned on during setup/install, Synapse endpoints are restricted to a VNET in the the same region. Thus, you need to create a Windows jumpbox inside the VNET to connect to Synapse Studio. For more details, refer to [synapse-security-diagram](https://docs.microsoft.com/en-us/azure/synapse-analytics/guidance/media/security-white-paper-network-security/private-endpoints.png) and [synapse-network-securiy-section](https://docs.microsoft.com/en-us/azure/synapse-analytics/guidance/security-white-paper-network-security). And, in order to furthur restriction network access to the VNET of your cluster, you also need to restrict the public access of your Synapse workspace. We may achieve this task from azure portal (i.e. Synapse workspace -> Networking -> Public network access -> select 'Disabled'). +- Once security feature is enabled, azure-cli that goes to Azure Synapse like `az synapse managed-private-endpoints` may only work inside the custom VNET and Azure VM attached to it. +- Once public access to your Synapse workspace is disabled, you may get 'PublicNetworkAccessDenied' error. Enable the public network access for re-deployment and restrict public access afterwards. # Cleanup Script diff --git a/deploy/install.sh b/deploy/install.sh index 7e974dd..2f8ffa6 100755 --- a/deploy/install.sh +++ b/deploy/install.sh @@ -48,7 +48,7 @@ DEPLOYMENT_SCRIPT="az deployment sub create -l $location -n $deploymentName \ preventDataExfiltration=$PREVENT_DATA_EXFILTRATION" $DEPLOYMENT_SCRIPT -if [[ $SECURITY_ENABLED ]] +if [[ "$SECURITY_ENABLED" == "true" ]] then # make sure to add synapse managed private endpoints before adding # private endpoints for synapse to custom vnet