Skip to content

Commit e0335ed

Browse files
sjyang18sjyang18
committed
adding synapse managed vnet feature
1 parent ef4a02a commit e0335ed

File tree

8 files changed

+382
-25
lines changed

8 files changed

+382
-25
lines changed

‎deploy/addManagedPE.sh‎

Lines changed: 116 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,116 @@
1+
#!/usr/bin/env bash
2+
3+
# Copyright (c) Microsoft Corporation.
4+
# Licensed under the MIT license.
5+
6+
set +x
7+
if [[ -z "$1" ]]
8+
then
9+
echo "Environment Code value not supplied"
10+
exit 1
11+
fi
12+
ENVCODE=$1
13+
14+
create_synapase_managed_private_endpoint() {
15+
local tmpfile=$(mktemp)
16+
local synpaseWorkspace=$1
17+
local peName=$2
18+
local groupId=$3
19+
local privateLinkResourceId=$4
20+
21+
echo "creating MPE if not exist for $peName"
22+
# check if peName exists
23+
local checkPeExists=$(az synapse managed-private-endpoints show \
24+
--pe-name $peName -otsv --query "id" --workspace-name $synpaseWorkspace 2>/dev/null || echo '')
25+
26+
if [[ -z $checkPeExists ]];
27+
then
28+
jq -n -r \
29+
--arg groupId "$groupId" \
30+
--arg privateLinkResourceId "$privateLinkResourceId" \
31+
'{groupId:$groupId, privateLinkResourceId:$privateLinkResourceId}' > $tmpfile
32+
33+
az synapse managed-private-endpoints create \
34+
--file @$tmpfile \
35+
--pe-name $peName \
36+
--workspace-name $1
37+
38+
sleep 60
39+
local provisioningState=$(az synapse managed-private-endpoints show --pe-name $peName \
40+
--workspace-name $synpaseWorkspace -o tsv --query "properties.provisioningState")
41+
while [[ $provisioningState != "Succeeded" ]];
42+
do
43+
sleep 10
44+
provisioningState=$(az synapse managed-private-endpoints show --pe-name $peName \
45+
--workspace-name $synpaseWorkspace -o tsv --query "properties.provisioningState")
46+
echo "provisioningState of $peName: $provisioningState"
47+
done
48+
fi
49+
}
50+
51+
approve_synapase_managed_private_endpoint() {
52+
local resourceGroup=$1
53+
local resourceName=$2
54+
local resourceType=$3
55+
56+
local PE_CONNECTION=$(az network private-endpoint-connection list -g $resourceGroup -n $resourceName \
57+
--type $resourceType --query "[0]" -ojson 2>/dev/null || echo '')
58+
if [[ -n $PE_CONNECTION ]];
59+
then
60+
local PE_CONNECTION_ID=$(echo $PE_CONNECTION | jq -r '.id')
61+
local PE_CONNECTION_APPROVAL_STATUS=$(echo $PE_CONNECTION | jq -r '.properties.privateLinkServiceConnectionState.status')
62+
63+
if [[ $PE_CONNECTION_APPROVAL_STATUS != "Approved" ]];
64+
then
65+
az network private-endpoint-connection approve \
66+
--id $PE_CONNECTION_ID --description "Approved by script"
67+
echo "$PE_CONNECTION_ID got approved"
68+
fi
69+
fi
70+
}
71+
72+
# wait for SYNAPSE_STORAGE_ACCT showing up in azcli and approve its managed private endpoint first.
73+
SYNAPSE_STORAGE_ACCT=$(az storage account list --query "[?tags.store && tags.store == 'synapse'].name" -o tsv -g $ENVCODE-pipeline-rg)
74+
while [[ -z $SYNAPSE_STORAGE_ACCT ]];
75+
do
76+
sleep 30
77+
SYNAPSE_STORAGE_ACCT=$(az storage account list --query "[?tags.store && tags.store == 'synapse'].name" -o tsv -g $ENVCODE-pipeline-rg)
78+
done
79+
approve_synapase_managed_private_endpoint $ENVCODE-pipeline-rg $SYNAPSE_STORAGE_ACCT "Microsoft.Storage/storageAccounts"
80+
81+
# Create Managed Private Endpoints (PE) if not exist
82+
PIPELINE_KV=$(az keyvault list --query "[?tags.usage && tags.usage == 'linkedService']" -ojson -g $ENVCODE-pipeline-rg)
83+
while [[ $PIPELINE_KV == '[]' ]];
84+
do
85+
sleep 30
86+
PIPELINE_KV=$(az keyvault list --query "[?tags.usage && tags.usage == 'linkedService']" -ojson -g $ENVCODE-pipeline-rg)
87+
done
88+
PIPELINE_KV_NAME=$(echo $PIPELINE_KV | jq -r '.[0].name')
89+
PIPELINE_KV_ID=$(echo $PIPELINE_KV | jq -r '.[0].id')
90+
create_synapase_managed_private_endpoint "$ENVCODE-pipeline-syn-ws" "$ENVCODE-mpe-pipeline-kv" "vault" "$PIPELINE_KV_ID"
91+
92+
DATA_STORAGE_ACCT=$(az storage account list --query "[?tags.store && tags.store == 'raw']" -ojson -g $ENVCODE-data-rg)
93+
while [[ $DATA_STORAGE_ACCT == '[]' ]]
94+
do
95+
sleep 30
96+
DATA_STORAGE_ACCT=$(az storage account list --query "[?tags.store && tags.store == 'raw']" -ojson -g $ENVCODE-data-rg)
97+
done
98+
DATA_STORAGE_ACCT_NAME=$(echo $DATA_STORAGE_ACCT | jq -r '.[0].name')
99+
DATA_STORAGE_ACCT_ID=$(echo $DATA_STORAGE_ACCT | jq -r '.[0].id')
100+
create_synapase_managed_private_endpoint "$ENVCODE-pipeline-syn-ws" "$ENVCODE-mpe-data-raw" "dfs" "$DATA_STORAGE_ACCT_ID"
101+
102+
DATA_KV=$(az keyvault list --query "[?tags.usage && tags.usage == 'general']" -ojson -g $ENVCODE-data-rg)
103+
while [[ $DATA_KV == '[]' ]];
104+
do
105+
sleep 30
106+
DATA_KV=$(az keyvault list --query "[?tags.usage && tags.usage == 'general']" -ojson -g $ENVCODE-data-rg)
107+
done
108+
DATA_KV_NAME=$(echo $DATA_KV | jq -r '.[0].name')
109+
DATA_KV_ID=$(echo $DATA_KV | jq -r '.[0].id')
110+
create_synapase_managed_private_endpoint "$ENVCODE-pipeline-syn-ws" "$ENVCODE-mpe-data-kv" "vault" "$DATA_KV_ID"
111+
112+
113+
# Approve remaining Managed Private Endpoints (PE)
114+
approve_synapase_managed_private_endpoint $ENVCODE-pipeline-rg $PIPELINE_KV_NAME "Microsoft.Keyvault/vaults"
115+
approve_synapase_managed_private_endpoint $ENVCODE-data-rg $DATA_STORAGE_ACCT_NAME "Microsoft.Storage/storageAccounts"
116+
approve_synapase_managed_private_endpoint $ENVCODE-data-rg $DATA_KV_NAME "Microsoft.Keyvault/vaults"

‎deploy/infra/groups/pipeline.bicep‎

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -84,6 +84,8 @@ param synapseMIStorageAccountRoles array = [
8484
]
8585

8686
param logAnalyticsWorkspaceId string
87+
param securityEnabled bool = false
88+
param preventDataExfiltration bool = false
8789

8890
var namingPrefix = '${environmentCode}-${projectName}'
8991
var synapseResourceGroupNameVar = empty(synapseResourceGroupName) ? '${namingPrefix}-rg' : synapseResourceGroupName
@@ -129,8 +131,6 @@ module synapseHnsStorageAccount '../modules/storage.hns.bicep' = {
129131
}
130132
}
131133

132-
133-
134134
module synapseWorkspace '../modules/synapse.workspace.bicep' = {
135135
name: '${namingPrefix}-workspace'
136136
params:{
@@ -153,6 +153,8 @@ module synapseWorkspace '../modules/synapse.workspace.bicep' = {
153153
gitRepoRootFolder: synapseGitRepoRootFolder
154154
gitRepoVstsTenantId: synapseGitRepoVstsTenantId
155155
gitRepoType: synapseGitRepoType
156+
createManagedVnet: securityEnabled
157+
preventDataExfiltration: preventDataExfiltration
156158
}
157159
dependsOn: [
158160
synapseHnsStorageAccount

‎deploy/infra/main.bicep‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -14,6 +14,12 @@ param environmentCode string
1414
@description('Environment will be used as Tag on the resource group')
1515
param environment string
1616

17+
@description('Flag to set whether security resources such as Synapse managed vnet, NSG, etc are created or not')
18+
param securityEnabled bool = false
19+
20+
@description('preventDataExfiltration for Synapse managed vnet')
21+
param preventDataExfiltration bool = false
22+
1723
@description('Used for naming of the network resource group and its resources')
1824
param networkModulePrefix string = 'network'
1925

@@ -105,6 +111,8 @@ module pipelineModule 'groups/pipeline.bicep' = {
105111
environmentCode: environmentCode
106112
environmentTag: environment
107113
logAnalyticsWorkspaceId: monitorModule.outputs.workspaceId
114+
securityEnabled: securityEnabled
115+
preventDataExfiltration: preventDataExfiltration
108116
}
109117
dependsOn: [
110118
networkModule

‎deploy/infra/modules/privateendpoints.bicep‎

Lines changed: 60 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,60 @@
1+
// Copyright (c) Microsoft Corporation.
2+
// Licensed under the MIT license.
3+
4+
param environmentCode string
5+
param location string
6+
param subnetId string
7+
param privateLinkServiceId string
8+
param privateDnsZoneName string
9+
param groupIds array
10+
11+
resource privateDnsZone 'Microsoft.Network/privateDnsZones@2020-06-01' existing = {
12+
name: privateDnsZoneName
13+
}
14+
15+
resource privateDnsZoneLink 'Microsoft.Network/privateDnsZones/virtualNetworkLinks@2020-06-01' existing = {
16+
parent: privateDnsZone
17+
name: privateDnsZoneName
18+
}
19+
20+
resource privateEndpoints 'Microsoft.Network/privateEndpoints@2021-05-01' = {
21+
name: guid(environmentCode, privateLinkServiceId, groupIds[0])
22+
location: location
23+
dependsOn: [
24+
privateDnsZoneLink
25+
]
26+
properties: {
27+
subnet: {
28+
id: subnetId
29+
}
30+
privateLinkServiceConnections: [
31+
{
32+
name: guid(environmentCode, privateLinkServiceId, groupIds[0])
33+
properties: {
34+
privateLinkServiceId: privateLinkServiceId
35+
groupIds: groupIds
36+
privateLinkServiceConnectionState: {
37+
status: 'Approved'
38+
description: 'Auto-Approved'
39+
actionsRequired: 'None'
40+
}
41+
}
42+
}
43+
]
44+
}
45+
}
46+
47+
resource privateDnsZoneGroup 'Microsoft.Network/privateEndpoints/privateDnsZoneGroups@2021-05-01' = {
48+
parent: privateEndpoints
49+
name: guid(environmentCode, privateLinkServiceId, groupIds[0])
50+
properties: {
51+
privateDnsZoneConfigs: [
52+
{
53+
name: privateDnsZone.name
54+
properties: {
55+
privateDnsZoneId: privateDnsZone.id
56+
}
57+
}
58+
]
59+
}
60+
}

‎deploy/infra/modules/privatelink.bicep‎

Lines changed: 23 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,23 @@
1+
// Copyright (c) Microsoft Corporation.
2+
// Licensed under the MIT license.
3+
4+
param privateDnsZoneName string
5+
param customVnetId string
6+
7+
resource privateDnsZone 'Microsoft.Network/privateDnsZones@2020-06-01' = {
8+
name: privateDnsZoneName
9+
location: 'global'
10+
}
11+
12+
resource privateDnsZoneLink 'Microsoft.Network/privateDnsZones/virtualNetworkLinks@2020-06-01' = {
13+
parent: privateDnsZone
14+
name: privateDnsZoneName
15+
location: 'global'
16+
properties : {
17+
registrationEnabled: false
18+
virtualNetwork: {
19+
id: customVnetId
20+
}
21+
}
22+
}
23+

‎deploy/infra/modules/synapse.workspace.bicep‎

Lines changed: 41 additions & 20 deletions
Original file line numberDiff line numberDiff line change
@@ -26,6 +26,46 @@ param synapseSqlAdminPasswordSecretName string = 'synapse-sqladmin-password'
2626
param utcValue string = utcNow()
2727
param workspaceId string = 'default'
2828

29+
param createManagedVnet bool = false
30+
@allowed([
31+
'default'
32+
''
33+
])
34+
param managedVirtualNetwork string = 'default'
35+
param preventDataExfiltration bool = false
36+
param managedVirtualNetworkSettings object = {
37+
managedVirtualNetworkSettings : {
38+
allowedAadTenantIdsForLinking: []
39+
preventDataExfiltration: preventDataExfiltration
40+
}
41+
managedVirtualNetwork : managedVirtualNetwork
42+
}
43+
44+
var defaultDataLakeStorageSettings = {
45+
resourceId: hnsStorage.id
46+
accountUrl: hnsStorage.properties.primaryEndpoints.dfs
47+
filesystem: hnsStorageFileSystem
48+
createManagedPrivateEndpoint: createManagedVnet
49+
}
50+
51+
var synapseCommonProperties = {
52+
defaultDataLakeStorage: defaultDataLakeStorageSettings
53+
sqlAdministratorLogin: sqlAdminLogin
54+
sqlAdministratorLoginPassword: sqlAdminLoginPassword
55+
workspaceRepositoryConfiguration:(empty(gitRepoType))? {}: {
56+
accountName: gitRepoAccountName
57+
collaborationBranch: gitRepoCollaborationBranch
58+
hostName: gitRepoHostName
59+
lastCommitId: gitRepoLastCommitId
60+
projectName: gitRepoVstsProjectName
61+
repositoryName: gitRepoRepositoryName
62+
rootFolder: gitRepoRootFolder
63+
tenantId: gitRepoVstsTenantId
64+
type: gitRepoType
65+
}
66+
}
67+
var selectedSynapseProperties = createManagedVnet ? union(synapseCommonProperties, managedVirtualNetworkSettings) : synapseCommonProperties
68+
2969
resource hnsStorage 'Microsoft.Storage/storageAccounts@2021-08-01' existing = {
3070
name: hnsStorageAccountName
3171
}
@@ -40,26 +80,7 @@ resource synapseWorspace 'Microsoft.Synapse/workspaces@2021-06-01' = {
4080
identity: {
4181
type: 'SystemAssigned'
4282
}
43-
properties: {
44-
defaultDataLakeStorage: {
45-
resourceId: hnsStorage.id
46-
accountUrl: hnsStorage.properties.primaryEndpoints.dfs
47-
filesystem: hnsStorageFileSystem
48-
}
49-
sqlAdministratorLogin: sqlAdminLogin
50-
sqlAdministratorLoginPassword: sqlAdminLoginPassword
51-
workspaceRepositoryConfiguration:(empty(gitRepoType))? {}: {
52-
accountName: gitRepoAccountName
53-
collaborationBranch: gitRepoCollaborationBranch
54-
hostName: gitRepoHostName
55-
lastCommitId: gitRepoLastCommitId
56-
projectName: gitRepoVstsProjectName
57-
repositoryName: gitRepoRepositoryName
58-
rootFolder: gitRepoRootFolder
59-
tenantId: gitRepoVstsTenantId
60-
type: gitRepoType
61-
}
62-
}
83+
properties: selectedSynapseProperties
6384
}
6485

6586
resource synapseWorkspaceFwRules 'Microsoft.Synapse/workspaces/firewallRules@2021-06-01' = {

0 commit comments

Comments
 (0)