Draft PR: Private endpoint support for container apps

infra/abbreviations.json

@@ -110,6 +110,7 @@
   "privateEndpoint": "pe-",
   "privateLink": "pl-",
   "purviewAccounts": "pview-",
+  "privateDnsResolver": "pdr-",
   "recoveryServicesVaults": "rsv-",
   "resourcesResourceGroups": "rg-",
   "searchSearchServices": "srch-",

infra/core/host/container-app-upsert.bicep

@@ -15,7 +15,7 @@ param containerMaxReplicas int = 10
 param containerMemory string = '1.0Gi'
 
 @description('The minimum number of replicas to run. Must be at least 1 for non-consumption workloads.')
-param containerMinReplicas int = 1
+param containerMinReplicas int = 0
 
 @description('The name of the container')
 param containerName string = 'main'

infra/core/host/container-app.bicep

@@ -20,7 +20,7 @@ param containerMaxReplicas int = 10
 param containerMemory string = '1.0Gi'
 
 @description('The minimum number of replicas to run. Must be at least 1.')
-param containerMinReplicas int = 1
+param containerMinReplicas int = 0
 
 @description('The name of the container')
 param containerName string = 'main'

infra/core/host/container-apps.bicep

@@ -8,8 +8,11 @@ param containerRegistryName string
 param containerRegistryResourceGroupName string = ''
 param containerRegistryAdminUserEnabled bool = false
 param logAnalyticsWorkspaceResourceId string
-param applicationInsightsName string = '' // Not used here, was used for DAPR
-param virtualNetworkSubnetId string = ''
+
+param subnetResourceId string = ''
+
+param usePrivateIngress bool = true
+
 @allowed(['Consumption', 'D4', 'D8', 'D16', 'D32', 'E4', 'E8', 'E16', 'E32', 'NC24-A100', 'NC48-A100', 'NC96-A100'])
 param workloadProfile string
 
@@ -26,7 +29,7 @@ var workloadProfiles = workloadProfile == 'Consumption'
         workloadProfileType: 'Consumption'
       }
       {
-        minimumCount: 0
+        minimumCount: 1
         maximumCount: 2
         name: workloadProfile
         workloadProfileType: workloadProfile
@@ -51,7 +54,8 @@ module containerAppsEnvironment 'br/public:avm/res/app/managed-environment:0.8.0
     name: containerAppsEnvironmentName
     // Non-required parameters
     infrastructureResourceGroupName: containerRegistryResourceGroupName
-    infrastructureSubnetId: virtualNetworkSubnetId
+    infrastructureSubnetId: subnetResourceId
+    internal: usePrivateIngress
     location: location
     tags: tags
     zoneRedundant: false
@@ -67,7 +71,9 @@ module containerRegistry 'br/public:avm/res/container-registry/registry:0.5.1' =
   params: {
     name: containerRegistryName
     location: location
+    acrSku: usePrivateIngress ? 'Premium' : 'Standard'
     acrAdminUserEnabled: containerRegistryAdminUserEnabled
+    publicNetworkAccess: usePrivateIngress ? 'Disabled' : 'Enabled'
     tags: tags
   }
 }

infra/core/search/search-services.bicep

@@ -55,6 +55,7 @@ resource search 'Microsoft.Search/searchServices@2023-11-01' = {
   }
   sku: sku
 
+  // https://github.com/Azure/bicep-types-az/issues/2421
   resource sharedPrivateLinkResource 'sharedPrivateLinkResources@2023-11-01' = [for (resourceId, i) in sharedPrivateLinkStorageAccounts: {
     name: 'search-shared-private-link-${i}'
     properties: {

infra/main.bicep

@@ -244,6 +244,9 @@ param publicNetworkAccess string = 'Enabled'
 @description('Add a private endpoints for network connectivity')
 param usePrivateEndpoint bool = false
 
+@description('Use a P2S VPN Gateway for secure access to the private endpoints')
+param useVpnGateway bool = false
+
 @description('Id of the user or app to assign application roles')
 param principalId string = ''
 
@@ -488,7 +491,7 @@ module backend 'core/host/appservice.bicep' = if (deploymentTarget == 'appservic
     appCommandLine: 'python3 -m gunicorn main:app'
     scmDoBuildDuringDeployment: true
     managedIdentity: true
-    virtualNetworkSubnetId: isolation.outputs.appSubnetId
+    virtualNetworkSubnetId: usePrivateEndpoint ? isolation.outputs.appSubnetId : ''
     publicNetworkAccess: publicNetworkAccess
     allowedOrigins: allowedOrigins
     clientAppId: clientAppId
@@ -529,6 +532,8 @@ module containerApps 'core/host/container-apps.bicep' = if (deploymentTarget ==
     containerAppsEnvironmentName: acaManagedEnvironmentName
     containerRegistryName: '${containerRegistryName}${resourceToken}'
     logAnalyticsWorkspaceResourceId: useApplicationInsights ? monitoring.outputs.logAnalyticsWorkspaceId : ''
+    subnetResourceId: usePrivateEndpoint ? isolation.outputs.appSubnetId : ''
+    usePrivateIngress: usePrivateEndpoint
   }
 }
 
@@ -541,8 +546,8 @@ module acaBackend 'core/host/container-app-upsert.bicep' = if (deploymentTarget
     acaIdentity
   ]
   params: {
-    name: !empty(backendServiceName) ? backendServiceName : '${abbrs.webSitesContainerApps}backend-${resourceToken}'
-    location: location
+    name: !empty(backendServiceName) ? backendServiceName : '${abbrs.webSitesContainerApps}backend${resourceToken}'
+    location: 'westus2'
     identityName: (deploymentTarget == 'containerapps') ? acaIdentityName : ''
     exists: webAppExists
     workloadProfile: azureContainerAppsWorkloadProfile
@@ -553,7 +558,7 @@ module acaBackend 'core/host/container-app-upsert.bicep' = if (deploymentTarget
     targetPort: 8000
     containerCpuCoreCount: '1.0'
     containerMemory: '2Gi'
-    containerMinReplicas: 0
+    containerMinReplicas: 1
     allowedOrigins: allowedOrigins
     env: union(appEnvVariables, {
       // For using managed identity to access Azure resources. See https://github.com/microsoft/azure-container-apps/issues/442
@@ -789,7 +794,7 @@ module searchService 'core/search/search-services.bicep' = {
     publicNetworkAccess: publicNetworkAccess == 'Enabled'
       ? 'enabled'
       : (publicNetworkAccess == 'Disabled' ? 'disabled' : null)
-    sharedPrivateLinkStorageAccounts: usePrivateEndpoint ? [storage.outputs.id] : []
+    sharedPrivateLinkStorageAccounts: (usePrivateEndpoint && useIntegratedVectorization) ? [storage.outputs.id] : []
   }
 }
 
@@ -1156,23 +1161,24 @@ module cosmosDbRoleBackend 'core/security/documentdb-sql-role.bicep' = if (useAu
   }
 }
 
-module isolation 'network-isolation.bicep' = {
+module isolation 'network-isolation.bicep' = if (usePrivateEndpoint) {
   name: 'networks'
   scope: resourceGroup
   params: {
-    deploymentTarget: deploymentTarget
     location: location
     tags: tags
     vnetName: '${abbrs.virtualNetworks}${resourceToken}'
+    usePrivateEndpoint: usePrivateEndpoint
+    deploymentTarget: deploymentTarget
     // Need to check deploymentTarget due to https://github.com/Azure/bicep/issues/3990
     appServicePlanName: deploymentTarget == 'appservice' ? appServicePlan.outputs.name : ''
-    usePrivateEndpoint: usePrivateEndpoint
+    containerAppsEnvName: deploymentTarget == 'containerapps' ? acaManagedEnvironmentName : ''
   }
 }
 
 var environmentData = environment()
 
-var openAiPrivateEndpointConnection = (isAzureOpenAiHost && deployAzureOpenAi && deploymentTarget == 'appservice')
+var openAiPrivateEndpointConnection = (isAzureOpenAiHost && deployAzureOpenAi)
   ? [
       {
         groupId: 'account'
@@ -1186,7 +1192,7 @@ var openAiPrivateEndpointConnection = (isAzureOpenAiHost && deployAzureOpenAi &&
       }
     ]
   : []
-var otherPrivateEndpointConnections = (usePrivateEndpoint && deploymentTarget == 'appservice')
+var otherPrivateEndpointConnections = (usePrivateEndpoint)
   ? [
       {
         groupId: 'blob'
@@ -1199,9 +1205,9 @@ var otherPrivateEndpointConnections = (usePrivateEndpoint && deploymentTarget ==
         resourceIds: [searchService.outputs.id]
       }
       {
-        groupId: 'sites'
-        dnsZoneName: 'privatelink.azurewebsites.net'
-        resourceIds: [backend.outputs.id]
+        groupId: 'managedEnvironments'
+        dnsZoneName: 'privatelink.${location}.azurecontainerapps.io'
+        resourceIds: [containerApps.outputs.environmentId]
       }
       {
         groupId: 'sql'
@@ -1213,7 +1219,7 @@ var otherPrivateEndpointConnections = (usePrivateEndpoint && deploymentTarget ==
 
 var privateEndpointConnections = concat(otherPrivateEndpointConnections, openAiPrivateEndpointConnection)
 
-module privateEndpoints 'private-endpoints.bicep' = if (usePrivateEndpoint && deploymentTarget == 'appservice') {
+module privateEndpoints 'private-endpoints.bicep' = if (usePrivateEndpoint) {
   name: 'privateEndpoints'
   scope: resourceGroup
   params: {
@@ -1228,6 +1234,51 @@ module privateEndpoints 'private-endpoints.bicep' = if (usePrivateEndpoint && de
   }
 }
 
+// Based on https://luke.geek.nz/azure/azure-point-to-site-vpn-and-private-dns-resolver/
+// Manual step required of updating azurevpnconfig.xml to use the correct DNS server IP address
+module dnsResolver 'br/public:avm/res/network/dns-resolver:0.5.3' = if (useVpnGateway) {
+  name: 'dnsResolverDeployment'
+  scope: resourceGroup
+  params: {
+    name: '${abbrs.privateDnsResolver}${resourceToken}'
+    location: location
+    virtualNetworkResourceId: isolation.outputs.vnetId
+    inboundEndpoints: [
+      {
+        name: 'inboundEndpoint'
+        subnetResourceId: useVpnGateway ? isolation.outputs.privateDnsResolverSubnetId : ''
+      }
+    ]
+  }
+}
+
+module virtualNetworkGateway 'br/public:avm/res/network/virtual-network-gateway:0.6.1' = if (useVpnGateway) {
+  name: 'virtualNetworkGatewayDeployment'
+  scope: resourceGroup
+  params: {
+    name: '${abbrs.networkVpnGateways}${resourceToken}'
+    clusterSettings: {
+      clusterMode: 'activePassiveNoBgp'
+    }
+    gatewayType: 'Vpn'
+    virtualNetworkResourceId: isolation.outputs.vnetId
+    vpnGatewayGeneration: 'Generation2'
+    vpnClientAddressPoolPrefix: '172.16.201.0/24'
+    skuName: 'VpnGw2'
+    vpnClientAadConfiguration: {
+      aadAudience: 'c632b3df-fb67-4d84-bdcf-b95ad541b5c8' // Azure VPN client
+      aadIssuer: 'https://sts.windows.net/${tenant().tenantId}/'
+      aadTenant: '${environment().authentication.loginEndpoint}${tenant().tenantId}'
+      vpnAuthenticationTypes: [
+        'AAD'
+      ]
+      vpnClientProtocols: [
+        'OpenVPN'
+      ]
+    }
+  }
+}
+
 // Used to read index definitions (required when using authentication)
 // https://learn.microsoft.com/azure/search/search-security-rbac
 module searchReaderRoleBackend 'core/security/role.bicep' = if (useAuthentication) {

infra/network-isolation.bicep

@@ -9,69 +9,101 @@ param location string = resourceGroup().location
 @description('The tags to apply to all resources')
 param tags object = {}
 
-@description('The name of an existing App Service Plan to connect to the VNet')
-param appServicePlanName string
-
 param usePrivateEndpoint bool = false
 
 @allowed(['appservice', 'containerapps'])
 param deploymentTarget string
 
+@description('The name of an existing App Service Plan to connect to the VNet')
+param appServicePlanName string
+
+@description('The name of an existing Container Apps Environment to connect to the VNet')
+param containerAppsEnvName string
+
+param deployVpnGateway bool = false
+
 resource appServicePlan 'Microsoft.Web/serverfarms@2022-03-01' existing = if (deploymentTarget == 'appservice') {
   name: appServicePlanName
 }
 
-module vnet './core/networking/vnet.bicep' = if (usePrivateEndpoint) {
-  name: 'vnet'
-  params: {
-    name: vnetName
-    location: location
-    tags: tags
-    subnets: [
-      {
+resource containerAppsEnvironment 'Microsoft.App/managedEnvironments@2023-05-01' existing = if (deploymentTarget == 'containerapps') {
+  name: containerAppsEnvName
+}
+
+// Always need this one
+var backendSubnet =  {
         name: 'backend-subnet'
         properties: {
           addressPrefix: '10.0.1.0/24'
           privateEndpointNetworkPolicies: 'Enabled'
           privateLinkServiceNetworkPolicies: 'Enabled'
         }
       }
-      {
-        name: 'AzureBastionSubnet'
-        properties: {
-          addressPrefix: '10.0.2.0/24'
-          privateEndpointNetworkPolicies: 'Enabled'
-          privateLinkServiceNetworkPolicies: 'Enabled'
-        }
-      }
-      {
-        name: 'app-int-subnet'
-        properties: {
-          addressPrefix: '10.0.3.0/24'
-          privateEndpointNetworkPolicies: 'Enabled'
-          privateLinkServiceNetworkPolicies: 'Enabled'
-          delegations: [
-            {
-              id: appServicePlan.id
-              name: appServicePlan.name
-              properties: {
-                serviceName: 'Microsoft.Web/serverFarms'
-              }
+
+var appServiceSubnet = {
+      name: 'app-int-subnet'
+      properties: {
+        addressPrefix: '10.0.3.0/24'
+        privateEndpointNetworkPolicies: 'Enabled'
+        privateLinkServiceNetworkPolicies: 'Enabled'
+        delegations: [
+          {
+            id: appServicePlan.id
+            name: appServicePlan.name
+            properties: {
+              serviceName: 'Microsoft.Web/serverFarms'
             }
-          ]
-        }
+          }
+        ]
       }
-      {
-        name: 'vm-subnet'
-        properties: {
-          addressPrefix: '10.0.4.0/24'
-        }
+    }
+
+var containerAppsSubnet = {
+      name: 'app-int-subnet'
+      properties: {
+        addressPrefix: '10.0.4.0/23'
+        privateEndpointNetworkPolicies: 'Enabled'
+        privateLinkServiceNetworkPolicies: 'Enabled'
+        delegations: [
+          {
+            id: containerAppsEnvironment.id
+            name: containerAppsEnvironment.name
+            properties: {
+              serviceName: 'Microsoft.App/environments'
+            }
+          }
+        ]
       }
-    ]
-  }
 }
 
+var gatewaySubnet = {
+    name: 'GatewaySubnet' // Required name for Gateway subnet
+    addressPrefix: '10.0.255.0/27' // Using a /27 subnet size which is minimal required size for gateway subnet
+  }
+
+var privateDnsResolverSubnet = {
+    name: 'dns-resolver-subnet' // Dedicated subnet for Azure Private DNS Resolver
+    addressPrefix: '10.0.11.0/28' // Original value kept as requested
+    delegation: 'Microsoft.Network/dnsResolvers'
+  }
+
+var subnets = union(
+  [backendSubnet, deploymentTarget == 'appservice' ? appServiceSubnet : containerAppsSubnet],
+  deployVpnGateway ? [gatewaySubnet, privateDnsResolverSubnet] : [])
+
+module vnet './core/networking/vnet.bicep' = if (usePrivateEndpoint) {
+  name: 'vnet'
+  params: {
+    name: vnetName
+    location: location
+    tags: tags
+    subnets: subnets
+  }
+}
 
-output appSubnetId string = usePrivateEndpoint ? vnet.outputs.vnetSubnets[2].id : ''
+output appSubnetId string = usePrivateEndpoint ? vnet.outputs.vnetSubnets[1].id : ''
+output appSubnetName string = usePrivateEndpoint ? vnet.outputs.vnetSubnets[1].name : ''
 output backendSubnetId string = usePrivateEndpoint ? vnet.outputs.vnetSubnets[0].id : ''
+output privateDnsResolverSubnetId string = deployVpnGateway ? vnet.outputs.vnetSubnets[3].id : ''
 output vnetName string = usePrivateEndpoint ? vnet.outputs.name : ''
+output vnetId string = usePrivateEndpoint ? vnet.outputs.id : ''