Merge branch 'main' into myao/sync-sb3

# Conflicts:
#	.github/CODEOWNERS
#	sdk/appcomplianceautomation/azure-resourcemanager-appcomplianceautomation/pom.xml
#	sdk/cognitiveservices/ms-azure-cs-textanalytics/pom.xml
#	sdk/keyvault/microsoft-azure-keyvault-cryptography/pom.xml
#	sdk/keyvault/microsoft-azure-keyvault-webkey/pom.xml
#	sdk/keyvault/microsoft-azure-keyvault/pom.xml
#	sdk/loganalytics/microsoft-azure-loganalytics/pom.xml
#	sdk/maps/azure-maps-elevation/pom.xml
#	sdk/mobilenetwork/azure-resourcemanager-mobilenetwork/pom.xml
#	sdk/purview/azure-analytics-purview-catalog/pom.xml
#	sdk/spring/ci.monitor.yml
#	sdk/spring/ci.yml
#	sdk/storage/microsoft-azure-storage-blob/pom.xml
#	sdk/webpubsub/azure-messaging-webpubsub/pom.xml

Author: Netyyyy

.github/CODEOWNERS_baseline_errors.txt

@@ -50,14 +50,14 @@ Merging Pull Requests (for project contributors with write access)
   - add `MAVEN_HOME` to environment variables
 
 >**Note:** If you ran into "long path" issue on `Windows`, enable paths longer than 260 characters by: <br><br>
-1.- Run this as Administrator on a command prompt:<br> 
+1.- Run this as Administrator on a command prompt:<br>
 `REG ADD HKLM\SYSTEM\CurrentControlSet\Control\FileSystem /v LongPathsEnabled /t REG_DWORD /d 1`<br>*(might need to type `yes` to override key if it already exists)*<br><br>
 2.- Set up `git` by running:<br> `git config --system core.longpaths true`
 
 ### Building and Unit Testing
 
 Refer to the [build wiki](https://github.com/Azure/azure-sdk-for-java/wiki/Building) for learning how to build Java SDKs
-and the [unit testing wiki](https://github.com/Azure/azure-sdk-for-java/wiki/Unit-Testing) for guidelines on unit 
+and the [unit testing wiki](https://github.com/Azure/azure-sdk-for-java/wiki/Unit-Testing) for guidelines on unit
 testing.
 
 ### Live testing
@@ -93,7 +93,7 @@ When building locally you might run into a Checkstyle such as the following:
 
 ```
 Execution default of goal org.apache.maven.plugins:maven-checkstyle-plugin:3.1.0:check failed:
-Plugin org.apache.maven.plugins:maven-checkstyle-plugin:3.1.0 or one of its dependencies could not be resolved: 
+Plugin org.apache.maven.plugins:maven-checkstyle-plugin:3.1.0 or one of its dependencies could not be resolved:
 Could not find artifact com.azure:sdk-build-tools:jar:1.0.0 in ossrh (https://oss.sonatype.org/content/repositories/snapshots/)
 ```
 
@@ -115,6 +115,29 @@ The dependency-version should be set to the most recent released version and the
 
 Note: In the case of a new artifact both versions will be the same. In the case of a released artifact, the dependecny version should be the latest released version.
 
+### Supported GroupIds and publishing
+
+We no longer publish directly to Maven Central, any publishing is done through ESRP. Libraries published need to conform to one of the groupIds that we can publish to. Here is the list of accepted groupdIs:
+
+- "com.microsoft"
+- "com.azure.*"
+- "com.azure"
+- "com.microsoft.azure"
+- "com.microsoft.azure.cognitiveservices"
+- "com.microsoft.azure.functions"
+- "com.microsoft.azure.kusto"
+- "com.microsoft.azure.sdk.iot"
+- "com.microsoft.azure.sdk.iot.provisioning"
+- "com.microsoft.commondatamodel"
+- "com.windowsazure"
+- "com.microsoft.sqlserver"
+- "com.microsoft.spring"
+- "com.microsoft.servicefabric"
+- "com.microsoft.rest"
+- "io.clientcore"
+
+Note: This list will be replaced with a link to a page created by ESRP providing a UI where GroupIds can be cheked and new ones requested if needed.
+
 ### Libraries vs External Dependencies
 
 Libraries refer to things that are built and released as part of the Azure SDK. Libraries have a current version and a dependency version.
@@ -193,7 +216,7 @@ After the unreleased version of `com.azure:azure-core` was released but before `
 
 For general packaging versioning policies see [Package Versioning](https://azure.github.io/azure-sdk/policies_releases.html#package-versioning) and see [Java](https://azure.github.io/azure-sdk/policies_releases.html#java) for specific rules used in this repo.
 
-While some Java projects use SNAPSHOT versions for nightly builds, we have opted not to use that convention because it has proven to be very unreliable in our scenarios. For example, if we use SNAPSHOT versions in our pom.xml files that usually ends up becoming viral throughout our entire repo and we want to more tightly control our versioning, especially our dependency versioning. On top of the viral nature, we have experienced a lot of network reliability issues when consuming SNAPSHOT versions from Maven central so we want to avoid this reliability issue in our build pipelines. 
+While some Java projects use SNAPSHOT versions for nightly builds, we have opted not to use that convention because it has proven to be very unreliable in our scenarios. For example, if we use SNAPSHOT versions in our pom.xml files that usually ends up becoming viral throughout our entire repo and we want to more tightly control our versioning, especially our dependency versioning. On top of the viral nature, we have experienced a lot of network reliability issues when consuming SNAPSHOT versions from Maven central so we want to avoid this reliability issue in our build pipelines.
 
 Given we don't use SNAPSHOT versions in our pom.xml files we generally have the version currently under development committed to the repo in the pom.xml file. This means if you are looking at our active development code for the version you will see a version that is not yet published to Maven central. If you want to try out our packages under development, you should look for our latest alpha build (see Dev Feed section below).
 

CONTRIBUTING.md

@@ -56,15 +56,15 @@ jobs:
         Java 8 MacOS (AzureCloud):
           JavaTestVersion: '1.8'
           Pool: Azure Pipelines
-          OSVmImage: macos-11
+          OSVmImage: macos-latest
           SubscriptionConfiguration: $(sub-config-azure-cloud-test-resources)
           ArmTemplateParameters: $(azureCloudArmParameters)
           PomFile: common/smoke-tests/pom.xml
           ServiceDirectory: '$(Build.SourcesDirectory)/common/smoke-tests/'
         Java LTS MacOS (AzureCloud):
           JavaTestVersion: '1.11'
           Pool: Azure Pipelines
-          OSVmImage: macos-11
+          OSVmImage: macos-latest
           SubscriptionConfiguration: $(sub-config-azure-cloud-test-resources)
           ArmTemplateParameters: $(azureCloudArmParameters)
           PomFile: common/smoke-tests/pom.xml

common/smoke-tests/smoke-test-dependencies.yml

@@ -511,7 +511,7 @@
         {
           "regex": true,
           "code": "java\\.annotation\\.(attributeRemoved|attributeAdded)",
-          "old": ".*? com\\.azure\\.resourcemanager\\.(cosmos|network)\\.models.*",
+          "old": ".*? com\\.azure\\.resourcemanager\\.(cosmos|network|mobilenetwork)\\.models.*",
           "justification": "Jackson annotation changed."
         },
         {
@@ -549,6 +549,27 @@
           "code": "java.method.removed",
           "old": "method com.azure.resourcemanager.cosmos.models.ServiceResourceCreateUpdateParameters com.azure.resourcemanager.cosmos.models.ServiceResourceCreateUpdateParameters::withServiceType(com.azure.resourcemanager.cosmos.models.ServiceType)",
           "justification": "The property flatten disabled due to model changed to polymorphic."
+        },
+        {
+          "regex": true,
+          "code" : "java\\.annotation\\.removed",
+          "old" : ".*? com\\.azure\\.communication\\.phonenumbers\\.models.*",
+          "new" : ".*? com\\.azure\\.communication\\.phonenumbers\\.models.*",
+          "justification": "Migration to azure-json"
+        },
+        {
+          "regex": true,
+          "code" : "java\\.annotation\\.removed",
+          "old" : ".*? com\\.azure\\.messaging\\.webpubsub\\.models.*",
+          "new" : ".*? com\\.azure\\.messaging\\.webpubsub\\.models.*",
+          "justification": "Migration to azure-json"
+        },
+        {
+          "regex": true,
+          "code" : "java\\.annotation\\.removed",
+          "old" : ".*? com\\.azure\\.messaging\\.webpubsub\\.client\\.models.*",
+          "new" : ".*? com\\.azure\\.messaging\\.webpubsub\\.client\\.models.*",
+          "justification": "Migration to azure-json"
         }
       ]
     }

eng/code-quality-reports/src/main/resources/revapi/revapi.json

@@ -106,6 +106,15 @@ param (
 
 . $PSScriptRoot/SubConfig-Helpers.ps1
 
+$azsdkPipelineVnet = "/subscriptions/a18897a6-7e44-457d-9260-f2854c0aca42/resourceGroups/azsdk-pools/providers/Microsoft.Network/virtualNetworks/azsdk-pipeline-vnet-wus"
+$azsdkPipelineSubnets = @(
+    ($azsdkPipelineVnet + "/subnets/pipeline-subnet-ubuntu-1804-general"),
+    ($azsdkPipelineVnet + "/subnets/pipeline-subnet-ubuntu-2004-general"),
+    ($azsdkPipelineVnet + "/subnets/pipeline-subnet-ubuntu-2204-general"),
+    ($azsdkPipelineVnet + "/subnets/pipeline-subnet-win-2019-general"),
+    ($azsdkPipelineVnet + "/subnets/pipeline-subnet-win-2022-general")
+)
+
 if (!$ServicePrincipalAuth) {
     # Clear secrets if not using Service Principal auth. This prevents secrets
     # from being passed to pre- and post-scripts.
@@ -743,13 +752,15 @@ try {
     if ($ProvisionerApplicationOid) {
         $templateParameters["provisionerApplicationOid"] = "$ProvisionerApplicationOid"
     }
-
     if ($TenantId) {
         $templateParameters.Add('tenantId', $TenantId)
     }
     if ($TestApplicationSecret -and $ServicePrincipalAuth) {
         $templateParameters.Add('testApplicationSecret', $TestApplicationSecret)
     }
+    if ($CI -and $Environment -eq 'AzureCloud') {
+        $templateParameters.Add('azsdkPipelineSubnetList', $azsdkPipelineSubnets)
+    }
 
     $defaultCloudParameters = LoadCloudConfig $Environment
     MergeHashes $defaultCloudParameters $(Get-Variable templateParameters)

eng/common/TestResources/New-TestResources.ps1

@@ -103,7 +103,7 @@ function ShouldMarkValueAsSecret([string]$serviceName, [string]$key, [string]$va
 
 function SetSubscriptionConfiguration([object]$subscriptionConfiguration)
 {
-    foreach($pair in $subscriptionConfiguration.GetEnumerator()) {
+    foreach ($pair in $subscriptionConfiguration.GetEnumerator()) {
         if ($pair.Value -is [Hashtable]) {
             foreach($nestedPair in $pair.Value.GetEnumerator()) {
                 # Mark values as secret so we don't print json blobs containing secrets in the logs.
@@ -126,36 +126,32 @@ function SetSubscriptionConfiguration([object]$subscriptionConfiguration)
         }
     }
 
-    Write-Host ($subscriptionConfiguration | ConvertTo-Json)
-    $serialized = $subscriptionConfiguration | ConvertTo-Json -Compress
-    Write-Host "##vso[task.setvariable variable=SubscriptionConfiguration;]$serialized"
+    return $subscriptionConfiguration
 }
 
 function UpdateSubscriptionConfiguration([object]$subscriptionConfigurationBase, [object]$subscriptionConfiguration)
 {
-        foreach ($pair in $subscriptionConfiguration.GetEnumerator()) {
-            if ($pair.Value -is [Hashtable]) {
-                if (!$subscriptionConfigurationBase.ContainsKey($pair.Name)) {
-                    $subscriptionConfigurationBase[$pair.Name] = @{}
-                }
-                foreach($nestedPair in $pair.Value.GetEnumerator()) {
-                    # Mark values as secret so we don't print json blobs containing secrets in the logs.
-                    # Prepend underscore to the variable name, so we can still access the variable names via environment
-                    # variables if they get set subsequently.
-                    if (ShouldMarkValueAsSecret "AZURE_" $nestedPair.Name $nestedPair.Value) {
-                        Write-Host "##vso[task.setvariable variable=_$($nestedPair.Name);issecret=true;]$($nestedPair.Value)"
-                    }
-                    $subscriptionConfigurationBase[$pair.Name][$nestedPair.Name] = $nestedPair.Value
-                }
-            } else {
-                if (ShouldMarkValueAsSecret "AZURE_" $pair.Name $pair.Value) {
-                    Write-Host "##vso[task.setvariable variable=_$($pair.Name);issecret=true;]$($pair.Value)"
+    foreach ($pair in $subscriptionConfiguration.GetEnumerator()) {
+        if ($pair.Value -is [Hashtable]) {
+            if (!$subscriptionConfigurationBase.ContainsKey($pair.Name)) {
+                $subscriptionConfigurationBase[$pair.Name] = @{}
+            }
+            foreach($nestedPair in $pair.Value.GetEnumerator()) {
+                # Mark values as secret so we don't print json blobs containing secrets in the logs.
+                # Prepend underscore to the variable name, so we can still access the variable names via environment
+                # variables if they get set subsequently.
+                if (ShouldMarkValueAsSecret "AZURE_" $nestedPair.Name $nestedPair.Value) {
+                    Write-Host "##vso[task.setvariable variable=_$($nestedPair.Name);issecret=true;]$($nestedPair.Value)"
                 }
-                $subscriptionConfigurationBase[$pair.Name] = $pair.Value
+                $subscriptionConfigurationBase[$pair.Name][$nestedPair.Name] = $nestedPair.Value
             }
+        } else {
+            if (ShouldMarkValueAsSecret "AZURE_" $pair.Name $pair.Value) {
+                Write-Host "##vso[task.setvariable variable=_$($pair.Name);issecret=true;]$($pair.Value)"
+            }
+            $subscriptionConfigurationBase[$pair.Name] = $pair.Value
         }
+    }
 
-        $serialized = $subscriptionConfigurationBase | ConvertTo-Json -Compress
-        Write-Host ($subscriptionConfigurationBase | ConvertTo-Json)
-        Write-Host "##vso[task.setvariable variable=SubscriptionConfiguration;]$serialized"
+    return $subscriptionConfigurationBase
 }

eng/common/TestResources/SubConfig-Helpers.ps1

@@ -9,38 +9,52 @@ parameters:
   - name: EnvVars
     type: object
     default: null
+  - name: SubscriptionConfigurationFilePaths
+    type: object
+    default: null
 
 steps:
-  - ${{ if parameters.SubscriptionConfiguration }}:
-    - pwsh: |
-        $config = @'
-          ${{ parameters.SubscriptionConfiguration }}
-        '@ | ConvertFrom-Json -AsHashtable
-
-        . ./eng/common/TestResources/SubConfig-Helpers.ps1
-        SetSubscriptionConfiguration $config
-      displayName: Initialize SubscriptionConfiguration variable
-      ${{ if parameters.EnvVars }}:
-        env: ${{ parameters.EnvVars }}
-
-  - ${{ if parameters.SubscriptionConfigurations }}:
-    - pwsh: |
-        Write-Host "##vso[task.setvariable variable=SubscriptionConfiguration;]{}"
-      displayName: Initialize SubscriptionConfiguration variable for merging
-      condition: eq(variables['SubscriptionConfiguration'], '')
-
-    - ${{ each config in parameters.SubscriptionConfigurations }}:
-      - pwsh: |
-          $configBase = @'
-            $(SubscriptionConfiguration)
-          '@ | ConvertFrom-Json -AsHashtable
-          $config = @'
-            ${{ config }}
-          '@ | ConvertFrom-Json -AsHashtable
-
-          . ./eng/common/TestResources/SubConfig-Helpers.ps1
-          UpdateSubscriptionConfiguration $configBase $config
-
-        displayName: Merge Test Resource Configurations
-        ${{ if parameters.EnvVars }}:
-          env: ${{ parameters.EnvVars }}
+  - pwsh: |
+      . ./eng/common/TestResources/SubConfig-Helpers.ps1
+
+      $finalConfig = @{}
+      $baseSubConfigRaw = @'
+        ${{ parameters.SubscriptionConfiguration }}
+      '@.Trim()
+      if ($baseSubConfigRaw) {
+        $baseSubConfig = $baseSubConfigRaw | ConvertFrom-Json -AsHashtable
+
+        Write-Host "Setting base sub config"
+        $finalConfig = SetSubscriptionConfiguration $baseSubConfig
+      }
+
+      $subConfigJsonsRaw = @'
+        ${{ convertToJson(parameters.SubscriptionConfigurations) }}
+      '@.Trim() -replace '"{', '{' -replace '}"', '}'
+
+      if ($subConfigJsonsRaw) {
+        $subConfigs = $subConfigJsonsRaw | ConvertFrom-Json -AsHashtable
+
+        foreach ($subConfig in $subConfigs) {
+          Write-Host "Merging sub config from list"
+          $finalConfig = UpdateSubscriptionConfiguration $finalConfig $subConfig
+        }
+      }
+
+      $subConfigFilesRaw = @'
+        ${{ convertToJson(parameters.SubscriptionConfigurationFilePaths) }}
+      '@.Trim()
+
+      if ($subConfigFilesRaw) {
+        $subConfigFiles = $subConfigFilesRaw | ConvertFrom-Json -AsHashtable
+        foreach ($file in $subConfigFiles) {
+          Write-Host "Merging sub config from file: $file"
+          $subConfig = Get-Content $file | ConvertFrom-Json -AsHashtable
+          $finalConfig = UpdateSubscriptionConfiguration $finalConfig $subConfig
+        }
+      }
+
+      Write-Host ($finalConfig | ConvertTo-Json)
+      $serialized = $finalConfig | ConvertTo-Json -Compress
+      Write-Host "##vso[task.setvariable variable=SubscriptionConfiguration;]$serialized"
+    displayName: Merge subscription configurations

eng/common/TestResources/build-test-resource-config.yml

@@ -8,8 +8,6 @@ parameters:
   ServiceConnection: not-specified
   ResourceType: test
   UseFederatedAuth: false
-  SubscriptionConfigurationFilePath: ''
-
 
 # SubscriptionConfiguration will be splatted into the parameters of the test
 # resources script. It should be JSON in the form:
@@ -56,18 +54,9 @@ steps:
         ScriptType: InlineScript
         Inline: |
           eng/common/scripts/Import-AzModules.ps1
-
-          if ('${{ parameters.SubscriptionConfigurationFilePath }}' -ne '') {
-            $subscriptionConfiguration = `
-              Get-Content '${{ parameters.SubscriptionConfigurationFilePath }}' `
-              | ConvertFrom-Json -AsHashtable;
-          } else {
-            # Multiline string termination ('@) needs to be at the beginning
-            # of the line
-            $subscriptionConfiguration = @'
-              ${{ parameters.SubscriptionConfiguration }}
+          $subscriptionConfiguration = @'
+            ${{ parameters.SubscriptionConfiguration }}
           '@ | ConvertFrom-Json -AsHashtable;
-          }
 
           # The subscriptionConfiguration may have ArmTemplateParameters defined, so
           # pass those in via the ArmTemplateParameters flag, and handle any
@@ -86,7 +75,6 @@ steps:
   - ${{ else }}:
     - pwsh: |
         eng/common/scripts/Import-AzModules.ps1
-
         $subscriptionConfiguration = @'
           ${{ parameters.SubscriptionConfiguration }}
         '@ | ConvertFrom-Json -AsHashtable;

eng/common/TestResources/deploy-test-resources.yml

@@ -8,7 +8,6 @@ parameters:
   ResourceType: test
   EnvVars: {}
   UseFederatedAuth: false
-  SubscriptionConfigurationFilePath: ''
 
 # SubscriptionConfiguration will be splat into the parameters of the test
 # resources script. It should be JSON in the form:
@@ -39,20 +38,10 @@ steps:
         Inline: |
           eng/common/scripts/Import-AzModules.ps1
 
-
-          if ('${{ parameters.SubscriptionConfigurationFilePath }}' -ne '') {
-            $subscriptionConfiguration = `
-              Get-Content '${{ parameters.SubscriptionConfigurationFilePath }}' `
-              | ConvertFrom-Json -AsHashtable;
-          } else {
-            # Multiline string termination ("@) needs to be at the beginning
-            # of the line
-            $subscriptionConfiguration = @"
-              ${{ parameters.SubscriptionConfiguration }}
+          $subscriptionConfiguration = @"
+            ${{ parameters.SubscriptionConfiguration }}
           "@ | ConvertFrom-Json -AsHashtable;
 
-          }
-
           eng/common/TestResources/Remove-TestResources.ps1 `
             @subscriptionConfiguration `
             -ResourceType '${{ parameters.ResourceType }}' `

eng/common/TestResources/remove-test-resources.yml

@@ -2,8 +2,10 @@
     "SubscriptionId": "2cd617ea-1866-46b1-90e3-fffb087ebf9b",
     "TenantId": "72f988bf-86f1-41af-91ab-2d7cd011db47",
     "TestApplicationId": "f850650c-1fcf-4489-b46f-71af2e30d360",
+    "TestApplicationSecret": "",
     "TestApplicationOid": "30511c9d-ba1a-4c7b-b422-5b543da11b3f",
     "ProvisionerApplicationId": "f850650c-1fcf-4489-b46f-71af2e30d360",
+    "ProvisionerApplicationSecret": "",
     "ProvisionerApplicationOid": "30511c9d-ba1a-4c7b-b422-5b543da11b3f",
     "Environment": "AzureCloud",
     "AzureSubscription": "Azure SDK Test Resources"