Fix test proxy network calls

eng/pipelines/templates/jobs/ci.tests.yml

@@ -150,11 +150,7 @@ jobs:
           VersionOverride: ${{ parameters.VersionOverride }}
 
       - template: /eng/pipelines/templates/steps/post-job-cleanup.yml
-
-      # Shut down proxy to prevent file locks on the log file for auto-injected credscan
-      # steps on windows
-      - template: /eng/common/testproxy/test-proxy-tool-shutdown.yml
-
+
       - pwsh: |
           $files = Get-ChildItem -Path $(Build.SourcesDirectory) -Filter test-proxy.log
           foreach($file in $files){
@@ -174,3 +170,11 @@ jobs:
           }
         displayName: Dump Test-Proxy Error Logs
         condition: succeededOrFailed()
+
+      - pwsh: |
+          $content = (Invoke-WebRequest -Uri "http://localhost:5000/Audit/Logs")
+          Write-Host $content.Content
+        displayName: Dump Test-Proxy Audit Logs
+        condition: succeededOrFailed()
+
+      - template: /eng/common/testproxy/test-proxy-tool-shutdown.yml

sdk/core/azure-core-test/src/main/java/com/azure/core/test/InterceptorManager.java

@@ -3,6 +3,7 @@
 package com.azure.core.test;
 
 import com.azure.core.http.HttpClient;
+import com.azure.core.http.okhttp.OkHttpAsyncHttpClientBuilder;
 import com.azure.core.http.policy.HttpPipelinePolicy;
 import com.azure.core.test.http.PlaybackClient;
 import com.azure.core.test.http.TestProxyPlaybackClient;
@@ -331,7 +332,10 @@ public HttpClient getPlaybackClient() {
                 throw new IllegalStateException("A playback client can only be requested in PLAYBACK mode.");
             }
             if (testProxyPlaybackClient == null) {
-                testProxyPlaybackClient = new TestProxyPlaybackClient(httpClient, skipRecordingRequestBody);
+//                reactor.netty.http.client.HttpClient client = reactor.netty.http.client.HttpClient.create().wiretap("reactor.netty.http.client.HttpClient", LogLevel.TRACE, AdvancedByteBufFormat.TEXTUAL);
+//                HttpClient localClient = new NettyAsyncHttpClientBuilder(client).build();
+                HttpClient localClient = new OkHttpAsyncHttpClientBuilder().build();
+                testProxyPlaybackClient = new TestProxyPlaybackClient(localClient, skipRecordingRequestBody);
                 proxyVariableQueue
                     .addAll(testProxyPlaybackClient.startPlayback(getTestProxyRecordFile(), testClassPath));
                 xRecordingFileLocation = testProxyPlaybackClient.getRecordingFileLocation();

sdk/core/azure-core-test/src/main/java/com/azure/core/test/http/TestProxyPlaybackClient.java

@@ -4,10 +4,12 @@
 package com.azure.core.test.http;
 
 import com.azure.core.http.HttpClient;
+import com.azure.core.http.HttpHeader;
 import com.azure.core.http.HttpHeaderName;
 import com.azure.core.http.HttpMethod;
 import com.azure.core.http.HttpRequest;
 import com.azure.core.http.HttpResponse;
+import com.azure.core.test.models.CustomMatcher;
 import com.azure.core.test.models.RecordFilePayload;
 import com.azure.core.test.models.TestProxyRequestMatcher;
 import com.azure.core.test.models.TestProxySanitizer;
@@ -29,6 +31,7 @@
 import java.nio.file.Path;
 import java.util.ArrayList;
 import java.util.Base64;
+import java.util.Collections;
 import java.util.Comparator;
 import java.util.HashMap;
 import java.util.LinkedList;
@@ -93,15 +96,17 @@ public Queue<String> startPlayback(File recordFile, Path testClassPath) {
                 .setBody(new RecordFilePayload(recordFile.toString(), assetJsonPath).toJsonString())
                 .setHeader(HttpHeaderName.ACCEPT, "application/json")
                 .setHeader(HttpHeaderName.CONTENT_TYPE, "application/json");
-            HttpResponse response = sendRequestWithRetries(request);
-            checkForTestProxyErrors(response);
-            xRecordingId = response.getHeaderValue(X_RECORDING_ID);
-            xRecordingFileLocation
-                = new String(Base64.getUrlDecoder().decode(response.getHeaders().getValue(X_RECORDING_FILE_LOCATION)),
+            String body;
+            try (HttpResponse response = sendRequestWithRetries(request)) {
+                checkForTestProxyErrors(response);
+                xRecordingId = response.getHeaderValue(X_RECORDING_ID);
+                xRecordingFileLocation = new String(
+                    Base64.getUrlDecoder().decode(response.getHeaders().getValue(X_RECORDING_FILE_LOCATION)),
                     StandardCharsets.UTF_8);
-            addProxySanitization(this.sanitizers);
-            addMatcherRequests(this.matchers);
-            String body = response.getBodyAsString().block();
+                addProxySanitization(this.sanitizers);
+                addMatcherRequests(this.matchers);
+                body = response.getBodyAsString().block();
+            }
             // The test proxy stores variables in a map with no guaranteed order.
             // The Java implementation of recording did not use a map, but relied on the order
             // of the variables as they were stored. Our scheme instead sets an increasing integer
@@ -134,14 +139,30 @@ private HttpResponse sendRequestWithRetries(HttpRequest request) {
         int retries = 0;
         while (true) {
             try {
+                String message = "Sending request to test proxy. RecordingID " + xRecordingId
+                    + " request: " + request.getUrl();
+                for(HttpHeader entry : request.getHeaders()) {
+                    message += " " + entry.getName() + ": " + entry.getValue();
+                }
+                if (request.getBody() != null) {
+                    message += " body: " + request.getBodyAsBinaryData().toString();
+                }
+                LOGGER.verbose(message);
                 HttpResponse response = client.sendSync(request, Context.NONE);
                 if (response.getStatusCode() / 100 != 2) {
-                    throw new RuntimeException("Test proxy returned a non-successful status code. "
-                        + response.getStatusCode() + "; response: " + response.getBodyAsString().block());
+                    String body = response.getBodyAsString().block();
+                    int statusCode = response.getStatusCode();
+                    // We don't generally want to close the response here as the caller will need it,
+                    // but if we're throwing we should clean it up.
+                    response.close();
+                    throw new RuntimeException(
+                        "Test proxy returned a non-successful status code. " + statusCode + "; response: " + body);
                 }
                 return response;
             } catch (Exception e) {
                 retries++;
+
+                LOGGER.error("Failed in the retry loop. RecordingID " + xRecordingId + " message: " + e.getMessage());
                 if (retries >= 3) {
                     throw e;
                 }
@@ -165,7 +186,9 @@ private void sleep(int durationInSeconds) {
     public void stopPlayback() {
         HttpRequest request
             = new HttpRequest(HttpMethod.POST, proxyUrl + "/playback/stop").setHeader(X_RECORDING_ID, xRecordingId);
-        sendRequestWithRetries(request);
+        try (HttpResponse response = sendRequestWithRetries(request)) {
+            checkForTestProxyErrors(response);
+        }
     }
 
     /**
@@ -224,7 +247,9 @@ public void addProxySanitization(List<TestProxySanitizer> sanitizers) {
             HttpRequest request
                 = createAddSanitizersRequest(sanitizers, proxyUrl).setHeader(X_RECORDING_ID, xRecordingId);
 
-            sendRequestWithRetries(request);
+            try (HttpResponse response = sendRequestWithRetries(request)) {
+                checkForTestProxyErrors(response);
+            }
         } else {
             this.sanitizers.addAll(sanitizers);
         }
@@ -250,7 +275,9 @@ public void removeProxySanitization(List<String> sanitizers) {
                 throw new RuntimeException(e);
             }
 
-            sendRequestWithRetries(request);
+            try (HttpResponse response = sendRequestWithRetries(request)) {
+                checkForTestProxyErrors(response);
+            }
         }
     }
 
@@ -260,13 +287,18 @@ public void removeProxySanitization(List<String> sanitizers) {
      */
     public void addMatcherRequests(List<TestProxyRequestMatcher> matchers) {
         if (isPlayingBack()) {
+            if (matchers.isEmpty()) {
+                matchers.add(new CustomMatcher().setExcludedHeaders(Collections.singletonList("Connect")));
+            }
             List<HttpRequest> matcherRequests = getMatcherRequests(matchers, proxyUrl);
             if (skipRecordingRequestBody) {
                 matcherRequests.add(TestProxyUtils.setCompareBodiesMatcher());
             }
             matcherRequests.forEach(request -> {
                 request.setHeader(X_RECORDING_ID, xRecordingId);
-                sendRequestWithRetries(request);
+                try (HttpResponse response = sendRequestWithRetries(request)) {
+                    checkForTestProxyErrors(response);
+                }
             });
         } else {
             this.matchers.addAll(matchers);

sdk/core/azure-core-test/src/main/java/com/azure/core/test/http/TestProxyTestServer.java

@@ -74,6 +74,8 @@ public int port() {
 
     @Override
     public void close() {
-        server.disposeNow();
+        if (server != null) {
+            server.disposeNow();
+        }
     }
 }

sdk/core/azure-core-test/src/main/java/com/azure/core/test/policy/TestProxyRecordPolicy.java

@@ -18,6 +18,7 @@
 import com.azure.core.test.utils.HttpURLConnectionHttpClient;
 import com.azure.core.test.utils.TestProxyUtils;
 import com.azure.core.util.Context;
+import com.azure.core.util.logging.ClientLogger;
 import com.azure.json.JsonProviders;
 import com.azure.json.JsonWriter;
 import reactor.core.publisher.Mono;
@@ -32,6 +33,7 @@
 import java.util.List;
 import java.util.Map;
 import java.util.Queue;
+import java.util.concurrent.TimeUnit;
 
 import static com.azure.core.test.utils.TestProxyUtils.checkForTestProxyErrors;
 import static com.azure.core.test.utils.TestProxyUtils.createAddSanitizersRequest;
@@ -43,6 +45,7 @@
  * A {@link HttpPipelinePolicy} for redirecting traffic through the test proxy for recording.
  */
 public class TestProxyRecordPolicy implements HttpPipelinePolicy {
+    private static final ClientLogger LOGGER = new ClientLogger(TestProxyRecordPolicy.class);
     private static final HttpHeaderName X_RECORDING_ID = HttpHeaderName.fromString("x-recording-id");
     private final HttpClient client;
     private final URL proxyUrl;
@@ -79,7 +82,7 @@ public void startRecording(File recordFile, Path testClassPath) {
                 .setBody(new RecordFilePayload(recordFile.toString(), assetJsonPath).toJsonString())
                 .setHeader(HttpHeaderName.CONTENT_TYPE, "application/json");
 
-            try (HttpResponse response = client.sendSync(request, Context.NONE)) {
+            try (HttpResponse response = sendRequestWithRetries(request)) {
                 checkForTestProxyErrors(response);
                 if (response.getStatusCode() != 200) {
                     throw new RuntimeException(response.getBodyAsBinaryData().toString());
@@ -99,7 +102,9 @@ private void setDefaultRecordingOptions() {
         HttpRequest request = new HttpRequest(HttpMethod.POST, proxyUrl + "/Admin/SetRecordingOptions")
             .setBody("{\"HandleRedirects\": false}")
             .setHeader(HttpHeaderName.CONTENT_TYPE, "application/json");
-        client.sendSync(request, Context.NONE).close();
+        try (HttpResponse response = sendRequestWithRetries(request)) {
+            checkForTestProxyErrors(response);
+        }
     }
 
     /**
@@ -113,7 +118,7 @@ public void stopRecording(Queue<String> variables) {
             .setHeader(X_RECORDING_ID, xRecordingId)
             .setBody(serializeVariables(variables));
 
-        try (HttpResponse response = client.sendSync(request, Context.NONE)) {
+        try (HttpResponse response = sendRequestWithRetries(request)) {
             checkForTestProxyErrors(response);
             if (response.getStatusCode() != 200) {
                 throw new RuntimeException(response.getBodyAsBinaryData().toString());
@@ -197,7 +202,9 @@ public void addProxySanitization(List<TestProxySanitizer> sanitizers) {
             HttpRequest request
                 = createAddSanitizersRequest(sanitizers, proxyUrl).setHeader(X_RECORDING_ID, xRecordingId);
 
-            client.sendSync(request, Context.NONE).close();
+            try (HttpResponse response = sendRequestWithRetries(request)) {
+                checkForTestProxyErrors(response);
+            }
         } else {
             this.sanitizers.addAll(sanitizers);
         }
@@ -227,7 +234,9 @@ public void removeProxySanitization(List<String> sanitizers) {
                 throw new RuntimeException(e);
             }
 
-            client.sendSync(request, Context.NONE).close();
+            try (HttpResponse response = sendRequestWithRetries(request)) {
+                checkForTestProxyErrors(response);
+            }
         }
     }
 
@@ -241,9 +250,48 @@ public void setRecordingOptions(TestProxyRecordingOptions testProxyRecordingOpti
             HttpRequest request = new HttpRequest(HttpMethod.POST, proxyUrl + "/admin/setrecordingoptions")
                 .setBody(testProxyRecordingOptions.toJsonString())
                 .setHeader(HttpHeaderName.CONTENT_TYPE, "application/json");
-            client.sendSync(request, Context.NONE).close();
+            try (HttpResponse response = sendRequestWithRetries(request)) {
+                checkForTestProxyErrors(response);
+            }
         } catch (IOException ex) {
             throw new IllegalArgumentException("Failed to process JSON input", ex);
         }
     }
+
+    private HttpResponse sendRequestWithRetries(HttpRequest request) {
+        int retries = 0;
+        while (true) {
+            try {
+                HttpResponse response = client.sendSync(request, Context.NONE);
+                if (response.getStatusCode() / 100 != 2) {
+                    String body = response.getBodyAsString().block();
+                    int statusCode = response.getStatusCode();
+                    // We don't generally want to close the response here as the caller will need it,
+                    // but if we're throwing we should clean it up.
+                    response.close();
+                    throw new RuntimeException(
+                        "Test proxy returned a non-successful status code. " + statusCode + "; response: " + body);
+                }
+                return response;
+            } catch (Exception e) {
+
+                LOGGER.error("Failed in the retry loop. RecordingID " + xRecordingId + " message: " + e.getMessage());
+
+                retries++;
+                if (retries >= 3) {
+                    throw e;
+                }
+                sleep(1);
+                LOGGER.warning("Retrying request to test proxy. Retry attempt: " + retries);
+            }
+        }
+    }
+
+    private void sleep(int durationInSeconds) {
+        try {
+            TimeUnit.SECONDS.sleep(durationInSeconds);
+        } catch (InterruptedException e) {
+            throw new RuntimeException(e);
+        }
+    }
 }

sdk/core/azure-core-test/src/main/java/com/azure/core/test/utils/TestProxyManager.java

@@ -112,12 +112,13 @@ private static boolean checkAlive(int loops, Duration waitTime, Process proxy) t
                 return false;
             }
 
-            try {
-                HttpResponse response = client.sendSync(request, Context.NONE);
-                if (response != null && response.getStatusCode() == 200) {
-                    return true;
+            try (HttpResponse response = client.sendSync(request, Context.NONE)) {
+                if (response != null) {
+                    if (response.getStatusCode() == 200) {
+                        return true;
+                    }
+                    TestProxyUtils.checkForTestProxyErrors(response);
                 }
-                TestProxyUtils.checkForTestProxyErrors(response);
             } catch (Exception ignored) {
             }
 

sdk/core/azure-core-test/src/main/java/com/azure/core/test/utils/TestProxyUtils.java

@@ -290,7 +290,7 @@ public static List<TestProxySanitizer> loadSanitizers() {
 
     private static String createCustomMatcherRequestBody(CustomMatcher customMatcher) {
         return String.format(
-            "{\"ignoredHeaders\":\"%s\",\"excludedHeaders\":\"%s\",\"compareBodies\":%s,\"ignoredQueryParameters\":\"%s\",\"ignoreQueryOrdering\":%s}",
+            "{\"ignoredHeaders\":\"%s\",\"excludedHeaders\":\"Accept, Accept-Encoding, Connect, Connection, %s\",\"compareBodies\":%s,\"ignoredQueryParameters\":\"%s\",\"ignoreQueryOrdering\":%s}",
             getCommaSeperatedString(customMatcher.getHeadersKeyOnlyMatch()),
             getCommaSeperatedString(customMatcher.getExcludedHeaders()), customMatcher.isComparingBodies(),
             getCommaSeperatedString(customMatcher.getIgnoredQueryParameters()), customMatcher.isQueryOrderingIgnored());
@@ -476,8 +476,11 @@ public static List<HttpRequest> getMatcherRequests(List<TestProxyRequestMatcher>
                     break;
 
                 case BODILESS:
-                    matcherType = TestProxyRequestMatcher.TestProxyRequestMatcherType.BODILESS.getName();
-                    request = new HttpRequest(HttpMethod.POST, proxyUrl + "/Admin/setmatcher");
+//                    matcherType = TestProxyRequestMatcher.TestProxyRequestMatcherType.BODILESS.getName();
+                    CustomMatcher fakeBodilessMatcher  = new CustomMatcher().setComparingBodies(false);
+                    String fakeRequestBody = createCustomMatcherRequestBody(fakeBodilessMatcher);
+                    matcherType = TestProxyRequestMatcher.TestProxyRequestMatcherType.CUSTOM.getName();
+                    request = new HttpRequest(HttpMethod.POST, proxyUrl + "/Admin/setmatcher").setBody(fakeRequestBody);
                     break;
 
                 case CUSTOM: