test doubles to record & replay server interactions in example tests

Author: ccreutzi

+llms/+internal/callAzureChatAPI.m

@@ -64,7 +64,7 @@
 
 parameters = buildParametersCall(messages, functions, nvp);
 
-[response, streamedText] = llms.internal.sendRequest(parameters,nvp.APIKey, URL, nvp.TimeOut, nvp.StreamFun);
+[response, streamedText] = llms.internal.sendRequestWrapper(parameters,nvp.APIKey, URL, nvp.TimeOut, nvp.StreamFun);
 
 % If call errors, "choices" will not be part of response.Body.Data, instead
 % we get response.Body.Data.error

+llms/+internal/callOllamaChatAPI.m

@@ -53,7 +53,7 @@
 
 parameters = buildParametersCall(model, messages, nvp);
 
-[response, streamedText] = llms.internal.sendRequest(parameters,[],URL,nvp.TimeOut,nvp.StreamFun);
+[response, streamedText] = llms.internal.sendRequestWrapper(parameters,[],URL,nvp.TimeOut,nvp.StreamFun);
 
 % If call errors, "choices" will not be part of response.Body.Data, instead
 % we get response.Body.Data.error

+llms/+internal/callOpenAIChatAPI.m

@@ -62,7 +62,7 @@
 
 parameters = buildParametersCall(messages, functions, nvp);
 
-[response, streamedText] = llms.internal.sendRequest(parameters,nvp.APIKey, END_POINT, nvp.TimeOut, nvp.StreamFun);
+[response, streamedText] = llms.internal.sendRequestWrapper(parameters,nvp.APIKey, END_POINT, nvp.TimeOut, nvp.StreamFun);
 
 % If call errors, "choices" will not be part of response.Body.Data, instead
 % we get response.Body.Data.error

+llms/+internal/sendRequestWrapper.m

@@ -0,0 +1,5 @@
+function [response, streamedText] = sendRequestWrapper(varargin)
+% This function is undocumented and will change in a future release
+
+% A wrapper around sendRequest to have a test seam
+[response, streamedText] = llms.internal.sendRequest(varargin{:});

.gitignore

@@ -1,6 +1,7 @@
 *.env
 *.asv
 *.mat
+!tests/recordings/*.mat
 startup.m
 papers_to_read.csv
 data/*

extractOpenAIEmbeddings.m

@@ -47,7 +47,7 @@
 end
 
 
-response = llms.internal.sendRequest(parameters,key, END_POINT, nvp.TimeOut);
+response = llms.internal.sendRequestWrapper(parameters,key, END_POINT, nvp.TimeOut);
 
 if isfield(response.Body.Data, "data")
     emb = [response.Body.Data.data.embedding];

tests/recording-doubles/+llms/+internal/sendRequestWrapper.m

@@ -0,0 +1,40 @@
+function [response, streamedText] = sendRequestWrapper(parameters, token, varargin)
+% This function is undocumented and will change in a future release
+
+% A wrapper around sendRequest to have a test seam
+persistent seenCalls
+if isempty(seenCalls)
+    seenCalls = cell(0,2);
+end
+
+persistent filename
+
+if nargin == 1 && isequal(parameters,"close")
+    save(filename+".mat","seenCalls");
+    seenCalls = cell(0,2);
+    return
+end
+
+if nargin==2 && isequal(parameters,"open")
+    filename = token;
+    return
+end
+
+streamFunCalls = {};
+hasCallback = nargin >= 5 && isa(varargin{3},'function_handle');
+if hasCallback
+    streamFun = varargin{3};
+end
+function wrappedStreamFun(varargin)
+    streamFunCalls(end+1) = varargin;
+    streamFun(varargin{:});
+end
+if hasCallback
+    varargin{3} = @wrappedStreamFun;
+end
+
+
+[response, streamedText] = llms.internal.sendRequest(parameters, token, varargin{:});
+
+seenCalls(end+1,:) = {{parameters},{response,streamFunCalls,streamedText}};
+end

tests/recording-doubles/addpath.m

@@ -0,0 +1,2 @@
+function addpath(~)
+% ignore addpath calls in examples

tests/recordings/AnalyzeScientificPapersUsingFunctionCalls.mat

@@ -0,0 +1,12 @@
+# Test Double Recordings
+
+Testing the examples typically takes a long time and tends to have false negatives relatively often, mostly due to timeout errors.
+
+The point of testing the examples is not to test that we can connect to the servers. We have other test points for that. Hence, we insert a “test double” while testing the examples that keeps recordings of previous interactions with the servers and just replays the responses.
+
+This directory contains those recordings.
+
+## Generating Recordings
+
+To generate or re-generate recordings (e.g., after changing an example, or making relevant software changes), open [`texampleTests.m`](../texampleTests.m) and in `setUpAndTearDowns`, change `capture = false;` to `capture = true;`. Then, run the test points relevant to the example(s) in question, and change `capture` back to `false`.
+

tests/recordings/AnalyzeSentimentinTextUsingChatGPTinJSONMode.mat

@@ -0,0 +1,30 @@
+function [response, streamedText] = sendRequestWrapper(parameters, token, varargin)
+% This function is undocumented and will change in a future release
+
+% A wrapper around sendRequest to have a test seam
+persistent seenCalls
+if isempty(seenCalls)
+    seenCalls = cell(0,2);
+end
+
+if nargin == 1 && isequal(parameters,"close")
+    seenCalls = cell(0,2);
+    return
+end
+
+if nargin==2 && isequal(parameters,"open")
+    load(token+".mat","seenCalls");
+    return
+end
+
+result = seenCalls{1,2};
+response = result{1};
+streamFunCalls = result{2};
+streamedText = result{3};
+
+if nargin >= 5 && isa(varargin{3},'function_handle')
+    streamFun = varargin{3};
+    cellfun(streamFun, streamFunCalls);
+end
+
+seenCalls(1,:) = [];

tests/recordings/AnalyzeTextDataUsingParallelFunctionCallwithChatGPT.mat

@@ -0,0 +1,2 @@
+function addpath(~)
+% ignore addpath calls in examples

tests/recordings/CreateSimpleChatBot.mat

@@ -8,9 +8,25 @@
         ChatBotExample = {"CreateSimpleChatBot", "CreateSimpleOllamaChatBot"};
     end
 
+    properties
+        TestDir;
+    end
 
     methods (TestClassSetup)
         function setUpAndTearDowns(testCase)
+            % Capture and replay server interactions
+            testCase.TestDir = fileparts(mfilename("fullpath"));
+            import matlab.unittest.fixtures.PathFixture
+            capture = false; % run in capture or replay mode, cf. recordings/README.md
+
+            if capture
+                testCase.applyFixture(PathFixture( ...
+                    fullfile(testCase.TestDir,"recording-doubles")));
+            else
+                testCase.applyFixture(PathFixture( ...
+                    fullfile(testCase.TestDir,"replaying-doubles")));
+            end
+
             import matlab.unittest.fixtures.CurrentFolderFixture
             testCase.applyFixture(CurrentFolderFixture("../examples/mlx-scripts"));
 
@@ -29,22 +45,39 @@ function setUpAndTearDowns(testCase)
             testCase.addTeardown(@() iCloseAll());
         end
     end
-    
+
+    methods
+        function startCapture(testCase,testName)
+            llms.internal.sendRequestWrapper("open", ...
+                fullfile(testCase.TestDir,"recordings",testName));
+        end
+    end
+
+    methods(TestMethodTeardown)
+        function closeCapture(~)
+            llms.internal.sendRequestWrapper("close");
+        end
+    end
+
     methods(Test)
-        function testAnalyzeScientificPapersUsingFunctionCalls(~)
+        function testAnalyzeScientificPapersUsingFunctionCalls(testCase)
+            testCase.startCapture("AnalyzeScientificPapersUsingFunctionCalls");
             AnalyzeScientificPapersUsingFunctionCalls;
         end
 
         function testAnalyzeSentimentinTextUsingChatGPTinJSONMode(testCase)
+            testCase.startCapture("AnalyzeSentimentinTextUsingChatGPTinJSONMode");
             testCase.verifyWarning(@AnalyzeSentimentinTextUsingChatGPTinJSONMode,...
                 "llms:warningJsonInstruction");
         end
 
-        function testAnalyzeTextDataUsingParallelFunctionCallwithChatGPT(~)
+        function testAnalyzeTextDataUsingParallelFunctionCallwithChatGPT(testCase)
+            testCase.startCapture("AnalyzeTextDataUsingParallelFunctionCallwithChatGPT");
             AnalyzeTextDataUsingParallelFunctionCallwithChatGPT;
         end
 
         function testCreateSimpleChatBot(testCase,ChatBotExample)
+            testCase.startCapture(ChatBotExample);
             % set up a fake input command, returning canned user prompts
             count = 0;
             prompts = [
@@ -85,43 +118,51 @@ function testCreateSimpleChatBot(testCase,ChatBotExample)
             testCase.verifySize(messages.Messages,[1 2*(count-1)]);
         end
 
-        function testDescribeImagesUsingChatGPT(~)
+        function testDescribeImagesUsingChatGPT(testCase)
+            testCase.startCapture("DescribeImagesUsingChatGPT");
             DescribeImagesUsingChatGPT;
         end
 
-        function testInformationRetrievalUsingOpenAIDocumentEmbedding(~)
+        function testInformationRetrievalUsingOpenAIDocumentEmbedding(testCase)
+            testCase.startCapture("InformationRetrievalUsingOpenAIDocumentEmbedding");
             InformationRetrievalUsingOpenAIDocumentEmbedding;
         end
 
-        function testProcessGeneratedTextinRealTimebyUsingChatGPTinStreamingMode(~)
+        function testProcessGeneratedTextinRealTimebyUsingChatGPTinStreamingMode(testCase)
+            testCase.startCapture("ProcessGeneratedTextinRealTimebyUsingChatGPTinStreamingMode");
             ProcessGeneratedTextinRealTimebyUsingChatGPTinStreamingMode;
         end
 
-        function testProcessGeneratedTextInRealTimeByUsingOllamaInStreamingMode(~)
+        function testProcessGeneratedTextInRealTimeByUsingOllamaInStreamingMode(testCase)
+            testCase.startCapture("ProcessGeneratedTextInRealTimeByUsingOllamaInStreamingMode");
             ProcessGeneratedTextInRealTimeByUsingOllamaInStreamingMode;
         end
 
-        function testRetrievalAugmentedGenerationUsingChatGPTandMATLAB(~)
+        function testRetrievalAugmentedGenerationUsingChatGPTandMATLAB(testCase)
+            testCase.startCapture("RetrievalAugmentedGenerationUsingChatGPTandMATLAB");
             RetrievalAugmentedGenerationUsingChatGPTandMATLAB;
         end
 
-        function testRetrievalAugmentedGenerationUsingOllamaAndMATLAB(~)
+        function testRetrievalAugmentedGenerationUsingOllamaAndMATLAB(testCase)
+            testCase.startCapture("RetrievalAugmentedGenerationUsingOllamaAndMATLAB");
             RetrievalAugmentedGenerationUsingOllamaAndMATLAB;
         end
 
-        function testSummarizeLargeDocumentsUsingChatGPTandMATLAB(~)
+        function testSummarizeLargeDocumentsUsingChatGPTandMATLAB(testCase)
+            testCase.startCapture("SummarizeLargeDocumentsUsingChatGPTandMATLAB");
             SummarizeLargeDocumentsUsingChatGPTandMATLAB;
         end
 
-        function testUsingDALLEToEditImages(~)
+        function testUsingDALLEToEditImages(testCase)
+            testCase.startCapture("UsingDALLEToEditImages");
             UsingDALLEToEditImages;
         end
 
-        function testUsingDALLEToGenerateImages(~)
+        function testUsingDALLEToGenerateImages(testCase)
+            testCase.startCapture("UsingDALLEToGenerateImages");
             UsingDALLEToGenerateImages;
         end
-    end
-    
+    end    
 end
 
 function iCloseAll()