Trace/replay llms.internal.sendRequest

To decouple `texampleTests.m` from availability and speed of external servers,
record calls to `llms.internal.sendRequest` (on dev machine) and replay
(during most test runs, including CI). See tests/recording/README.md for
instructions.

Author: ccreutzi

+llms/+internal/callAzureChatAPI.m

@@ -64,7 +64,7 @@
 
 parameters = buildParametersCall(messages, functions, nvp);
 
-[response, streamedText] = llms.internal.sendRequest(parameters,nvp.APIKey, URL, nvp.TimeOut, nvp.StreamFun);
+[response, streamedText] = llms.internal.sendRequestWrapper(parameters,nvp.APIKey, URL, nvp.TimeOut, nvp.StreamFun);
 
 % If call errors, "choices" will not be part of response.Body.Data, instead
 % we get response.Body.Data.error

+llms/+internal/callOllamaChatAPI.m

@@ -53,7 +53,7 @@
 
 parameters = buildParametersCall(model, messages, nvp);
 
-[response, streamedText] = llms.internal.sendRequest(parameters,[],URL,nvp.TimeOut,nvp.StreamFun);
+[response, streamedText] = llms.internal.sendRequestWrapper(parameters,[],URL,nvp.TimeOut,nvp.StreamFun);
 
 % If call errors, "choices" will not be part of response.Body.Data, instead
 % we get response.Body.Data.error

+llms/+internal/callOpenAIChatAPI.m

@@ -62,7 +62,7 @@
 
 parameters = buildParametersCall(messages, functions, nvp);
 
-[response, streamedText] = llms.internal.sendRequest(parameters,nvp.APIKey, END_POINT, nvp.TimeOut, nvp.StreamFun);
+[response, streamedText] = llms.internal.sendRequestWrapper(parameters,nvp.APIKey, END_POINT, nvp.TimeOut, nvp.StreamFun);
 
 % If call errors, "choices" will not be part of response.Body.Data, instead
 % we get response.Body.Data.error

+llms/+internal/sendRequestWrapper.m

@@ -0,0 +1,5 @@
+function [response, streamedText] = sendRequestWrapper(varargin)
+% This function is undocumented and will change in a future release
+
+% A wrapper around sendRequest to have a test seam
+[response, streamedText] = llms.internal.sendRequest(varargin{:});

.gitignore

@@ -1,6 +1,7 @@
 *.env
 *.asv
 *.mat
+!tests/recordings/*.mat
 startup.m
 papers_to_read.csv
 data/*

extractOpenAIEmbeddings.m

@@ -47,7 +47,7 @@
 end
 
 
-response = llms.internal.sendRequest(parameters,key, END_POINT, nvp.TimeOut);
+response = llms.internal.sendRequestWrapper(parameters,key, END_POINT, nvp.TimeOut);
 
 if isfield(response.Body.Data, "data")
     emb = [response.Body.Data.data.embedding];

tests/code-coverage/coverage.xml

@@ -0,0 +1,40 @@
+function [response, streamedText] = sendRequestWrapper(parameters, token, varargin)
+% This function is undocumented and will change in a future release
+
+% A wrapper around sendRequest to have a test seam
+persistent seenCalls
+if isempty(seenCalls)
+    seenCalls = cell(0,2);
+end
+
+persistent filename
+
+if nargin == 1 && isequal(parameters,"close")
+    save(filename+".mat","seenCalls");
+    seenCalls = cell(0,2);
+    return
+end
+
+if nargin==2 && isequal(parameters,"open")
+    filename = token;
+    return
+end
+
+streamFunCalls = {};
+hasCallback = nargin >= 5 && isa(varargin{3},'function_handle');
+if hasCallback
+    streamFun = varargin{3};
+end
+function wrappedStreamFun(varargin)
+    streamFunCalls(end+1) = varargin;
+    streamFun(varargin{:});
+end
+if hasCallback
+    varargin{3} = @wrappedStreamFun;
+end
+
+
+[response, streamedText] = llms.internal.sendRequest(parameters, token, varargin{:});
+
+seenCalls(end+1,:) = {{parameters},{response,streamFunCalls,streamedText}};
+end

tests/private/recording-doubles/+llms/+internal/sendRequestWrapper.m

@@ -0,0 +1,2 @@
+function addpath(~)
+% ignore addpath calls in examples

tests/private/recording-doubles/addpath.m

@@ -0,0 +1,30 @@
+function [response, streamedText] = sendRequestWrapper(parameters, token, varargin)
+% This function is undocumented and will change in a future release
+
+% A wrapper around sendRequest to have a test seam
+persistent seenCalls
+if isempty(seenCalls)
+    seenCalls = cell(0,2);
+end
+
+if nargin == 1 && isequal(parameters,"close")
+    seenCalls = cell(0,2);
+    return
+end
+
+if nargin==2 && isequal(parameters,"open")
+    load(token+".mat","seenCalls");
+    return
+end
+
+result = seenCalls{1,2};
+response = result{1};
+streamFunCalls = result{2};
+streamedText = result{3};
+
+if nargin >= 5 && isa(varargin{3},'function_handle')
+    streamFun = varargin{3};
+    cellfun(streamFun, streamFunCalls);
+end
+
+seenCalls(1,:) = [];

tests/private/replaying-doubles/+llms/+internal/sendRequestWrapper.m

@@ -0,0 +1,2 @@
+function addpath(~)
+% ignore addpath calls in examples

tests/private/replaying-doubles/addpath.m

@@ -0,0 +1,12 @@
+# Test Double Recordings
+
+Testing the examples typically takes a long time and tends to have false negatives relatively often, mostly due to timeout errors.
+
+The point of testing the examples is not to test that we can connect to the servers. We have other test points for that. Hence, we insert a “test double” while testing the examples that keeps recordings of previous interactions with the servers and just replays the responses.
+
+This directory contains those recordings.
+
+## Generating Recordings
+
+To generate or re-generate recordings (e.g., after changing an example, or making relevant software changes), open [`texampleTests.m`](../texampleTests.m) and in `setUpAndTearDowns`, change `capture = false;` to `capture = true;`. Then, run the test points relevant to the example(s) in question, and change `capture` back to `false`.
+