Skip to content

Commit 156f369

Browse files
AsakusaRinneAsakusaRinne
authored
Merge pull request #609 from SignalRT/LlavaExecutor
Feature: LLava executor
2 parents 5e17e0f + bc487de commit 156f369

15 files changed

+447
-81
lines changed

‎.github/workflows/compile.yml

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -367,13 +367,22 @@ jobs:
367367
cp artifacts/llava-bin-osx-x64.dylib/libllava_shared.dylib deps/osx-x64/libllava_shared.dylib
368368
369369
cp artifacts/llama-bin-win-cublas-cu11.7.1-x64.dll/llama.dll deps/cu11.7.1/llama.dll
370+
cp artifacts/llava-bin-win-cublas-cu11.7.1-x64.dll/llava_shared.dll deps/cu11.7.1/llava_shared.dll
371+
370372
cp artifacts/llama-bin-linux-cublas-cu11.7.1-x64.so/libllama.so deps/cu11.7.1/libllama.so
373+
cp artifacts/llava-bin-linux-cublas-cu11.7.1-x64.so/libllava_shared.so deps/cu11.7.1/libllama_shared.so
374+
371375
cp artifacts/llama-bin-win-cublas-cu12.1.0-x64.dll/llama.dll deps/cu12.1.0/llama.dll
376+
cp artifacts/llava-bin-win-cublas-cu12.1.0-x64.dll/llava_shared.dll deps/cu12.1.0/llava_shared.dll
377+
372378
cp artifacts/llama-bin-linux-cublas-cu12.1.0-x64.so/libllama.so deps/cu12.1.0/libllama.so
379+
cp artifacts/llava-bin-linux-cublas-cu12.1.0-x64.so/libllava_shared.so deps/cu12.1.0/libllava_shared.so
373380
374381
cp artifacts/llama-bin-win-clblast-x64.dll/{llama,clblast}.dll deps/clblast/
382+
375383
cp artifacts/llama-bin-linux-clblast-x64.so/libllama.so deps/clblast/
376384
385+
377386
- name: Upload artifacts
378387
uses: actions/upload-artifact@v3
379388
with:

‎LLama.Examples/ExampleRunner.cs

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -13,6 +13,7 @@ public class ExampleRunner
1313
{ "Chat Session: Automatic conversation", TalkToYourself.Run },
1414
{ "Chat Session: Chinese characters", ChatChineseGB2312.Run },
1515
{ "Executor: Interactive mode chat", InteractiveModeExecute.Run },
16+
{ "Executor: Llava Interactive mode chat", LlavaInteractiveModeExecute.Run },
1617
{ "Executor: Instruct mode chat", InstructModeExecute.Run },
1718
{ "Executor: Stateless mode chat", StatelessModeExecute.Run },
1819
{ "Save and Load: chat session", SaveAndLoadSession.Run },

‎LLama.Examples/Examples/LlavaInteractiveModeExecute.cs

Lines changed: 121 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,121 @@
1+
using System.Text.RegularExpressions;
2+
using LLama.Batched;
3+
using LLama.Common;
4+
using Spectre.Console;
5+
6+
namespace LLama.Examples.Examples
7+
{
8+
public class LlavaInteractiveModeExecute
9+
{
10+
public static async Task Run()
11+
{
12+
string multiModalProj = UserSettings.GetMMProjPath();
13+
string modelPath = UserSettings.GetModelPath();
14+
string modelImage = UserSettings.GetImagePath();
15+
const int maxTokens = 1024;
16+
17+
var prompt = $"{{{modelImage}}}\nUSER:\nProvide a full description of the image.\nASSISTANT:\n";
18+
19+
var parameters = new ModelParams(modelPath)
20+
{
21+
ContextSize = 4096,
22+
Seed = 1337,
23+
};
24+
using var model = LLamaWeights.LoadFromFile(parameters);
25+
using var context = model.CreateContext(parameters);
26+
27+
// Llava Init
28+
using var clipModel = LLavaWeights.LoadFromFile(multiModalProj);
29+
30+
var ex = new InteractiveExecutor(context, clipModel );
31+
32+
Console.ForegroundColor = ConsoleColor.Yellow;
33+
Console.WriteLine("The executor has been enabled. In this example, the prompt is printed, the maximum tokens is set to {0} and the context size is {1}.", maxTokens, parameters.ContextSize );
34+
Console.WriteLine("To send an image, enter its filename in curly braces, like this {c:/image.jpg}.");
35+
36+
var inferenceParams = new InferenceParams() { Temperature = 0.1f, AntiPrompts = new List<string> { "\nUSER:" }, MaxTokens = maxTokens };
37+
38+
do
39+
{
40+
41+
// Evaluate if we have images
42+
//
43+
var imageMatches = Regex.Matches(prompt, "{([^}]*)}").Select(m => m.Value);
44+
var imageCount = imageMatches.Count();
45+
var hasImages = imageCount > 0;
46+
byte[][] imageBytes = null;
47+
48+
if (hasImages)
49+
{
50+
var imagePathsWithCurlyBraces = Regex.Matches(prompt, "{([^}]*)}").Select(m => m.Value);
51+
var imagePaths = Regex.Matches(prompt, "{([^}]*)}").Select(m => m.Groups[1].Value);
52+
53+
try
54+
{
55+
imageBytes = imagePaths.Select(File.ReadAllBytes).ToArray();
56+
}
57+
catch (IOException exception)
58+
{
59+
Console.ForegroundColor = ConsoleColor.Red;
60+
Console.Write(
61+
$"Could not load your {(imageCount == 1 ? "image" : "images")}:");
62+
Console.Write($"{exception.Message}");
63+
Console.ForegroundColor = ConsoleColor.Yellow;
64+
Console.WriteLine("Please try again.");
65+
break;
66+
}
67+
68+
69+
int index = 0;
70+
foreach (var path in imagePathsWithCurlyBraces)
71+
{
72+
// First image replace to tag <image, the rest of the images delete the tag
73+
if (index++ == 0)
74+
prompt = prompt.Replace(path, "<image>");
75+
else
76+
prompt = prompt.Replace(path, "");
77+
}
78+
79+
80+
Console.ForegroundColor = ConsoleColor.Yellow;
81+
Console.WriteLine($"Here are the images, that are sent to the chat model in addition to your message.");
82+
Console.WriteLine();
83+
84+
foreach (var consoleImage in imageBytes?.Select(bytes => new CanvasImage(bytes)))
85+
{
86+
consoleImage.MaxWidth = 50;
87+
AnsiConsole.Write(consoleImage);
88+
}
89+
90+
Console.WriteLine();
91+
Console.ForegroundColor = ConsoleColor.Yellow;
92+
Console.WriteLine($"The images were scaled down for the console only, the model gets full versions.");
93+
Console.WriteLine($"Write /exit or press Ctrl+c to return to main menu.");
94+
Console.WriteLine();
95+
96+
97+
// Initilize Images in executor
98+
//
99+
ex.ImagePaths = imagePaths.ToList();
100+
}
101+
102+
Console.ForegroundColor = Color.White;
103+
await foreach (var text in ex.InferAsync(prompt, inferenceParams))
104+
{
105+
Console.Write(text);
106+
}
107+
Console.Write(" ");
108+
Console.ForegroundColor = ConsoleColor.Green;
109+
prompt = Console.ReadLine();
110+
Console.WriteLine();
111+
112+
// let the user finish with exit
113+
//
114+
if (prompt.Equals("/exit", StringComparison.OrdinalIgnoreCase))
115+
break;
116+
117+
}
118+
while(true);
119+
}
120+
}
121+
}

‎LLama.Examples/Examples/StatelessModeExecute.cs

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -21,7 +21,7 @@ public static async Task Run()
2121
Console.ForegroundColor = ConsoleColor.Yellow;
2222
Console.WriteLine("The executor has been enabled. In this example, the inference is an one-time job. That says, the previous input and response has " +
2323
"no impact on the current response. Now you can ask it questions. Note that in this example, no prompt was set for LLM and the maximum response tokens is 50. " +
24-
"It may not perform well because of lack of prompt. This is also an example that could indicate the improtance of prompt in LLM. To improve it, you can add " +
24+
"It may not perform well because of lack of prompt. This is also an example that could indicate the importance of prompt in LLM. To improve it, you can add " +
2525
"a prompt for it yourself!");
2626
Console.ForegroundColor = ConsoleColor.White;
2727

‎LLama.Examples/LLama.Examples.csproj

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -19,6 +19,7 @@
1919
<PackageReference Include="Microsoft.SemanticKernel" Version="1.6.2" />
2020
<PackageReference Include="Microsoft.SemanticKernel.Plugins.Memory" Version="1.6.2-alpha" />
2121
<PackageReference Include="Spectre.Console" Version="0.48.0" />
22+
<PackageReference Include="Spectre.Console.ImageSharp" Version="0.48.0" />
2223
</ItemGroup>
2324

2425
<ItemGroup>

‎LLama.Examples/UserSettings.cs

Lines changed: 44 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -4,54 +4,84 @@ namespace LLama.Examples;
44

55
internal static class UserSettings
66
{
7-
private static readonly string SettingsFilePath = Path.Join(AppContext.BaseDirectory, "DefaultModel.env");
7+
private static readonly string SettingsModelPath = Path.Join(AppContext.BaseDirectory, "DefaultModel.env");
8+
private static readonly string SettingsMMprojPath = Path.Join(AppContext.BaseDirectory, "DefaultMMProj.env");
9+
private static readonly string SettingsImagePath = Path.Join(AppContext.BaseDirectory, "DefaultImage.env");
810

9-
private static string? ReadDefaultModelPath()
11+
private static string? ReadDefaultPath(string file)
1012
{
11-
if (!File.Exists(SettingsFilePath))
13+
if (!File.Exists(file))
1214
return null;
1315

14-
string path = File.ReadAllText(SettingsFilePath).Trim();
16+
string path = File.ReadAllText(file).Trim();
1517
if (!File.Exists(path))
1618
return null;
1719

1820
return path;
1921
}
2022

21-
private static void WriteDefaultModelPath(string path)
23+
private static void WriteDefaultPath(string settings, string path)
2224
{
23-
File.WriteAllText(SettingsFilePath, path);
25+
File.WriteAllText(settings, path);
2426
}
2527

2628
public static string GetModelPath(bool alwaysPrompt = false)
2729
{
28-
var defaultPath = ReadDefaultModelPath();
30+
var defaultPath = ReadDefaultPath(SettingsModelPath);
2931
var path = defaultPath is null || alwaysPrompt
3032
? PromptUserForPath()
3133
: PromptUserForPathWithDefault(defaultPath);
3234

3335
if (File.Exists(path))
34-
WriteDefaultModelPath(path);
36+
WriteDefaultPath(SettingsModelPath, path);
3537

3638
return path;
3739
}
40+
41+
// TODO: Refactorize
42+
public static string GetMMProjPath(bool alwaysPrompt = false)
43+
{
44+
var defaultPath = ReadDefaultPath(SettingsMMprojPath);
45+
var path = defaultPath is null || alwaysPrompt
46+
? PromptUserForPath("MMProj")
47+
: PromptUserForPathWithDefault(defaultPath, "MMProj");
48+
49+
if (File.Exists(path))
50+
WriteDefaultPath(SettingsMMprojPath, path);
51+
52+
return path;
53+
}
54+
55+
// TODO: Refactorize
56+
public static string GetImagePath(bool alwaysPrompt = false)
57+
{
58+
var defaultPath = ReadDefaultPath(SettingsImagePath);
59+
var path = defaultPath is null || alwaysPrompt
60+
? PromptUserForPath("image")
61+
: PromptUserForPathWithDefault(defaultPath, "image");
62+
63+
if (File.Exists(path))
64+
WriteDefaultPath(SettingsImagePath, path);
65+
66+
return path;
67+
}
3868

39-
private static string PromptUserForPath()
69+
private static string PromptUserForPath(string text = "model")
4070
{
4171
return AnsiConsole.Prompt(
42-
new TextPrompt<string>("Please input your model path:")
72+
new TextPrompt<string>(string.Format("Please input your {0} path:", text) )
4373
.PromptStyle("white")
44-
.Validate(File.Exists, "[red]ERROR: invalid model file path - file does not exist[/]")
74+
.Validate(File.Exists, string.Format("[red]ERROR: invalid {0} file path - file does not exist[/]", text) )
4575
);
4676
}
4777

48-
private static string PromptUserForPathWithDefault(string defaultPath)
78+
private static string PromptUserForPathWithDefault(string defaultPath, string text = "model")
4979
{
5080
return AnsiConsole.Prompt(
51-
new TextPrompt<string>("Please input your model path (or ENTER for default):")
81+
new TextPrompt<string>(string.Format("Please input your {0} path (or ENTER for default):", text) )
5282
.DefaultValue(defaultPath)
5383
.PromptStyle("white")
54-
.Validate(File.Exists, "[red]ERROR: invalid model file path - file does not exist[/]")
84+
.Validate(File.Exists, string.Format("[red]ERROR: invalid {0} file path - file does not exist[/]", text))
5585
);
5686
}
5787
}

‎LLama.Unittest/LLavaWeightsTests.cs

Lines changed: 9 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -31,23 +31,23 @@ public void Dispose()
3131
_llamaWeights.Dispose();
3232
_lLavaWeights.Dispose();
3333
}
34-
3534

36-
37-
[Fact(Skip = "Very slow in CI")]
35+
[Fact(Skip = "Very very slow in CI")]
3836
public void EmbedImageAsFileName()
3937
{
4038
int n_past = 0;
41-
Assert.True( _lLavaWeights.EmbedImage( _context, Constants.LLavaImage, ref n_past ) );
42-
}
43-
44-
[Fact(Skip = "Very slow in CI")]
39+
SafeLlavaImageEmbedHandle emb = _lLavaWeights.CreateImageEmbeddings(_context, Constants.LLavaImage);
40+
Assert.True( _lLavaWeights.EvalImageEmbed( _context, emb, ref n_past ) );
41+
}
42+
43+
[Fact(Skip = "Very very slow in CI")]
4544
public void EmbedImageAsBinary()
4645
{
4746
int n_past = 0;
4847
byte[] image = System.IO.File.ReadAllBytes(Constants.LLavaImage);
49-
Assert.True( _lLavaWeights.EmbedImage( _context, image, ref n_past ) );
50-
}
48+
SafeLlavaImageEmbedHandle emb = _lLavaWeights.CreateImageEmbeddings(_context, image);
49+
Assert.True( _lLavaWeights.EvalImageEmbed( _context, emb, ref n_past ) );
50+
}
5151

5252
}
5353
}

‎LLama/Abstractions/ILLamaExecutor.cs

Lines changed: 18 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -12,7 +12,24 @@ public interface ILLamaExecutor
1212
/// The loaded context for this executor.
1313
/// </summary>
1414
public LLamaContext Context { get; }
15-
15+
16+
// LLava Section
17+
//
18+
/// <summary>
19+
/// Identify if it's a multi-modal model and there is a image to process.
20+
/// </summary>
21+
public bool IsMultiModal { get; }
22+
/// <summary>
23+
/// Muti-Modal Projections / Clip Model weights
24+
/// </summary>
25+
public LLavaWeights? ClipModel { get; }
26+
27+
/// <summary>
28+
/// List of images: Image filename and path (jpeg images).
29+
/// </summary>
30+
public List<string> ImagePaths { get; set; }
31+
32+
1633
/// <summary>
1734
/// Asynchronously infers a response from the model.
1835
/// </summary>

‎LLama/LLamaExecutorBase.cs

Lines changed: 24 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -64,6 +64,23 @@ public abstract class StatefulExecutorBase : ILLamaExecutor
6464
/// </summary>
6565
public LLamaContext Context { get; }
6666

67+
// LLava Section
68+
//
69+
/// <inheritdoc />
70+
public bool IsMultiModal
71+
{
72+
get
73+
{
74+
return ClipModel != null;
75+
}
76+
}
77+
78+
/// <inheritdoc />
79+
public LLavaWeights? ClipModel { get; }
80+
81+
/// <inheritdoc />
82+
public List<string> ImagePaths { get; set; }
83+
6784
/// <summary>
6885
/// Current "mu" value for mirostat sampling
6986
/// </summary>
@@ -78,6 +95,7 @@ public abstract class StatefulExecutorBase : ILLamaExecutor
7895
/// <param name="logger"></param>
7996
protected StatefulExecutorBase(LLamaContext context, ILogger? logger = null)
8097
{
98+
ImagePaths = new List<string>();
8199
_logger = logger;
82100
Context = context;
83101
_pastTokensCount = 0;
@@ -86,6 +104,12 @@ protected StatefulExecutorBase(LLamaContext context, ILogger? logger = null)
86104
_last_n_tokens = new FixedSizeQueue<LLamaToken>((int)Context.ContextSize);
87105
_decoder = new StreamingTokenDecoder(context);
88106
}
107+
108+
public StatefulExecutorBase(LLamaContext context, LLavaWeights lLavaWeights, ILogger? logger = null) :
109+
this( context, logger )
110+
{
111+
ClipModel = lLavaWeights;
112+
}
89113

90114
/// <summary>
91115
/// This API is currently not verified.

0 commit comments

Comments
 (0)