Skip to content

add snippets tests & response_format support #1524

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Draft
wants to merge 2 commits into
base: main
Choose a base branch
from
Draft

add snippets tests & response_format support #1524

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
6fb1214
add snippets tests
TGlide Jun 11, 2025
dd504e3
add response_format support
TGlide Jun 11, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
72 changes: 70 additions & 2 deletions packages/inference/src/snippets/getInferenceSnippets.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -305,16 +305,44 @@ const prepareConversationalInput = (
temperature?: GenerationParameters["temperature"];
max_tokens?: GenerationParameters["max_new_tokens"];
top_p?: GenerationParameters["top_p"];
response_format?: Record<string, unknown>;
}
): object => {
return {
messages: opts?.messages ?? getModelInputSnippet(model),
...(opts?.temperature ? { temperature: opts?.temperature } : undefined),
...(opts?.max_tokens ? { max_tokens: opts?.max_tokens } : undefined),
...(opts?.top_p ? { top_p: opts?.top_p } : undefined),
...(opts?.response_format ? { response_format: opts?.response_format } : undefined),
};
};

const prepareTextGenerationInput = (
model: ModelDataMinimal,
opts?: {
streaming?: boolean;
temperature?: GenerationParameters["temperature"];
max_tokens?: GenerationParameters["max_new_tokens"];
top_p?: GenerationParameters["top_p"];
response_format?: Record<string, unknown>;
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I think there is a confusion between the Chat Completion API that accepts a response_format and the Text Generation API that accepts a grammar input.

}
): object => {
const base = { inputs: getModelInputSnippet(model) };
const parameters: Record<string, unknown> = {};

if (opts?.temperature !== undefined) parameters.temperature = opts.temperature;
if (opts?.max_tokens !== undefined) parameters.max_new_tokens = opts.max_tokens;
if (opts?.top_p !== undefined) parameters.top_p = opts.top_p;
if (opts?.response_format !== undefined) parameters.response_format = opts.response_format;

// Only add parameters if there are any
if (Object.keys(parameters).length > 0) {
return { ...base, parameters };
}

return base;
};

const prepareQuestionAnsweringInput = (model: ModelDataMinimal): object => {
const data = JSON.parse(getModelInputSnippet(model) as string);
return { question: data.question, context: data.context };
Expand Down Expand Up @@ -355,7 +383,7 @@ const snippets: Partial<
"tabular-regression": snippetGenerator("tabular"),
"table-question-answering": snippetGenerator("tableQuestionAnswering", prepareTableQuestionAnsweringInput),
"text-classification": snippetGenerator("basic"),
"text-generation": snippetGenerator("basic"),
"text-generation": snippetGenerator("basic", prepareTextGenerationInput),
"text-to-audio": snippetGenerator("textToAudio"),
"text-to-image": snippetGenerator("textToImage"),
"text-to-speech": snippetGenerator("textToSpeech"),
Expand Down Expand Up @@ -393,7 +421,7 @@ function formatBody(obj: object, format: "curl" | "json" | "python" | "ts"): str
return indentString(
Object.entries(obj)
.map(([key, value]) => {
const formattedValue = JSON.stringify(value, null, 4).replace(/"/g, '"');
const formattedValue = formatPythonValue(value, 1);
return `${key}=${formattedValue},`;
})
.join("\n")
Expand All @@ -408,6 +436,46 @@ function formatBody(obj: object, format: "curl" | "json" | "python" | "ts"): str
}
}

function formatPythonValue(obj: unknown, depth?: number): string {
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I start to really think we should first generate the snippets and then format it. Seems that there are a few solutions although not very popular (blackjs, prettier/plugin-python). For now, let's keep it like this.

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Out of curiosity, was this code auto-generate or written manually? (if yes, better to mention it in docstring)

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

(note to myself, maybe a single-file python formatter would be enough given our small requirements. Here's an example)

depth = depth ?? 0;

/// Case boolean - convert to Python format
if (typeof obj === "boolean") {
return obj ? "True" : "False";
}

/// Case null - convert to Python format
if (obj === null) {
return "None";
}

/// Case number or string
if (typeof obj !== "object") {
return JSON.stringify(obj);
}

/// Case array
if (Array.isArray(obj)) {
const items = obj
.map((item) => {
const formatted = formatPythonValue(item, depth + 1);
return `${" ".repeat(4 * (depth + 1))}${formatted},`;
})
.join("\n");
return `[\n${items}\n${" ".repeat(4 * depth)}]`;
}

/// Case mapping (object)
const entries = Object.entries(obj);
const lines = entries
.map(([key, value]) => {
const formattedValue = formatPythonValue(value, depth + 1);
return `${" ".repeat(4 * (depth + 1))}"${key}": ${formattedValue},`;
})
.join("\n");
return `{\n${lines}\n${" ".repeat(4 * depth)}}`;
}

function formatTsObject(obj: unknown, depth?: number): string {
depth = depth ?? 0;

Expand Down
Loading
Loading