Fix system prompts (#43)

src/lib/components/InferencePlayground/InferencePlayground.svelte

@@ -20,16 +20,17 @@
 
 	export let models: ModelEntryWithTokenizer[];
 
-	const startMessage: ChatCompletionInputMessage = { role: "user", content: "" };
+	const startMessageUser: ChatCompletionInputMessage = { role: "user", content: "" };
+	const startMessageSystem: ChatCompletionInputMessage = { role: "system", content: "" };
 
 	let conversation: Conversation = {
 		model: models[0],
 		config: defaultGenerationConfig,
-		messages: [{ ...startMessage }],
+		messages: [{ ...startMessageUser }],
+		systemMessage: startMessageSystem,
 		streaming: true,
 	};
 
-	let systemMessage: ChatCompletionInputMessage = { role: "system", content: "" };
 	let hfToken: string | undefined = import.meta.env.VITE_HF_TOKEN;
 	let viewCode = false;
 	let showTokenModal = false;
@@ -39,11 +40,6 @@
 	let waitForNonStreaming = true;
 
 	$: systemPromptSupported = isSystemPromptSupported(conversation.model);
-	$: {
-		if (!systemPromptSupported) {
-			systemMessage = { role: "system", content: "" };
-		}
-	}
 
 	function addMessage() {
 		conversation.messages = [
@@ -61,8 +57,8 @@
 	}
 
 	function reset() {
-		systemMessage.content = "";
-		conversation.messages = [{ ...startMessage }];
+		conversation.systemMessage.content = "";
+		conversation.messages = [{ ...startMessageUser }];
 	}
 
 	function abort() {
@@ -98,12 +94,11 @@
 							conversation.messages = [...conversation.messages];
 						}
 					},
-					abortController,
-					systemMessage
+					abortController
 				);
 			} else {
 				waitForNonStreaming = true;
-				const newMessage = await handleNonStreamingResponse(hf, conversation, systemMessage);
+				const newMessage = await handleNonStreamingResponse(hf, conversation);
 				// check if the user did not abort the request
 				if (waitForNonStreaming) {
 					conversation.messages = [...conversation.messages, newMessage];
@@ -162,7 +157,8 @@
 				placeholder={systemPromptSupported
 					? "Enter a custom prompt"
 					: "System prompt is not supported with the chosen model."}
-				bind:value={systemMessage.content}
+				value={systemPromptSupported ? conversation.systemMessage.content : ""}
+				on:input={e => (conversation.systemMessage.content = e.currentTarget.value)}
 				class="absolute inset-x-0 bottom-0 h-full resize-none bg-transparent px-3 pt-10 text-sm outline-none"
 			></textarea>
 		</div>

src/lib/components/InferencePlayground/InferencePlaygroundCodeSnippets.svelte

@@ -8,6 +8,7 @@
 	import http from "highlight.js/lib/languages/http";
 
 	import IconCopyCode from "../Icons/IconCopyCode.svelte";
+	import { isSystemPromptSupported } from "./inferencePlaygroundUtils";
 
 	hljs.registerLanguage("javascript", javascript);
 	hljs.registerLanguage("python", python);
@@ -46,10 +47,17 @@
 
 	function getMessages() {
 		const placeholder = [{ role: "user", content: "Tell me a story" }];
-		let messages = conversation.messages;
+
+		let messages = [...conversation.messages];
 		if (messages.length === 1 && messages[0].role === "user" && !messages[0].content) {
 			messages = placeholder;
 		}
+
+		const { model, systemMessage } = conversation;
+		if (isSystemPromptSupported(model) && systemMessage.content?.length) {
+			messages.unshift(systemMessage);
+		}
+
 		return messages;
 	}
 

src/lib/components/InferencePlayground/inferencePlaygroundUtils.ts

@@ -11,17 +11,17 @@ export async function handleStreamingResponse(
 	hf: HfInference,
 	conversation: Conversation,
 	onChunk: (content: string) => void,
-	abortController: AbortController,
-	systemMessage?: ChatCompletionInputMessage
+	abortController: AbortController
 ): Promise<void> {
+	const { model, systemMessage } = conversation;
 	const messages = [
-		...(isSystemPromptSupported(conversation.model) && systemMessage?.content?.length ? [systemMessage] : []),
+		...(isSystemPromptSupported(model) && systemMessage.content?.length ? [systemMessage] : []),
 		...conversation.messages,
 	];
 	let out = "";
 	for await (const chunk of hf.chatCompletionStream(
 		{
-			model: conversation.model.id,
+			model: model.id,
 			messages,
 			temperature: conversation.config.temperature,
 			max_tokens: conversation.config.maxTokens,
@@ -37,16 +37,16 @@ export async function handleStreamingResponse(
 
 export async function handleNonStreamingResponse(
 	hf: HfInference,
-	conversation: Conversation,
-	systemMessage?: ChatCompletionInputMessage
+	conversation: Conversation
 ): Promise<ChatCompletionInputMessage> {
+	const { model, systemMessage } = conversation;
 	const messages = [
-		...(isSystemPromptSupported(conversation.model) && systemMessage?.content?.length ? [systemMessage] : []),
+		...(isSystemPromptSupported(model) && systemMessage.content?.length ? [systemMessage] : []),
 		...conversation.messages,
 	];
 
 	const response = await hf.chatCompletion({
-		model: conversation.model,
+		model: model.id,
 		messages,
 		temperature: conversation.config.temperature,
 		max_tokens: conversation.config.maxTokens,

src/lib/components/InferencePlayground/types.ts

@@ -6,6 +6,7 @@ export type Conversation = {
 	model: ModelEntryWithTokenizer;
 	config: GenerationConfig;
 	messages: ChatCompletionInputMessage[];
+	systemMessage: ChatCompletionInputMessage;
 	streaming: boolean;
 };