Ollama support

.example.env

@@ -1,4 +1,8 @@
 TOGETHER_API_KEY=
+OLLAMA_BASE_URL=http://localhost:11434
+
+# You must provide a postgresql database url (this project uses neon database provider)
+DATABASE_URL=
 
 # Optional, if you want observability
 HELICONE_API_KEY=

.gitpod.yml

@@ -0,0 +1,25 @@
+# This configuration file was automatically generated by Gitpod.
+tasks:
+  - name: Setup Node.js and Project
+    init: npm install && npm run build
+    command: npm run start
+
+  - name: Setup and Run Ollama
+    init: |
+      if ! command -v ollama &> /dev/null; then
+        echo "Ollama not found, installing..."
+        curl -fsSL https://ollama.com/install.sh | sh
+      else
+        echo "Ollama already installed, skipping installation"
+      fi
+    command: ollama serve
+
+  - name: Run a Model
+    command: |
+      echo "Waiting for Ollama server to start..."
+      while ! curl -s http://localhost:11434/api/version &>/dev/null; do
+        sleep 2
+        echo "Still waiting for Ollama server..."
+      done
+      echo "Ollama server is running, starting model..."
+      ollama run gemma3:12b

README.md

@@ -15,12 +15,14 @@
 - Next.js app router with Tailwind
 - Helicone for observability
 - Plausible for website analytics
+- WIP: Add the possibility to use Ollama server (dev)
 
 ## Cloning & running
 
 1. Clone the repo: `git clone https://github.com/Nutlope/llamacoder`
 2. Create a `.env` file and add your [Together AI API key](https://togetherai.link/?utm_source=example-app&utm_medium=llamacoder&utm_campaign=llamacoder-app-signup): `TOGETHER_API_KEY=`
 3. Run `npm install` and `npm run dev` to install dependencies and run locally
+4. If you want to use Ollama: set `OLLAMA_BASE_URL` and install and run a model.
 
 ## Contributing
 

app/(main)/actions.ts

@@ -9,6 +9,53 @@ import {
 import { notFound } from "next/navigation";
 import Together from "together-ai";
 
+// Helper function to determine if a model is an Ollama model
+function isOllamaModel() {
+  return !!process.env.OLLAMA_BASE_URL;//model.includes(':') || model.startsWith('gemma');
+}
+
+// Function to call Ollama API
+async function callOllamaAPI(model: string, messages: any[], options: any = {}) {
+  const ollamaBaseUrl = process.env.OLLAMA_BASE_URL || 'http://localhost:11434';
+
+  try {
+    const response = await fetch(`${ollamaBaseUrl}/api/chat`, {
+      method: 'POST',
+      headers: {
+        'Content-Type': 'application/json',
+      },
+      body: JSON.stringify({
+        model,
+        messages,
+        temperature: options.temperature || 0.7,
+        max_tokens: options.max_tokens,
+        stream: false,
+      }),
+    });
+
+    if (!response.ok) {
+      throw new Error(`Ollama API error: ${response.status}`);
+    }
+
+    const data = await response.json();
+
+    // Transform Ollama response to match Together.ai format
+    return {
+      choices: [
+        {
+          message: {
+            content: data.message?.content || '',
+            role: data.message?.role || 'assistant',
+          },
+        },
+      ],
+    };
+  } catch (error) {
+    console.error('Error calling Ollama API:', error);
+    throw error;
+  }
+}
+
 export async function createChat(
   prompt: string,
   model: string,
@@ -39,48 +86,68 @@ export async function createChat(
 
   const together = new Together(options);
 
+  // Use a fallback model for title generation if using Ollama
+  const titleModel = isOllamaModel() ? 
+    "meta-llama/Meta-Llama-3.1-8B-Instruct-Turbo" : 
+    "meta-llama/Meta-Llama-3.1-8B-Instruct-Turbo";
+
   async function fetchTitle() {
-    const responseForChatTitle = await together.chat.completions.create({
-      model: "meta-llama/Meta-Llama-3.1-8B-Instruct-Turbo",
-      messages: [
-        {
-          role: "system",
-          content:
-            "You are a chatbot helping the user create a simple app or script, and your current job is to create a succinct title, maximum 3-5 words, for the chat given their initial prompt. Please return only the title.",
-        },
-        {
-          role: "user",
-          content: prompt,
-        },
-      ],
-    });
-    const title = responseForChatTitle.choices[0].message?.content || prompt;
+    const titleMessages = [
+      {
+        role: "system",
+        content:
+          "You are a chatbot helping the user create a simple app or script, and your current job is to create a succinct title, maximum 3-5 words, for the chat given their initial prompt. Please return only the title.",
+      },
+      {
+        role: "user",
+        content: prompt,
+      },
+    ];
+
+    let titleResponse;
+    if (isOllamaModel()) {
+      titleResponse = await callOllamaAPI(titleModel, titleMessages);
+    } else {
+      titleResponse = await together.chat.completions.create({
+        model: titleModel,
+        messages: titleMessages,
+      });
+    }
+
+    const title = titleResponse.choices[0].message?.content || prompt;
     return title;
   }
 
   async function fetchTopExample() {
-    const findSimilarExamples = await together.chat.completions.create({
-      model: "meta-llama/Meta-Llama-3.1-8B-Instruct-Turbo",
-      messages: [
-        {
-          role: "system",
-          content: `You are a helpful bot. Given a request for building an app, you match it to the most similar example provided. If the request is NOT similar to any of the provided examples, return "none". Here is the list of examples, ONLY reply with one of them OR "none":
-
-          - landing page
-          - blog app
-          - quiz app
-          - pomodoro timer
-          `,
-        },
-        {
-          role: "user",
-          content: prompt,
-        },
-      ],
-    });
+    const exampleMessages = [
+      {
+        role: "system",
+        content: `You are a helpful bot. Given a request for building an app, you match it to the most similar example provided. If the request is NOT similar to any of the provided examples, return "none". Here is the list of examples, ONLY reply with one of them OR "none":
+
+        - landing page
+        - blog app
+        - quiz app
+        - pomodoro timer
+        `,
+      },
+      {
+        role: "user",
+        content: prompt,
+      },
+    ];
+
+    let exampleResponse;
+    if (isOllamaModel()) {
+      exampleResponse = await callOllamaAPI(titleModel, exampleMessages);
+    } else {
+      exampleResponse = await together.chat.completions.create({
+        model: titleModel,
+        messages: exampleMessages,
+      });
+    }
 
     const mostSimilarExample =
-      findSimilarExamples.choices[0].message?.content || "none";
+      exampleResponse.choices[0].message?.content || "none";
     return mostSimilarExample;
   }
 
@@ -91,6 +158,7 @@ export async function createChat(
 
   let fullScreenshotDescription;
   if (screenshotUrl) {
+    // For screenshots, always use Together.ai as Ollama may not support vision
     const screenshotResponse = await together.chat.completions.create({
       model: "meta-llama/Llama-3.2-90B-Vision-Instruct-Turbo",
       temperature: 0.2,
@@ -116,23 +184,34 @@ export async function createChat(
 
   let userMessage: string;
   if (quality === "high") {
-    let initialRes = await together.chat.completions.create({
-      model: "Qwen/Qwen2.5-Coder-32B-Instruct",
-      messages: [
-        {
-          role: "system",
-          content: softwareArchitectPrompt,
-        },
-        {
-          role: "user",
-          content: fullScreenshotDescription
-            ? fullScreenshotDescription + prompt
-            : prompt,
-        },
-      ],
-      temperature: 0.2,
-      max_tokens: 3000,
-    });
+    const highQualityModel = isOllamaModel() ? model : "Qwen/Qwen2.5-Coder-32B-Instruct";
+    const highQualityMessages = [
+      {
+        role: "system",
+        content: softwareArchitectPrompt,
+      },
+      {
+        role: "user",
+        content: fullScreenshotDescription
+          ? fullScreenshotDescription + prompt
+          : prompt,
+      },
+    ];
+
+    let initialRes;
+    if (isOllamaModel()) {
+      initialRes = await callOllamaAPI(highQualityModel, highQualityMessages, {
+        temperature: 0.2,
+        max_tokens: 3000,
+      });
+    } else {
+      initialRes = await together.chat.completions.create({
+        model: highQualityModel,
+        messages: highQualityMessages,
+        temperature: 0.2,
+        max_tokens: 3000,
+      });
+    }
 
     userMessage = initialRes.choices[0].message?.content ?? prompt;
   } else if (fullScreenshotDescription) {

app/(main)/chats/[id]/page.client.tsx

@@ -14,6 +14,73 @@ import CodeViewerLayout from "./code-viewer-layout";
 import type { Chat } from "./page";
 import { Context } from "../../providers";
 
+// Custom stream handler that works with both Together API and Ollama API
+function createStreamHandler(stream: ReadableStream, callbacks: {
+  onContent: (delta: string, content: string) => void;
+  onFinalContent: (content: string) => void;
+}) {
+  // Try to use Together's ChatCompletionStream if available
+  try {
+    return ChatCompletionStream.fromReadableStream(stream)
+      .on("content", callbacks.onContent)
+      .on("finalContent", callbacks.onFinalContent);
+  } catch (error) {
+    // Fallback for Ollama or other stream sources
+    console.log("Using fallback stream handler");
+
+    const reader = stream.getReader();
+    const decoder = new TextDecoder();
+    let content = '';
+
+    const read = async () => {
+      try {
+        const { done, value } = await reader.read();
+
+        if (done) {
+          callbacks.onFinalContent(content);
+          return;
+        }
+
+        const chunk = decoder.decode(value, { stream: true });
+        const lines = chunk.split('\n').filter(line => line.trim() && line.startsWith('data: '));
+
+        for (const line of lines) {
+          try {
+            const data = line.substring(6); // Remove 'data: ' prefix
+
+            if (data === '[DONE]') {
+              callbacks.onFinalContent(content);
+              return;
+            }
+
+            const parsed = JSON.parse(data);
+            const delta = parsed.choices?.[0]?.delta?.content || '';
+
+            if (delta) {
+              content += delta;
+              callbacks.onContent(delta, content);
+            }
+          } catch (e) {
+            console.error('Error parsing stream chunk:', e);
+          }
+        }
+
+        return read();
+      } catch (error) {
+        console.error('Stream reading error:', error);
+        callbacks.onFinalContent(content);
+      }
+    };
+
+    read();
+
+    // Return a dummy object with cancel method to match Together's API
+    return {
+      cancel: () => reader.cancel(),
+    };
+  }
+}
+
 export default function PageClient({ chat }: { chat: Chat }) {
   const context = use(Context);
   const [streamPromise, setStreamPromise] = useState<
@@ -41,8 +108,8 @@ export default function PageClient({ chat }: { chat: Chat }) {
       let didPushToCode = false;
       let didPushToPreview = false;
 
-      ChatCompletionStream.fromReadableStream(stream)
-        .on("content", (delta, content) => {
+      createStreamHandler(stream, {
+        onContent: (delta, content) => {
           setStreamText((text) => text + delta);
 
           if (
@@ -66,8 +133,8 @@ export default function PageClient({ chat }: { chat: Chat }) {
             setIsShowingCodeViewer(true);
             setActiveTab("preview");
           }
-        })
-        .on("finalContent", async (finalText) => {
+        },
+        onFinalContent: async (finalText) => {
           startTransition(async () => {
             const message = await createMessage(
               chat.id,
@@ -83,12 +150,14 @@ export default function PageClient({ chat }: { chat: Chat }) {
               router.refresh();
             });
           });
-        });
+        }
+      });
     }
 
     f();
   }, [chat.id, router, streamPromise, context]);
 
+  // Rest of the component remains unchanged
   return (
     <div className="h-dvh">
       <div className="flex h-full">