From 8b00fd4ba37b8b2577a05daf0aba446c01c25d66 Mon Sep 17 00:00:00 2001 From: Ryan Toronto Date: Tue, 15 Jul 2025 10:47:07 -0400 Subject: [PATCH 1/7] log errors --- app/(main)/chats/[id]/page.client.tsx | 3 +++ 1 file changed, 3 insertions(+) diff --git a/app/(main)/chats/[id]/page.client.tsx b/app/(main)/chats/[id]/page.client.tsx index 5464b1ca9..67df119a2 100644 --- a/app/(main)/chats/[id]/page.client.tsx +++ b/app/(main)/chats/[id]/page.client.tsx @@ -67,6 +67,9 @@ export default function PageClient({ chat }: { chat: Chat }) { setActiveTab("preview"); } }) + .on("error", (error) => { + console.error("Stream error:", error); + }) .on("finalContent", async (finalText) => { startTransition(async () => { const message = await createMessage( From 245e1aeaba354ef7348eac024bf60526a296823e Mon Sep 17 00:00:00 2001 From: Ryan Toronto Date: Tue, 15 Jul 2025 12:07:42 -0400 Subject: [PATCH 2/7] dont use edge for this route --- app/api/get-next-completion-stream-promise/route.ts | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/app/api/get-next-completion-stream-promise/route.ts b/app/api/get-next-completion-stream-promise/route.ts index 4d897d9a1..08ada9641 100644 --- a/app/api/get-next-completion-stream-promise/route.ts +++ b/app/api/get-next-completion-stream-promise/route.ts @@ -60,5 +60,4 @@ export async function POST(req: Request) { return new Response(res.toReadableStream()); } -export const runtime = "edge"; -export const maxDuration = 45; +export const maxDuration = 60; From 95bca5e494a3c1157e1d66011ded45f5033b70a4 Mon Sep 17 00:00:00 2001 From: Ryan Toronto Date: Tue, 15 Jul 2025 12:19:11 -0400 Subject: [PATCH 3/7] verbose logging --- .../route.ts | 19 ++++++++++++++++++- 1 file changed, 18 insertions(+), 1 deletion(-) diff --git a/app/api/get-next-completion-stream-promise/route.ts b/app/api/get-next-completion-stream-promise/route.ts index 08ada9641..80c2efc72 100644 --- a/app/api/get-next-completion-stream-promise/route.ts +++ b/app/api/get-next-completion-stream-promise/route.ts @@ -3,6 +3,7 @@ import { PrismaNeon } from "@prisma/adapter-neon"; import { Pool } from "@neondatabase/serverless"; import { z } from "zod"; import Together from "together-ai"; +import { ChatCompletionStream } from "together-ai/lib/ChatCompletionStream.mjs"; export async function POST(req: Request) { const neon = new Pool({ connectionString: process.env.DATABASE_URL }); @@ -57,7 +58,23 @@ export async function POST(req: Request) { max_tokens: 9000, }); - return new Response(res.toReadableStream()); + const [s1, s2] = res.tee(); + + ChatCompletionStream.fromReadableStream(s2.toReadableStream()) + .on("content", (delta) => { + console.log("Stream content:", delta); + }) + .on("error", (error) => { + console.error("Stream error:", error); + }) + .on("finalContent", (finalText) => { + console.log("Final content:", finalText); + }) + .on("end", () => { + console.log("Stream ended"); + }); + + return new Response(s1.toReadableStream()); } export const maxDuration = 60; From 7e84fb30fabd886276abf2f9418b7c268fcb30b1 Mon Sep 17 00:00:00 2001 From: Ryan Toronto Date: Tue, 15 Jul 2025 12:30:00 -0400 Subject: [PATCH 4/7] debug with after --- .../get-next-completion-stream-promise/route.ts | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/app/api/get-next-completion-stream-promise/route.ts b/app/api/get-next-completion-stream-promise/route.ts index 80c2efc72..11268246b 100644 --- a/app/api/get-next-completion-stream-promise/route.ts +++ b/app/api/get-next-completion-stream-promise/route.ts @@ -4,6 +4,7 @@ import { Pool } from "@neondatabase/serverless"; import { z } from "zod"; import Together from "together-ai"; import { ChatCompletionStream } from "together-ai/lib/ChatCompletionStream.mjs"; +import { after } from "next/server"; export async function POST(req: Request) { const neon = new Pool({ connectionString: process.env.DATABASE_URL }); @@ -60,20 +61,34 @@ export async function POST(req: Request) { const [s1, s2] = res.tee(); + let unlock: () => void; + const promise = new Promise((resolve) => { + unlock = () => { + resolve(); + }; + }); + ChatCompletionStream.fromReadableStream(s2.toReadableStream()) .on("content", (delta) => { console.log("Stream content:", delta); }) .on("error", (error) => { console.error("Stream error:", error); + unlock(); }) .on("finalContent", (finalText) => { console.log("Final content:", finalText); }) .on("end", () => { console.log("Stream ended"); + unlock(); }); + after(async () => { + await promise; + console.log("exiting after hook"); + }); + return new Response(s1.toReadableStream()); } From 471fb505ad0b34ee3099a368a136eb19c0ef9dcd Mon Sep 17 00:00:00 2001 From: Ryan Toronto Date: Tue, 15 Jul 2025 12:36:04 -0400 Subject: [PATCH 5/7] need to keep logs small --- app/api/get-next-completion-stream-promise/route.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/app/api/get-next-completion-stream-promise/route.ts b/app/api/get-next-completion-stream-promise/route.ts index 11268246b..547b23992 100644 --- a/app/api/get-next-completion-stream-promise/route.ts +++ b/app/api/get-next-completion-stream-promise/route.ts @@ -70,14 +70,14 @@ export async function POST(req: Request) { ChatCompletionStream.fromReadableStream(s2.toReadableStream()) .on("content", (delta) => { - console.log("Stream content:", delta); + // console.log("Stream content:", delta); }) .on("error", (error) => { console.error("Stream error:", error); unlock(); }) .on("finalContent", (finalText) => { - console.log("Final content:", finalText); + console.log("Final content hook called"); }) .on("end", () => { console.log("Stream ended"); From e2244d3d8b4676eec53740832f2cdf3156904c5d Mon Sep 17 00:00:00 2001 From: Ryan Toronto Date: Tue, 15 Jul 2025 12:45:56 -0400 Subject: [PATCH 6/7] more logs --- app/api/get-next-completion-stream-promise/route.ts | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/app/api/get-next-completion-stream-promise/route.ts b/app/api/get-next-completion-stream-promise/route.ts index 547b23992..3061963db 100644 --- a/app/api/get-next-completion-stream-promise/route.ts +++ b/app/api/get-next-completion-stream-promise/route.ts @@ -68,6 +68,8 @@ export async function POST(req: Request) { }; }); + console.log("last message:", messages.at(-1)); + ChatCompletionStream.fromReadableStream(s2.toReadableStream()) .on("content", (delta) => { // console.log("Stream content:", delta); @@ -76,6 +78,9 @@ export async function POST(req: Request) { console.error("Stream error:", error); unlock(); }) + .on("finalChatCompletion", (finalChunk) => { + console.log("Final Chat Completion:", finalChunk); + }) .on("finalContent", (finalText) => { console.log("Final content hook called"); }) @@ -92,4 +97,4 @@ export async function POST(req: Request) { return new Response(s1.toReadableStream()); } -export const maxDuration = 60; +export const maxDuration = 240; From 8319ed6e9fcf2843872703b6f4b7438dc905d9d9 Mon Sep 17 00:00:00 2001 From: Ryan Toronto Date: Tue, 15 Jul 2025 13:39:21 -0400 Subject: [PATCH 7/7] more debugging --- app/(main)/chats/[id]/chat-box.tsx | 1 - app/api/get-next-completion-stream-promise/route.ts | 8 +++++--- package-lock.json | 9 +++++---- package.json | 2 +- 4 files changed, 11 insertions(+), 9 deletions(-) diff --git a/app/(main)/chats/[id]/chat-box.tsx b/app/(main)/chats/[id]/chat-box.tsx index bbd73ba1c..1c56a725e 100644 --- a/app/(main)/chats/[id]/chat-box.tsx +++ b/app/(main)/chats/[id]/chat-box.tsx @@ -2,7 +2,6 @@ import ArrowRightIcon from "@/components/icons/arrow-right"; import Spinner from "@/components/spinner"; -import assert from "assert"; import { useRouter } from "next/navigation"; import { useEffect, useRef, useState, useTransition } from "react"; import { createMessage } from "../../actions"; diff --git a/app/api/get-next-completion-stream-promise/route.ts b/app/api/get-next-completion-stream-promise/route.ts index 3061963db..5f9346f5c 100644 --- a/app/api/get-next-completion-stream-promise/route.ts +++ b/app/api/get-next-completion-stream-promise/route.ts @@ -59,7 +59,9 @@ export async function POST(req: Request) { max_tokens: 9000, }); - const [s1, s2] = res.tee(); + const stream = res.toReadableStream(); + + const [s1, s2] = stream.tee(); let unlock: () => void; const promise = new Promise((resolve) => { @@ -70,7 +72,7 @@ export async function POST(req: Request) { console.log("last message:", messages.at(-1)); - ChatCompletionStream.fromReadableStream(s2.toReadableStream()) + ChatCompletionStream.fromReadableStream(s1) .on("content", (delta) => { // console.log("Stream content:", delta); }) @@ -94,7 +96,7 @@ export async function POST(req: Request) { console.log("exiting after hook"); }); - return new Response(s1.toReadableStream()); + return new Response(s2); } export const maxDuration = 240; diff --git a/package-lock.json b/package-lock.json index fa16af236..930bf74da 100644 --- a/package-lock.json +++ b/package-lock.json @@ -40,7 +40,7 @@ "sonner": "^1.7.2", "tailwind-merge": "^2.6.0", "tailwindcss-animate": "^1.0.7", - "together-ai": "^0.11.1", + "together-ai": "^0.20.0", "use-stick-to-bottom": "^1.0.43", "vaul": "^1.1.2", "zod": "^3.24.1" @@ -10566,9 +10566,10 @@ } }, "node_modules/together-ai": { - "version": "0.11.1", - "resolved": "https://registry.npmjs.org/together-ai/-/together-ai-0.11.1.tgz", - "integrity": "sha512-SK+sOT1qPnaNxWFUGy2CHD588ALSmPKd4RgNSD2FXIVAy69R/JrgISV/gj+1LqBnuBDCSBIQMxX2j20bJSs/UQ==", + "version": "0.20.0", + "resolved": "https://registry.npmjs.org/together-ai/-/together-ai-0.20.0.tgz", + "integrity": "sha512-TPijoMHi/OY+IcNtzTlrkte0hT/OHnSaQCeCVTgn8LyQ9T7Y4rnl39h91w/x9WbZHirDK1gmYaFjvoAan3a+3g==", + "license": "Apache-2.0", "dependencies": { "@types/node": "^18.11.18", "@types/node-fetch": "^2.6.4", diff --git a/package.json b/package.json index 52d72f498..e296c75ac 100644 --- a/package.json +++ b/package.json @@ -41,7 +41,7 @@ "sonner": "^1.7.2", "tailwind-merge": "^2.6.0", "tailwindcss-animate": "^1.0.7", - "together-ai": "^0.11.1", + "together-ai": "^0.20.0", "use-stick-to-bottom": "^1.0.43", "vaul": "^1.1.2", "zod": "^3.24.1"