From 8f70d83785570973bcde3249fb243ca0a593f342 Mon Sep 17 00:00:00 2001 From: Jack Merrill Date: Fri, 18 Apr 2025 00:29:19 -0400 Subject: [PATCH] Enhance document processing by updating Markdown handling, adding processing status to uploads, and improving error messaging in the upload component --- app/dashboard/documents/[id]/page.tsx | 2 +- app/dashboard/upload/page.tsx | 3 +- bun.lockb | Bin 196712 -> 196712 bytes components/UploadZone.tsx | 2 +- components/nav-favorites.tsx | 12 ++++-- .../index.ts} | 37 ++++++++++++------ 6 files changed, 39 insertions(+), 17 deletions(-) rename supabase/functions/{process-document.ts => process-document/index.ts} (89%) diff --git a/app/dashboard/documents/[id]/page.tsx b/app/dashboard/documents/[id]/page.tsx index efe15c2..7309060 100644 --- a/app/dashboard/documents/[id]/page.tsx +++ b/app/dashboard/documents/[id]/page.tsx @@ -68,7 +68,7 @@ export default async function DocumentPage(props: { params: { id: string } }) { const processedContent = await remark() .use(remarkHtml) - .process(pages.join(" ")); + .process(pages.join("\n")); return ( diff --git a/app/dashboard/upload/page.tsx b/app/dashboard/upload/page.tsx index 9ccf45b..38cfa4a 100644 --- a/app/dashboard/upload/page.tsx +++ b/app/dashboard/upload/page.tsx @@ -30,7 +30,7 @@ export default async function Page() { const { data: documents, error } = await supabase .from("documents") - .select("id, file_name, created_at, owner") + .select("id, file_name, created_at, owner, is_processing") .eq("owner", user.id) .order("created_at", { ascending: false }); @@ -44,6 +44,7 @@ export default async function Page() { { return { + disabled: d.is_processing, name: d.file_name, url: `/dashboard/documents/${d.id}`, emoji: "📄", diff --git a/bun.lockb b/bun.lockb index da4b3e2e8364d6e82331ab971a745616f2aa3836..bacc2ec0544caecfd63bde37e2e070efc5339692 100755 GIT binary patch delta 1014 zcmXZYc}P@I6vy#<$0(j!wisqwU^bAI<2Y(AnL=rQCNx@LVF^uTj+0Fi1;%K*kXokQ z{w&HCA?S~4+O#r7%@&FvT1dxn#tABONi*BNWDh>v-#NU?JMZ|)0)1tHW!L44u9?%j zrTF)_CQkoU)z>9sRd~1jq#6G=i`5L5>~}Qz3Go2;(TEP65N&8j3tG{Po4A8}aN-tj z;5Hg@1y|94bErctYEXp=9ETb6ahK7qNueA)Qtz-+?8YvXU@da64rZ)E238^ys}Y89 zbn~ZO=!FL^^uP-@KHv@B;XPjCEnXo8Rw2?w5y6H+m=KRRbg+@f=)@B|#X~gX5n9md zr&8Qg3CTziLIrrFrfsJvzz&t*k&*MVi8)x0XYArRUXt55+togg zG&_=rDM&^lGT2QjCgTF1tw%K~QH9eug<71zNz|bRMX(_YiC73dBCr6F2*5NLF%JdR%>Vb z$UHu0d7s=Fq|F(fsIAVi5m`EEip(<^mKluB(s3EEDcDl9$6_~R?k%y{ZTVJniM_zK SO`XZrEn)!SY|7O&)c*wujJBr$ delta 1078 zcmXZbSxA&o6u|L&N2x~53M|{SCKFoSOD&tAtfx}QG?=22vNV@!X3&CcroD;Wm-`%L zSrGJ4O`Bz=sM$ghL<>3NG@}ohYg*a%7d!CdoWq6toqOQxvifyd{j4|TtM+Nr9a48%1B@{2a^|LGy6b;=$~otF>~@d)*3)(FvpRy3js4Y-Bd zu;UKy;vOEL4p(sv*Kq+hoJTRrQHd&~AQ<H5f*G5Tf>fko4b~zN z>#!cPFdJ`q@jJZ72YkdQw8H@>I^e=*xZ%MYgdpfzIEhm@jWftc0rnyjyODuC*no{vSVR&<5P~rWGZ27o-u)Fl=tUp; zF@QmQ!w`n?9Y63BBlv~i_=CR?+$W*Ia@ttLAzmmSxAfPYrMBS!GLeOS*o;(cQ98E_ z(Y;`eFVTz^yyhsD9EW(UQ0Z=&Vo7A32^fbaC`JiNQHF95(Pc+D>Ys*M7KD?w_QcJ3Q8#?R`V?i1&DI91Xj9MEdTS!mpo| bXELYf { console.error("SSE Error:", event); - toast.error("An error occurred while processing the document.", { + toast.error("An error occurred while processing the document", { description: event.data || "Unknown error", }); setUploading(false); diff --git a/components/nav-favorites.tsx b/components/nav-favorites.tsx index d1eaf4a..ddaa4c6 100644 --- a/components/nav-favorites.tsx +++ b/components/nav-favorites.tsx @@ -4,6 +4,7 @@ import { ArrowUpRight, FileText, Link, + LoaderCircle, MoreHorizontal, StarOff, Trash2, @@ -30,6 +31,7 @@ export function NavDocuments({ documents, }: { documents: { + disabled?: boolean; name: string; url: string; emoji?: string; @@ -42,10 +44,14 @@ export function NavDocuments({ Documents {documents.map((item) => ( - - + + - {item.emoji ? item.emoji : } + {item.disabled ? ( + + ) : ( + {item.emoji ? item.emoji : } + )} {item.name} diff --git a/supabase/functions/process-document.ts b/supabase/functions/process-document/index.ts similarity index 89% rename from supabase/functions/process-document.ts rename to supabase/functions/process-document/index.ts index 5103b76..d1388fe 100644 --- a/supabase/functions/process-document.ts +++ b/supabase/functions/process-document/index.ts @@ -20,18 +20,27 @@ Do not return the Markdown as a code block, only as a raw string, without any ne No data or information should ever be removed, it should only be processed and formatted. -There are in-text citations/references in the text, remove them from the text and put them into an object where the key is the reference number and the value is the text. +There are in-text citations/references in the text, remove them from the text (**but most importantly, keep the reference number in the text. use a tag**) and put them into an object where the key is the reference number and the value is the text. -The Markdown should be human-readable and well-formatted. +The Markdown should be human-readable and well-formatted. The markdown string should properly sanitized and should not break a JSON parser when returned as the final format. + +Return the final result as a text object with the following structure (without code block formatting): + +""" + + +--------- -Return the final result as a JSON object with the following structure: { - "markdown": "", "citations": { - "1": "", - "2": "" + "1": "Citation text for reference 1", + "2": "Citation text for reference 2", + // ... more citations } } +""" + +Do not return the text object as a code block, only as a raw string. `; Deno.serve(async (req) => { if (req.method === "OPTIONS") { @@ -276,16 +285,22 @@ Deno.serve(async (req) => { } if (response.choices[0].message.content) { - const markdownResponse = JSON.parse( - response.choices[0].message.content.toString() + // remove any potential code block formatting from the content + console.log( + `[${page.index}] ${response.choices[0].message.content}` ); - const citations = markdownResponse.citations; - const markdown = markdownResponse.markdown; + const split = + response.choices[0].message.content.split("---------"); + + const content = split[0].trim(); + const citationsStr = split[1]?.trim() || "{}"; + const citations = JSON.parse(citationsStr).citations || {}; + console.log("Generating Markdown for page:", page.index); sendEvent("status", { message: `Generating Markdown for page ${page.index}`, }); - const markdown = replaceImagesInMarkdown(markdown, imageData); + const markdown = replaceImagesInMarkdown(content, imageData); return { ...page,