up

src/app/main.tsx

@@ -39,6 +39,7 @@ export function Main() {
   const mainCharacterVoice = useStore(s => s.mainCharacterVoice)
   const orientation = useStore(s => s.orientation)
   const status = useStore(s => s.status)
+  const parseGenerationStatus = useStore(s => s.parseGenerationStatus)
   const storyGenerationStatus = useStore(s => s.storyGenerationStatus)
   const assetGenerationStatus = useStore(s => s.assetGenerationStatus)
   const voiceGenerationStatus = useStore(s => s.voiceGenerationStatus)
@@ -55,6 +56,7 @@ export function Main() {
   const toggleOrientation = useStore(s => s.toggleOrientation)
   const error = useStore(s => s.error)
   const setError = useStore(s => s.setError)
+  const setParseGenerationStatus = useStore(s => s.setParseGenerationStatus)
   const setStoryGenerationStatus = useStore(s => s.setStoryGenerationStatus)
   const setAssetGenerationStatus = useStore(s => s.setAssetGenerationStatus)
   const setVoiceGenerationStatus = useStore(s => s.setVoiceGenerationStatus)
@@ -78,24 +80,93 @@ export function Main() {
 
   const isBusy = status === "generating" || hasPendingTasks
 
-
-  const { openFilePicker, filesContent, loading } = useFilePicker({
+  const { openFilePicker, filesContent } = useFilePicker({
     accept: '.clap',
     readAs: "ArrayBuffer"
   })
-
   const fileData = filesContent[0]
 
   useEffect(() => {
     const fn = async () => {
-      if (fileData?.name) {
-        try {
-          const blob = new Blob([fileData.content])
-          await loadClap(blob, fileData.name)
-        } catch (err) {
-          console.error("failed to load the Clap file:", err)
-        }
+      if (!fileData?.name) { return }
+
+      const {
+        setStatus,
+        setProgress,
+        setParseGenerationStatus,
+        setVideoGenerationStatus,
+        setGeneratedVideo
+      } = useStore.getState()
+
+      let clap: ClapProject | undefined = undefined
+
+      setStatus("generating")
+      setProgress(25)
+      setParseGenerationStatus("generating")
+
+      try {
+        const blob = new Blob([fileData.content])
+        clap = await loadClap(blob, fileData.name)
+      } catch (err) {
+        console.error("failed to load the Clap file:", err)
+        setError(`${err}`)
+      }
+
+      if (!clap) {
+        setParseGenerationStatus("error")
+        setStatus("error")
+        setProgress(0)
+        return
+      }
+
+      setParseGenerationStatus("finished")
+
+      try {
+        setProgress(60)
+        setVoiceGenerationStatus("generating")
+        clap = await editClapDialogues({ clap })
+
+        if (!clap) { throw new Error(`failed to edit the dialogues`) }
+
+        console.log(`handleSubmit(): received a clap with dialogues = `, clap)
+        setCurrentClap(clap)
+        setVoiceGenerationStatus("finished")
+      } catch (err) {
+        setVoiceGenerationStatus("error")
+        setStatus("error")
+        setError(`${err}`)
+        return
       }
+      if (!clap) {
+        return
+      }
+
+      setVideoGenerationStatus("generating")
+  
+      let assetUrl = ""
+      try {
+        assetUrl = await exportClapToVideo({ clap })
+      } catch (err) {
+        console.error("failed to render the Clap file:", err)
+        setError(`${err}`)
+      }
+    
+      if (!assetUrl) {
+        setVideoGenerationStatus("error")
+        setStatus("error")
+        setProgress(0)
+        return
+      }
+
+      setVideoGenerationStatus("finished")
+
+      setProgress(80)
+  
+      console.log(`loadClap(): generated a video: ${assetUrl.slice(0, 60)}...`)
+    
+      setGeneratedVideo(assetUrl)
+      setStatus("finished")
+      setError("")
     }
     fn()
   }, [fileData?.name])
@@ -456,6 +527,29 @@ export function Main() {
                       </Button>
                     */}
 
+                    <div className="
+               
+                    flex flex-row
+                    justify-between items-center
+                    space-x-3">
+                    {canSeeBetaFeatures ?
+                    <Button
+                      onClick={openFilePicker}
+                      disabled={isBusy}
+                      // variant="ghost"
+                      className={cn(
+                        `text-sm md:text-base lg:text-lg`,
+                        `bg-stone-800/90 text-amber-400/100 dark:bg-stone-800/90 dark:text-amber-400/100`,
+                        `font-bold`,
+                        `hover:bg-stone-800/100 hover:text-amber-300/100 dark:hover:bg-stone-800/100 dark:hover:text-amber-300/100`,
+                        storyPromptDraft ? "opacity-100" : "opacity-80"
+                      )}
+                    >
+                      <span className="hidden xl:inline mr-1">Load project</span>
+                      <span className="inline xl:hidden mr-1">Load</span>
+                    </Button> : <div></div>
+                    }
+  
                     {canSeeBetaFeatures ?
                     <Button
                       onClick={() => saveClap()}
@@ -469,9 +563,11 @@ export function Main() {
                         storyPromptDraft ? "opacity-100" : "opacity-80"
                       )}
                     >
-                     <span className="mr-1">Save project</span>
+                      <span className="hidden xl:inline mr-1">Save project</span>
+                      <span className="inline xl:hidden mr-1">Save</span>
                     </Button> : <div></div>
                     }
+                    </div>
   
                     <div className=" 
                     flex flex-row
@@ -486,9 +582,12 @@ export function Main() {
                       cursor-pointer
                       "
                       onClick={() => toggleOrientation()}>
-                        <div>Orientation:</div>
+                        <div>
+                        <span className="hidden xl:inline mr-1">Orientation:</span>
+                        <span className="inline xl:hidden mr-1"></span>
+                        </div>
                         <div className="
-                        w-10 h-10
+                        w-8 h-8
                         flex flex-row items-center justify-center
                         "
                         >
@@ -506,7 +605,7 @@ export function Main() {
                         disabled={!storyPromptDraft || isBusy}
                         // variant="ghost"
                         className={cn(
-                          `text-lg md:text-xl lg:text-2xl`,
+                          `text-base md:text-lg lg:text-xl xl:text-2xl`,
                           `bg-stone-800/90 text-amber-400/100 dark:bg-stone-800/90 dark:text-amber-400/100`,
                           `font-bold`,
                           `hover:bg-stone-800/100 hover:text-amber-300/100 dark:hover:bg-stone-800/100 dark:hover:text-amber-300/100`,
@@ -566,10 +665,11 @@ export function Main() {
                     <p className="text-2xl font-bold">{progress}%</p> 
                     <p className="text-base text-white/70">{isBusy
                       ? (
-                        storyGenerationStatus === "generating" ? "Enhancing the story.."
-                        : assetGenerationStatus === "generating" ? "Creating characters.."
-                        : imageGenerationStatus === "generating" ? "Generating storyboards.."
-                        : voiceGenerationStatus === "generating" ? "Generating voices.."
+                        storyGenerationStatus === "generating" ? "Writing the story.."
+                        : parseGenerationStatus === "generating" ? "Loading the project.."
+                        : assetGenerationStatus === "generating" ? "Casting characters.."
+                        : imageGenerationStatus === "generating" ? "Creating storyboards.."
+                        : voiceGenerationStatus === "generating" ? "Recording voices.."
                         : videoGenerationStatus === "generating" ? "Assembling final video.."
                         : "Please wait.."
                       )
@@ -603,7 +703,7 @@ export function Main() {
                   font-sans
                   -mb-0
                   `,
-                  isLandscape ? 'h-4' : 'h-16'
+                  isLandscape ? 'h-4' : 'h-14'
                  )}
                   style={{ width: isPortrait ? 288 : 512 }}>
                   <span className="text-stone-100/50 text-4xs"

src/app/server/aitube/config.ts

@@ -1,2 +1,7 @@
 
 export const serverHuggingfaceApiKey = `${process.env.HF_API_TOKEN || ""}`
+
+// initially I used 1024x512 (a 2:1 ratio)
+// but that is a bit too extreme, most phones only take 16:9
+export const RESOLUTION_LONG = 1024
+export const RESOLUTION_SHORT = 576

src/app/server/aitube/createClap.ts

@@ -4,12 +4,9 @@ import { ClapProject } from "@aitube/clap"
 import { createClap as apiCreateClap } from "@aitube/client"
 
 import { VideoOrientation } from "../../types"
-import { getToken } from "./getToken"
 
-// initially I used 1024x512 (a 2:1 ratio)
-// but that is a bit too extreme, most phones only take 16:9
-const RESOLUTION_LONG = 1024
-const RESOLUTION_SHORT = 576
+import { getToken } from "./getToken"
+import { RESOLUTION_LONG, RESOLUTION_SHORT } from "./config"
 
 export async function createClap({
   prompt = "",

src/app/store.ts

@@ -4,9 +4,11 @@ import { ClapProject, parseClap, serializeClap } from "@aitube/clap"
 import { create } from "zustand"
 
 import { GlobalStatus, TaskStatus } from "@/types"
+import { getVideoOrientation } from "@/lib/utils/getVideoOrientation"
+import { parseVideoOrientation } from "@/lib/utils/parseVideoOrientation"
 
 import { VideoOrientation } from "./types"
-import { getVideoOrientation } from "@/lib/utils/getVideoOrientation"
+import { RESOLUTION_LONG, RESOLUTION_SHORT } from "./server/aitube/config"
 
 export const useStore = create<{
   mainCharacterImage: string
@@ -19,6 +21,7 @@ export const useStore = create<{
   orientation: VideoOrientation
 
   status: GlobalStatus
+  parseGenerationStatus: TaskStatus
   storyGenerationStatus: TaskStatus
   assetGenerationStatus: TaskStatus
   voiceGenerationStatus: TaskStatus
@@ -39,6 +42,7 @@ export const useStore = create<{
   setStoryPromptDraft: (storyPromptDraft: string) => void
   setStoryPrompt: (storyPrompt: string) => void
   setStatus: (status: GlobalStatus) => void
+  setParseGenerationStatus: (parseGenerationStatus: TaskStatus) => void
   setStoryGenerationStatus: (storyGenerationStatus: TaskStatus) => void
   setAssetGenerationStatus: (assetGenerationStatus: TaskStatus) => void
   setVoiceGenerationStatus: (voiceGenerationStatus: TaskStatus) => void
@@ -51,8 +55,8 @@ export const useStore = create<{
 
   setProgress: (progress: number) => void
   setError: (error: string) => void
-  saveClap: (fileName?: string) => Promise<void>
-  loadClap: (blob: Blob, fileName?: string) => Promise<void>
+  saveClap: () => Promise<void>
+  loadClap: (blob: Blob, fileName?: string) => Promise<ClapProject>
 }>((set, get) => ({
   mainCharacterImage: "",
   mainCharacterVoice: "",
@@ -60,6 +64,7 @@ export const useStore = create<{
   storyPrompt: "",
   orientation: VideoOrientation.PORTRAIT,
   status: "idle",
+  parseGenerationStatus: "idle",
   storyGenerationStatus: "idle",
   assetGenerationStatus: "idle",
   voiceGenerationStatus: "idle",
@@ -93,6 +98,7 @@ export const useStore = create<{
   setStoryPromptDraft: (storyPromptDraft: string) => { set({ storyPromptDraft }) },
   setStoryPrompt: (storyPrompt: string) => { set({ storyPrompt }) },
   setStatus: (status: GlobalStatus) => { set({ status }) },
+  setParseGenerationStatus: (parseGenerationStatus: TaskStatus) => { set({ parseGenerationStatus }) },
   setStoryGenerationStatus: (storyGenerationStatus: TaskStatus) => { set({ storyGenerationStatus }) },
   setAssetGenerationStatus: (assetGenerationStatus: TaskStatus) => { set({ assetGenerationStatus }) },
   setVoiceGenerationStatus: (voiceGenerationStatus: TaskStatus) => { set({ voiceGenerationStatus }) },
@@ -109,8 +115,8 @@ export const useStore = create<{
   },
   setProgress: (progress: number) => { set({ progress }) },
   setError: (error: string) => { set({ error }) },
-  saveClap: async (fileName: string = "untitled_story.clap"): Promise<void> => {
-    const { currentClap } = get()
+  saveClap: async (): Promise<void> => {
+    const { currentClap , storyPrompt } = get()
 
     if (!currentClap) { throw new Error(`cannot save a clap.. if there is no clap`) }
 
@@ -123,7 +129,13 @@ export const useStore = create<{
     const anchor = document.createElement("a")
     anchor.href = objectUrl
 
-    anchor.download = fileName
+    const firstPartOfStoryPrompt = storyPrompt // .split(",").shift() || ""
+
+    const cleanStoryPrompt = firstPartOfStoryPrompt.replace(/([^a-z0-9, ]+)/gi, "_")
+
+    const cleanName = `${cleanStoryPrompt.slice(0, 50)}`
+
+    anchor.download = `${cleanName}.clap`
 
     document.body.appendChild(anchor) // Append to the body (could be removed once clicked)
     anchor.click() // Trigger the download
@@ -132,15 +144,34 @@ export const useStore = create<{
     URL.revokeObjectURL(objectUrl)
     document.body.removeChild(anchor)
   },
-  loadClap: async (blob: Blob, fileName: string = "untitled_story.clap"): Promise<void> => {
+  loadClap: async (blob: Blob, fileName: string = "untitled_story.clap"): Promise<ClapProject> => {
     if (!blob) {
       throw new Error(`missing blob`)
     }
 
-    const currentClap: ClapProject = await parseClap(blob)
+    const currentClap: ClapProject | undefined = await parseClap(blob)
+
+    if (!currentClap) { throw new Error(`failed to import the clap`) }
+
+
+    const storyPrompt = currentClap.meta.description.split("||").pop() || ""
+
+    // TODO: parseVideoOrientation should be put inside @aitube/clap (in the utils)
+    // const orientation = parseVideoOrientation(currentClap.meta.orientation)
+    // let's use the UI settings for now
+    const { orientation } = get()
+
+    currentClap.meta.height = orientation === VideoOrientation.LANDSCAPE ? RESOLUTION_SHORT : RESOLUTION_LONG
+    currentClap.meta.width = orientation === VideoOrientation.PORTRAIT ? RESOLUTION_SHORT : RESOLUTION_LONG
 
     set({
       currentClap,
+      storyPrompt,
+      storyPromptDraft: storyPrompt,
+      orientation,
+      currentVideoOrientation: orientation,
     })
+
+    return currentClap
   },
 }))

src/lib/utils/parseVideoOrientation.ts

@@ -0,0 +1,11 @@
+import { VideoOrientation } from "@/app/types"
+
+export function parseVideoOrientation(input?: any): VideoOrientation {
+  const orientation = `${input || ""}`.trim().toLowerCase() || "square"
+
+  return (
+    (orientation === "vertical" || orientation === "portrait") ? VideoOrientation.PORTRAIT
+    : (orientation === "horizontal" || orientation === "landscape") ? VideoOrientation.LANDSCAPE
+    : VideoOrientation.SQUARE
+  )
+}