"use client" import React, { useEffect, useRef, useTransition } from 'react' import { IoMdPhonePortrait } from 'react-icons/io' import { GiRollingDices } from 'react-icons/gi' import { FaCloudDownloadAlt, FaDiscord } from "react-icons/fa" import { useLocalStorage } from "usehooks-ts" import { ClapProject, ClapMediaOrientation, ClapSegmentCategory, updateClap } from '@aitube/clap' import Image from 'next/image' import { useSearchParams } from "next/navigation" import { useFilePicker } from 'use-file-picker' import { DeviceFrameset } from 'react-device-frameset' import 'react-device-frameset/styles/marvel-devices.min.css' import { Card, CardContent, CardHeader } from '@/components/ui/card' import { Button } from '@/components/ui/button' import { Toaster } from '@/components/ui/sonner' import { TextareaField } from '@/components/form/textarea-field' import { cn } from '@/lib/utils/cn' import { createClap } from './server/aitube/createClap' import { editClapEntities } from './server/aitube/editClapEntities' import { editClapDialogues } from './server/aitube/editClapDialogues' import { editClapStoryboards } from './server/aitube/editClapStoryboards' import { editClapSounds } from './server/aitube/editClapSounds' import { editClapMusic } from './server/aitube/editClapMusic' import { editClapVideos } from './server/aitube/editClapVideos' import { exportClapToVideo } from './server/aitube/exportClapToVideo' import { useStore } from './store' import HFLogo from "./hf-logo.svg" import { Input } from '@/components/ui/input' import { Field } from '@/components/form/field' import { Label } from '@/components/form/label' import { getParam } from '@/lib/utils/getParam' import { GenerationStage } from '@/types' import { FileContent } from 'use-file-picker/dist/interfaces' import { generateRandomStory } from '@/lib/utils/generateRandomStory' import { logImage } from '@/lib/utils/logImage' import { defaultPrompt } from './config' export function Main() { const [storyPromptDraft, setStoryPromptDraft] = useLocalStorage( "AI_STORIES_FACTORY_STORY_PROMPT_DRAFT", defaultPrompt ) const promptDraftRef = useRef("") promptDraftRef.current = storyPromptDraft const [_isPending, startTransition] = useTransition() const storyPrompt = useStore(s => s.storyPrompt) const mainCharacterImage = useStore(s => s.mainCharacterImage) const mainCharacterVoice = useStore(s => s.mainCharacterVoice) const orientation = useStore(s => s.orientation) const setOrientation = useStore(s => s.setOrientation) const status = useStore(s => s.status) const parseGenerationStatus = useStore(s => s.parseGenerationStatus) const storyGenerationStatus = useStore(s => s.storyGenerationStatus) const assetGenerationStatus = useStore(s => s.assetGenerationStatus) const soundGenerationStatus = useStore(s => s.soundGenerationStatus) const musicGenerationStatus = useStore(s => s.musicGenerationStatus) const voiceGenerationStatus = useStore(s => s.voiceGenerationStatus) const imageGenerationStatus = useStore(s => s.imageGenerationStatus) const videoGenerationStatus = useStore(s => s.videoGenerationStatus) const finalGenerationStatus = useStore(s => s.finalGenerationStatus) const currentClap = useStore(s => s.currentClap) const currentVideo = useStore(s => s.currentVideo) const currentVideoOrientation = useStore(s => s.currentVideoOrientation) const setStoryPrompt = useStore(s => s.setStoryPrompt) const setMainCharacterImage = useStore(s => s.setMainCharacterImage) const setMainCharacterVoice = useStore(s => s.setMainCharacterVoice) const setStatus = useStore(s => s.setStatus) const toggleOrientation = useStore(s => s.toggleOrientation) const error = useStore(s => s.error) const setError = useStore(s => s.setError) const setParseGenerationStatus = useStore(s => s.setParseGenerationStatus) const setStoryGenerationStatus = useStore(s => s.setStoryGenerationStatus) const setAssetGenerationStatus = useStore(s => s.setAssetGenerationStatus) const setSoundGenerationStatus = useStore(s => s.setSoundGenerationStatus) const setMusicGenerationStatus = useStore(s => s.setMusicGenerationStatus) const setVoiceGenerationStatus = useStore(s => s.setVoiceGenerationStatus) const setImageGenerationStatus = useStore(s => s.setImageGenerationStatus) const setVideoGenerationStatus = useStore(s => s.setVideoGenerationStatus) const setFinalGenerationStatus = useStore(s => s.setFinalGenerationStatus) const setCurrentClap = useStore(s => s.setCurrentClap) const setCurrentVideo = useStore(s => s.setCurrentVideo) const progress = useStore(s => s.progress) const setProgress = useStore(s => s.setProgress) const saveVideo = useStore(s => s.saveVideo) const saveClap = useStore(s => s.saveClap) const loadClap = useStore(s => s.loadClap) // let's disable this for now const canSeeBetaFeatures = true // getParam("beta", false) const isBusy = useStore(s => s.isBusy) const busyRef = useRef(isBusy) busyRef.current = isBusy const importStory = async (fileData: FileContent): Promise => { if (!fileData?.name) { throw new Error(`invalid file (missing file name)`) } const { setStatus, setProgress, setParseGenerationStatus, } = useStore.getState() let clap: ClapProject | undefined = undefined setParseGenerationStatus("generating") try { const blob = new Blob([fileData.content]) clap = await loadClap(blob, fileData.name) if (!clap) { throw new Error(`failed to load the clap file`) } setParseGenerationStatus("finished") setCurrentClap(clap) return clap } catch (err) { console.error("failed to load the Clap file:", err) setParseGenerationStatus("error") throw err } } const generateStory = async (): Promise => { let clap: ClapProject | undefined = undefined try { setProgress(0) setStatus("generating") setStoryGenerationStatus("generating") setStoryPrompt(promptDraftRef.current) clap = await createClap({ prompt: promptDraftRef.current, orientation: useStore.getState().orientation, turbo: true, }) if (!clap) { throw new Error(`failed to create the clap`) } if (clap.segments.length <= 1) { throw new Error(`failed to generate more than one segments`) } console.log(`handleSubmit(): received a clap = `, clap) setCurrentClap(clap) setStoryGenerationStatus("finished") console.log("---------------- GENERATED STORY ----------------") console.table(clap.segments, [ // 'startTimeInMs', 'endTimeInMs', // 'track', 'category', 'prompt' ]) return clap } catch (err) { setStoryGenerationStatus("error") throw err } } const generateEntities = async (clap: ClapProject): Promise => { try { // setProgress(20) setAssetGenerationStatus("generating") clap = await editClapEntities({ clap, // generating entities requires a "smart" LLM turbo: false, // turbo: true, }).then(r => r.promise) if (!clap) { throw new Error(`failed to edit the entities`) } console.log(`handleSubmit(): received a clap with entities = `, clap) setCurrentClap(clap) setAssetGenerationStatus("finished") console.log("---------------- GENERATED ENTITIES ----------------") console.table(clap.entities, [ 'category', 'label', 'imagePrompt', 'appearance' ]) return clap } catch (err) { setAssetGenerationStatus("error") throw err } } const generateSounds = async (clap: ClapProject): Promise => { try { // setProgress(30) setSoundGenerationStatus("generating") clap = await editClapSounds({ clap, turbo: true }).then(r => r.promise) if (!clap) { throw new Error(`failed to edit the sound`) } console.log(`handleSubmit(): received a clap with sound = `, clap) setCurrentClap(clap) setSoundGenerationStatus("finished") console.log("---------------- GENERATED SOUND ----------------") console.table(clap.segments.filter(s => s.category === ClapSegmentCategory.SOUND), [ 'endTimeInMs', 'prompt', 'entityId', ]) return clap } catch (err) { setSoundGenerationStatus("error") throw err } } const generateMusic = async (clap: ClapProject): Promise => { try { // setProgress(30) setMusicGenerationStatus("generating") clap = await editClapMusic({ clap, turbo: true }).then(r => r.promise) if (!clap) { throw new Error(`failed to edit the music`) } console.log(`handleSubmit(): received a clap with music = `, clap) setCurrentClap(clap) setMusicGenerationStatus("finished") console.log("---------------- GENERATED MUSIC ----------------") console.table(clap.segments.filter(s => s.category === ClapSegmentCategory.MUSIC), [ 'endTimeInMs', 'prompt', 'entityId', ]) return clap } catch (err) { setMusicGenerationStatus("error") throw err } } const generateStoryboards = async (clap: ClapProject): Promise => { try { // setProgress(40) setImageGenerationStatus("generating") clap = await editClapStoryboards({ clap, // if we use entities, then we MUST use turbo // that's because turbo uses PulID, // but SDXL doesn't turbo: true, }).then(r => r.promise) if (!clap) { throw new Error(`failed to edit the storyboards`) } // const fusion = console.log(`handleSubmit(): received a clap with images = `, clap) setCurrentClap(clap) setImageGenerationStatus("finished") console.log("---------------- GENERATED STORYBOARDS ----------------") clap.segments .filter(s => s.category === ClapSegmentCategory.STORYBOARD) .forEach((s, i) => { if (s.status === "completed" && s.assetUrl) { // console.log(` [${i}] storyboard: ${s.prompt}`) logImage(s.assetUrl, 0.35) } else { console.log(` [${i}] failed to generate storyboard`) } // console.log(`------------------`) }) console.table(clap.segments.filter(s => s.category === ClapSegmentCategory.STORYBOARD), [ 'endTimeInMs', 'prompt', 'assetUrl' ]) return clap } catch (err) { setImageGenerationStatus("error") throw err } } const generateVideos = async (clap: ClapProject): Promise => { try { // setProgress(50) setVideoGenerationStatus("generating") clap = await editClapVideos({ clap, turbo: false }).then(r => r.promise) if (!clap) { throw new Error(`failed to edit the videos`) } console.log(`handleSubmit(): received a clap with videos = `, clap) setCurrentClap(clap) setVideoGenerationStatus("finished") console.log("---------------- GENERATED VIDEOS ----------------") console.table(clap.segments.filter(s => s.category === ClapSegmentCategory.VIDEO), [ 'endTimeInMs', 'prompt', 'entityId', ]) return clap } catch (err) { setVideoGenerationStatus("error") throw err } } const generateStoryboardsThenVideos = async (clap: ClapProject): Promise => { clap = await generateStoryboards(clap) clap = await generateVideos(clap) return clap } const generateDialogues = async (clap: ClapProject): Promise => { try { // setProgress(70) setVoiceGenerationStatus("generating") clap = await editClapDialogues({ clap, turbo: true }).then(r => r.promise) if (!clap) { throw new Error(`failed to edit the dialogues`) } console.log(`handleSubmit(): received a clap with dialogues = `, clap) setCurrentClap(clap) setVoiceGenerationStatus("finished") console.log("---------------- GENERATED DIALOGUES ----------------") console.table(clap.segments.filter(s => s.category === ClapSegmentCategory.DIALOGUE), [ 'endTimeInMs', 'prompt', 'entityId', ]) return clap } catch (err) { setVoiceGenerationStatus("error") throw err } } const generateFinalVideo = async (clap: ClapProject): Promise => { let assetUrl = "" try { // setProgress(85) setFinalGenerationStatus("generating") assetUrl = await exportClapToVideo({ clap, turbo: true }) setCurrentVideo(assetUrl) if (assetUrl.length < 128) { throw new Error(`handleSubmit(): the generated video is too small, so we failed`) } console.log(`handleSubmit(): received a video: ${assetUrl.slice(0, 120)}...`) setFinalGenerationStatus("finished") return assetUrl } catch (err) { setFinalGenerationStatus("error") throw err } } const handleSubmit = async () => { setStatus("generating") busyRef.current = true startTransition(async () => { setStatus("generating") busyRef.current = true console.log(`handleSubmit(): generating a clap using prompt = "${promptDraftRef.current}" `) try { let clap = await generateStory() const tasks = [ generateMusic(clap), generateStoryboardsThenVideos(clap) ] const claps = await Promise.all(tasks) console.log(`finished processing ${tasks.length} tasks in parallel`) for (const newerClap of claps) { clap = await updateClap(clap, newerClap, { overwriteMeta: false, inlineReplace: true, }) } /* clap = await claps.reduce(async (existingClap, newerClap) => updateClap(existingClap, newerClap, { overwriteMeta: false, inlineReplace: true, }) , Promise.resolve(clap) */ // We can't have consistent characters with video (yet) // clap = await generateEntities(clap) /* if (mainCharacterImage) { console.log("handleSubmit(): User specified a main character image") // various strategies here, for instance we can assume that the first character is the main character, // or maybe a more reliable way is to count the number of occurrences. // there is a risk of misgendering, so ideally we should add some kind of UI to do this, // such as a list of characters. } */ // let's skip storyboards for now // clap = await generateStoryboards(clap) // clap = await generateVideos(clap) // clap = await generateDialogues(clap) console.log("final clap: ", clap) await generateFinalVideo(clap) setStatus("finished") setError("") } catch (err) { console.error(`failed to generate: `, err) setStatus("error") setError(`Error, please contact an admin on Discord (${err})`) } }) } const { openFilePicker, filesContent } = useFilePicker({ accept: '.clap', readAs: "ArrayBuffer" }) const fileData = filesContent[0] useEffect(() => { const fn = async () => { if (!fileData?.name) { return } const { setStatus, setProgress } = useStore.getState() setProgress(0) setStatus("generating") try { let clap = await importStory(fileData) const claps = await Promise.all([ generateMusic(clap), generateVideos(clap) ]) // console.log("finished processing the 2 tasks in parallel") for (const newerClap of claps) { clap = await updateClap(clap, newerClap, { overwriteMeta: false, inlineReplace: true, }) } await generateFinalVideo(clap) setStatus("finished") setProgress(100) setError("") } catch (err) { console.error(`failed to import: `, err) setStatus("error") setError(`${err}`) } } fn() }, [fileData?.name]) // note: we are interested in the *current* video orientation, // not the requested video orientation requested for the next video const isLandscape = currentVideoOrientation === ClapMediaOrientation.LANDSCAPE const isPortrait = currentVideoOrientation === ClapMediaOrientation.PORTRAIT const isSquare = currentVideoOrientation === ClapMediaOrientation.SQUARE const runningRef = useRef(false) const timerRef = useRef() const timerFn = async () => { const { isBusy, progress, stage } = useStore.getState() clearTimeout(timerRef.current) if (!isBusy || stage === "idle") { return } /* console.log("progress function:", { stage, delay: progressDelayInMsPerStage[stage], progress, }) */ useStore.setState({ // progress: Math.min(maxProgressPerStage[stage], progress + 1) progress: Math.min(100, progress + 1) }) // timerRef.current = setTimeout(timerFn, progressDelayInMsPerStage[stage]) timerRef.current = setTimeout(timerFn, 1200) } useEffect(() => { timerFn() clearTimeout(timerRef.current) if (!isBusy) { return } timerRef.current = setTimeout(timerFn, 0) }, [isBusy]) // this is how we support query string parameters // ?prompt=hello <- set a default prompt // ?prompt=hello&autorun=true <- automatically run the app // ?orientation=landscape <- can be "landscape" or "portrait" (default) const searchParams = useSearchParams() const queryStringPrompt = (searchParams?.get('prompt') as string) || "" const queryStringAutorun = (searchParams?.get('autorun') as string) || "" const queryStringOrientation = (searchParams?.get('orientation') as string) || "" useEffect(() => { if (queryStringOrientation?.length > 1) { console.log(`orientation = "${queryStringOrientation}"`) const orientation = queryStringOrientation.trim().toLowerCase() === "landscape" ? ClapMediaOrientation.LANDSCAPE : ClapMediaOrientation.PORTRAIT setOrientation(orientation) } if (queryStringPrompt?.length > 1) { console.log(`prompt = "${queryStringPrompt}"`) if (queryStringPrompt !== promptDraftRef.current) { setStoryPromptDraft(queryStringPrompt) } const maybeAutorun = queryStringAutorun.trim().toLowerCase() console.log(`autorun = "${maybeAutorun}"`) // note: during development we will be called twice, // which is why we have a guard on busyRef.current if (maybeAutorun === "true" || maybeAutorun === "1" && !busyRef.current) { handleSubmit() } } }, [queryStringPrompt, queryStringAutorun, queryStringOrientation]) return (
AI
Stories Factory

Make video stories using AI ✨

{/* LEFT MENU BUTTONS + MAIN PROMPT INPUT */}
{/* TODO: To finish by Julian a bit later
) => { if (e.target.files && e.target.files.length > 0) { const file = e.target.files[0]; const newImageBase64 = await fileToBase64(file) setMainCharacterImage(newImageBase64) } }} accept="image/*" />
*/} {/* MAIN PROMPT INPUT */}
{ setStoryPromptDraft(e.target.value) promptDraftRef.current = e.target.value }} placeholder={defaultPrompt} inputClassName=" transition-all duration-200 ease-in-out h-32 md:h-56 lg:h-64 " disabled={isBusy} value={storyPromptDraft} /> {/* END OF MAIN PROMPT INPUT */}
{/* END OF LEFT MENU BUTTONS + MAIN PROMPT INPUT */}
{/* ACTION BAR */}
{/* */}
{/* */} {canSeeBetaFeatures ? :
}
{/* RANDOMNESS SWITCH */}
{ const randomStory = generateRandomStory() setStoryPromptDraft(randomStory) promptDraftRef.current = randomStory }}>
{/* END OF RANDOMNESS SWITCH */} {/* ORIENTATION SWITCH */}
toggleOrientation()}>
{/* END OF ORIENTATION SWITCH */}
{/* END OF ACTION BAR */}
{isBusy ?

{progress}%

{isBusy ? ( // note: some of those tasks are running in parallel, // and some are super-slow (like music or video) // by carefully selecting in which order we set the ternaries, // we can create the illusion that we just have a succession of reasonably-sized tasks storyGenerationStatus === "generating" ? "Writing story.." : parseGenerationStatus === "generating" ? "Loading the project.." : assetGenerationStatus === "generating" ? "Casting characters.." : imageGenerationStatus === "generating" ? "Creating storyboards.." : soundGenerationStatus === "generating" ? "Recording sounds.." : videoGenerationStatus === "generating" ? "Filming shots.." : musicGenerationStatus === "generating" ? "Producing music.." : voiceGenerationStatus === "generating" ? "Recording dialogues.." : finalGenerationStatus === "generating" ? "Editing final cut.." : "Please wait.." ) : status === "error" ? {error || ""} : {error ? error :  } // to prevent layout changes }

: (currentVideo && currentVideo?.length > 128) ?
Powered by Hugging Face Hugging Face
{(currentVideo && currentVideo.length > 128) ?
 Download
: null}
Join us on Discord Discord
); }