Spaces:
Running
on
CPU Upgrade
Running
on
CPU Upgrade
File size: 3,977 Bytes
6215321 0bf0c48 6215321 0bf0c48 6215321 0bf0c48 6215321 0bf0c48 6215321 0bf0c48 6215321 0bf0c48 6215321 0bf0c48 6215321 0bf0c48 6215321 0bf0c48 6215321 |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 |
import { NextResponse, NextRequest } from "next/server"
import { serializeClap } from "@/lib/clap/serializeClap"
import { parseClap } from "@/lib/clap/parseClap"
import { startOfSegment1IsWithinSegment2 } from "@/lib/utils/startOfSegment1IsWithinSegment2"
import { getVideoPrompt } from "@/components/interface/latent-engine/core/prompts/getVideoPrompt"
import { newSegment } from "@/lib/clap/newSegment"
import { generateImage } from "@/components/interface/latent-engine/resolvers/image/generateImage"
import { getToken } from "@/app/api/auth/getToken"
// a helper to generate storyboards for a Clap
// this is mostly used by external apps such as the Stories Factory
// this function will:
//
// - add missing storyboards to the shots
// - add missing storyboard prompts
// - add missing storyboard images
export async function POST(req: NextRequest) {
const jwtToken = await getToken({ user: "anonymous" })
const blob = await req.blob()
const clap = await parseClap(blob)
if (!clap?.segments) { throw new Error(`no segment found in the provided clap!`) }
console.log(`[api/generate/storyboards] detected ${clap.segments} segments`)
const shotsSegments = clap.segments.filter(s => s.category === "camera")
console.log(`[api/generate/storyboards] detected ${shotsSegments.length} shots`)
if (shotsSegments.length > 32) {
throw new Error(`Error, this endpoint being synchronous, it is designed for short stories only (max 32 shots).`)
}
for (const shotSegment of shotsSegments) {
const shotSegments = clap.segments.filter(s =>
startOfSegment1IsWithinSegment2(s, shotSegment)
)
const shotStoryboardSegments = shotSegments.filter(s =>
s.category === "storyboard"
)
let shotStoryboardSegment = shotStoryboardSegments.at(0)
console.log(`[api/generate/storyboards] shot [${shotSegment.startTimeInMs}:${shotSegment.endTimeInMs}] has ${shotSegments.length} segments (${shotStoryboardSegments.length} storyboards)`)
// TASK 1: GENERATE MISSING STORYBOARD SEGMENT
if (!shotStoryboardSegment) {
shotStoryboardSegment = newSegment({
track: 1,
startTimeInMs: shotSegment.startTimeInMs,
endTimeInMs: shotSegment.endTimeInMs,
assetDurationInMs: shotSegment.assetDurationInMs,
category: "storyboard",
prompt: "",
assetUrl: "",
outputType: "image"
})
console.log(`[api/generate/storyboards] generated storyboard segment [${shotSegment.startTimeInMs}:${shotSegment.endTimeInMs}]`)
}
// TASK 2: GENERATE MISSING STORYBOARD PROMPT
if (!shotStoryboardSegment.prompt) {
// storyboard is missing, let's generate it
shotStoryboardSegment.prompt = getVideoPrompt(shotSegments, {}, [])
console.log(`[api/generate/storyboards] generating storyboard prompt: ${shotStoryboardSegment.prompt}`)
}
// TASK 3: GENERATE MISSING STORYBOARD BITMAP
if (!shotStoryboardSegment.assetUrl) {
console.log(`[api/generate/storyboards] generating image..`)
// note this will do a fetch to AiTube API
// which is a bit weird since we are already inside the API, but it works
//TODO Julian: maybe we could use an internal function call instead?
shotStoryboardSegment.assetUrl = await generateImage({
prompt: shotStoryboardSegment.prompt,
width: clap.meta.width,
height: clap.meta.height,
token: jwtToken,
})
console.log(`[api/generate/storyboards] generated storyboard image: ${shotStoryboardSegment.assetUrl.slice(0, 50)}...`)
} else {
console.log(`[api/generate/storyboards] there is already a storyboard image: ${shotStoryboardSegment.assetUrl.slice(0, 50)}...`)
}
}
console.log(`[api/generate/storyboards] returning the clap augmented with storyboards`)
return new NextResponse(await serializeClap(clap), {
status: 200,
headers: new Headers({ "content-type": "application/x-gzip" }),
})
}
|