File size: 13,496 Bytes
1f122c3
3d4392e
1f122c3
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
8101ed0
 
1f122c3
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
54bc655
 
 
 
1f122c3
 
 
 
 
 
 
 
 
 
 
3d4392e
1f122c3
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
f62b8d3
1f122c3
 
 
 
 
f62b8d3
 
1f122c3
 
f62b8d3
f276512
 
 
 
 
 
 
 
f62b8d3
 
 
 
 
 
 
 
 
 
 
 
 
 
1f122c3
f62b8d3
 
 
1f122c3
f62b8d3
4c34e70
761239a
 
 
 
 
 
 
4c34e70
 
f62b8d3
 
 
1f122c3
f62b8d3
1f122c3
f62b8d3
 
 
 
0f35d4c
 
 
 
 
f62b8d3
 
 
1185ec1
 
 
f62b8d3
1185ec1
 
f62b8d3
 
 
 
 
 
 
 
 
 
 
 
1f1caeb
 
 
f62b8d3
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1185ec1
 
 
 
 
f62b8d3
 
 
 
 
 
 
 
 
 
4c34e70
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
f62b8d3
 
 
 
0f35d4c
 
 
 
 
 
 
 
 
 
1f122c3
 
0f35d4c
 
 
 
f27679f
ac7030c
c5b101c
 
ac7030c
f42b4a1
c5b101c
 
ac7030c
 
f62b8d3
 
 
1f122c3
f62b8d3
 
ac7030c
f62b8d3
 
 
 
ac7030c
f62b8d3
1f122c3
f62b8d3
 
ac7030c
f62b8d3
 
 
 
ac7030c
f62b8d3
 
 
 
ac7030c
f62b8d3
1f122c3
f62b8d3
1185ec1
 
 
 
 
f62b8d3
 
1185ec1
 
 
 
 
 
 
f62b8d3
1f122c3
f62b8d3
1185ec1
ac7030c
1185ec1
 
 
f62b8d3
 
 
 
 
1f122c3
 
f62b8d3
 
 
 
 
 
 
8f2b05f
 
 
 
 
 
 
f62b8d3
ac7030c
f62b8d3
 
 
 
 
 
 
 
4c34e70
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
f62b8d3
 
 
1f122c3
0f35d4c
 
ac7030c
0f35d4c
 
 
 
ac7030c
0f35d4c
 
 
 
ac7030c
0f35d4c
 
 
 
ac7030c
0f35d4c
 
c5b101c
 
ac7030c
c5b101c
ac7030c
1f122c3
 
e3d26ad
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
ac7030c
e3d26ad
 
38d787b
e4d3d8a
 
38d787b
e4d3d8a
 
 
38d787b
 
e4d3d8a
 
 
 
38d787b
 
 
e4d3d8a
 
38d787b
e4d3d8a
 
38d787b
 
 
 
e4d3d8a
 
 
 
 
 
 
 
 
 
38d787b
e4d3d8a
38d787b
 
 
 
 
e4d3d8a
 
38d787b
 
4c34e70
 
 
 
3d4392e
 
4c34e70
1f122c3
 
 
 
f27679f
e40bd21
f27679f
 
 
 
 
 
 
 
 
1f122c3
 
f62b8d3
 
f42b4a1
 
f62b8d3
 
 
09a7c47
f42b4a1
63769e0
09a7c47
 
ac7030c
 
 
df83860
f62b8d3
e4d3d8a
f62b8d3
 
16891a6
f62b8d3
 
 
 
 
4c34e70
761239a
 
93f8352
 
4c34e70
f62b8d3
93f8352
f62b8d3
 
0f35d4c
f62b8d3
 
 
 
 
 
 
 
 
 
 
 
 
 
 
4c34e70
 
 
 
93f8352
4c34e70
 
0f35d4c
f62b8d3
 
 
 
 
 
 
 
 
 
9cea1bb
 
 
 
 
 
3d4392e
 
 
 
 
 
 
 
6215321
3d4392e
 
9cea1bb
 
8f2b05f
 
 
 
 
 
 
29f166e
8f2b05f
ac7030c
8f2b05f
 
 
 
f62b8d3
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732


export type ProjectionMode = 'cartesian' | 'spherical'

export type MouseEventType = "hover" | "click"

export type MouseEventHandler = (type: MouseEventType, x: number, y: number) => Promise<void>

export type CacheMode = "use" | "renew" | "ignore"

export interface RenderRequest {
  prompt: string

  // whether to use video segmentation
  // disabled (default)
  // firstframe: we only analyze the first frame
  // allframes: we analyze all the frames
  segmentation: 'disabled' | 'firstframe' | 'allframes'

  // segmentation will only be executed if we have a non-empty list of actionnables
  // actionnables are names of things like "chest", "key", "tree", "chair" etc
  actionnables: string[]

  nbFrames: number
  nbFPS: number

  nbSteps: number // min: 1, max: 50

  seed: number

  width: number // fixed at 1024 for now
  height: number // fixed at 512 for now

  // upscaling factor
  // 0: no upscaling
  // 1: no upscaling
  // 2: 2x larger
  // 3: 3x larger
  // 4x: 4x larger, up to 4096x4096 (warning: a PNG of this size can be 50 Mb!)
  upscalingFactor: number

  projection: ProjectionMode

  /**
   * Use turbo mode
   * 
   * At the time of writing this will use SSD-1B + LCM
   * https://huggingface.co/spaces/jbilcke-hf/fast-image-server
   */
  turbo: boolean

  cache: CacheMode

  wait: boolean // wait until the job is completed

  analyze: boolean // analyze the image to generate a caption (optional)
}

export interface ImageSegment {
  id: number
  box: number[]
  color: number[]
  label: string
  score: number 
}

export type RenderedSceneStatus =
  | "pending" // for retro-compatibility only, but this is deprecated
  | "to_generate"
  | "to_interpolate"
  | "to_upscale"
  | "completed"
  | "error"

export interface RenderedScene {
  renderId: string
  status: RenderedSceneStatus
  assetUrl: string 
  alt: string
  error: string
  maskUrl: string
  segments: ImageSegment[]
  durationInMs?: number
}

export interface ImageAnalysisRequest {
  image: string // in base64
  prompt: string
}

export interface ImageAnalysisResponse {
  result: string
  error?: string
}

export type RenderingEngine =
  | "VIDEOCHAIN"
  | "OPENAI"
  | "REPLICATE"

export type PostVisibility =
  | "featured" // featured by admins
  | "trending" // top trending / received more than 10 upvotes
  | "normal" // default visibility

export type Post = {
  postId: string
  appId: string
  prompt: string
  model: string
  previewUrl: string
  assetUrl: string
  createdAt: string
  visibility: PostVisibility
  upvotes: number
  downvotes: number
}

export type CreatePostResponse = {
  success?: boolean
  error?: string
  post: Post
}

export type GetAppPostsResponse = {
  success?: boolean
  error?: string
  posts: Post[]
}

export type GetAppPostResponse = {
  success?: boolean
  error?: string
  post: Post
}

// vendor-specific types

export type HotshotImageInferenceSize =
| '320x768'
| '384x672'
| '416x608'
| '512x512'
| '608x416'
| '672x384'
| '768x320'
| '1024x1024' // custom ratio - this isn't supported / supposed to work properly
| '1024x512' // custom panoramic ratio - this isn't supported / supposed to work properly
| '1024x576' // movie ratio (16:9) this isn't supported / supposed to work properly
| '576x1024' // tiktok ratio (9:16) this isn't supported / supposed to work properly

export type VideoOptions = {
  positivePrompt: string

  negativePrompt?: string

  size?: HotshotImageInferenceSize
  
  /**
   * Must be a model *name*
   */
  huggingFaceLora?: string

  replicateLora?: string

  triggerWord?: string
  
  nbFrames?: number // FPS (eg. 8)
  duration?: number // in milliseconds

  steps?: number

  key?: string // a semi-unique key to prevent abuse from some users
}

/**
 * A channel is a video generator
 * 
 * Video will be uploaded to a dataset
 */
export type ChannelInfo = {
  /**
   * We actually use the dataset ID for the channel ID.
   * 
   */
  id: string

  /**
   * The name used in the URL for the channel
   * 
   * eg: my-time-travel-journeys
   */
  slug: string

  /**
   * username id of the Hugging Face dataset
   * 
   * ex: f9a38286ec3436a45edd2cca
   */
  // DISABLED FOR NOW
  // datasetUserId: string

  /**
   * username slug of the Hugging Face dataset
   * 
   * eg: jbilcke-hf
   */
  datasetUser: string

  /**
   * dataset slug of the Hugging Face dataset
   * 
   * eg: ai-tube-my-time-travel-journeys
   */
  datasetName: string

  label: string

  description: string

  thumbnail: string

  model: VideoGenerationModel

  lora: string
  
  style: string

  voice: string

  music: string

  /**
   * The system prompt
   */
  prompt: string

  likes: number

  tags: string[]

  updatedAt: string

  /**
   * Default video orientation
   */
  orientation: VideoOrientation
}

export type VideoStatus =
  | "draft" // the video should be ignored for now
  | "submitted" // the video has been submitted, but is not added to the index queue yet
  | "queued" // the video has been added to the index queue, but is not processed yet. Once queued it cannot be modified.
  | "generating" // the video is being generated
  // TODO add a state to indicate the audio is being generated
  // this will be useful in case generation fails
  | "published" // success!
  | "error" // video failed to generate

/**
 * A video request, made by a user or robot on a channel
 */
export type VideoRequest = {
  /**
   * UUID (v4)
   */
  id: string

  // skip generation (mostly used for debugging)
  // skip?: boolean

  /**
   * Human readable title for the video
   */
  label: string

  /**
   * Human readable description for the video
   */
  description: string

  /**
   * Video prompt
   */
  prompt: string

  /**
   * URL to the video thumbnail
   */
  thumbnailUrl: string

  /**
   * URL to a clap file
   */
  clapUrl: string

  /**
   * When was the video updated
   */
  updatedAt: string

  /**
   * Arbotrary string tags to label the content
   */
  tags: string[]

  /**
   * Model name
   */
  model: VideoGenerationModel

  /**
   * LoRA name
   */
  lora: string

  /**
   * style name
   */
  style: string

  /**
   * Music prompt
   */
  music: string

  /**
   * Voice prompt
   */
  voice: string

  /**
   * ID of the channel
   */
  channel: ChannelInfo

  /**
   * Video orientation
   */
  orientation: VideoOrientation

  /**
   * Video duration
   */
  duration: number
}

export type VideoOrientation =
  | "portrait"
  | "landscape"
  | "square"

export type MediaProjection =
  | "cartesian" // this is the default
  | "equirectangular"
  | "gaussian"
  | "latent"


// TODO: renamed this to MediaInfo
export type MediaInfo = {
  /**
   * UUID (v4)
   */
  id: string

  /**
   * Status of the media
   */
  status: VideoStatus

  /**
   * Human readable title for the media
   */
  label: string

  /**
   * Human readable description for the media
   */
  description: string

  /**
   * Content prompt
   */
  prompt: string

  /**
   * URL to the media thumbnail
   */
  thumbnailUrl: string

  /**
   * URL to a clap file
   */
  clapUrl: string
  
  /**
   * URL to the binary file
   * 
   * This is the standard format
   * 
   * see width and height for details
   * 
   * (it will be something like 1024x576, 576x1024, 1024x512)
   * 
   */
  assetUrl: string

  /**
   * This is contain the storage URL of the higher-resolution content
   */
  assetUrlHd: string

  /**
   * Counter for the number of views
   * 
   * Note: should be managed by the index to prevent cheating
   */
  numberOfViews: number

  /**
   * Counter for the number of likes
   * 
   * Note: should be managed by the index to prevent cheating
   */
  numberOfLikes: number

  /**
   * Counter for the number of dislikes
   * 
   * Note: should be managed by the index to prevent cheating
   */
  numberOfDislikes: number

  /**
   * When was the media updated
   */
  updatedAt: string

  /**
   * Arbotrary string tags to label the content
   */
  tags: string[]

  /**
   * Model name
   */
  model: VideoGenerationModel

  /**
   * LoRA name
   */
  lora: string

  /**
   * style name
   */
  style: string

  /**
   * Music prompt
   */
  music: string

  /**
   * Voice prompt
   */
  voice: string

  /**
   * The channel
   */
  channel: ChannelInfo

  /**
   * Media duration
   */
  duration: number

  /**
   * Media width (eg. 1024)
   */
  width: number

  /**
   * Media height (eg. 576)
   */
  height: number

  /**
   * General media aspect ratio
   */
  orientation: VideoOrientation

  /**
   * Media projection (cartesian by default)
   */
  projection: MediaProjection
}

export type CollectionInfo = {
  /**
   * UUID (v4)
   */
  id: string

  /**
   * Human readable title for the video
   */
  label: string

  /**
   * Human readable description for the video
   */
  description: string

  /**
   * URL to the video thumbnail
   */
  thumbnailUrl: string

  /**
   * Counter for the number of views
   * 
   * Note: should be managed by the index to prevent cheating
   */
  numberOfViews: number

  /**
   * Counter for the number of likes
   * 
   * Note: should be managed by the index to prevent cheating
   */
  numberOfLikes: number

  /**
   * When was the video updated
   */
  updatedAt: string

  /**
   * Arbotrary string tags to label the content
   */
  tags: string[]

  /**
   * The owner channel
   */
  channel: ChannelInfo

  /**
   * Collection duration
   */
  duration: number

  items: Array<MediaInfo>[]
}

export type UserInfo = {
  id: string

  type: "creator" | "normal" | "admin"

  userName: string

  fullName: string
  
  thumbnail: string

  channels: ChannelInfo[]

  // the Hugging Face API token is confidential,
  // and will only be available for the current user
  hfApiToken?: string
}

export type CommentInfo = {
  id: string
  
  userId: string

  // only populated when rendering
  userInfo?: UserInfo

  // if the video comment is in response to another comment,
  // then "inReplyTo" will contain the other video comment id
  inReplyTo?: string

  createdAt: string
  updatedAt: string
  message: string

  // how many likes did the comment receive
  numberOfLikes: number

  // how many replies did the comment receive
  numberOfReplies: number

  // if the comment was appreciated by the original content poster
  likedByOriginalPoster: boolean
}

export type StoredCommentInfo = Omit<CommentInfo, "userInfo">

export type VideoGenerationModel =
  | "HotshotXL"
  | "SVD"
  | "LaVie"
  | "AnimateDiff"
  | "SDXL" // yep, we can use images!

export type InterfaceDisplayMode =
  | "desktop"
  | "tv"

export type InterfaceHeaderMode =
  | "hidden"
  | "normal"
  | "compact"

export type InterfaceMenuMode =
  | "slider_hidden"
  | "slider_text"
  | "normal_icon"
  | "normal_text"

export type InterfaceView =
  | "home"
  | "user_channels"
  | "user_channel" // for a user to admin their channels
  | "user_medias"
  | "user_media"
  | "user_account"
  | "public_channels"
  | "public_channel" // public view of a channel
  | "public_media" // public view of an individual media (video, gaussian splat, clap video)
  | "public_media_embed" // for integration into twitter etc
  | "public_music_videos" // public music videos - it's a special category, because music is *cool*
  | "public_latent_media" // public view of an individual dream (a latent media, so it's not a "real" file)
  | "public_latent_media_embed" // for integration into twitter etc (which would be hardcore for our server load.. so maybe not)
  | "public_gaming" // for AiTube Gaming
  | "public_4d" // for AiTube 4D
  | "public_live" // for AiTube Live
  | "not_found"


export type Settings = {
  huggingfaceApiKey: string
  huggingfaceTemporaryApiKey: string
}

export type ParsedDatasetReadme = {
  license: string
  pretty_name: string
  model: VideoGenerationModel
  lora: string
  style: string
  thumbnail: string
  voice: string
  music: string
  tags: string[]
  hf_tags: string[]
  description: string
  prompt: string
  orientation: VideoOrientation
}

export type ParsedMetadataAndContent = {
  metadata: {
    license: string,
    pretty_name: string,
    tags: string[]
  }
  content: string
}

export type ParsedDatasetPrompt = {
  title: string
  description: string
  prompt: string
  tags: string[]
  model: VideoGenerationModel
  lora: string
  style: string
  thumbnail: string
  voice: string
  music: string
  orientation: VideoOrientation
}

export type UpdateQueueRequest = {
  channel?: ChannelInfo
  apiKey: string
}

export type UpdateQueueResponse = {
  error?: string
  nbUpdated: number
}


export type AppQueryProps = {
  params: { id: string }
  searchParams: {
    v?: string | string[], // video id (deprecated)
    m?: string | string[], // media id (a better term)
    c?: string | string[], // clap id (do we still need this?)
    [key: string]: string | string[] | undefined
  }
}

export type LatentQueryProps = {
  params: { jwtToken: string }
  searchParams: {
    l?: string | string[], // latent content (serialized to a base64 json)
    [key: string]: string | string[] | undefined
  }
}

export type MediaDisplayLayout =
  | "grid" // default mode, items goas back to the line
  | "horizontal" // will be used for a "Netflix" horizontal sliding mode
  | "vertical" // used in the right recommendation panel
  | "table" // used when shown in a table mode
  | "micro"

export type MediaRating = {
  isLikedByUser: boolean
  isDislikedByUser: boolean
  numberOfLikes: number
  numberOfDislikes: number
}