Trying new video tile layout algorithm (#355)
Co-authored-by: Matt Lorentz <5+mplorentz@noreply.coracle.social>
This commit is contained in:
Matt Lorentz
hodlbod
parent
9d8be25bae
commit
3d2ec8c030
2 changed files with 247 additions and 19 deletions
210
src/app/call.ts
210
src/app/call.ts
|
|
@ -31,6 +31,216 @@ export const LIVEKIT_PARTICIPANTS = 39004
|
|||
|
||||
export {supportsAudioOutputSelection}
|
||||
|
||||
/**
|
||||
* Aspect ratio constraints for tiles. The lower bound is dynamic
|
||||
* (1:1 on landscape, 3:4 on portrait); the upper bound is 16:9.
|
||||
*/
|
||||
const TILE_ASPECT_PORTRAIT = 3 / 4
|
||||
const TILE_ASPECT_LANDSCAPE = 16 / 9
|
||||
const TILE_GAP = 8
|
||||
|
||||
/**
|
||||
* Minimum pixel height for a tile before we allow the grid to
|
||||
* overflow (scroll) instead of forcing tiles into portrait mode.
|
||||
* Calibrated so ~6-8 tiles on a standard portrait phone fit
|
||||
* without scrolling; beyond that, scroll is acceptable.
|
||||
*/
|
||||
const MIN_TILE_HEIGHT = 120
|
||||
|
||||
/**
|
||||
* A single row in an adaptive tile grid.
|
||||
*/
|
||||
export type TileRow = {
|
||||
columnCount: number
|
||||
tileWidth: number
|
||||
tileHeight: number
|
||||
/** Total width of this row including gaps — for the template's max-width */
|
||||
rowWidth: number
|
||||
aspectRatio: number
|
||||
}
|
||||
|
||||
/**
|
||||
* Adaptive tile grid: all tiles share the same dimensions. Full rows
|
||||
* fill the container width; partial rows are centered.
|
||||
*
|
||||
* Example (3 tiles, 2 columns):
|
||||
* row 0: [ square ] [ square ]
|
||||
* row 1: [ square ] (centered, same size)
|
||||
*/
|
||||
export type AdaptiveTileGrid = {
|
||||
rows: TileRow[]
|
||||
totalWidth: number
|
||||
totalHeight: number
|
||||
}
|
||||
|
||||
/**
|
||||
* Score for comparing candidate layouts. Lower is better.
|
||||
* Uses named fields instead of opaque array "keys" (per review feedback).
|
||||
*/
|
||||
type LayoutScore = {
|
||||
/** Penalty for vertical overflow: 0 if none, else huge */
|
||||
verticalOverflowPenalty: number
|
||||
/** Penalty for horizontal overflow: 0 if none, else huge */
|
||||
horizontalOverflowPenalty: number
|
||||
/** Penalty when tiles fall below MIN_TILE_HEIGHT even while fitting */
|
||||
minTileHeightPenalty: number
|
||||
/** Negative tile area (we want to maximize area, so negate) */
|
||||
negativeArea: number
|
||||
/** Leftover whitespace (underflow) in px */
|
||||
whitespace: number
|
||||
/** Average absolute deviation from 16:9 across all tiles */
|
||||
aspectDeviation: number
|
||||
}
|
||||
|
||||
const compareScores = (a: LayoutScore, b: LayoutScore): number => {
|
||||
const fields: (keyof LayoutScore)[] = [
|
||||
"verticalOverflowPenalty",
|
||||
"horizontalOverflowPenalty",
|
||||
"minTileHeightPenalty",
|
||||
"negativeArea",
|
||||
"whitespace",
|
||||
"aspectDeviation",
|
||||
]
|
||||
for (const field of fields) {
|
||||
if (a[field] !== b[field]) return a[field] - b[field]
|
||||
}
|
||||
return 0
|
||||
}
|
||||
|
||||
/**
|
||||
* Compute the largest tile size that fits within a given width and height,
|
||||
* bounded by [minAspect, TILE_ASPECT_LANDSCAPE]. The tile is shrunk to fit
|
||||
* whichever dimension is more constraining, so it never overflows.
|
||||
*/
|
||||
const fitTile = (availWidth: number, availHeight: number, minAspect: number) => {
|
||||
const fillAspect = availWidth / availHeight
|
||||
const aspectRatio = Math.max(minAspect, Math.min(TILE_ASPECT_LANDSCAPE, fillAspect))
|
||||
if (fillAspect >= aspectRatio) {
|
||||
const tileWidth = availHeight * aspectRatio
|
||||
return {tileWidth, tileHeight: availHeight, aspectRatio}
|
||||
}
|
||||
const tileHeight = availWidth / aspectRatio
|
||||
return {tileWidth: availWidth, tileHeight, aspectRatio}
|
||||
}
|
||||
|
||||
/**
|
||||
* Build a candidate grid for a given column count. All tiles share the
|
||||
* same dimensions; the partial last row (if any) is centered with the
|
||||
* same tile size as full rows.
|
||||
*/
|
||||
const buildCandidate = (
|
||||
tileCount: number,
|
||||
columnCount: number,
|
||||
containerWidth: number,
|
||||
containerHeight: number,
|
||||
minAspect: number,
|
||||
): AdaptiveTileGrid | undefined => {
|
||||
const gap = TILE_GAP
|
||||
const fullRowCount = Math.floor(tileCount / columnCount)
|
||||
const remainder = tileCount % columnCount
|
||||
const totalRowCount = fullRowCount + (remainder > 0 ? 1 : 0)
|
||||
|
||||
const availHeight = (containerHeight - (totalRowCount - 1) * gap) / totalRowCount
|
||||
const availWidth = (containerWidth - (columnCount - 1) * gap) / columnCount
|
||||
if (availWidth <= 0 || availHeight <= 0) return undefined
|
||||
|
||||
const {tileWidth, tileHeight, aspectRatio} = fitTile(availWidth, availHeight, minAspect)
|
||||
|
||||
const rows: TileRow[] = []
|
||||
for (let r = 0; r < fullRowCount; r++) {
|
||||
rows.push({
|
||||
columnCount,
|
||||
tileWidth,
|
||||
tileHeight,
|
||||
rowWidth: columnCount * tileWidth + (columnCount - 1) * gap,
|
||||
aspectRatio,
|
||||
})
|
||||
}
|
||||
if (remainder > 0) {
|
||||
rows.push({
|
||||
columnCount: remainder,
|
||||
tileWidth,
|
||||
tileHeight,
|
||||
rowWidth: remainder * tileWidth + (remainder - 1) * gap,
|
||||
aspectRatio,
|
||||
})
|
||||
}
|
||||
|
||||
const totalHeight = totalRowCount * tileHeight + (totalRowCount - 1) * gap
|
||||
const totalWidth = columnCount * tileWidth + (columnCount - 1) * gap
|
||||
|
||||
return {rows, totalWidth, totalHeight}
|
||||
}
|
||||
|
||||
/**
|
||||
* Compute an adaptive tile grid. All tiles share the same dimensions;
|
||||
* partial rows are centered. Tiles flex between a minimum aspect
|
||||
* (1:1 on landscape, 3:4 on portrait) and 16:9, capped so they never
|
||||
* overflow the container.
|
||||
*
|
||||
* Only allows overflow (scroll) when tiles would be below
|
||||
* MIN_TILE_HEIGHT (~6-8 tiles on portrait phone).
|
||||
*
|
||||
* Prioritises:
|
||||
* 1. No overflow (tiles shrink to fit within aspect bounds)
|
||||
* 2. Minimal whitespace
|
||||
* 3. Aspect ratios close to 16:9
|
||||
* 4. Larger tiles
|
||||
*/
|
||||
export const computeAdaptiveGrid = (
|
||||
tileCount: number,
|
||||
containerWidth: number,
|
||||
containerHeight: number,
|
||||
): AdaptiveTileGrid | undefined => {
|
||||
if (tileCount <= 0 || containerWidth <= 0 || containerHeight <= 0) return undefined
|
||||
|
||||
const minAspect = containerWidth / containerHeight >= 4 / 3 ? 1 : TILE_ASPECT_PORTRAIT
|
||||
|
||||
let best: AdaptiveTileGrid | undefined
|
||||
let bestScore: LayoutScore | undefined
|
||||
|
||||
for (let columnCount = 1; columnCount <= tileCount; columnCount++) {
|
||||
const candidate = buildCandidate(
|
||||
tileCount,
|
||||
columnCount,
|
||||
containerWidth,
|
||||
containerHeight,
|
||||
minAspect,
|
||||
)
|
||||
if (!candidate) continue
|
||||
|
||||
const {tileWidth, tileHeight, aspectRatio} = candidate.rows[0]
|
||||
const verticalOverflow = Math.max(0, candidate.totalHeight - containerHeight)
|
||||
const horizontalOverflow = Math.max(0, candidate.totalWidth - containerWidth)
|
||||
const whitespace =
|
||||
Math.max(0, containerWidth - candidate.totalWidth) +
|
||||
Math.max(0, containerHeight - candidate.totalHeight)
|
||||
const totalArea = tileWidth * tileHeight * tileCount
|
||||
|
||||
const overflowPenaltyMultiplier = tileHeight >= MIN_TILE_HEIGHT ? 1_000_000 : 1
|
||||
const verticalOverflowPenalty = verticalOverflow * overflowPenaltyMultiplier
|
||||
const horizontalOverflowPenalty = horizontalOverflow * overflowPenaltyMultiplier
|
||||
const minTileHeightPenalty =
|
||||
tileHeight < MIN_TILE_HEIGHT ? (MIN_TILE_HEIGHT - tileHeight) * 1000 : 0
|
||||
|
||||
const score: LayoutScore = {
|
||||
verticalOverflowPenalty,
|
||||
horizontalOverflowPenalty,
|
||||
minTileHeightPenalty,
|
||||
negativeArea: -totalArea,
|
||||
whitespace,
|
||||
aspectDeviation: Math.abs(aspectRatio - TILE_ASPECT_LANDSCAPE),
|
||||
}
|
||||
|
||||
if (!bestScore || compareScores(score, bestScore) < 0) {
|
||||
bestScore = score
|
||||
best = candidate
|
||||
}
|
||||
}
|
||||
|
||||
return best
|
||||
}
|
||||
|
||||
const LIVEKIT_DEFAULT_DEVICE_ID = "default"
|
||||
const RECONNECT_DELAYS = [1000, 2000, 4000, 8000, 16000]
|
||||
|
||||
|
|
|
|||
|
|
@ -19,6 +19,8 @@
|
|||
participantMediaState,
|
||||
pubkeyFromLiveKitIdentity,
|
||||
videoTrackRevision,
|
||||
computeAdaptiveGrid,
|
||||
type AdaptiveTileGrid,
|
||||
} from "@app/call"
|
||||
import {profiles} from "@app/core"
|
||||
|
||||
|
|
@ -38,7 +40,7 @@
|
|||
source: Track.Source.Camera | Track.Source.ScreenShare
|
||||
}
|
||||
|
||||
type TileLayout = "spotlight" | "default" | "strip"
|
||||
type TileLayoutVariant = "spotlight" | "default" | "strip"
|
||||
|
||||
const {layout, mobile = false, url, h, class: className = ""}: Props = $props()
|
||||
|
||||
|
|
@ -149,13 +151,15 @@
|
|||
return videoTiles.filter(t => tileKey(t) !== pk)
|
||||
})
|
||||
|
||||
let gridWidth = $state(0)
|
||||
let gridHeight = $state(0)
|
||||
|
||||
const useSpotlightLayout = $derived(primaryTile !== undefined)
|
||||
// 2 tiles stacked full-width felt like they were "overlapping" each other rather
|
||||
// than tiling — put 2+ side by side in a grid, same as everyone else's call UI.
|
||||
// Chat (Split) floats over the video as an overlay card rather than shrinking
|
||||
// this panel's width, so the tile grid doesn't need to collapse for it either.
|
||||
const useMultiGrid = $derived(!useSpotlightLayout && videoTiles.length > 1)
|
||||
const multiGridClass = "grid-cols-1 sm:grid-cols-2"
|
||||
const useMultiGrid = $derived(!useSpotlightLayout)
|
||||
|
||||
const tileGrid = $derived<AdaptiveTileGrid | undefined>(
|
||||
useMultiGrid ? computeAdaptiveGrid(videoTiles.length, gridWidth, gridHeight) : undefined,
|
||||
)
|
||||
|
||||
$effect(() => {
|
||||
const k = $videoPrimaryTileKey
|
||||
|
|
@ -195,7 +199,7 @@
|
|||
)
|
||||
</script>
|
||||
|
||||
{#snippet videoTile(tile: VideoTileData, layout: TileLayout)}
|
||||
{#snippet videoTile(tile: VideoTileData, layout: TileLayoutVariant)}
|
||||
{@const media = $mediaStateByIdentity(tile.liveKitIdentity)}
|
||||
{@const label = labelFor(tile.liveKitIdentity, tile.source)}
|
||||
<div
|
||||
|
|
@ -204,7 +208,7 @@
|
|||
// camera-off tile reads as its own card instead of blending into the panel.
|
||||
"relative isolate overflow-hidden rounded-2xl border border-line shadow-sm",
|
||||
layout === "spotlight" && "min-h-0 flex-1",
|
||||
layout === "default" && "aspect-video w-full min-h-0",
|
||||
layout === "default" && "min-h-0 h-full w-full",
|
||||
layout === "strip" && "aspect-video w-44 shrink-0",
|
||||
tile.source === Track.Source.ScreenShare ? "bg-black" : "bg-surface-more",
|
||||
)}>
|
||||
|
|
@ -265,16 +269,30 @@
|
|||
{/if}
|
||||
</div>
|
||||
{:else if useMultiGrid}
|
||||
<div class={cx("grid min-h-0 flex-1 content-start gap-2 overflow-y-auto", multiGridClass)}>
|
||||
{#each videoTiles as tile (tileKey(tile))}
|
||||
{@render videoTile(tile, "default")}
|
||||
{/each}
|
||||
</div>
|
||||
{:else}
|
||||
<div class="flex min-h-0 flex-1 flex-col gap-2 overflow-y-auto">
|
||||
{#each videoTiles as tile (tileKey(tile))}
|
||||
{@render videoTile(tile, "default")}
|
||||
{/each}
|
||||
<div
|
||||
bind:clientWidth={gridWidth}
|
||||
bind:clientHeight={gridHeight}
|
||||
class="min-h-0 flex-1 overflow-y-auto [scrollbar-gutter:stable]">
|
||||
{#if tileGrid}
|
||||
<div class="flex flex-col items-center gap-2">
|
||||
{#each tileGrid.rows as row, rowIndex}
|
||||
{@const offset = tileGrid.rows
|
||||
.slice(0, rowIndex)
|
||||
.reduce((sum, r) => sum + r.columnCount, 0)}
|
||||
<div
|
||||
class="flex flex-nowrap justify-center gap-2"
|
||||
style={`max-width: ${row.rowWidth}px`}>
|
||||
{#each videoTiles.slice(offset, offset + row.columnCount) as tile (tileKey(tile))}
|
||||
<div
|
||||
class="overflow-hidden rounded-2xl shrink-0"
|
||||
style={`width: ${row.tileWidth}px; height: ${row.tileHeight}px`}>
|
||||
{@render videoTile(tile, "default")}
|
||||
</div>
|
||||
{/each}
|
||||
</div>
|
||||
{/each}
|
||||
</div>
|
||||
{/if}
|
||||
</div>
|
||||
{/if}
|
||||
{:else}
|
||||
|
|
|
|||
Loading…
Reference in a new issue