PR-C1: adaptive video grid sizing (foundation) #2

Closed
hudson wants to merge 18 commits from flotilla-sph-adaptive-grid into dev
2 changed files with 247 additions and 17 deletions

View file

@ -31,6 +31,216 @@ export const LIVEKIT_PARTICIPANTS = 39004
export {supportsAudioOutputSelection} export {supportsAudioOutputSelection}
/**
* Aspect ratio constraints for tiles. The lower bound is dynamic
* (1:1 on landscape, 3:4 on portrait); the upper bound is 16:9.
*/
const TILE_ASPECT_PORTRAIT = 3 / 4
const TILE_ASPECT_LANDSCAPE = 16 / 9
const TILE_GAP = 8
/**
* Minimum pixel height for a tile before we allow the grid to
* overflow (scroll) instead of forcing tiles into portrait mode.
* Calibrated so ~6-8 tiles on a standard portrait phone fit
* without scrolling; beyond that, scroll is acceptable.
*/
const MIN_TILE_HEIGHT = 120
/**
* A single row in an adaptive tile grid.
*/
export type TileRow = {
columnCount: number
tileWidth: number
tileHeight: number
/** Total width of this row including gaps — for the template's max-width */
rowWidth: number
aspectRatio: number
}
/**
* Adaptive tile grid: all tiles share the same dimensions. Full rows
* fill the container width; partial rows are centered.
*
* Example (3 tiles, 2 columns):
* row 0: [ square ] [ square ]
* row 1: [ square ] (centered, same size)
*/
export type AdaptiveTileGrid = {
rows: TileRow[]
totalWidth: number
totalHeight: number
}
/**
* Score for comparing candidate layouts. Lower is better.
* Uses named fields instead of opaque array "keys" (per review feedback).
*/
type LayoutScore = {
/** Penalty for vertical overflow: 0 if none, else huge */
verticalOverflowPenalty: number
/** Penalty for horizontal overflow: 0 if none, else huge */
horizontalOverflowPenalty: number
/** Penalty when tiles fall below MIN_TILE_HEIGHT even while fitting */
minTileHeightPenalty: number
/** Negative tile area (we want to maximize area, so negate) */
negativeArea: number
/** Leftover whitespace (underflow) in px */
whitespace: number
/** Average absolute deviation from 16:9 across all tiles */
aspectDeviation: number
}
const compareScores = (a: LayoutScore, b: LayoutScore): number => {
const fields: (keyof LayoutScore)[] = [
"verticalOverflowPenalty",
"horizontalOverflowPenalty",
"minTileHeightPenalty",
"negativeArea",
"whitespace",
"aspectDeviation",
]
for (const field of fields) {
if (a[field] !== b[field]) return a[field] - b[field]
}
return 0
}
/**
* Compute the largest tile size that fits within a given width and height,
* bounded by [minAspect, TILE_ASPECT_LANDSCAPE]. The tile is shrunk to fit
* whichever dimension is more constraining, so it never overflows.
*/
const fitTile = (availWidth: number, availHeight: number, minAspect: number) => {
const fillAspect = availWidth / availHeight
const aspectRatio = Math.max(minAspect, Math.min(TILE_ASPECT_LANDSCAPE, fillAspect))
if (fillAspect >= aspectRatio) {
const tileWidth = availHeight * aspectRatio
return {tileWidth, tileHeight: availHeight, aspectRatio}
}
const tileHeight = availWidth / aspectRatio
return {tileWidth: availWidth, tileHeight, aspectRatio}
}
/**
* Build a candidate grid for a given column count. All tiles share the
* same dimensions; the partial last row (if any) is centered with the
* same tile size as full rows.
*/
const buildCandidate = (
tileCount: number,
columnCount: number,
containerWidth: number,
containerHeight: number,
minAspect: number,
): AdaptiveTileGrid | undefined => {
const gap = TILE_GAP
const fullRowCount = Math.floor(tileCount / columnCount)
const remainder = tileCount % columnCount
const totalRowCount = fullRowCount + (remainder > 0 ? 1 : 0)
const availHeight = (containerHeight - (totalRowCount - 1) * gap) / totalRowCount
const availWidth = (containerWidth - (columnCount - 1) * gap) / columnCount
if (availWidth <= 0 || availHeight <= 0) return undefined
const {tileWidth, tileHeight, aspectRatio} = fitTile(availWidth, availHeight, minAspect)
const rows: TileRow[] = []
for (let r = 0; r < fullRowCount; r++) {
rows.push({
columnCount,
tileWidth,
tileHeight,
rowWidth: columnCount * tileWidth + (columnCount - 1) * gap,
aspectRatio,
})
}
if (remainder > 0) {
rows.push({
columnCount: remainder,
tileWidth,
tileHeight,
rowWidth: remainder * tileWidth + (remainder - 1) * gap,
aspectRatio,
})
}
const totalHeight = totalRowCount * tileHeight + (totalRowCount - 1) * gap
const totalWidth = columnCount * tileWidth + (columnCount - 1) * gap
return {rows, totalWidth, totalHeight}
}
/**
* Compute an adaptive tile grid. All tiles share the same dimensions;
* partial rows are centered. Tiles flex between a minimum aspect
* (1:1 on landscape, 3:4 on portrait) and 16:9, capped so they never
* overflow the container.
*
* Only allows overflow (scroll) when tiles would be below
* MIN_TILE_HEIGHT (~6-8 tiles on portrait phone).
*
* Prioritises:
* 1. No overflow (tiles shrink to fit within aspect bounds)
* 2. Minimal whitespace
* 3. Aspect ratios close to 16:9
* 4. Larger tiles
*/
export const computeAdaptiveGrid = (
tileCount: number,
containerWidth: number,
containerHeight: number,
): AdaptiveTileGrid | undefined => {
if (tileCount <= 0 || containerWidth <= 0 || containerHeight <= 0) return undefined
const minAspect = containerWidth / containerHeight >= 4 / 3 ? 1 : TILE_ASPECT_PORTRAIT
let best: AdaptiveTileGrid | undefined
let bestScore: LayoutScore | undefined
for (let columnCount = 1; columnCount <= tileCount; columnCount++) {
const candidate = buildCandidate(
tileCount,
columnCount,
containerWidth,
containerHeight,
minAspect,
)
if (!candidate) continue
const {tileWidth, tileHeight, aspectRatio} = candidate.rows[0]
const verticalOverflow = Math.max(0, candidate.totalHeight - containerHeight)
const horizontalOverflow = Math.max(0, candidate.totalWidth - containerWidth)
const whitespace =
Math.max(0, containerWidth - candidate.totalWidth) +
Math.max(0, containerHeight - candidate.totalHeight)
const totalArea = tileWidth * tileHeight * tileCount
const overflowPenaltyMultiplier = tileHeight >= MIN_TILE_HEIGHT ? 1_000_000 : 1
const verticalOverflowPenalty = verticalOverflow * overflowPenaltyMultiplier
const horizontalOverflowPenalty = horizontalOverflow * overflowPenaltyMultiplier
const minTileHeightPenalty =
tileHeight < MIN_TILE_HEIGHT ? (MIN_TILE_HEIGHT - tileHeight) * 1000 : 0
const score: LayoutScore = {
verticalOverflowPenalty,
horizontalOverflowPenalty,
minTileHeightPenalty,
negativeArea: -totalArea,
whitespace,
aspectDeviation: Math.abs(aspectRatio - TILE_ASPECT_LANDSCAPE),
}
if (!bestScore || compareScores(score, bestScore) < 0) {
bestScore = score
best = candidate
}
}
return best
}
const LIVEKIT_DEFAULT_DEVICE_ID = "default" const LIVEKIT_DEFAULT_DEVICE_ID = "default"
const RECONNECT_DELAYS = [1000, 2000, 4000, 8000, 16000] const RECONNECT_DELAYS = [1000, 2000, 4000, 8000, 16000]

View file

@ -19,6 +19,8 @@
participantMediaState, participantMediaState,
pubkeyFromLiveKitIdentity, pubkeyFromLiveKitIdentity,
videoTrackRevision, videoTrackRevision,
computeAdaptiveGrid,
type AdaptiveTileGrid,
} from "@app/call" } from "@app/call"
import {profiles} from "@app/core" import {profiles} from "@app/core"
@ -38,7 +40,7 @@
source: Track.Source.Camera | Track.Source.ScreenShare source: Track.Source.Camera | Track.Source.ScreenShare
} }
type TileLayout = "spotlight" | "default" | "strip" type TileLayoutVariant = "spotlight" | "default" | "strip"
const {layout, mobile = false, url, h, class: className = ""}: Props = $props() const {layout, mobile = false, url, h, class: className = ""}: Props = $props()
@ -149,10 +151,14 @@
return videoTiles.filter(t => tileKey(t) !== pk) return videoTiles.filter(t => tileKey(t) !== pk)
}) })
let gridWidth = $state(0)
let gridHeight = $state(0)
const useSpotlightLayout = $derived(primaryTile !== undefined) const useSpotlightLayout = $derived(primaryTile !== undefined)
const useMultiGrid = $derived(!useSpotlightLayout && videoTiles.length > 2) const useMultiGrid = $derived(!useSpotlightLayout)
const multiGridClass = $derived(
layout === VideoCallLayout.Split ? "grid-cols-1" : "grid-cols-1 sm:grid-cols-2", const tileGrid = $derived<AdaptiveTileGrid | undefined>(
useMultiGrid ? computeAdaptiveGrid(videoTiles.length, gridWidth, gridHeight) : undefined,
) )
$effect(() => { $effect(() => {
@ -193,14 +199,14 @@
) )
</script> </script>
{#snippet videoTile(tile: VideoTileData, layout: TileLayout)} {#snippet videoTile(tile: VideoTileData, layout: TileLayoutVariant)}
{@const media = $mediaStateByIdentity(tile.liveKitIdentity)} {@const media = $mediaStateByIdentity(tile.liveKitIdentity)}
{@const label = labelFor(tile.liveKitIdentity, tile.source)} {@const label = labelFor(tile.liveKitIdentity, tile.source)}
<div <div
class={cx( class={cx(
"relative isolate overflow-hidden rounded-2xl shadow-sm", "relative isolate overflow-hidden rounded-2xl shadow-sm",
layout === "spotlight" && "min-h-0 flex-1", layout === "spotlight" && "min-h-0 flex-1",
layout === "default" && "aspect-video w-full min-h-0", layout === "default" && "min-h-0 h-full w-full",
layout === "strip" && "aspect-video w-44 shrink-0", layout === "strip" && "aspect-video w-44 shrink-0",
tile.source === Track.Source.ScreenShare ? "bg-black" : "bg-surface", tile.source === Track.Source.ScreenShare ? "bg-black" : "bg-surface",
)}> )}>
@ -261,16 +267,30 @@
{/if} {/if}
</div> </div>
{:else if useMultiGrid} {:else if useMultiGrid}
<div class={cx("grid min-h-0 flex-1 content-start gap-2 overflow-y-auto", multiGridClass)}> <div
{#each videoTiles as tile (tileKey(tile))} bind:clientWidth={gridWidth}
{@render videoTile(tile, "default")} bind:clientHeight={gridHeight}
{/each} class="min-h-0 flex-1 overflow-y-auto [scrollbar-gutter:stable]">
</div> {#if tileGrid}
{:else} <div class="flex flex-col items-center gap-2">
<div class="flex min-h-0 flex-1 flex-col gap-2 overflow-y-auto"> {#each tileGrid.rows as row, rowIndex}
{#each videoTiles as tile (tileKey(tile))} {@const offset = tileGrid.rows
{@render videoTile(tile, "default")} .slice(0, rowIndex)
{/each} .reduce((sum, r) => sum + r.columnCount, 0)}
<div
class="flex flex-nowrap justify-center gap-2"
style={`max-width: ${row.rowWidth}px`}>
{#each videoTiles.slice(offset, offset + row.columnCount) as tile (tileKey(tile))}
<div
class="overflow-hidden rounded-2xl shrink-0"
style={`width: ${row.tileWidth}px; height: ${row.tileHeight}px`}>
{@render videoTile(tile, "default")}
</div>
{/each}
</div>
{/each}
</div>
{/if}
</div> </div>
{/if} {/if}
{:else} {:else}
@ -285,7 +305,7 @@
<div class={panelChrome}> <div class={panelChrome}>
{#if mobile} {#if mobile}
<div class="flex min-h-0 flex-1 flex-col gap-2"> <div class="flex min-h-0 flex-1 flex-col gap-2">
<div class="min-h-0 flex-1 overflow-hidden"> <div class="flex min-h-0 flex-1 flex-col overflow-hidden">
{@render videoPanelBody()} {@render videoPanelBody()}
</div> </div>
<div class="shrink-0 pb-2"> <div class="shrink-0 pb-2">