PR-C1: adaptive video grid sizing (foundation) #2

Closed
hudson wants to merge 18 commits from flotilla-sph-adaptive-grid into dev
2 changed files with 278 additions and 74 deletions
Showing only changes of commit 972425a4b7 - Show all commits

View file

@ -31,93 +31,290 @@ export const LIVEKIT_PARTICIPANTS = 39004
export {supportsAudioOutputSelection}
/** Aspect ratio constraints — tiles can flex between 3:4 (portrait) and 16:9 (landscape). */
const TILE_ASPECT_MIN = 3 / 4
const TILE_ASPECT_MAX = 16 / 9
/**
* Aspect ratio constraints — tiles can flex between 3:4 (portrait)
* and 16:9 (landscape). Tiles are ALLOWED to be wider than 16:9
* when the container is very wide, to prevent overflow (review #2
* bug: n=2 second tile half off screen). The lower bound (3:4)
* prevents tiles from being too tall.
*/
const TILE_ASPECT_PORTRAIT = 3 / 4
const TILE_ASPECT_LANDSCAPE = 16 / 9
const TILE_GAP = 8
/**
* Uniform-size tile grid: all tiles are identical px dimensions, arranged
* in a CSS grid. Column count is chosen to maximize tile area while
* respecting aspect ratio bounds [3:4 .. 16:9].
* Orphan rows (fewer tiles than a full row) center at the same tile size.
* Minimum pixel height for a tile before we allow the grid to
* overflow (scroll) instead of forcing tiles into portrait mode.
* Calibrated so ~6-8 tiles on a standard portrait phone fit
* without scrolling; beyond that, scroll is acceptable.
*/
export type UniformTileGrid = {
cols: number
rows: number
const MIN_TILE_HEIGHT = 120
/**
* A single row in an adaptive tile grid: tiles in this row share
* the same dimensions but may differ from other rows.
*/
export type TileRow = {
columnCount: number
tileWidth: number
tileHeight: number
aspect: number
aspectRatio: number
}
/**
/** Numeric lexicographic comparison for scoring arrays. */
const compareKeys = (a: number[], b: number[]): number => {
for (let i = 0; i < a.length; i++) {
if (a[i] !== b[i]) return a[i] - b[i]
* Adaptive tile grid: rows can have different tile sizes, allowing
* a Jitsi-style layout where tiles flex between 3:4 and 16:9 and
* partial rows use wider tiles to fill whitespace.
*
* Example (3 tiles, 2 columns):
* row 0: [ square ] [ square ]
* row 1: [ 16:9 ] (centered)
*/
export type AdaptiveTileGrid = {
rows: TileRow[]
totalWidth: number
totalHeight: number
/** True when the grid overflows — tiles are below MIN_TILE_HEIGHT */
overflows: boolean
}
/**
* Score for comparing candidate layouts. Lower is better.
* Uses named fields instead of opaque array "keys" (per review feedback).
*/
type LayoutScore = {
/** Penalty for vertical overflow: 0 if none, else huge */
verticalOverflowPenalty: number
/** Penalty for horizontal overflow: 0 if none, else huge */
horizontalOverflowPenalty: number
/** Penalty when tiles fall below MIN_TILE_HEIGHT even while fitting */
minTileHeightPenalty: number
/** Negative tile area (we want to maximize area, so negate) */
negativeArea: number
/** Leftover whitespace (underflow) in px */
whitespace: number
/** Average absolute deviation from 16:9 across all tiles */
aspectDeviation: number
}
const compareScores = (a: LayoutScore, b: LayoutScore): number => {
const fields: (keyof LayoutScore)[] = [
"verticalOverflowPenalty",
"horizontalOverflowPenalty",
"minTileHeightPenalty",
"negativeArea",
"whitespace",
"aspectDeviation",
]
for (const field of fields) {
if (a[field] !== b[field]) return a[field] - b[field]
}
return 0
}
/**
* Pick the column count that best fills the viewport. Prioritises:
* 1. No vertical overflow (avoids unnecessary scroll)
* 2. No horizontal overflow (avoids sideways clip/scroll)
* 3. Minimal leftover whitespace (h-pad + v-pad)
* 4. Aspect ratio closest to 16:9
* 5. Larger tiles
* Build a single-row TileRow for the uniform case where all rows
* have the same number of tiles. No upper-bound clamping on aspect
* ratio — tiles can be wider than 16:9 to avoid overflow.
*/
export const computeUniformGrid = (
const buildUniformRow = (
columnCount: number,
containerWidth: number,
containerHeight: number,
rowCount: number,
): TileRow => {
const gap = TILE_GAP
const availableHeight = containerHeight - (rowCount - 1) * gap
const rowHeight = availableHeight / rowCount
const tileWidth = (containerWidth - (columnCount - 1) * gap) / columnCount
const fillAspect = tileWidth / rowHeight
// Cap at 16:9 to avoid horizontally stretching video (review feedback)
const aspectRatio = Math.max(TILE_ASPECT_PORTRAIT, Math.min(TILE_ASPECT_LANDSCAPE, fillAspect))
const tileHeight = tileWidth / aspectRatio
return {columnCount, tileWidth, tileHeight, aspectRatio}
}
/**
* Build a mixed-size layout candidate for a given column count.
* Uses a "match heights" approach: the last (partial) row is at 16:9
* (per review #1: "single centered 16:9 tile") and full-row tiles
* match that same height. This creates a cohesive Jitsi-style look
* where all tiles in all rows are the same height, with different
* widths per row (full-row tiles are narrower/squarer, last-row
* tiles are wider).
*
* If the full-row aspect would go below the portrait bound (3:4),
* the height diverges: full rows clamp to 3:4 portrait.
*/
const buildMixedCandidate = (
tileCount: number,
cw: number,
ch: number,
): UniformTileGrid | undefined => {
if (tileCount <= 0 || cw <= 0 || ch <= 0) return undefined
columnCount: number,
containerWidth: number,
containerHeight: number,
): AdaptiveTileGrid | undefined => {
const gap = TILE_GAP
const fullRowCount = Math.floor(tileCount / columnCount)
const remainder = tileCount % columnCount
const totalRowCount = fullRowCount + 1
let best: UniformTileGrid | undefined
let bestKey: number[] | undefined
const tileWidthFull = (containerWidth - (columnCount - 1) * gap) / columnCount
if (tileWidthFull <= 0) return undefined
for (let cols = 1; cols <= tileCount; cols++) {
const rows = Math.ceil(tileCount / cols)
const availH = ch - (rows - 1) * TILE_GAP
const widthLimit = (cw - (cols - 1) * TILE_GAP) / cols
if (availH <= 0 || widthLimit <= 0) continue
const tileWidthLast = (containerWidth - (remainder - 1) * gap) / remainder
if (tileWidthLast <= 0) return undefined
const rowH = availH / rows
const fillAspect = widthLimit / rowH
const aspect = Math.max(TILE_ASPECT_MIN, Math.min(TILE_ASPECT_MAX, fillAspect))
// Find a common height that fits both row types and container
const maxPossibleHeight = Math.min(
tileWidthFull / TILE_ASPECT_PORTRAIT,
tileWidthLast / TILE_ASPECT_PORTRAIT,
(containerHeight - (totalRowCount - 1) * gap) / totalRowCount,
)
let tileW: number, tileH: number
if (fillAspect > TILE_ASPECT_MAX) {
// Too wide: cap at 16:9, tiles taller than row -> vertical overflow
tileW = widthLimit
tileH = widthLimit / TILE_ASPECT_MAX
} else if (fillAspect < TILE_ASPECT_MIN) {
// Too tall: cap at 3:4, tiles shorter than row -> v-pad (letterbox)
tileW = widthLimit
tileH = widthLimit / TILE_ASPECT_MIN
// Target 16:9 for the last row if possible
const targetHeight = tileWidthLast / TILE_ASPECT_LANDSCAPE
const commonHeight = Math.min(maxPossibleHeight, targetHeight)
const tileHeightFull = commonHeight
const tileHeightLast = commonHeight
const fullRowAspectRatio = tileWidthFull / commonHeight
const lastRowAspectRatio = tileWidthLast / commonHeight
const totalHeight = fullRowCount * tileHeightFull + tileHeightLast + (totalRowCount - 1) * gap
const totalWidth = Math.max(
columnCount * tileWidthFull + (columnCount - 1) * gap,
remainder * tileWidthLast + (remainder - 1) * gap,
)
const rows: TileRow[] = []
for (let r = 0; r < fullRowCount; r++) {
rows.push({
columnCount,
tileWidth: tileWidthFull,
tileHeight: tileHeightFull,
aspectRatio: fullRowAspectRatio,
})
}
rows.push({
columnCount: remainder,
tileWidth: tileWidthLast,
tileHeight: tileHeightLast,
aspectRatio: lastRowAspectRatio,
})
return {
rows,
totalWidth,
totalHeight,
overflows: totalHeight > containerHeight,
}
}
/**
* Compute an adaptive tile grid using Jitsi-style mixed row sizes.
*
* - Tiles flex between 3:4 (portrait) and wider than 16:9 if needed
* to avoid overflow. No upper bound on aspect ratio — tiles can
* be wider than 16:9 when the container is very wide.
* - When the last row is partial (fewer tiles than a full row),
* those tiles are wider (16:9) while full rows use square-ish
* tiles, filling whitespace per review #1.
* - Only allows overflow (scroll) when tiles would be below
* MIN_TILE_HEIGHT (review #2: ~6-8 tiles on portrait phone).
*
* Prioritises:
* 1. No overflow (tiles shrink to fit within aspect bounds)
* 2. Minimal whitespace
* 3. Aspect ratios close to 16:9
* 4. Larger tiles
*/
export const computeAdaptiveGrid = (
tileCount: number,
containerWidth: number,
containerHeight: number,
): AdaptiveTileGrid | undefined => {
if (tileCount <= 0 || containerWidth <= 0 || containerHeight <= 0) return undefined
let best: AdaptiveTileGrid | undefined
let bestScore: LayoutScore | undefined
for (let columnCount = 1; columnCount <= tileCount; columnCount++) {
const fullRowCount = Math.floor(tileCount / columnCount)
const remainder = tileCount % columnCount
const totalRowCount = fullRowCount + (remainder > 0 ? 1 : 0)
let candidate: AdaptiveTileGrid | undefined
if (remainder === 0) {
// Uniform layout — all rows identical
const row = buildUniformRow(columnCount, containerWidth, containerHeight, totalRowCount)
const totalHeight = totalRowCount * row.tileHeight + (totalRowCount - 1) * TILE_GAP
const totalWidth = columnCount * row.tileWidth + (columnCount - 1) * TILE_GAP
const rows: TileRow[] = []
for (let r = 0; r < totalRowCount; r++) {
rows.push({...row})
}
candidate = {
rows,
totalWidth,
totalHeight,
overflows: totalHeight > containerHeight,
}
} else {
// In range: fills both dims exactly
tileW = widthLimit
tileH = rowH
// Mixed layout — last row has fewer tiles, make them wider
candidate = buildMixedCandidate(tileCount, columnCount, containerWidth, containerHeight)
}
const bboxW = cols * tileW + (cols - 1) * TILE_GAP
const bboxH = rows * tileH + (rows - 1) * TILE_GAP
const vOverflow = Math.max(0, bboxH - ch)
const hOverflow = Math.max(0, bboxW - cw)
const waste = Math.max(0, cw - bboxW) + Math.max(0, ch - bboxH)
const key = [
Math.round(vOverflow),
Math.round(hOverflow),
Math.round(waste),
Number(Math.abs(aspect - TILE_ASPECT_MAX).toFixed(3)),
-Math.round(tileW * tileH),
]
if (!candidate) continue
if (!bestKey || compareKeys(key, bestKey) < 0) {
bestKey = key
best = {cols, rows, tileWidth: tileW, tileHeight: tileH, aspect}
// Compute score
const verticalOverflow = Math.max(0, candidate.totalHeight - containerHeight)
const horizontalOverflow = Math.max(0, candidate.totalWidth - containerWidth)
const whitespace =
Math.max(0, containerWidth - candidate.totalWidth) +
Math.max(0, containerHeight - candidate.totalHeight)
const aspectDeviation =
candidate.rows.reduce(
(sum, row) => sum + Math.abs(row.aspectRatio - TILE_ASPECT_LANDSCAPE),
0,
) / candidate.rows.length
const totalArea = candidate.rows.reduce(
(sum, row) => sum + row.tileWidth * row.tileHeight * row.columnCount,
0,
)
// For overflow: if tiles are above MIN_TILE_HEIGHT, overflow is
// unacceptable — heavily penalize. If tiles are below MIN_TILE_HEIGHT,
// overflow is acceptable (tiles are already too small to shrink further).
const minTileHeightInCandidate = Math.min(...candidate.rows.map(r => r.tileHeight))
const verticalOverflowPenalty =
verticalOverflow > 0 && minTileHeightInCandidate >= MIN_TILE_HEIGHT
? verticalOverflow * 1_000_000
: verticalOverflow
const horizontalOverflowPenalty =
horizontalOverflow > 0 && minTileHeightInCandidate >= MIN_TILE_HEIGHT
? horizontalOverflow * 1_000_000
: horizontalOverflow
// Min-tile-height penalty: even when fitting, discourage layouts
// where tiles are below the minimum acceptable size.
const minTileHeightPenalty =
minTileHeightInCandidate < MIN_TILE_HEIGHT
? (MIN_TILE_HEIGHT - minTileHeightInCandidate) * 1000
: 0
const score: LayoutScore = {
verticalOverflowPenalty,
horizontalOverflowPenalty,
minTileHeightPenalty,
negativeArea: -totalArea,
whitespace,
aspectDeviation,
}
if (!bestScore || compareScores(score, bestScore) < 0) {
bestScore = score
best = candidate
}
}

View file

@ -19,8 +19,8 @@
participantMediaState,
pubkeyFromLiveKitIdentity,
videoTrackRevision,
computeUniformGrid,
type UniformTileGrid,
computeAdaptiveGrid,
type AdaptiveTileGrid,
} from "@app/call"
import {profiles} from "@app/core"
@ -157,8 +157,8 @@
const useSpotlightLayout = $derived(primaryTile !== undefined)
const useMultiGrid = $derived(!useSpotlightLayout)
const tileGrid = $derived<UniformTileGrid | undefined>(
useMultiGrid ? computeUniformGrid(videoTiles.length, gridWidth, gridHeight) : undefined,
const tileGrid = $derived<AdaptiveTileGrid | undefined>(
useMultiGrid ? computeAdaptiveGrid(videoTiles.length, gridWidth, gridHeight) : undefined,
)
$effect(() => {
@ -272,14 +272,21 @@
bind:clientHeight={gridHeight}
class="min-h-0 flex-1 overflow-y-auto">
{#if tileGrid}
<div
class="mx-auto flex flex-wrap justify-center gap-2"
style={`max-width: ${tileGrid.cols * tileGrid.tileWidth + (tileGrid.cols - 1) * 8}px`}>
{#each videoTiles as tile (tileKey(tile))}
<div class="flex flex-col items-center gap-2">
{#each tileGrid.rows as row, rowIndex}
{@const offset = tileGrid.rows
.slice(0, rowIndex)
.reduce((sum, r) => sum + r.columnCount, 0)}
<div
class="overflow-hidden rounded-2xl shrink-0"
style={`width: ${tileGrid.tileWidth}px; height: ${tileGrid.tileHeight}px`}>
{@render videoTile(tile, "default")}
class="flex flex-wrap justify-center gap-2"
style={`max-width: ${row.columnCount * row.tileWidth + (row.columnCount - 1) * 8}px`}>
{#each videoTiles.slice(offset, offset + row.columnCount) as tile (tileKey(tile))}
<div
class="overflow-hidden rounded-2xl shrink-0"
style={`width: ${row.tileWidth}px; height: ${row.tileHeight}px`}>
{@render videoTile(tile, "default")}
</div>
{/each}
</div>
{/each}
</div>