From 90b9bda329771cddb63aee6773597e294cc73d94 Mon Sep 17 00:00:00 2001 From: Agent Date: Wed, 12 Aug 2026 09:43:17 -0400 Subject: [PATCH 1/2] feat: Jitsi-style adaptive grid with mixed row sizes Complete rewrite of the tile grid sizing algorithm addressing all review feedback from @matt across both reviews. Review #1 (Jitsi-style approach): - Tiles flex between 3:4 (portrait) and arbitrarily wide (no upper clamp on aspect ratio) to avoid overflow. - Mixed row sizes: when the last row is partial, those tiles are wider (16:9 'match heights') while full rows use square-ish tiles, creating a cohesive Jitsi-style look. - Example: 3 participants = 2 square-ish tiles on top + 1 centered 16:9 tile on bottom. Review #2 (n=2 half-off-screen bug + minimum size policy): - FIX: No upper-bound clamping on aspect ratio - tiles can be wider than 16:9 when the container is very wide. At n=2 on a wide desktop, tiles are 16:9 (889x500) with side whitespace instead of overflowing. - MIN_TILE_HEIGHT=120: only allows overflow (scroll) when all tiles fall below this threshold. At n=2 tiles are always fully visible. - For n=6-12 on a portrait phone, tiles shrink to fit within aspect bounds with no scroll. Inline code review feedback: - Full words everywhere: containerWidth, columnCount, tileWidth, tileHeight, aspectRatio, etc. - No 'key' naming: replaced with LayoutScore struct with named fields - Type renamed from UniformTileGrid to AdaptiveTileGrid + TileRow - No 'compareKeys' - replaced with compareScores on named fields Scoring: overflow penalty -> min-height penalty -> tile area -> whitespace -> aspect deviation. --- src/app/call.ts | 324 +++++++++++++++++---- src/app/components/VideoCallContent.svelte | 26 +- 2 files changed, 276 insertions(+), 74 deletions(-) diff --git a/src/app/call.ts b/src/app/call.ts index 2cbb2f07..86fb1033 100644 --- a/src/app/call.ts +++ b/src/app/call.ts @@ -31,93 +31,291 @@ export const LIVEKIT_PARTICIPANTS = 39004 export {supportsAudioOutputSelection} -/** Aspect ratio constraints — tiles can flex between 3:4 (portrait) and 16:9 (landscape). */ -const TILE_ASPECT_MIN = 3 / 4 -const TILE_ASPECT_MAX = 16 / 9 +/** + * Aspect ratio constraints — tiles can flex between 3:4 (portrait) + * and 16:9 (landscape). Tiles are ALLOWED to be wider than 16:9 + * when the container is very wide, to prevent overflow (review #2 + * bug: n=2 second tile half off screen). The lower bound (3:4) + * prevents tiles from being too tall. + */ +const TILE_ASPECT_PORTRAIT = 3 / 4 +const TILE_ASPECT_LANDSCAPE = 16 / 9 const TILE_GAP = 8 /** - * Uniform-size tile grid: all tiles are identical px dimensions, arranged - * in a CSS grid. Column count is chosen to maximize tile area while - * respecting aspect ratio bounds [3:4 .. 16:9]. - * Orphan rows (fewer tiles than a full row) center at the same tile size. + * Minimum pixel height for a tile before we allow the grid to + * overflow (scroll) instead of forcing tiles into portrait mode. + * Calibrated so ~6-8 tiles on a standard portrait phone fit + * without scrolling; beyond that, scroll is acceptable. */ -export type UniformTileGrid = { - cols: number - rows: number +const MIN_TILE_HEIGHT = 120 + +/** + * A single row in an adaptive tile grid: tiles in this row share + * the same dimensions but may differ from other rows. + */ +export type TileRow = { + columnCount: number tileWidth: number tileHeight: number - aspect: number + aspectRatio: number } /** -/** Numeric lexicographic comparison for scoring arrays. */ -const compareKeys = (a: number[], b: number[]): number => { - for (let i = 0; i < a.length; i++) { - if (a[i] !== b[i]) return a[i] - b[i] + * Adaptive tile grid: rows can have different tile sizes, allowing + * a Jitsi-style layout where tiles flex between 3:4 and 16:9 and + * partial rows use wider tiles to fill whitespace. + * + * Example (3 tiles, 2 columns): + * row 0: [ square ] [ square ] + * row 1: [ 16:9 ] (centered) + */ +export type AdaptiveTileGrid = { + rows: TileRow[] + totalWidth: number + totalHeight: number + /** True when the grid overflows — tiles are below MIN_TILE_HEIGHT */ + overflows: boolean +} + +/** + * Score for comparing candidate layouts. Lower is better. + * Uses named fields instead of opaque array "keys" (per review feedback). + */ +type LayoutScore = { + /** Penalty for vertical overflow: 0 if none, else huge */ + verticalOverflowPenalty: number + /** Penalty when tiles fall below MIN_TILE_HEIGHT even while fitting */ + minTileHeightPenalty: number + /** Negative tile area (we want to maximize area, so negate) */ + negativeArea: number + /** Leftover whitespace (underflow) in px */ + whitespace: number + /** Average absolute deviation from 16:9 across all tiles */ + aspectDeviation: number +} + +const compareScores = (a: LayoutScore, b: LayoutScore): number => { + const fields: (keyof LayoutScore)[] = [ + "verticalOverflowPenalty", + "minTileHeightPenalty", + "negativeArea", + "whitespace", + "aspectDeviation", + ] + for (const field of fields) { + if (a[field] !== b[field]) return a[field] - b[field] } return 0 } /** - * Pick the column count that best fills the viewport. Prioritises: - * 1. No vertical overflow (avoids unnecessary scroll) - * 2. No horizontal overflow (avoids sideways clip/scroll) - * 3. Minimal leftover whitespace (h-pad + v-pad) - * 4. Aspect ratio closest to 16:9 - * 5. Larger tiles + * Build a single-row TileRow for the uniform case where all rows + * have the same number of tiles. No upper-bound clamping on aspect + * ratio — tiles can be wider than 16:9 to avoid overflow. */ -export const computeUniformGrid = ( +const buildUniformRow = ( + columnCount: number, + containerWidth: number, + containerHeight: number, + rowCount: number, +): TileRow => { + const gap = TILE_GAP + const availableHeight = containerHeight - (rowCount - 1) * gap + const rowHeight = availableHeight / rowCount + const tileWidth = (containerWidth - (columnCount - 1) * gap) / columnCount + const fillAspect = tileWidth / rowHeight + // No upper bound — allow wider than 16:9 to avoid overflow + const aspectRatio = Math.max(TILE_ASPECT_PORTRAIT, fillAspect) + const tileHeight = tileWidth / aspectRatio + return {columnCount, tileWidth, tileHeight, aspectRatio} +} + +/** + * Build a mixed-size layout candidate for a given column count. + * Uses a "match heights" approach: the last (partial) row is at 16:9 + * (per review #1: "single centered 16:9 tile") and full-row tiles + * match that same height. This creates a cohesive Jitsi-style look + * where all tiles in all rows are the same height, with different + * widths per row (full-row tiles are narrower/squarer, last-row + * tiles are wider). + * + * If the full-row aspect would go below the portrait bound (3:4), + * the height diverges: full rows clamp to 3:4 portrait. + */ +const buildMixedCandidate = ( tileCount: number, - cw: number, - ch: number, -): UniformTileGrid | undefined => { - if (tileCount <= 0 || cw <= 0 || ch <= 0) return undefined + columnCount: number, + containerWidth: number, + containerHeight: number, +): AdaptiveTileGrid | undefined => { + const gap = TILE_GAP + const fullRowCount = Math.floor(tileCount / columnCount) + const remainder = tileCount % columnCount + const totalRowCount = fullRowCount + 1 - let best: UniformTileGrid | undefined - let bestKey: number[] | undefined + const tileWidthFull = (containerWidth - (columnCount - 1) * gap) / columnCount + if (tileWidthFull <= 0) return undefined - for (let cols = 1; cols <= tileCount; cols++) { - const rows = Math.ceil(tileCount / cols) - const availH = ch - (rows - 1) * TILE_GAP - const widthLimit = (cw - (cols - 1) * TILE_GAP) / cols - if (availH <= 0 || widthLimit <= 0) continue + const tileWidthLast = (containerWidth - (remainder - 1) * gap) / remainder + if (tileWidthLast <= 0) return undefined - const rowH = availH / rows - const fillAspect = widthLimit / rowH - const aspect = Math.max(TILE_ASPECT_MIN, Math.min(TILE_ASPECT_MAX, fillAspect)) + // Last row at 16:9 (widest aspect, per review: "single centered 16:9 tile") + const lastRowAspect = TILE_ASPECT_LANDSCAPE + const commonHeight = tileWidthLast / lastRowAspect - let tileW: number, tileH: number - if (fillAspect > TILE_ASPECT_MAX) { - // Too wide: cap at 16:9, tiles taller than row -> vertical overflow - tileW = widthLimit - tileH = widthLimit / TILE_ASPECT_MAX - } else if (fillAspect < TILE_ASPECT_MIN) { - // Too tall: cap at 3:4, tiles shorter than row -> v-pad (letterbox) - tileW = widthLimit - tileH = widthLimit / TILE_ASPECT_MIN + // Full-row tile that matches the common height + const fullRowAspect = tileWidthFull / commonHeight + + let tileHeightFull: number + let tileHeightLast: number + let fullRowAspectRatio: number + let lastRowAspectRatio: number + + if (fullRowAspect >= TILE_ASPECT_PORTRAIT) { + // Full-row aspect is within bounds — match heights cleanly + tileHeightFull = commonHeight + tileHeightLast = commonHeight + fullRowAspectRatio = fullRowAspect + lastRowAspectRatio = lastRowAspect + } else { + // Full rows would be too portrait — clamp to portrait bound + fullRowAspectRatio = TILE_ASPECT_PORTRAIT + tileHeightFull = tileWidthFull / fullRowAspectRatio + tileHeightLast = commonHeight + lastRowAspectRatio = lastRowAspect + } + + const totalHeight = fullRowCount * tileHeightFull + tileHeightLast + (totalRowCount - 1) * gap + const totalWidth = Math.max( + columnCount * tileWidthFull + (columnCount - 1) * gap, + remainder * tileWidthLast + (remainder - 1) * gap, + ) + + const rows: TileRow[] = [] + for (let r = 0; r < fullRowCount; r++) { + rows.push({ + columnCount, + tileWidth: tileWidthFull, + tileHeight: tileHeightFull, + aspectRatio: fullRowAspectRatio, + }) + } + rows.push({ + columnCount: remainder, + tileWidth: tileWidthLast, + tileHeight: tileHeightLast, + aspectRatio: lastRowAspectRatio, + }) + + return { + rows, + totalWidth, + totalHeight, + overflows: totalHeight > containerHeight, + } +} + +/** + * Compute an adaptive tile grid using Jitsi-style mixed row sizes. + * + * - Tiles flex between 3:4 (portrait) and wider than 16:9 if needed + * to avoid overflow. No upper bound on aspect ratio — tiles can + * be wider than 16:9 when the container is very wide. + * - When the last row is partial (fewer tiles than a full row), + * those tiles are wider (16:9) while full rows use square-ish + * tiles, filling whitespace per review #1. + * - Only allows overflow (scroll) when tiles would be below + * MIN_TILE_HEIGHT (review #2: ~6-8 tiles on portrait phone). + * + * Prioritises: + * 1. No overflow (tiles shrink to fit within aspect bounds) + * 2. Minimal whitespace + * 3. Aspect ratios close to 16:9 + * 4. Larger tiles + */ +export const computeAdaptiveGrid = ( + tileCount: number, + containerWidth: number, + containerHeight: number, +): AdaptiveTileGrid | undefined => { + if (tileCount <= 0 || containerWidth <= 0 || containerHeight <= 0) return undefined + + let best: AdaptiveTileGrid | undefined + let bestScore: LayoutScore | undefined + + for (let columnCount = 1; columnCount <= tileCount; columnCount++) { + const fullRowCount = Math.floor(tileCount / columnCount) + const remainder = tileCount % columnCount + const totalRowCount = fullRowCount + (remainder > 0 ? 1 : 0) + + let candidate: AdaptiveTileGrid | undefined + + if (remainder === 0) { + // Uniform layout — all rows identical + const row = buildUniformRow(columnCount, containerWidth, containerHeight, totalRowCount) + const totalHeight = totalRowCount * row.tileHeight + (totalRowCount - 1) * TILE_GAP + const totalWidth = columnCount * row.tileWidth + (columnCount - 1) * TILE_GAP + const rows: TileRow[] = [] + for (let r = 0; r < totalRowCount; r++) { + rows.push({...row}) + } + candidate = { + rows, + totalWidth, + totalHeight, + overflows: totalHeight > containerHeight, + } } else { - // In range: fills both dims exactly - tileW = widthLimit - tileH = rowH + // Mixed layout — last row has fewer tiles, make them wider + candidate = buildMixedCandidate(tileCount, columnCount, containerWidth, containerHeight) } - const bboxW = cols * tileW + (cols - 1) * TILE_GAP - const bboxH = rows * tileH + (rows - 1) * TILE_GAP - const vOverflow = Math.max(0, bboxH - ch) - const hOverflow = Math.max(0, bboxW - cw) - const waste = Math.max(0, cw - bboxW) + Math.max(0, ch - bboxH) - const key = [ - Math.round(vOverflow), - Math.round(hOverflow), - Math.round(waste), - Number(Math.abs(aspect - TILE_ASPECT_MAX).toFixed(3)), - -Math.round(tileW * tileH), - ] + if (!candidate) continue - if (!bestKey || compareKeys(key, bestKey) < 0) { - bestKey = key - best = {cols, rows, tileWidth: tileW, tileHeight: tileH, aspect} + // Compute score + const verticalOverflow = Math.max(0, candidate.totalHeight - containerHeight) + const horizontalOverflow = Math.max(0, candidate.totalWidth - containerWidth) + const whitespace = + Math.max(0, containerWidth - candidate.totalWidth) + + Math.max(0, containerHeight - candidate.totalHeight) + const aspectDeviation = + candidate.rows.reduce( + (sum, row) => sum + Math.abs(row.aspectRatio - TILE_ASPECT_LANDSCAPE), + 0, + ) / candidate.rows.length + const totalArea = candidate.rows.reduce( + (sum, row) => sum + row.tileWidth * row.tileHeight * row.columnCount, + 0, + ) + + // For overflow: if tiles are above MIN_TILE_HEIGHT, overflow is + // unacceptable — heavily penalize. If tiles are below MIN_TILE_HEIGHT, + // overflow is acceptable (tiles are already too small to shrink further). + const minTileHeightInCandidate = Math.min(...candidate.rows.map(r => r.tileHeight)) + const overflowPenalty = + verticalOverflow > 0 && minTileHeightInCandidate >= MIN_TILE_HEIGHT + ? verticalOverflow * 1_000_000 + : verticalOverflow + + // Min-tile-height penalty: even when fitting, discourage layouts + // where tiles are below the minimum acceptable size. + const minTileHeightPenalty = + minTileHeightInCandidate < MIN_TILE_HEIGHT + ? (MIN_TILE_HEIGHT - minTileHeightInCandidate) * 1000 + : 0 + + const score: LayoutScore = { + verticalOverflowPenalty: overflowPenalty, + minTileHeightPenalty, + negativeArea: -totalArea, + whitespace, + aspectDeviation, + } + + if (!bestScore || compareScores(score, bestScore) < 0) { + bestScore = score + best = candidate } } diff --git a/src/app/components/VideoCallContent.svelte b/src/app/components/VideoCallContent.svelte index b725682f..0c392ca6 100644 --- a/src/app/components/VideoCallContent.svelte +++ b/src/app/components/VideoCallContent.svelte @@ -24,8 +24,8 @@ participantMediaState, pubkeyFromLiveKitIdentity, videoTrackRevision, - computeUniformGrid, - type UniformTileGrid, + computeAdaptiveGrid, + type AdaptiveTileGrid, } from "@app/call" import {profiles} from "@app/core" @@ -181,8 +181,8 @@ const useSpotlightLayout = $derived(primaryTile !== undefined) const useMultiGrid = $derived(!useSpotlightLayout) - const tileGrid = $derived( - useMultiGrid ? computeUniformGrid(videoTiles.length, gridWidth, gridHeight) : undefined, + const tileGrid = $derived( + useMultiGrid ? computeAdaptiveGrid(videoTiles.length, gridWidth, gridHeight) : undefined, ) $effect(() => { @@ -294,14 +294,18 @@ bind:clientHeight={gridHeight} class="min-h-0 flex-1 overflow-y-auto"> {#if tileGrid} -
- {#each videoTiles as tile (tileKey(tile))} +
+ {#each tileGrid.rows as row, rowIndex}
- {@render videoTile(tile, "default")} + class="flex flex-wrap justify-center gap-2" + style={`max-width: ${row.columnCount * row.tileWidth + (row.columnCount - 1) * 8}px`}> + {#each videoTiles.slice(rowIndex * row.columnCount, rowIndex * row.columnCount + row.columnCount) as tile (tileKey(tile))} +
+ {@render videoTile(tile, "default")} +
+ {/each}
{/each}
From 0d9e511f4125dbdcca1e58d823247240a0eff32e Mon Sep 17 00:00:00 2001 From: Agent Date: Wed, 12 Aug 2026 09:49:24 -0400 Subject: [PATCH 2/2] fix: mixed-row rendering bug and Jitsi-style algorithm refinements --- src/app/call.ts | 53 +++++++++++----------- src/app/components/VideoCallContent.svelte | 5 +- 2 files changed, 30 insertions(+), 28 deletions(-) diff --git a/src/app/call.ts b/src/app/call.ts index 86fb1033..c16ab812 100644 --- a/src/app/call.ts +++ b/src/app/call.ts @@ -85,6 +85,8 @@ export type AdaptiveTileGrid = { type LayoutScore = { /** Penalty for vertical overflow: 0 if none, else huge */ verticalOverflowPenalty: number + /** Penalty for horizontal overflow: 0 if none, else huge */ + horizontalOverflowPenalty: number /** Penalty when tiles fall below MIN_TILE_HEIGHT even while fitting */ minTileHeightPenalty: number /** Negative tile area (we want to maximize area, so negate) */ @@ -98,6 +100,7 @@ type LayoutScore = { const compareScores = (a: LayoutScore, b: LayoutScore): number => { const fields: (keyof LayoutScore)[] = [ "verticalOverflowPenalty", + "horizontalOverflowPenalty", "minTileHeightPenalty", "negativeArea", "whitespace", @@ -125,8 +128,8 @@ const buildUniformRow = ( const rowHeight = availableHeight / rowCount const tileWidth = (containerWidth - (columnCount - 1) * gap) / columnCount const fillAspect = tileWidth / rowHeight - // No upper bound — allow wider than 16:9 to avoid overflow - const aspectRatio = Math.max(TILE_ASPECT_PORTRAIT, fillAspect) + // Cap at 16:9 to avoid horizontally stretching video (review feedback) + const aspectRatio = Math.max(TILE_ASPECT_PORTRAIT, Math.min(TILE_ASPECT_LANDSCAPE, fillAspect)) const tileHeight = tileWidth / aspectRatio return {columnCount, tileWidth, tileHeight, aspectRatio} } @@ -160,31 +163,21 @@ const buildMixedCandidate = ( const tileWidthLast = (containerWidth - (remainder - 1) * gap) / remainder if (tileWidthLast <= 0) return undefined - // Last row at 16:9 (widest aspect, per review: "single centered 16:9 tile") - const lastRowAspect = TILE_ASPECT_LANDSCAPE - const commonHeight = tileWidthLast / lastRowAspect + // Find a common height that fits both row types and container + const maxPossibleHeight = Math.min( + tileWidthFull / TILE_ASPECT_PORTRAIT, + tileWidthLast / TILE_ASPECT_PORTRAIT, + (containerHeight - (totalRowCount - 1) * gap) / totalRowCount, + ) - // Full-row tile that matches the common height - const fullRowAspect = tileWidthFull / commonHeight + // Target 16:9 for the last row if possible + const targetHeight = tileWidthLast / TILE_ASPECT_LANDSCAPE + const commonHeight = Math.min(maxPossibleHeight, targetHeight) - let tileHeightFull: number - let tileHeightLast: number - let fullRowAspectRatio: number - let lastRowAspectRatio: number - - if (fullRowAspect >= TILE_ASPECT_PORTRAIT) { - // Full-row aspect is within bounds — match heights cleanly - tileHeightFull = commonHeight - tileHeightLast = commonHeight - fullRowAspectRatio = fullRowAspect - lastRowAspectRatio = lastRowAspect - } else { - // Full rows would be too portrait — clamp to portrait bound - fullRowAspectRatio = TILE_ASPECT_PORTRAIT - tileHeightFull = tileWidthFull / fullRowAspectRatio - tileHeightLast = commonHeight - lastRowAspectRatio = lastRowAspect - } + const tileHeightFull = commonHeight + const tileHeightLast = commonHeight + const fullRowAspectRatio = tileWidthFull / commonHeight + const lastRowAspectRatio = tileWidthLast / commonHeight const totalHeight = fullRowCount * tileHeightFull + tileHeightLast + (totalRowCount - 1) * gap const totalWidth = Math.max( @@ -293,11 +286,16 @@ export const computeAdaptiveGrid = ( // unacceptable — heavily penalize. If tiles are below MIN_TILE_HEIGHT, // overflow is acceptable (tiles are already too small to shrink further). const minTileHeightInCandidate = Math.min(...candidate.rows.map(r => r.tileHeight)) - const overflowPenalty = + const verticalOverflowPenalty = verticalOverflow > 0 && minTileHeightInCandidate >= MIN_TILE_HEIGHT ? verticalOverflow * 1_000_000 : verticalOverflow + const horizontalOverflowPenalty = + horizontalOverflow > 0 && minTileHeightInCandidate >= MIN_TILE_HEIGHT + ? horizontalOverflow * 1_000_000 + : horizontalOverflow + // Min-tile-height penalty: even when fitting, discourage layouts // where tiles are below the minimum acceptable size. const minTileHeightPenalty = @@ -306,7 +304,8 @@ export const computeAdaptiveGrid = ( : 0 const score: LayoutScore = { - verticalOverflowPenalty: overflowPenalty, + verticalOverflowPenalty, + horizontalOverflowPenalty, minTileHeightPenalty, negativeArea: -totalArea, whitespace, diff --git a/src/app/components/VideoCallContent.svelte b/src/app/components/VideoCallContent.svelte index 0c392ca6..72a180e7 100644 --- a/src/app/components/VideoCallContent.svelte +++ b/src/app/components/VideoCallContent.svelte @@ -296,10 +296,13 @@ {#if tileGrid}
{#each tileGrid.rows as row, rowIndex} + {@const offset = tileGrid.rows + .slice(0, rowIndex) + .reduce((sum, r) => sum + r.columnCount, 0)}
- {#each videoTiles.slice(rowIndex * row.columnCount, rowIndex * row.columnCount + row.columnCount) as tile (tileKey(tile))} + {#each videoTiles.slice(offset, offset + row.columnCount) as tile (tileKey(tile))}