fix(player): composite multi-zone layouts in screenshot/stream capture

captureAndSend() grabbed a single querySelector('video'|'img') and stretched it
across a fixed 960x540 canvas, so multi-zone Now-Playing screenshots and the 1fps
remote stream showed one zone stretched fullscreen instead of the actual layout.

- Multi-zone layouts now composite each zone from its REAL rendered geometry
  (getBoundingClientRect relative to the container, scaled proportionally onto the
  canvas), so positions/sizes stay true to the layout.
- Canvas height derives from the container aspect (not a hardcoded 540); media is
  drawn honouring its object-fit (cover/contain/fill) instead of being stretched.
- Cross-origin / iframe zones (YouTube, widgets) can't be read back without
  CORS-tainting the whole canvas (which makes toDataURL throw and kills the entire
  capture). They now get a deliberate, labelled placeholder ("YouTube"/"Widget"/
  "Video") so the shot still shows the layout structure with that zone marked,
  instead of a transparent hole or a failed capture.
- Split rendering into renderCaptureCanvas() (socket-free, headlessly verifiable)
  and captureAndSend() (encode + emit). One full-quality path serves BOTH the
  on-demand screenshot and the 1fps stream — the composite is only a few drawImage
  calls over already-decoded media, so no separate low-quality stream path.

Web player only; Android (view.draw already composites correctly) untouched.
Verified headlessly on a 3-zone device: red/green image zones render in their
correct positions, the YouTube zone shows a labelled placeholder, and the capture
succeeds with no CORS taint.

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
ScreenTinker 2026-06-22 23:12:21 -05:00
parent 184f07c272
commit 0ebbd20968

View file

@ -1880,69 +1880,168 @@
} }
// ==================== Screenshots ==================== // ==================== Screenshots ====================
function captureAndSend() { // Draw a media element into a destination rect honouring its object-fit, so the capture
if (!socket?.connected) return; // matches what's on screen instead of stretching the source to a fixed size (the old
// bug). 'cover' crops the source; 'contain' letterboxes; 'fill' stretches.
function drawMediaFit(ctx, el, ew, eh, dx, dy, dw, dh, fit) {
if (!ew || !eh) { ctx.drawImage(el, dx, dy, dw, dh); return; }
if (fit === 'fill') { ctx.drawImage(el, dx, dy, dw, dh); return; }
const er = ew / eh, dr = dw / dh;
if (fit === 'contain') {
let w = dw, h = dh;
if (er > dr) h = dw / er; else w = dh * er;
ctx.drawImage(el, dx + (dw - w) / 2, dy + (dh - h) / 2, w, h);
} else { // 'cover' (zone default) and anything unexpected -> crop to fill, no distortion
let sw = ew, sh = eh, sx = 0, sy = 0;
if (er > dr) { sw = eh * dr; sx = (ew - sw) / 2; }
else { sh = ew / dr; sy = (eh - sh) / 2; }
ctx.drawImage(el, sx, sy, sw, sh, dx, dy, dw, dh);
}
}
// A cross-origin <img>/<video> drawn onto the canvas without CORS taints the WHOLE
// canvas, making toDataURL() throw and killing the entire capture. Only same-origin
// media (served by us) or media explicitly loaded with crossOrigin is safe to read back.
function isMediaReadable(el) {
const url = el.currentSrc || el.src || '';
if (!url) return false;
if (el.crossOrigin) return true;
try { return new URL(url, location.href).origin === location.origin; }
catch (e) { return false; }
}
function zonePlaceholderLabel(el) {
if (!el) return 'Live';
if (el.tagName === 'IFRAME') {
const s = el.src || '';
if (/youtube|ytimg|youtu\.be/i.test(s)) return 'YouTube';
if (/\/widgets\//i.test(s)) return 'Widget';
return 'Web';
}
if (el.tagName === 'VIDEO') return 'Video';
return 'Live';
}
// Deliberate, labelled placeholder for a zone we can't read back (cross-origin iframe
// like YouTube/widgets, or cross-origin media). The shot still shows the layout
// structure with this zone clearly marked — never a transparent hole.
function drawZonePlaceholder(ctx, dx, dy, dw, dh, label) {
ctx.save();
ctx.fillStyle = '#1f2937';
ctx.fillRect(dx, dy, dw, dh);
ctx.strokeStyle = 'rgba(148,163,184,0.35)';
ctx.lineWidth = 1;
ctx.strokeRect(dx + 0.5, dy + 0.5, Math.max(0, dw - 1), Math.max(0, dh - 1));
if (label) {
ctx.fillStyle = '#cbd5e1';
const fs = Math.max(11, Math.min(22, Math.round(dh * 0.16)));
ctx.font = `600 ${fs}px sans-serif`;
ctx.textAlign = 'center';
ctx.textBaseline = 'middle';
ctx.fillText(label, dx + dw / 2, dy + dh / 2);
}
ctx.restore();
}
// Composite a multi-zone layout zone-by-zone. Each zone's destination rect is derived
// from its REAL rendered geometry (getBoundingClientRect relative to the container) and
// scaled proportionally onto the canvas — so positions/sizes stay true to the layout
// rather than one element stretched across the frame.
function drawZoneComposite(ctx, container, cr, W, H) {
ctx.fillStyle = '#000';
ctx.fillRect(0, 0, W, H);
const zones = container.querySelectorAll('.zone');
if (!zones.length) return false;
zones.forEach((zd) => {
const r = zd.getBoundingClientRect();
const dx = ((r.left - cr.left) / cr.width) * W;
const dy = ((r.top - cr.top) / cr.height) * H;
const dw = (r.width / cr.width) * W;
const dh = (r.height / cr.height) * H;
const el = zd.querySelector('video, img, iframe');
let drawn = false;
if (el && el.tagName === 'IMG' && el.complete && el.naturalWidth > 0 && isMediaReadable(el)) {
try { drawMediaFit(ctx, el, el.naturalWidth, el.naturalHeight, dx, dy, dw, dh, getComputedStyle(el).objectFit); drawn = true; } catch (e) {}
} else if (el && el.tagName === 'VIDEO' && el.readyState >= 2 && el.videoWidth > 0 && isMediaReadable(el)) {
try { drawMediaFit(ctx, el, el.videoWidth, el.videoHeight, dx, dy, dw, dh, getComputedStyle(el).objectFit); drawn = true; } catch (e) {}
}
if (!drawn) drawZonePlaceholder(ctx, dx, dy, dw, dh, zonePlaceholderLabel(el));
});
return true;
}
// Build the screenshot/stream canvas and return it (caller encodes + sends). Exposed as
// a plain function so a headless render pass can verify the composite without a socket.
function renderCaptureCanvas() {
const container = document.getElementById('playerContainer');
const cr = container ? container.getBoundingClientRect() : null;
// ~960 on the long edge; height from the REAL container aspect, not a hardcoded 540,
// so non-16:9 layouts compose without distortion.
const aspect = (cr && cr.width > 0 && cr.height > 0) ? (cr.width / cr.height) : (16 / 9);
const canvas = document.createElement('canvas'); const canvas = document.createElement('canvas');
canvas.width = 960; canvas.width = 960;
canvas.height = 540; canvas.height = Math.max(1, Math.round(960 / aspect));
const ctx = canvas.getContext('2d'); const ctx = canvas.getContext('2d');
const W = canvas.width, H = canvas.height;
let captured = false; let captured = false;
try { try {
const container = document.getElementById('playerContainer'); const multiZone = !!(layout && Array.isArray(layout.zones) && layout.zones.length > 1 && !wallConfig);
const video = container?.querySelector('video'); if (multiZone && container) {
const img = container?.querySelector('img'); captured = drawZoneComposite(ctx, container, cr, W, H);
} else if (container) {
// Try video first // Single-zone / fullscreen fast path: one element, drawn with its own object-fit
if (video && video.readyState >= 2 && video.videoWidth > 0) { // (the old code stretched it to a fixed 960x540).
try { const video = container.querySelector('video');
ctx.drawImage(video, 0, 0, 960, 540); const img = container.querySelector('img');
captured = true; if (video && video.readyState >= 2 && video.videoWidth > 0 && isMediaReadable(video)) {
} catch (e) { try { drawMediaFit(ctx, video, video.videoWidth, video.videoHeight, 0, 0, W, H, getComputedStyle(video).objectFit); captured = true; } catch (e) { console.warn('Video capture failed (CORS?):', e.message); }
console.warn('Video capture failed (CORS?):', e.message);
} }
} if (!captured && img && img.complete && img.naturalWidth > 0 && isMediaReadable(img)) {
try { drawMediaFit(ctx, img, img.naturalWidth, img.naturalHeight, 0, 0, W, H, getComputedStyle(img).objectFit); captured = true; } catch (e) { console.warn('Image capture failed:', e.message); }
// Try image
if (!captured && img && img.complete && img.naturalWidth > 0) {
try {
ctx.drawImage(img, 0, 0, 960, 540);
captured = true;
} catch (e) {
console.warn('Image capture failed:', e.message);
} }
} }
// Fallback: draw status info // Fallback: draw status info
if (!captured) { if (!captured) {
ctx.fillStyle = '#111827'; ctx.fillStyle = '#111827';
ctx.fillRect(0, 0, 960, 540); ctx.fillRect(0, 0, W, H);
ctx.fillStyle = '#3b82f6'; ctx.fillStyle = '#3b82f6';
ctx.font = 'bold 28px sans-serif'; ctx.font = 'bold 28px sans-serif';
ctx.textAlign = 'center'; ctx.textAlign = 'center';
ctx.fillText('ScreenTinker Web Player', 480, 230); ctx.fillText('ScreenTinker Web Player', W / 2, H / 2 - 40);
ctx.fillStyle = '#94a3b8'; ctx.fillStyle = '#94a3b8';
ctx.font = '16px sans-serif'; ctx.font = '16px sans-serif';
const item = playlist[currentIndex]; const item = playlist[currentIndex];
ctx.fillText(item ? `Playing: ${item.filename}` : 'No content', 480, 270); ctx.fillText(item ? `Playing: ${item.filename}` : 'No content', W / 2, H / 2);
ctx.fillText(`${config.deviceName || 'Web Player'} | ${new Date().toLocaleTimeString()}`, 480, 310); ctx.fillText(`${config.deviceName || 'Web Player'} | ${new Date().toLocaleTimeString()}`, W / 2, H / 2 + 40);
} }
} catch (e) { } catch (e) {
// Even on error, draw something // Even on error, draw something
ctx.fillStyle = '#000'; ctx.fillStyle = '#000';
ctx.fillRect(0, 0, 960, 540); ctx.fillRect(0, 0, W, H);
ctx.fillStyle = '#ef4444'; ctx.fillStyle = '#ef4444';
ctx.font = '16px sans-serif'; ctx.font = '16px sans-serif';
ctx.textAlign = 'center'; ctx.textAlign = 'center';
ctx.fillText('Screenshot error: ' + e.message, 480, 270); ctx.fillText('Screenshot error: ' + e.message, W / 2, H / 2);
} }
return canvas;
}
function captureAndSend() {
if (!socket?.connected) return;
// Also drives the 1fps remote stream (startStreaming). The composite is just a handful
// of drawImage calls over already-decoded media, so one full-quality path serves both
// the on-demand screenshot and the 1fps stream — no separate low-quality stream path.
let canvas;
try { canvas = renderCaptureCanvas(); }
catch (e) { console.error('Screenshot render failed:', e); return; }
try { try {
const dataUrl = canvas.toDataURL('image/jpeg', 0.4); const dataUrl = canvas.toDataURL('image/jpeg', 0.4);
const base64 = dataUrl.split(',')[1]; const base64 = dataUrl.split(',')[1];
if (base64 && base64.length > 100) { if (base64 && base64.length > 100) {
socket.emit('device:screenshot', { device_id: config.deviceId, image_b64: base64 }); socket.emit('device:screenshot', { device_id: config.deviceId, image_b64: base64 });
console.log('Screenshot sent:', base64.length, 'chars', captured ? '(content)' : '(fallback)'); console.log('Screenshot sent:', base64.length, 'chars');
} }
} catch (e) { } catch (e) {
console.error('Screenshot encode/send failed:', e); console.error('Screenshot encode/send failed:', e);