mirror of
https://github.com/screentinker/screentinker.git
synced 2026-06-29 09:23:16 -06:00
fix(player): composite multi-zone layouts in screenshot/stream capture
captureAndSend() grabbed a single querySelector('video'|'img') and stretched it
across a fixed 960x540 canvas, so multi-zone Now-Playing screenshots and the 1fps
remote stream showed one zone stretched fullscreen instead of the actual layout.
- Multi-zone layouts now composite each zone from its REAL rendered geometry
(getBoundingClientRect relative to the container, scaled proportionally onto the
canvas), so positions/sizes stay true to the layout.
- Canvas height derives from the container aspect (not a hardcoded 540); media is
drawn honouring its object-fit (cover/contain/fill) instead of being stretched.
- Cross-origin / iframe zones (YouTube, widgets) can't be read back without
CORS-tainting the whole canvas (which makes toDataURL throw and kills the entire
capture). They now get a deliberate, labelled placeholder ("YouTube"/"Widget"/
"Video") so the shot still shows the layout structure with that zone marked,
instead of a transparent hole or a failed capture.
- Split rendering into renderCaptureCanvas() (socket-free, headlessly verifiable)
and captureAndSend() (encode + emit). One full-quality path serves BOTH the
on-demand screenshot and the 1fps stream — the composite is only a few drawImage
calls over already-decoded media, so no separate low-quality stream path.
Web player only; Android (view.draw already composites correctly) untouched.
Verified headlessly on a 3-zone device: red/green image zones render in their
correct positions, the YouTube zone shows a labelled placeholder, and the capture
succeeds with no CORS taint.
Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
parent
184f07c272
commit
0ebbd20968
|
|
@ -1880,69 +1880,168 @@
|
||||||
}
|
}
|
||||||
|
|
||||||
// ==================== Screenshots ====================
|
// ==================== Screenshots ====================
|
||||||
function captureAndSend() {
|
// Draw a media element into a destination rect honouring its object-fit, so the capture
|
||||||
if (!socket?.connected) return;
|
// matches what's on screen instead of stretching the source to a fixed size (the old
|
||||||
|
// bug). 'cover' crops the source; 'contain' letterboxes; 'fill' stretches.
|
||||||
|
function drawMediaFit(ctx, el, ew, eh, dx, dy, dw, dh, fit) {
|
||||||
|
if (!ew || !eh) { ctx.drawImage(el, dx, dy, dw, dh); return; }
|
||||||
|
if (fit === 'fill') { ctx.drawImage(el, dx, dy, dw, dh); return; }
|
||||||
|
const er = ew / eh, dr = dw / dh;
|
||||||
|
if (fit === 'contain') {
|
||||||
|
let w = dw, h = dh;
|
||||||
|
if (er > dr) h = dw / er; else w = dh * er;
|
||||||
|
ctx.drawImage(el, dx + (dw - w) / 2, dy + (dh - h) / 2, w, h);
|
||||||
|
} else { // 'cover' (zone default) and anything unexpected -> crop to fill, no distortion
|
||||||
|
let sw = ew, sh = eh, sx = 0, sy = 0;
|
||||||
|
if (er > dr) { sw = eh * dr; sx = (ew - sw) / 2; }
|
||||||
|
else { sh = ew / dr; sy = (eh - sh) / 2; }
|
||||||
|
ctx.drawImage(el, sx, sy, sw, sh, dx, dy, dw, dh);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// A cross-origin <img>/<video> drawn onto the canvas without CORS taints the WHOLE
|
||||||
|
// canvas, making toDataURL() throw and killing the entire capture. Only same-origin
|
||||||
|
// media (served by us) or media explicitly loaded with crossOrigin is safe to read back.
|
||||||
|
function isMediaReadable(el) {
|
||||||
|
const url = el.currentSrc || el.src || '';
|
||||||
|
if (!url) return false;
|
||||||
|
if (el.crossOrigin) return true;
|
||||||
|
try { return new URL(url, location.href).origin === location.origin; }
|
||||||
|
catch (e) { return false; }
|
||||||
|
}
|
||||||
|
|
||||||
|
function zonePlaceholderLabel(el) {
|
||||||
|
if (!el) return 'Live';
|
||||||
|
if (el.tagName === 'IFRAME') {
|
||||||
|
const s = el.src || '';
|
||||||
|
if (/youtube|ytimg|youtu\.be/i.test(s)) return 'YouTube';
|
||||||
|
if (/\/widgets\//i.test(s)) return 'Widget';
|
||||||
|
return 'Web';
|
||||||
|
}
|
||||||
|
if (el.tagName === 'VIDEO') return 'Video';
|
||||||
|
return 'Live';
|
||||||
|
}
|
||||||
|
|
||||||
|
// Deliberate, labelled placeholder for a zone we can't read back (cross-origin iframe
|
||||||
|
// like YouTube/widgets, or cross-origin media). The shot still shows the layout
|
||||||
|
// structure with this zone clearly marked — never a transparent hole.
|
||||||
|
function drawZonePlaceholder(ctx, dx, dy, dw, dh, label) {
|
||||||
|
ctx.save();
|
||||||
|
ctx.fillStyle = '#1f2937';
|
||||||
|
ctx.fillRect(dx, dy, dw, dh);
|
||||||
|
ctx.strokeStyle = 'rgba(148,163,184,0.35)';
|
||||||
|
ctx.lineWidth = 1;
|
||||||
|
ctx.strokeRect(dx + 0.5, dy + 0.5, Math.max(0, dw - 1), Math.max(0, dh - 1));
|
||||||
|
if (label) {
|
||||||
|
ctx.fillStyle = '#cbd5e1';
|
||||||
|
const fs = Math.max(11, Math.min(22, Math.round(dh * 0.16)));
|
||||||
|
ctx.font = `600 ${fs}px sans-serif`;
|
||||||
|
ctx.textAlign = 'center';
|
||||||
|
ctx.textBaseline = 'middle';
|
||||||
|
ctx.fillText(label, dx + dw / 2, dy + dh / 2);
|
||||||
|
}
|
||||||
|
ctx.restore();
|
||||||
|
}
|
||||||
|
|
||||||
|
// Composite a multi-zone layout zone-by-zone. Each zone's destination rect is derived
|
||||||
|
// from its REAL rendered geometry (getBoundingClientRect relative to the container) and
|
||||||
|
// scaled proportionally onto the canvas — so positions/sizes stay true to the layout
|
||||||
|
// rather than one element stretched across the frame.
|
||||||
|
function drawZoneComposite(ctx, container, cr, W, H) {
|
||||||
|
ctx.fillStyle = '#000';
|
||||||
|
ctx.fillRect(0, 0, W, H);
|
||||||
|
const zones = container.querySelectorAll('.zone');
|
||||||
|
if (!zones.length) return false;
|
||||||
|
zones.forEach((zd) => {
|
||||||
|
const r = zd.getBoundingClientRect();
|
||||||
|
const dx = ((r.left - cr.left) / cr.width) * W;
|
||||||
|
const dy = ((r.top - cr.top) / cr.height) * H;
|
||||||
|
const dw = (r.width / cr.width) * W;
|
||||||
|
const dh = (r.height / cr.height) * H;
|
||||||
|
const el = zd.querySelector('video, img, iframe');
|
||||||
|
let drawn = false;
|
||||||
|
if (el && el.tagName === 'IMG' && el.complete && el.naturalWidth > 0 && isMediaReadable(el)) {
|
||||||
|
try { drawMediaFit(ctx, el, el.naturalWidth, el.naturalHeight, dx, dy, dw, dh, getComputedStyle(el).objectFit); drawn = true; } catch (e) {}
|
||||||
|
} else if (el && el.tagName === 'VIDEO' && el.readyState >= 2 && el.videoWidth > 0 && isMediaReadable(el)) {
|
||||||
|
try { drawMediaFit(ctx, el, el.videoWidth, el.videoHeight, dx, dy, dw, dh, getComputedStyle(el).objectFit); drawn = true; } catch (e) {}
|
||||||
|
}
|
||||||
|
if (!drawn) drawZonePlaceholder(ctx, dx, dy, dw, dh, zonePlaceholderLabel(el));
|
||||||
|
});
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
|
||||||
|
// Build the screenshot/stream canvas and return it (caller encodes + sends). Exposed as
|
||||||
|
// a plain function so a headless render pass can verify the composite without a socket.
|
||||||
|
function renderCaptureCanvas() {
|
||||||
|
const container = document.getElementById('playerContainer');
|
||||||
|
const cr = container ? container.getBoundingClientRect() : null;
|
||||||
|
// ~960 on the long edge; height from the REAL container aspect, not a hardcoded 540,
|
||||||
|
// so non-16:9 layouts compose without distortion.
|
||||||
|
const aspect = (cr && cr.width > 0 && cr.height > 0) ? (cr.width / cr.height) : (16 / 9);
|
||||||
const canvas = document.createElement('canvas');
|
const canvas = document.createElement('canvas');
|
||||||
canvas.width = 960;
|
canvas.width = 960;
|
||||||
canvas.height = 540;
|
canvas.height = Math.max(1, Math.round(960 / aspect));
|
||||||
const ctx = canvas.getContext('2d');
|
const ctx = canvas.getContext('2d');
|
||||||
|
const W = canvas.width, H = canvas.height;
|
||||||
let captured = false;
|
let captured = false;
|
||||||
|
|
||||||
try {
|
try {
|
||||||
const container = document.getElementById('playerContainer');
|
const multiZone = !!(layout && Array.isArray(layout.zones) && layout.zones.length > 1 && !wallConfig);
|
||||||
const video = container?.querySelector('video');
|
if (multiZone && container) {
|
||||||
const img = container?.querySelector('img');
|
captured = drawZoneComposite(ctx, container, cr, W, H);
|
||||||
|
} else if (container) {
|
||||||
// Try video first
|
// Single-zone / fullscreen fast path: one element, drawn with its own object-fit
|
||||||
if (video && video.readyState >= 2 && video.videoWidth > 0) {
|
// (the old code stretched it to a fixed 960x540).
|
||||||
try {
|
const video = container.querySelector('video');
|
||||||
ctx.drawImage(video, 0, 0, 960, 540);
|
const img = container.querySelector('img');
|
||||||
captured = true;
|
if (video && video.readyState >= 2 && video.videoWidth > 0 && isMediaReadable(video)) {
|
||||||
} catch (e) {
|
try { drawMediaFit(ctx, video, video.videoWidth, video.videoHeight, 0, 0, W, H, getComputedStyle(video).objectFit); captured = true; } catch (e) { console.warn('Video capture failed (CORS?):', e.message); }
|
||||||
console.warn('Video capture failed (CORS?):', e.message);
|
|
||||||
}
|
}
|
||||||
}
|
if (!captured && img && img.complete && img.naturalWidth > 0 && isMediaReadable(img)) {
|
||||||
|
try { drawMediaFit(ctx, img, img.naturalWidth, img.naturalHeight, 0, 0, W, H, getComputedStyle(img).objectFit); captured = true; } catch (e) { console.warn('Image capture failed:', e.message); }
|
||||||
// Try image
|
|
||||||
if (!captured && img && img.complete && img.naturalWidth > 0) {
|
|
||||||
try {
|
|
||||||
ctx.drawImage(img, 0, 0, 960, 540);
|
|
||||||
captured = true;
|
|
||||||
} catch (e) {
|
|
||||||
console.warn('Image capture failed:', e.message);
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// Fallback: draw status info
|
// Fallback: draw status info
|
||||||
if (!captured) {
|
if (!captured) {
|
||||||
ctx.fillStyle = '#111827';
|
ctx.fillStyle = '#111827';
|
||||||
ctx.fillRect(0, 0, 960, 540);
|
ctx.fillRect(0, 0, W, H);
|
||||||
ctx.fillStyle = '#3b82f6';
|
ctx.fillStyle = '#3b82f6';
|
||||||
ctx.font = 'bold 28px sans-serif';
|
ctx.font = 'bold 28px sans-serif';
|
||||||
ctx.textAlign = 'center';
|
ctx.textAlign = 'center';
|
||||||
ctx.fillText('ScreenTinker Web Player', 480, 230);
|
ctx.fillText('ScreenTinker Web Player', W / 2, H / 2 - 40);
|
||||||
ctx.fillStyle = '#94a3b8';
|
ctx.fillStyle = '#94a3b8';
|
||||||
ctx.font = '16px sans-serif';
|
ctx.font = '16px sans-serif';
|
||||||
const item = playlist[currentIndex];
|
const item = playlist[currentIndex];
|
||||||
ctx.fillText(item ? `Playing: ${item.filename}` : 'No content', 480, 270);
|
ctx.fillText(item ? `Playing: ${item.filename}` : 'No content', W / 2, H / 2);
|
||||||
ctx.fillText(`${config.deviceName || 'Web Player'} | ${new Date().toLocaleTimeString()}`, 480, 310);
|
ctx.fillText(`${config.deviceName || 'Web Player'} | ${new Date().toLocaleTimeString()}`, W / 2, H / 2 + 40);
|
||||||
}
|
}
|
||||||
} catch (e) {
|
} catch (e) {
|
||||||
// Even on error, draw something
|
// Even on error, draw something
|
||||||
ctx.fillStyle = '#000';
|
ctx.fillStyle = '#000';
|
||||||
ctx.fillRect(0, 0, 960, 540);
|
ctx.fillRect(0, 0, W, H);
|
||||||
ctx.fillStyle = '#ef4444';
|
ctx.fillStyle = '#ef4444';
|
||||||
ctx.font = '16px sans-serif';
|
ctx.font = '16px sans-serif';
|
||||||
ctx.textAlign = 'center';
|
ctx.textAlign = 'center';
|
||||||
ctx.fillText('Screenshot error: ' + e.message, 480, 270);
|
ctx.fillText('Screenshot error: ' + e.message, W / 2, H / 2);
|
||||||
}
|
}
|
||||||
|
return canvas;
|
||||||
|
}
|
||||||
|
|
||||||
|
function captureAndSend() {
|
||||||
|
if (!socket?.connected) return;
|
||||||
|
// Also drives the 1fps remote stream (startStreaming). The composite is just a handful
|
||||||
|
// of drawImage calls over already-decoded media, so one full-quality path serves both
|
||||||
|
// the on-demand screenshot and the 1fps stream — no separate low-quality stream path.
|
||||||
|
let canvas;
|
||||||
|
try { canvas = renderCaptureCanvas(); }
|
||||||
|
catch (e) { console.error('Screenshot render failed:', e); return; }
|
||||||
try {
|
try {
|
||||||
const dataUrl = canvas.toDataURL('image/jpeg', 0.4);
|
const dataUrl = canvas.toDataURL('image/jpeg', 0.4);
|
||||||
const base64 = dataUrl.split(',')[1];
|
const base64 = dataUrl.split(',')[1];
|
||||||
if (base64 && base64.length > 100) {
|
if (base64 && base64.length > 100) {
|
||||||
socket.emit('device:screenshot', { device_id: config.deviceId, image_b64: base64 });
|
socket.emit('device:screenshot', { device_id: config.deviceId, image_b64: base64 });
|
||||||
console.log('Screenshot sent:', base64.length, 'chars', captured ? '(content)' : '(fallback)');
|
console.log('Screenshot sent:', base64.length, 'chars');
|
||||||
}
|
}
|
||||||
} catch (e) {
|
} catch (e) {
|
||||||
console.error('Screenshot encode/send failed:', e);
|
console.error('Screenshot encode/send failed:', e);
|
||||||
|
|
|
||||||
Loading…
Reference in a new issue