From 0ebbd209682f7ee146967ef8c5a4d2e39dfe3e30 Mon Sep 17 00:00:00 2001
From: ScreenTinker <hello@screentinker.com>
Date: Mon, 22 Jun 2026 23:12:21 -0500
Subject: [PATCH] fix(player): composite multi-zone layouts in
 screenshot/stream capture
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

captureAndSend() grabbed a single querySelector('video'|'img') and stretched it
across a fixed 960x540 canvas, so multi-zone Now-Playing screenshots and the 1fps
remote stream showed one zone stretched fullscreen instead of the actual layout.

- Multi-zone layouts now composite each zone from its REAL rendered geometry
  (getBoundingClientRect relative to the container, scaled proportionally onto the
  canvas), so positions/sizes stay true to the layout.
- Canvas height derives from the container aspect (not a hardcoded 540); media is
  drawn honouring its object-fit (cover/contain/fill) instead of being stretched.
- Cross-origin / iframe zones (YouTube, widgets) can't be read back without
  CORS-tainting the whole canvas (which makes toDataURL throw and kills the entire
  capture). They now get a deliberate, labelled placeholder ("YouTube"/"Widget"/
  "Video") so the shot still shows the layout structure with that zone marked,
  instead of a transparent hole or a failed capture.
- Split rendering into renderCaptureCanvas() (socket-free, headlessly verifiable)
  and captureAndSend() (encode + emit). One full-quality path serves BOTH the
  on-demand screenshot and the 1fps stream — the composite is only a few drawImage
  calls over already-decoded media, so no separate low-quality stream path.

Web player only; Android (view.draw already composites correctly) untouched.
Verified headlessly on a 3-zone device: red/green image zones render in their
correct positions, the YouTube zone shows a labelled placeholder, and the capture
succeeds with no CORS taint.

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
---
 server/player/index.html | 159 +++++++++++++++++++++++++++++++--------
 1 file changed, 129 insertions(+), 30 deletions(-)

diff --git a/server/player/index.html b/server/player/index.html
index f1a25ed..de236ff 100644
--- a/server/player/index.html
+++ b/server/player/index.html
@@ -1880,69 +1880,168 @@
     }
 
     // ==================== Screenshots ====================
-    function captureAndSend() {
-      if (!socket?.connected) return;
+    // Draw a media element into a destination rect honouring its object-fit, so the capture
+    // matches what's on screen instead of stretching the source to a fixed size (the old
+    // bug). 'cover' crops the source; 'contain' letterboxes; 'fill' stretches.
+    function drawMediaFit(ctx, el, ew, eh, dx, dy, dw, dh, fit) {
+      if (!ew || !eh) { ctx.drawImage(el, dx, dy, dw, dh); return; }
+      if (fit === 'fill') { ctx.drawImage(el, dx, dy, dw, dh); return; }
+      const er = ew / eh, dr = dw / dh;
+      if (fit === 'contain') {
+        let w = dw, h = dh;
+        if (er > dr) h = dw / er; else w = dh * er;
+        ctx.drawImage(el, dx + (dw - w) / 2, dy + (dh - h) / 2, w, h);
+      } else { // 'cover' (zone default) and anything unexpected -> crop to fill, no distortion
+        let sw = ew, sh = eh, sx = 0, sy = 0;
+        if (er > dr) { sw = eh * dr; sx = (ew - sw) / 2; }
+        else { sh = ew / dr; sy = (eh - sh) / 2; }
+        ctx.drawImage(el, sx, sy, sw, sh, dx, dy, dw, dh);
+      }
+    }
+
+    // A cross-origin <img>/<video> drawn onto the canvas without CORS taints the WHOLE
+    // canvas, making toDataURL() throw and killing the entire capture. Only same-origin
+    // media (served by us) or media explicitly loaded with crossOrigin is safe to read back.
+    function isMediaReadable(el) {
+      const url = el.currentSrc || el.src || '';
+      if (!url) return false;
+      if (el.crossOrigin) return true;
+      try { return new URL(url, location.href).origin === location.origin; }
+      catch (e) { return false; }
+    }
+
+    function zonePlaceholderLabel(el) {
+      if (!el) return 'Live';
+      if (el.tagName === 'IFRAME') {
+        const s = el.src || '';
+        if (/youtube|ytimg|youtu\.be/i.test(s)) return 'YouTube';
+        if (/\/widgets\//i.test(s)) return 'Widget';
+        return 'Web';
+      }
+      if (el.tagName === 'VIDEO') return 'Video';
+      return 'Live';
+    }
+
+    // Deliberate, labelled placeholder for a zone we can't read back (cross-origin iframe
+    // like YouTube/widgets, or cross-origin media). The shot still shows the layout
+    // structure with this zone clearly marked — never a transparent hole.
+    function drawZonePlaceholder(ctx, dx, dy, dw, dh, label) {
+      ctx.save();
+      ctx.fillStyle = '#1f2937';
+      ctx.fillRect(dx, dy, dw, dh);
+      ctx.strokeStyle = 'rgba(148,163,184,0.35)';
+      ctx.lineWidth = 1;
+      ctx.strokeRect(dx + 0.5, dy + 0.5, Math.max(0, dw - 1), Math.max(0, dh - 1));
+      if (label) {
+        ctx.fillStyle = '#cbd5e1';
+        const fs = Math.max(11, Math.min(22, Math.round(dh * 0.16)));
+        ctx.font = `600 ${fs}px sans-serif`;
+        ctx.textAlign = 'center';
+        ctx.textBaseline = 'middle';
+        ctx.fillText(label, dx + dw / 2, dy + dh / 2);
+      }
+      ctx.restore();
+    }
+
+    // Composite a multi-zone layout zone-by-zone. Each zone's destination rect is derived
+    // from its REAL rendered geometry (getBoundingClientRect relative to the container) and
+    // scaled proportionally onto the canvas — so positions/sizes stay true to the layout
+    // rather than one element stretched across the frame.
+    function drawZoneComposite(ctx, container, cr, W, H) {
+      ctx.fillStyle = '#000';
+      ctx.fillRect(0, 0, W, H);
+      const zones = container.querySelectorAll('.zone');
+      if (!zones.length) return false;
+      zones.forEach((zd) => {
+        const r = zd.getBoundingClientRect();
+        const dx = ((r.left - cr.left) / cr.width) * W;
+        const dy = ((r.top - cr.top) / cr.height) * H;
+        const dw = (r.width / cr.width) * W;
+        const dh = (r.height / cr.height) * H;
+        const el = zd.querySelector('video, img, iframe');
+        let drawn = false;
+        if (el && el.tagName === 'IMG' && el.complete && el.naturalWidth > 0 && isMediaReadable(el)) {
+          try { drawMediaFit(ctx, el, el.naturalWidth, el.naturalHeight, dx, dy, dw, dh, getComputedStyle(el).objectFit); drawn = true; } catch (e) {}
+        } else if (el && el.tagName === 'VIDEO' && el.readyState >= 2 && el.videoWidth > 0 && isMediaReadable(el)) {
+          try { drawMediaFit(ctx, el, el.videoWidth, el.videoHeight, dx, dy, dw, dh, getComputedStyle(el).objectFit); drawn = true; } catch (e) {}
+        }
+        if (!drawn) drawZonePlaceholder(ctx, dx, dy, dw, dh, zonePlaceholderLabel(el));
+      });
+      return true;
+    }
+
+    // Build the screenshot/stream canvas and return it (caller encodes + sends). Exposed as
+    // a plain function so a headless render pass can verify the composite without a socket.
+    function renderCaptureCanvas() {
+      const container = document.getElementById('playerContainer');
+      const cr = container ? container.getBoundingClientRect() : null;
+      // ~960 on the long edge; height from the REAL container aspect, not a hardcoded 540,
+      // so non-16:9 layouts compose without distortion.
+      const aspect = (cr && cr.width > 0 && cr.height > 0) ? (cr.width / cr.height) : (16 / 9);
       const canvas = document.createElement('canvas');
       canvas.width = 960;
-      canvas.height = 540;
+      canvas.height = Math.max(1, Math.round(960 / aspect));
       const ctx = canvas.getContext('2d');
+      const W = canvas.width, H = canvas.height;
       let captured = false;
 
       try {
-        const container = document.getElementById('playerContainer');
-        const video = container?.querySelector('video');
-        const img = container?.querySelector('img');
-
-        // Try video first
-        if (video && video.readyState >= 2 && video.videoWidth > 0) {
-          try {
-            ctx.drawImage(video, 0, 0, 960, 540);
-            captured = true;
-          } catch (e) {
-            console.warn('Video capture failed (CORS?):', e.message);
+        const multiZone = !!(layout && Array.isArray(layout.zones) && layout.zones.length > 1 && !wallConfig);
+        if (multiZone && container) {
+          captured = drawZoneComposite(ctx, container, cr, W, H);
+        } else if (container) {
+          // Single-zone / fullscreen fast path: one element, drawn with its own object-fit
+          // (the old code stretched it to a fixed 960x540).
+          const video = container.querySelector('video');
+          const img = container.querySelector('img');
+          if (video && video.readyState >= 2 && video.videoWidth > 0 && isMediaReadable(video)) {
+            try { drawMediaFit(ctx, video, video.videoWidth, video.videoHeight, 0, 0, W, H, getComputedStyle(video).objectFit); captured = true; } catch (e) { console.warn('Video capture failed (CORS?):', e.message); }
           }
-        }
-
-        // Try image
-        if (!captured && img && img.complete && img.naturalWidth > 0) {
-          try {
-            ctx.drawImage(img, 0, 0, 960, 540);
-            captured = true;
-          } catch (e) {
-            console.warn('Image capture failed:', e.message);
+          if (!captured && img && img.complete && img.naturalWidth > 0 && isMediaReadable(img)) {
+            try { drawMediaFit(ctx, img, img.naturalWidth, img.naturalHeight, 0, 0, W, H, getComputedStyle(img).objectFit); captured = true; } catch (e) { console.warn('Image capture failed:', e.message); }
           }
         }
 
         // Fallback: draw status info
         if (!captured) {
           ctx.fillStyle = '#111827';
-          ctx.fillRect(0, 0, 960, 540);
+          ctx.fillRect(0, 0, W, H);
           ctx.fillStyle = '#3b82f6';
           ctx.font = 'bold 28px sans-serif';
           ctx.textAlign = 'center';
-          ctx.fillText('ScreenTinker Web Player', 480, 230);
+          ctx.fillText('ScreenTinker Web Player', W / 2, H / 2 - 40);
           ctx.fillStyle = '#94a3b8';
           ctx.font = '16px sans-serif';
           const item = playlist[currentIndex];
-          ctx.fillText(item ? `Playing: ${item.filename}` : 'No content', 480, 270);
-          ctx.fillText(`${config.deviceName || 'Web Player'} | ${new Date().toLocaleTimeString()}`, 480, 310);
+          ctx.fillText(item ? `Playing: ${item.filename}` : 'No content', W / 2, H / 2);
+          ctx.fillText(`${config.deviceName || 'Web Player'} | ${new Date().toLocaleTimeString()}`, W / 2, H / 2 + 40);
         }
       } catch (e) {
         // Even on error, draw something
         ctx.fillStyle = '#000';
-        ctx.fillRect(0, 0, 960, 540);
+        ctx.fillRect(0, 0, W, H);
         ctx.fillStyle = '#ef4444';
         ctx.font = '16px sans-serif';
         ctx.textAlign = 'center';
-        ctx.fillText('Screenshot error: ' + e.message, 480, 270);
+        ctx.fillText('Screenshot error: ' + e.message, W / 2, H / 2);
       }
+      return canvas;
+    }
 
+    function captureAndSend() {
+      if (!socket?.connected) return;
+      // Also drives the 1fps remote stream (startStreaming). The composite is just a handful
+      // of drawImage calls over already-decoded media, so one full-quality path serves both
+      // the on-demand screenshot and the 1fps stream — no separate low-quality stream path.
+      let canvas;
+      try { canvas = renderCaptureCanvas(); }
+      catch (e) { console.error('Screenshot render failed:', e); return; }
       try {
         const dataUrl = canvas.toDataURL('image/jpeg', 0.4);
         const base64 = dataUrl.split(',')[1];
         if (base64 && base64.length > 100) {
           socket.emit('device:screenshot', { device_id: config.deviceId, image_b64: base64 });
-          console.log('Screenshot sent:', base64.length, 'chars', captured ? '(content)' : '(fallback)');
+          console.log('Screenshot sent:', base64.length, 'chars');
         }
       } catch (e) {
         console.error('Screenshot encode/send failed:', e);