From ce7b2948ae942c58331a913fc5a7d3a4f69e6e66 Mon Sep 17 00:00:00 2001 From: ScreenTinker Date: Fri, 19 Jun 2026 14:19:32 -0500 Subject: [PATCH] fix(#109): render Android PiP overlay above the YouTube WebView video plane MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The PiP overlay (#109) returned sent:1 and showed its title in `uiautomator dump`, but nothing painted on screen while YouTube was playing. By elimination (YouTube-specific, landscape so no off-screen transform, real on-screen bounds in the dump) the cause is surface occlusion: pipLayout sat as the last child of rootLayout — the SAME compositing band as R.id.youtubeWebView — so the playing video surface drew over it. Fix (task option 1a): reparent pipLayout out of rootLayout to the window content (android.R.id.content) as a top-level sibling drawn after rootLayout, so it composites above the WebView. MainActivity.mirrorTransformToPip() copies rootView's orientation/wall transform onto it so corner positions still track the rotated content (web/Tizen parity). show() also bringToFront()+ requestLayout()+invalidate() on attach (covers the cause-3 measure/visibility path). Remote-view screenshots now capture the content root so the PiP is still included. Instrumentation (Phase 1, default OFF): PipOverlay.pipDebug paints a solid magenta box + border with media on top (box paints even if media never loads) and logs box/pipLayout/rootView/youtubeWebView geometry over device:log tag "pip"; loadImageInto also logs on success. Toggled via device:command {type:"pip_debug"} (routed through MainActivity.onCommand). Server: POST /api/pip and the clear handler log one concise [pip] dispatch line (target + sent/offline) so journalctl shows PiP activity. Validated end-to-end on an emulator (pixel10/API34) paired to an isolated local server with YouTube playing: no crash, the PiP box composites above the live video frame (center + top-right), clear removes it, and the portrait transform mirror rotates the overlay with the stage (no off-screen). The Fire TV hardware-overlay punch-through still needs real hardware (emulator composites video inline); pipDebug + docs/109-android-pip-visibility.md cover that. Co-Authored-By: Claude Opus 4.8 (1M context) --- .../com/remotedisplay/player/MainActivity.kt | 65 +++++++- .../remotedisplay/player/player/PipOverlay.kt | 79 ++++++++- .../app/src/main/res/layout/activity_main.xml | 12 +- docs/109-android-pip-visibility.md | 151 ++++++++++++++++++ server/routes/pip.js | 8 +- 5 files changed, 297 insertions(+), 18 deletions(-) create mode 100644 docs/109-android-pip-visibility.md diff --git a/android/app/src/main/java/com/remotedisplay/player/MainActivity.kt b/android/app/src/main/java/com/remotedisplay/player/MainActivity.kt index e26dcd6..9c61f6b 100644 --- a/android/app/src/main/java/com/remotedisplay/player/MainActivity.kt +++ b/android/app/src/main/java/com/remotedisplay/player/MainActivity.kt @@ -57,6 +57,8 @@ class MainActivity : AppCompatActivity() { private lateinit var statusOverlay: View private lateinit var statusText: TextView private lateinit var rootView: View + private lateinit var pipLayout: FrameLayout // #109: reparented above rootView (see onCreate) + private lateinit var captureRoot: View // window content; capture source (includes pipLayout) private var currentOrientation: String? = null private val handler = Handler(Looper.getMainLooper()) @@ -134,9 +136,27 @@ class MainActivity : AppCompatActivity() { // Hide player controls playerView.useController = false - // #109: PiP overlay layer (top child of rootLayout; inherits the orientation - // transform). Reports show/clear over device:log (tag "pip"). - pipOverlay = PipOverlay(this, findViewById(R.id.pipLayout)) { level, message -> + val youtubeWebView = findViewById(R.id.youtubeWebView) + + // #109 fix (1): the PiP layer must render ABOVE the YouTube WebView's video plane. + // As the last child of rootLayout it sat in the SAME compositing band as the WebView + // and was occluded by the playing video surface. Reparent it OUT of rootLayout to the + // window content (android.R.id.content), as a top-level sibling drawn AFTER rootLayout + // — so it composites above the WebView. applyOrientation()/applyWallTransform() mirror + // rootView's transform onto it so corner positions still track the rotated content, + // and the remote-view screenshot captures `captureRoot` (content) so the PiP is still + // included. See docs/109-android-pip-visibility.md. + captureRoot = findViewById(android.R.id.content) + pipLayout = findViewById(R.id.pipLayout) + (pipLayout.parent as? ViewGroup)?.removeView(pipLayout) + (captureRoot as ViewGroup).addView( + pipLayout, + FrameLayout.LayoutParams(FrameLayout.LayoutParams.MATCH_PARENT, FrameLayout.LayoutParams.MATCH_PARENT) + ) + + // #109: PiP overlay layer. Reports show/clear over device:log (tag "pip"); rootView + + // youtubeWebView are passed for pipDebug geometry logging only. + pipOverlay = PipOverlay(this, pipLayout, rootView, youtubeWebView) { level, message -> wsService?.sendLog("pip", level, message) } @@ -155,7 +175,6 @@ class MainActivity : AppCompatActivity() { ) // Setup media player - val youtubeWebView = findViewById(R.id.youtubeWebView) mediaPlayer = MediaPlayerManager( context = this, playerView = playerView, @@ -252,9 +271,30 @@ class MainActivity : AppCompatActivity() { rootView.translationY = if (swap) (h - w) / 2f else 0f rootView.rotation = rot rootView.requestLayout() + mirrorTransformToPip() Log.i("MainActivity", "Applied orientation: $orientation (rotation=$rot, swap=$swap)") } + // #109: pipLayout was reparented out of rootView (to draw above the WebView), so it no + // longer inherits the orientation/wall transform. Copy rootView's current size + transform + // onto it verbatim so the PiP box still lands in the same rotated coordinate space as the + // visible content (mirrors the web/Tizen players, which apply the same transform to #pip + // as to #stage). Called after every rootView transform change. + private fun mirrorTransformToPip() { + if (!::pipLayout.isInitialized) return + val rl = rootView.layoutParams + val pl = pipLayout.layoutParams + pl.width = rl.width + pl.height = rl.height + pipLayout.layoutParams = pl + pipLayout.translationX = rootView.translationX + pipLayout.translationY = rootView.translationY + pipLayout.rotation = rootView.rotation + pipLayout.scaleX = rootView.scaleX + pipLayout.scaleY = rootView.scaleY + pipLayout.requestLayout() + } + private fun parseWallConfig(wc: JSONObject): WallController.WallConfig { fun rect(key: String): WallController.Rect { val o = wc.optJSONObject(key) @@ -291,6 +331,7 @@ class MainActivity : AppCompatActivity() { rootView.scaleX = 1f rootView.scaleY = 1f rootView.requestLayout() + mirrorTransformToPip() // Force the next playlist update to re-apply orientation (applyOrientation // early-returns when the value is unchanged). currentOrientation = null @@ -314,6 +355,7 @@ class MainActivity : AppCompatActivity() { rootView.scaleX = 1f rootView.scaleY = 1f rootView.requestLayout() + mirrorTransformToPip() // Orientation no longer reflects reality; ensure it re-applies after wall exit. currentOrientation = null Log.i("MainActivity", "Wall transform: size=${lp.width}x${lp.height} tx=${rootView.translationX} ty=${rootView.translationY}") @@ -483,9 +525,11 @@ class MainActivity : AppCompatActivity() { // Handled by service now } - // Provide screenshot callback to service (composite capture on main thread) + // Provide screenshot callback to service (composite capture on main thread). + // Capture the window content (not just rootView) so the reparented #109 PiP layer + // is included in remote-view screenshots. wsService?.onCaptureScreenshot = { - screenshotCapture.captureView(rootView, 40) + screenshotCapture.captureView(captureRoot, 40) } wsService?.onRemoteStop = { @@ -552,6 +596,13 @@ class MainActivity : AppCompatActivity() { "refresh" -> { wsService?.connect() } + // #109 debug: toggle the PiP magenta-box + geometry logging (default off). + // device:command {type:"pip_debug", payload:{enabled:true}}. + "pip_debug" -> { + val on = payload?.optBoolean("enabled", false) ?: false + if (::pipOverlay.isInitialized) pipOverlay.pipDebug = on + Log.i("MainActivity", "PiP debug ${if (on) "ENABLED" else "disabled"}") + } } } @@ -667,7 +718,7 @@ class MainActivity : AppCompatActivity() { private fun captureAndSendScreenshot() { Log.i("MainActivity", "Capturing screenshot") - val base64 = screenshotCapture.captureView(rootView, 40) + val base64 = screenshotCapture.captureView(captureRoot, 40) if (base64 != null) { Log.i("MainActivity", "Screenshot captured, size=${base64.length} chars, sending...") wsService?.sendScreenshot(base64) diff --git a/android/app/src/main/java/com/remotedisplay/player/player/PipOverlay.kt b/android/app/src/main/java/com/remotedisplay/player/player/PipOverlay.kt index 40a8802..39d06d0 100644 --- a/android/app/src/main/java/com/remotedisplay/player/player/PipOverlay.kt +++ b/android/app/src/main/java/com/remotedisplay/player/player/PipOverlay.kt @@ -2,12 +2,14 @@ package com.remotedisplay.player.player import android.content.Context import android.graphics.Color +import android.graphics.Rect import android.graphics.drawable.GradientDrawable import android.os.Handler import android.os.Looper import android.util.Log import android.view.Gravity import android.view.View +import android.view.ViewGroup import android.webkit.WebView import android.widget.FrameLayout import android.widget.ImageView @@ -33,6 +35,11 @@ import org.json.JSONObject class PipOverlay( private val context: Context, private val pipLayout: FrameLayout, + // #109 debug: refs used ONLY by pipDebug logging (the orientation transform that keeps + // pipLayout aligned with the content is applied in MainActivity). Both are nullable so + // the overlay stays constructable without them. + private val rootView: View? = null, + private val youtubeWebView: WebView? = null, private val log: (level: String, message: String) -> Unit = { _, _ -> } ) { private val handler = Handler(Looper.getMainLooper()) @@ -40,6 +47,16 @@ class PipOverlay( private var current: String? = null private var webView: WebView? = null + /** + * #109 debug flag (default OFF — no behavior change in prod). When on, [show] paints the + * box solid magenta with a magenta border and renders the media ON TOP, so the BOX itself + * is visible even if media never loads; it then posts a one-shot Runnable that logs the + * geometry/visibility of the box, pipLayout, rootView and the YouTube WebView over + * device:log tag "pip" — enough to tell surface-occlusion (1) from orientation (2) from + * a measure/visibility (3) failure. Toggled via device:command {type:"pip_debug"}. + */ + @Volatile var pipDebug: Boolean = false + fun show(p: JSONObject) { try { teardown() // single slot, last-show-wins @@ -56,10 +73,17 @@ class PipOverlay( clipToOutline = true val radius = p.optInt("border_radius", 0).toFloat() background = GradientDrawable().apply { - setColor(parseColor(p.optString("background_color", ""), Color.BLACK)) + if (pipDebug) { + // Solid magenta fill + 8px magenta border so the BOX paints even if + // media never loads (#CCFF00FF = 80%-opaque magenta). + setColor(0xCCFF00FF.toInt()) + setStroke(8, Color.MAGENTA) + } else { + setColor(parseColor(p.optString("background_color", ""), Color.BLACK)) + } cornerRadius = radius } - alpha = p.optDouble("opacity", 1.0).toFloat().coerceIn(0f, 1f) + alpha = if (pipDebug) 1f else p.optDouble("opacity", 1.0).toFloat().coerceIn(0f, 1f) } val title = p.optString("title", "") @@ -107,6 +131,17 @@ class PipOverlay( pipLayout.visibility = View.VISIBLE current = p.optString("pip_id", "(anon)") + // #109 fix (1)/(3): the pip layer is a top-level view above the WebView (reparented + // to the window content in MainActivity), but make sure it composites last and is + // laid out before the next frame — raise it to the front and force a layout/redraw + // so the box is never left attached-but-unpainted behind the playing content. + pipLayout.bringToFront() + pipLayout.requestLayout() + pipLayout.invalidate() + (pipLayout.parent as? View)?.invalidate() + + if (pipDebug) logGeometry(box) + val dur = p.optInt("duration", 0) if (dur > 0) { val id = current @@ -147,12 +182,48 @@ class PipOverlay( img.post { // Drop the result if this overlay was torn down / replaced while decoding. if (img.parent == null || token != current) return@post - if (bmp != null) img.setImageBitmap(bmp) - else { log("warn", "pip image failed to load"); clear(token) } + if (bmp != null) { + img.setImageBitmap(bmp) + if (pipDebug) log("info", "pip image loaded ${bmp.width}x${bmp.height}") + } else { log("warn", "pip image failed to load"); clear(token) } } }.start() } + /** + * #109 debug: dump the geometry/visibility that tells the three candidate causes apart. + * Posted onto the box so it runs after the first layout pass. Reads device:log tag "pip": + * - magenta box visible over an image but NOT over YouTube -> (1) surface occlusion + * - box visible rect empty / off the panel -> (2) orientation transform + * - box not shown / 0-size / pipLayout GONE or 0 children -> (3) measure/visibility + */ + private fun logGeometry(box: View) { + box.post { + try { + val r = Rect() + val boxVisible = box.getGlobalVisibleRect(r) + val parent = pipLayout.parent as? ViewGroup + val idx = parent?.indexOfChild(pipLayout) ?: -1 + log("info", "pip dbg box ${box.width}x${box.height} shown=${box.isShown} " + + "globalRect=$r($boxVisible)") + log("info", "pip dbg pipLayout ${pipLayout.width}x${pipLayout.height} " + + "vis=${pipLayout.visibility} children=${pipLayout.childCount} " + + "idxInParent=$idx parent=${parent?.javaClass?.simpleName}") + rootView?.let { + log("info", "pip dbg root rot=${it.rotation} tx=${it.translationX} " + + "ty=${it.translationY} sx=${it.scaleX} hwAccel=${it.isHardwareAccelerated}") + } + youtubeWebView?.let { + val yr = Rect() + val yv = it.getGlobalVisibleRect(yr) + log("info", "pip dbg youtube vis=${it.visibility} globalRect=$yr($yv)") + } + } catch (e: Throwable) { + log("warn", "pip dbg failed: ${e.message}") + } + } + } + private fun parseColor(hex: String, fallback: Int): Int = if (hex.matches(Regex("^#[0-9A-Fa-f]{6}$"))) try { Color.parseColor(hex) } catch (e: Throwable) { fallback } else fallback } diff --git a/android/app/src/main/res/layout/activity_main.xml b/android/app/src/main/res/layout/activity_main.xml index 5d18d0c..35c57a9 100644 --- a/android/app/src/main/res/layout/activity_main.xml +++ b/android/app/src/main/res/layout/activity_main.xml @@ -76,11 +76,13 @@ android:indeterminate="true" /> - + Note: on-device capture must be done on the real signage hardware (Fire TV / +> Android TV). The WebView hardware-video-overlay behaviour that drives cause (1) +> is device- and WebView-version-specific and does **not** reproduce on a stock +> emulator, so it cannot be captured from a CI/dev box with no device attached. + +## Which cause — and the fix + +By elimination the symptom points at **(1) surface occlusion**: + +- It is **YouTube-specific** (a WebView playing HTML5 video). An orientation (2) + or measure (3) fault would fail over images too, but the overlay is only + reported broken over YouTube. +- The repro is **landscape** (the default orientation → `rotation = 0`, no + translation), so the box cannot be transformed off-screen → not (2). +- The title shows with **real on-screen bounds** in `uiautomator dump`, so the + box is laid out at non-zero size and `pipLayout` is `VISIBLE` → not (3). + +That leaves the video surface compositing above the in-tree overlay. + +### Fix (cause 1) — file/line + +`pipLayout` previously lived as the **last child of `rootLayout`**, i.e. in the +**same compositing band** as `R.id.youtubeWebView`; the WebView's playing video +surface drew over it. The fix moves the PiP layer to a **top-level view above the +WebView** (the task's option 1a): + +- **`MainActivity.onCreate`** (`android/app/src/main/java/com/remotedisplay/player/MainActivity.kt`) + reparents `R.id.pipLayout` out of `rootLayout` up to the window content + (`android.R.id.content`), as a sibling drawn **after** `rootLayout` → it + composites above the WebView. +- **`MainActivity.mirrorTransformToPip()`** copies `rootView`'s current size + + rotation/translation/scale onto `pipLayout` after every transform change + (`applyOrientation` / `applyWallTransform`), so corner positions still track the + rotated content — mirroring how the web/Tizen players apply the same transform + to `#pip` as to `#stage`. +- **`PipOverlay.show()`** (`…/player/PipOverlay.kt`) raises the layer and forces a + layout/redraw on attach (`bringToFront()` + `requestLayout()` + `invalidate()`), + which also covers the cause-(3) measure/visibility path. +- The remote-view screenshot source moved from `rootView` to `captureRoot` + (the window content) so the reparented PiP is still captured. + +### Server dispatch logging + +`POST /api/pip` and the clear handler (`server/routes/pip.js`) now log one +concise `[pip] …` line each (target kind + id + sent/offline counts) so +`journalctl` shows PiP activity. + +## Emulator validation (landscape + portrait) + +The fix was exercised end-to-end on an Android emulator (pixel10, API 34) paired +to an isolated local server, with a YouTube item playing in +`R.id.youtubeWebView`: + +- **No crash** — provisioning → `MainActivity` → playback ran clean; the reparent + + `mirrorTransformToPip()` executed (`Applied orientation: landscape … / portrait + (rotation=90.0, swap=true)`). +- **PiP composites above the playing YouTube video** — a `POST /api/pip` box + (magenta via `background_color`) rendered on top of the live video frame + (center and top-right placements both correct, 4% inset honoured). +- **Clear** removed the overlay cleanly; the video kept playing. +- **Portrait** — the overlay rotated *with* the rotated stage and stayed inside + the frame (not off-screen), confirming the transform mirror. +- Server `[pip] show … 1 sent` / `[pip] clear (all) …` dispatch lines appeared. + +Caveat (unchanged): the emulator's WebView composites video **inline**, so it +confirms the reparent renders correctly and doesn't regress, but it does **not** +reproduce the Fire TV / Android TV hardware-overlay punch-through that is the +strongest form of cause (1). That still needs the real signage device — use the +`pipDebug` magenta box there to confirm. + +## If the magenta box is STILL hidden over YouTube on the test device + +Then it is the stronger form of cause (1): the WebView places its video on a +**hardware overlay / `SurfaceView` plane** that no in-window view can beat. +Escalate (task options 1b/1c), keeping the first that works on the device: + +- host the PiP box in a `SurfaceView` with `setZOrderMediaOverlay(true)` / + `setZOrderOnTop(true)`, or a small `WindowManager` panel sub-window; or +- when YouTube is active, render the PiP via the in-tree image path so no + competing WebView video surface is involved. + +The `pipDebug` instrumentation stays in place to make that determination. diff --git a/server/routes/pip.js b/server/routes/pip.js index 68a29e5..0d3cdcd 100644 --- a/server/routes/pip.js +++ b/server/routes/pip.js @@ -138,7 +138,9 @@ router.post('/', requireScope('full'), (req, res) => { if (b.background_color) payload.background_color = b.background_color; const results = emitToTargets(req, targets.devices, 'device:pip-show', payload); - res.json({ success: true, pip_id, target: targets.kind, ...summarize(results) }); + const summary = summarize(results); + console.log(`[pip] show ${pip_id} (${b.type}) -> ${targets.kind} ${b.device_id}: ${summary.sent} sent, ${summary.offline} offline`); + res.json({ success: true, pip_id, target: targets.kind, ...summary }); }); // Clear an overlay. DELETE /api/pip and POST /api/pip/clear are equivalent; an omitted @@ -150,7 +152,9 @@ function handleClear(req, res) { if (!targets) return res.status(404).json({ error: 'device or group not found in this workspace' }); const payload = b.pip_id ? { pip_id: String(b.pip_id) } : {}; const results = emitToTargets(req, targets.devices, 'device:pip-clear', payload); - res.json({ success: true, target: targets.kind, ...summarize(results) }); + const summary = summarize(results); + console.log(`[pip] clear ${b.pip_id || '(all)'} -> ${targets.kind} ${b.device_id}: ${summary.sent} sent, ${summary.offline} offline`); + res.json({ success: true, target: targets.kind, ...summary }); } router.post('/clear', requireScope('full'), handleClear);