diff --git a/android/app/src/main/java/com/remotedisplay/player/MainActivity.kt b/android/app/src/main/java/com/remotedisplay/player/MainActivity.kt index e26dcd6..9c61f6b 100644 --- a/android/app/src/main/java/com/remotedisplay/player/MainActivity.kt +++ b/android/app/src/main/java/com/remotedisplay/player/MainActivity.kt @@ -57,6 +57,8 @@ class MainActivity : AppCompatActivity() { private lateinit var statusOverlay: View private lateinit var statusText: TextView private lateinit var rootView: View + private lateinit var pipLayout: FrameLayout // #109: reparented above rootView (see onCreate) + private lateinit var captureRoot: View // window content; capture source (includes pipLayout) private var currentOrientation: String? = null private val handler = Handler(Looper.getMainLooper()) @@ -134,9 +136,27 @@ class MainActivity : AppCompatActivity() { // Hide player controls playerView.useController = false - // #109: PiP overlay layer (top child of rootLayout; inherits the orientation - // transform). Reports show/clear over device:log (tag "pip"). - pipOverlay = PipOverlay(this, findViewById(R.id.pipLayout)) { level, message -> + val youtubeWebView = findViewById(R.id.youtubeWebView) + + // #109 fix (1): the PiP layer must render ABOVE the YouTube WebView's video plane. + // As the last child of rootLayout it sat in the SAME compositing band as the WebView + // and was occluded by the playing video surface. Reparent it OUT of rootLayout to the + // window content (android.R.id.content), as a top-level sibling drawn AFTER rootLayout + // — so it composites above the WebView. applyOrientation()/applyWallTransform() mirror + // rootView's transform onto it so corner positions still track the rotated content, + // and the remote-view screenshot captures `captureRoot` (content) so the PiP is still + // included. See docs/109-android-pip-visibility.md. + captureRoot = findViewById(android.R.id.content) + pipLayout = findViewById(R.id.pipLayout) + (pipLayout.parent as? ViewGroup)?.removeView(pipLayout) + (captureRoot as ViewGroup).addView( + pipLayout, + FrameLayout.LayoutParams(FrameLayout.LayoutParams.MATCH_PARENT, FrameLayout.LayoutParams.MATCH_PARENT) + ) + + // #109: PiP overlay layer. Reports show/clear over device:log (tag "pip"); rootView + + // youtubeWebView are passed for pipDebug geometry logging only. + pipOverlay = PipOverlay(this, pipLayout, rootView, youtubeWebView) { level, message -> wsService?.sendLog("pip", level, message) } @@ -155,7 +175,6 @@ class MainActivity : AppCompatActivity() { ) // Setup media player - val youtubeWebView = findViewById(R.id.youtubeWebView) mediaPlayer = MediaPlayerManager( context = this, playerView = playerView, @@ -252,9 +271,30 @@ class MainActivity : AppCompatActivity() { rootView.translationY = if (swap) (h - w) / 2f else 0f rootView.rotation = rot rootView.requestLayout() + mirrorTransformToPip() Log.i("MainActivity", "Applied orientation: $orientation (rotation=$rot, swap=$swap)") } + // #109: pipLayout was reparented out of rootView (to draw above the WebView), so it no + // longer inherits the orientation/wall transform. Copy rootView's current size + transform + // onto it verbatim so the PiP box still lands in the same rotated coordinate space as the + // visible content (mirrors the web/Tizen players, which apply the same transform to #pip + // as to #stage). Called after every rootView transform change. + private fun mirrorTransformToPip() { + if (!::pipLayout.isInitialized) return + val rl = rootView.layoutParams + val pl = pipLayout.layoutParams + pl.width = rl.width + pl.height = rl.height + pipLayout.layoutParams = pl + pipLayout.translationX = rootView.translationX + pipLayout.translationY = rootView.translationY + pipLayout.rotation = rootView.rotation + pipLayout.scaleX = rootView.scaleX + pipLayout.scaleY = rootView.scaleY + pipLayout.requestLayout() + } + private fun parseWallConfig(wc: JSONObject): WallController.WallConfig { fun rect(key: String): WallController.Rect { val o = wc.optJSONObject(key) @@ -291,6 +331,7 @@ class MainActivity : AppCompatActivity() { rootView.scaleX = 1f rootView.scaleY = 1f rootView.requestLayout() + mirrorTransformToPip() // Force the next playlist update to re-apply orientation (applyOrientation // early-returns when the value is unchanged). currentOrientation = null @@ -314,6 +355,7 @@ class MainActivity : AppCompatActivity() { rootView.scaleX = 1f rootView.scaleY = 1f rootView.requestLayout() + mirrorTransformToPip() // Orientation no longer reflects reality; ensure it re-applies after wall exit. currentOrientation = null Log.i("MainActivity", "Wall transform: size=${lp.width}x${lp.height} tx=${rootView.translationX} ty=${rootView.translationY}") @@ -483,9 +525,11 @@ class MainActivity : AppCompatActivity() { // Handled by service now } - // Provide screenshot callback to service (composite capture on main thread) + // Provide screenshot callback to service (composite capture on main thread). + // Capture the window content (not just rootView) so the reparented #109 PiP layer + // is included in remote-view screenshots. wsService?.onCaptureScreenshot = { - screenshotCapture.captureView(rootView, 40) + screenshotCapture.captureView(captureRoot, 40) } wsService?.onRemoteStop = { @@ -552,6 +596,13 @@ class MainActivity : AppCompatActivity() { "refresh" -> { wsService?.connect() } + // #109 debug: toggle the PiP magenta-box + geometry logging (default off). + // device:command {type:"pip_debug", payload:{enabled:true}}. + "pip_debug" -> { + val on = payload?.optBoolean("enabled", false) ?: false + if (::pipOverlay.isInitialized) pipOverlay.pipDebug = on + Log.i("MainActivity", "PiP debug ${if (on) "ENABLED" else "disabled"}") + } } } @@ -667,7 +718,7 @@ class MainActivity : AppCompatActivity() { private fun captureAndSendScreenshot() { Log.i("MainActivity", "Capturing screenshot") - val base64 = screenshotCapture.captureView(rootView, 40) + val base64 = screenshotCapture.captureView(captureRoot, 40) if (base64 != null) { Log.i("MainActivity", "Screenshot captured, size=${base64.length} chars, sending...") wsService?.sendScreenshot(base64) diff --git a/android/app/src/main/java/com/remotedisplay/player/player/PipOverlay.kt b/android/app/src/main/java/com/remotedisplay/player/player/PipOverlay.kt index 40a8802..309636c 100644 --- a/android/app/src/main/java/com/remotedisplay/player/player/PipOverlay.kt +++ b/android/app/src/main/java/com/remotedisplay/player/player/PipOverlay.kt @@ -2,12 +2,14 @@ package com.remotedisplay.player.player import android.content.Context import android.graphics.Color +import android.graphics.Rect import android.graphics.drawable.GradientDrawable import android.os.Handler import android.os.Looper import android.util.Log import android.view.Gravity import android.view.View +import android.view.ViewGroup import android.webkit.WebView import android.widget.FrameLayout import android.widget.ImageView @@ -33,6 +35,11 @@ import org.json.JSONObject class PipOverlay( private val context: Context, private val pipLayout: FrameLayout, + // #109 debug: refs used ONLY by pipDebug logging (the orientation transform that keeps + // pipLayout aligned with the content is applied in MainActivity). Both are nullable so + // the overlay stays constructable without them. + private val rootView: View? = null, + private val youtubeWebView: WebView? = null, private val log: (level: String, message: String) -> Unit = { _, _ -> } ) { private val handler = Handler(Looper.getMainLooper()) @@ -40,6 +47,16 @@ class PipOverlay( private var current: String? = null private var webView: WebView? = null + /** + * #109 debug flag (default OFF — no behavior change in prod). When on, [show] paints the + * box solid magenta with a magenta border and renders the media ON TOP, so the BOX itself + * is visible even if media never loads; it then posts a one-shot Runnable that logs the + * geometry/visibility of the box, pipLayout, rootView and the YouTube WebView over + * device:log tag "pip" — enough to tell surface-occlusion (1) from orientation (2) from + * a measure/visibility (3) failure. Toggled via device:command {type:"pip_debug"}. + */ + @Volatile var pipDebug: Boolean = false + fun show(p: JSONObject) { try { teardown() // single slot, last-show-wins @@ -51,15 +68,29 @@ class PipOverlay( val w = p.optInt("width", 480).coerceIn(1, dm.widthPixels * 4) val h = p.optInt("height", 360).coerceIn(1, dm.heightPixels * 4) + // Set the slot token BEFORE building media: loadImageInto captures `current` as + // its drop-if-replaced token, and its decode finishes on a background thread that + // posts back AFTER show() returns. If current were still null here (teardown + // clears it), that guard would always fail and the decoded bitmap would be + // dropped — i.e. image PiPs never painted. (#109 follow-up.) + current = p.optString("pip_id", "(anon)") + val box = LinearLayout(context).apply { orientation = LinearLayout.VERTICAL clipToOutline = true val radius = p.optInt("border_radius", 0).toFloat() background = GradientDrawable().apply { - setColor(parseColor(p.optString("background_color", ""), Color.BLACK)) + if (pipDebug) { + // Solid magenta fill + 8px magenta border so the BOX paints even if + // media never loads (#CCFF00FF = 80%-opaque magenta). + setColor(0xCCFF00FF.toInt()) + setStroke(8, Color.MAGENTA) + } else { + setColor(parseColor(p.optString("background_color", ""), Color.BLACK)) + } cornerRadius = radius } - alpha = p.optDouble("opacity", 1.0).toFloat().coerceIn(0f, 1f) + alpha = if (pipDebug) 1f else p.optDouble("opacity", 1.0).toFloat().coerceIn(0f, 1f) } val title = p.optString("title", "") @@ -103,9 +134,49 @@ class PipOverlay( else -> { lp.rightMargin = mx; lp.topMargin = my; Gravity.TOP or Gravity.END } // top-right } - pipLayout.addView(box, lp) + // Optional close button (close_button:true). Render a tappable ✕ floating at the + // box's top-right corner; tapping clears THIS overlay (id-matched). It lives in a + // FrameLayout wrapper that is a SIBLING of the box — so it isn't clipped by the + // box outline and isn't dimmed by the box's opacity. Only the button is clickable; + // the rest of pipLayout stays touch-transparent so taps fall through to content. + val attach: View = if (p.optBoolean("close_button", false)) { + val token = current + FrameLayout(context).apply { + addView(box, FrameLayout.LayoutParams( + FrameLayout.LayoutParams.MATCH_PARENT, FrameLayout.LayoutParams.MATCH_PARENT)) + val d = dm.density + addView(TextView(context).apply { + text = "✕" // ✕ + setTextColor(Color.WHITE) + textSize = 16f + gravity = Gravity.CENTER + background = GradientDrawable().apply { + shape = GradientDrawable.OVAL; setColor(Color.argb(150, 0, 0, 0)) + } + isClickable = true + contentDescription = "Close" + setOnClickListener { clear(token) } + }, FrameLayout.LayoutParams((28 * d).toInt(), (28 * d).toInt()).apply { + gravity = Gravity.TOP or Gravity.END + val m = (6 * d).toInt(); topMargin = m; rightMargin = m + }) + } + } else box + + pipLayout.addView(attach, lp) pipLayout.visibility = View.VISIBLE - current = p.optString("pip_id", "(anon)") + // current was set above (before media build) so loadImageInto's token matches. + + // #109 fix (1)/(3): the pip layer is a top-level view above the WebView (reparented + // to the window content in MainActivity), but make sure it composites last and is + // laid out before the next frame — raise it to the front and force a layout/redraw + // so the box is never left attached-but-unpainted behind the playing content. + pipLayout.bringToFront() + pipLayout.requestLayout() + pipLayout.invalidate() + (pipLayout.parent as? View)?.invalidate() + + if (pipDebug) logGeometry(box) val dur = p.optInt("duration", 0) if (dur > 0) { @@ -147,12 +218,48 @@ class PipOverlay( img.post { // Drop the result if this overlay was torn down / replaced while decoding. if (img.parent == null || token != current) return@post - if (bmp != null) img.setImageBitmap(bmp) - else { log("warn", "pip image failed to load"); clear(token) } + if (bmp != null) { + img.setImageBitmap(bmp) + if (pipDebug) log("info", "pip image loaded ${bmp.width}x${bmp.height}") + } else { log("warn", "pip image failed to load"); clear(token) } } }.start() } + /** + * #109 debug: dump the geometry/visibility that tells the three candidate causes apart. + * Posted onto the box so it runs after the first layout pass. Reads device:log tag "pip": + * - magenta box visible over an image but NOT over YouTube -> (1) surface occlusion + * - box visible rect empty / off the panel -> (2) orientation transform + * - box not shown / 0-size / pipLayout GONE or 0 children -> (3) measure/visibility + */ + private fun logGeometry(box: View) { + box.post { + try { + val r = Rect() + val boxVisible = box.getGlobalVisibleRect(r) + val parent = pipLayout.parent as? ViewGroup + val idx = parent?.indexOfChild(pipLayout) ?: -1 + log("info", "pip dbg box ${box.width}x${box.height} shown=${box.isShown} " + + "globalRect=$r($boxVisible)") + log("info", "pip dbg pipLayout ${pipLayout.width}x${pipLayout.height} " + + "vis=${pipLayout.visibility} children=${pipLayout.childCount} " + + "idxInParent=$idx parent=${parent?.javaClass?.simpleName}") + rootView?.let { + log("info", "pip dbg root rot=${it.rotation} tx=${it.translationX} " + + "ty=${it.translationY} sx=${it.scaleX} hwAccel=${it.isHardwareAccelerated}") + } + youtubeWebView?.let { + val yr = Rect() + val yv = it.getGlobalVisibleRect(yr) + log("info", "pip dbg youtube vis=${it.visibility} globalRect=$yr($yv)") + } + } catch (e: Throwable) { + log("warn", "pip dbg failed: ${e.message}") + } + } + } + private fun parseColor(hex: String, fallback: Int): Int = if (hex.matches(Regex("^#[0-9A-Fa-f]{6}$"))) try { Color.parseColor(hex) } catch (e: Throwable) { fallback } else fallback } diff --git a/android/app/src/main/res/layout/activity_main.xml b/android/app/src/main/res/layout/activity_main.xml index 5d18d0c..35c57a9 100644 --- a/android/app/src/main/res/layout/activity_main.xml +++ b/android/app/src/main/res/layout/activity_main.xml @@ -76,11 +76,13 @@ android:indeterminate="true" /> - + Note: on-device capture must be done on the real signage hardware (Fire TV / +> Android TV). The WebView hardware-video-overlay behaviour that drives cause (1) +> is device- and WebView-version-specific and does **not** reproduce on a stock +> emulator, so it cannot be captured from a CI/dev box with no device attached. + +## Which cause — and the fix + +By elimination the symptom points at **(1) surface occlusion**: + +- It is **YouTube-specific** (a WebView playing HTML5 video). An orientation (2) + or measure (3) fault would fail over images too, but the overlay is only + reported broken over YouTube. +- The repro is **landscape** (the default orientation → `rotation = 0`, no + translation), so the box cannot be transformed off-screen → not (2). +- The title shows with **real on-screen bounds** in `uiautomator dump`, so the + box is laid out at non-zero size and `pipLayout` is `VISIBLE` → not (3). + +That leaves the video surface compositing above the in-tree overlay. + +### Fix (cause 1) — file/line + +`pipLayout` previously lived as the **last child of `rootLayout`**, i.e. in the +**same compositing band** as `R.id.youtubeWebView`; the WebView's playing video +surface drew over it. The fix moves the PiP layer to a **top-level view above the +WebView** (the task's option 1a): + +- **`MainActivity.onCreate`** (`android/app/src/main/java/com/remotedisplay/player/MainActivity.kt`) + reparents `R.id.pipLayout` out of `rootLayout` up to the window content + (`android.R.id.content`), as a sibling drawn **after** `rootLayout` → it + composites above the WebView. +- **`MainActivity.mirrorTransformToPip()`** copies `rootView`'s current size + + rotation/translation/scale onto `pipLayout` after every transform change + (`applyOrientation` / `applyWallTransform`), so corner positions still track the + rotated content — mirroring how the web/Tizen players apply the same transform + to `#pip` as to `#stage`. +- **`PipOverlay.show()`** (`…/player/PipOverlay.kt`) raises the layer and forces a + layout/redraw on attach (`bringToFront()` + `requestLayout()` + `invalidate()`), + which also covers the cause-(3) measure/visibility path. +- The remote-view screenshot source moved from `rootView` to `captureRoot` + (the window content) so the reparented PiP is still captured. + +### Server dispatch logging + +`POST /api/pip` and the clear handler (`server/routes/pip.js`) now log one +concise `[pip] …` line each (target kind + id + sent/offline counts) so +`journalctl` shows PiP activity. + +## Emulator validation (landscape + portrait) + +The fix was exercised end-to-end on an Android emulator (pixel10, API 34) paired +to an isolated local server, with a YouTube item playing in +`R.id.youtubeWebView`: + +- **No crash** — provisioning → `MainActivity` → playback ran clean; the reparent + + `mirrorTransformToPip()` executed (`Applied orientation: landscape … / portrait + (rotation=90.0, swap=true)`). +- **PiP composites above the playing YouTube video** — a `POST /api/pip` box + (magenta via `background_color`) rendered on top of the live video frame + (center and top-right placements both correct, 4% inset honoured). +- **Clear** removed the overlay cleanly; the video kept playing. +- **Portrait** — the overlay rotated *with* the rotated stage and stayed inside + the frame (not off-screen), confirming the transform mirror. +- Server `[pip] show … 1 sent` / `[pip] clear (all) …` dispatch lines appeared. + +Caveat (unchanged): the emulator's WebView composites video **inline**, so it +confirms the reparent renders correctly and doesn't regress, but it does **not** +reproduce the Fire TV / Android TV hardware-overlay punch-through that is the +strongest form of cause (1). That still needs the real signage device — use the +`pipDebug` magenta box there to confirm. + +### Follow-up bug found in emulator testing: image PiPs never painted the image + +Verifying an **image** PiP (a QR PNG) surfaced a separate, pre-existing defect +(present on `main`, unrelated to the occlusion reparent): the image area was +always blank — only the box background + title showed. Root cause in +`PipOverlay.show()`: `teardown()` clears `current` to null, then `loadImageInto` +captured `token = current` (null) as its drop-if-replaced guard, but `current` +was only set to the new `pip_id` *after* the media was built. The decode finishes +on a background thread and posts back **after** `show()` returns — so the guard +`token != current` (null ≠ pip_id) was always true and **every decoded bitmap was +dropped**. (Web PiPs and the box/title were unaffected, which masked it.) + +Fix: set `current = pip_id` **before** building the media (so `loadImageInto`'s +token matches). Confirmed on the emulator — the QR now renders in the PiP box +over both a static image and live YouTube. + +### Content types verified on the emulator (over live YouTube) + +- **image** PiP (a QR PNG) — renders after the token-ordering fix above. +- **web** PiP (an HTML page) — loads in the PiP WebView and **executes JS** (a + page that stamps `JS OK ·