Android player: video-wall (wall:sync) support

Ports the wall:sync protocol the web and Tizen players already ship to native
Kotlin/ExoPlayer, so the Android player can join a video wall.

- WallController (new): 4Hz leader broadcast; follower latency-compensated drift
  controller (hard-seek past 0.3s, gentle +/-3% playbackRate nudge past 0.05s);
  role handling with immediate align on entry and on wall:sync-request. Per-tile
  rotation intentionally not applied (web/Tizen parity; left as a TODO).
- MediaPlayerManager: expose position/duration/seekExact/setSpeed for the drift
  controller; RESIZE_MODE_FILL / ImageView FIT_XY in wall mode (object-fit:fill
  parity), restored to fit/fitCenter on exit. Follower mute (setWallMute) persists
  across leader-driven item switches, and followers loop (REPEAT_MODE_ONE) so they
  never freeze on the last frame if the leader's next index is late.
- PlaylistController: wallFollower flag suppresses auto-advance (leader drives the
  index); getIndex/gotoIndex for follower tracking; itemStartedAtMs for non-video
  sync position.
- WebSocketService: onWallSync/onWallSyncRequest handlers (posted to the main
  thread since they drive ExoPlayer) + emitWallSync/emitWallSyncRequest senders
  guarded on socket.connected() like sendPlaybackState.
- MainActivity: parse wall_config in onPlaylistUpdate and branch before the
  orientation + multi-zone paths; size/translate rootView to this screen's slice;
  exit() restores full screen.

Compiles clean (./gradlew :app:assembleDebug). NOT yet validated on a device or a
real wall — the ExoPlayer seek/speed sync and the slice transform need on-device
tuning before this is trusted.

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
ScreenTinker 2026-06-17 19:26:02 -05:00
parent 10726fde42
commit b903144456
5 changed files with 371 additions and 7 deletions

View file

@ -14,6 +14,7 @@ import android.os.Looper
import android.util.Log
import android.view.KeyEvent
import android.view.View
import android.view.ViewGroup
import android.view.WindowManager
import android.view.accessibility.AccessibilityManager
import android.widget.ImageView
@ -25,6 +26,7 @@ import com.remotedisplay.player.data.ServerConfig
import com.remotedisplay.player.player.MediaPlayerManager
import com.remotedisplay.player.player.PlaylistController
import com.remotedisplay.player.player.PlaylistItem
import com.remotedisplay.player.player.WallController
import com.remotedisplay.player.player.ZoneManager
import com.remotedisplay.player.remote.ScreenshotCapture
import com.remotedisplay.player.remote.TouchInjector
@ -46,6 +48,7 @@ class MainActivity : AppCompatActivity() {
private lateinit var playlistController: PlaylistController
private lateinit var updateChecker: UpdateChecker
private var zoneManager: ZoneManager? = null
private lateinit var wallController: WallController
private lateinit var playerView: PlayerView
private lateinit var imageView: ImageView
@ -157,6 +160,17 @@ class MainActivity : AppCompatActivity() {
}
)
// Video-wall controller. The emit lambdas read wsService lazily (it's bound after
// onCreate), and they no-op until the socket is connected (guarded in the service).
wallController = WallController(
media = mediaPlayer,
playlist = playlistController,
deviceId = { config.deviceId },
emitSync = { wallId, idx, contentId, posSec -> wsService?.emitWallSync(wallId, idx, contentId, posSec) },
emitSyncRequest = { wallId -> wsService?.emitWallSyncRequest(wallId) },
applyTransform = { cfg -> applyWallTransform(cfg) }
)
// Restore cached playlist for offline cold-start (play immediately from disk cache).
// Catch Throwable (not just Exception) so an OOM or corrupt entry can't kill the app
// before the WebSocket connects — that's the crash-loop scenario. If the cache is
@ -233,10 +247,75 @@ class MainActivity : AppCompatActivity() {
Log.i("MainActivity", "Applied orientation: $orientation (rotation=$rot, swap=$swap)")
}
private fun parseWallConfig(wc: JSONObject): WallController.WallConfig {
fun rect(key: String): WallController.Rect {
val o = wc.optJSONObject(key)
return WallController.Rect(
x = o?.optDouble("x", 0.0)?.toFloat() ?: 0f,
y = o?.optDouble("y", 0.0)?.toFloat() ?: 0f,
w = o?.optDouble("w", 0.0)?.toFloat() ?: 0f,
h = o?.optDouble("h", 0.0)?.toFloat() ?: 0f
)
}
return WallController.WallConfig(
wallId = wc.optString("wall_id", ""),
screen = rect("screen_rect"),
player = rect("player_rect"),
isLeader = wc.optBoolean("is_leader", false),
rotation = wc.optInt("rotation", 0)
)
}
// Video-wall slice transform. The content view represents the whole wall (player_rect);
// size + offset rootView so this screen's screen_rect fills the device viewport, content
// stretched to fill (object-fit:fill parity, set on the views via MediaPlayerManager).
// Mirrors the web player's vw/vh stage math. Per-tile rotation is intentionally not
// applied (web/Tizen parity). cfg == null restores full screen.
private fun applyWallTransform(cfg: WallController.WallConfig?) {
val lp = rootView.layoutParams
if (cfg == null) {
lp.width = ViewGroup.LayoutParams.MATCH_PARENT
lp.height = ViewGroup.LayoutParams.MATCH_PARENT
rootView.layoutParams = lp
rootView.translationX = 0f
rootView.translationY = 0f
rootView.rotation = 0f
rootView.scaleX = 1f
rootView.scaleY = 1f
rootView.requestLayout()
// Force the next playlist update to re-apply orientation (applyOrientation
// early-returns when the value is unchanged).
currentOrientation = null
Log.i("MainActivity", "Wall transform cleared (restored full screen)")
return
}
val s = cfg.screen
val p = cfg.player
if (s.w == 0f || s.h == 0f) {
Log.w("MainActivity", "Wall screen_rect has zero size; skipping transform")
return
}
val dw = resources.displayMetrics.widthPixels.toFloat()
val dh = resources.displayMetrics.heightPixels.toFloat()
lp.width = ((p.w / s.w) * dw).toInt()
lp.height = ((p.h / s.h) * dh).toInt()
rootView.layoutParams = lp
rootView.translationX = ((p.x - s.x) / s.w) * dw // negative for right/lower tiles
rootView.translationY = ((p.y - s.y) / s.h) * dh
rootView.rotation = 0f // per-tile rotation: TODO (parity = none)
rootView.scaleX = 1f
rootView.scaleY = 1f
rootView.requestLayout()
// Orientation no longer reflects reality; ensure it re-applies after wall exit.
currentOrientation = null
Log.i("MainActivity", "Wall transform: size=${lp.width}x${lp.height} tx=${rootView.translationX} ty=${rootView.translationY}")
}
private fun setupServiceCallbacks() {
wsService?.onPlaylistUpdate = { data ->
try {
applyOrientation(data.optString("orientation", "landscape"))
// Orientation is applied in the non-wall branch below; wall mode owns the
// root-view transform itself and must not be rotated.
// Check if device is suspended (trial expired / over limit)
if (data.optBoolean("suspended", false)) {
val message = data.optString("message", "Account Suspended")
@ -257,6 +336,20 @@ class MainActivity : AppCompatActivity() {
// Cache playlist JSON for offline cold-start
config.cachedPlaylist = data.toString()
// Video-wall mode takes precedence over orientation + multi-zone: the wall is
// fullscreen, and WallController owns the root-view slice transform and the
// leader/follower role. (We're on the main thread here — onPlaylistUpdate is
// posted to the main looper by WebSocketService.)
val wallObj = if (data.isNull("wall_config")) null else data.optJSONObject("wall_config")
if (wallObj != null) {
com.remotedisplay.player.util.DebugLog.i("Player", "Layout: VIDEO-WALL (${assignments.length()} assignments)")
if (zoneManager?.hasZones() == true) zoneManager?.cleanup()
wallController.apply(parseWallConfig(wallObj))
playlistController.updatePlaylist(assignments)
} else {
wallController.exit()
applyOrientation(data.optString("orientation", "landscape"))
// Check for multi-zone layout
val layoutObj = if (data.isNull("layout")) null else data.optJSONObject("layout")
val layoutZones = layoutObj?.optJSONArray("zones")
@ -301,8 +394,11 @@ class MainActivity : AppCompatActivity() {
if (zoneManager?.hasZones() == true) handler.post { zoneManager?.cleanup() }
playlistController.updatePlaylist(assignments)
}
} // end else (not a video wall)
// Download any missing local content (skip remote URLs)
// Download any missing local content (skip remote URLs).
// Runs for wall + single-zone; multi-zone drives its own rendering via ZoneManager
// (the startIfNeeded below is guarded so it won't run behind zones).
thread {
for (i in 0 until assignments.length()) {
val item = assignments.getJSONObject(i)
@ -451,6 +547,9 @@ class MainActivity : AppCompatActivity() {
}
}
wsService?.onWallSync = { data -> if (::wallController.isInitialized) wallController.onSync(data) }
wsService?.onWallSyncRequest = { data -> if (::wallController.isInitialized) wallController.onSyncRequest(data) }
wsService?.onRegistered = { _ ->
hideStatus()
}

View file

@ -7,9 +7,12 @@ import android.webkit.WebChromeClient
import android.webkit.WebView
import android.webkit.WebViewClient
import android.widget.ImageView
import androidx.media3.common.C
import androidx.media3.common.MediaItem
import androidx.media3.common.Player
import androidx.media3.exoplayer.ExoPlayer
import androidx.media3.exoplayer.SeekParameters
import androidx.media3.ui.AspectRatioFrameLayout
import androidx.media3.ui.PlayerView
import com.remotedisplay.player.util.ImageLoader
import java.io.File
@ -24,6 +27,9 @@ class MediaPlayerManager(
) {
private var exoPlayer: ExoPlayer? = null
private var currentType: MediaType = MediaType.NONE
// Wall mode: followers must stay muted even as the leader's sync switches them
// to a new (possibly unmuted) item, so the mute has to survive each playVideo.
private var wallMute = false
enum class MediaType { NONE, VIDEO, IMAGE, YOUTUBE, WIDGET }
@ -91,7 +97,7 @@ class MediaPlayerManager(
youtubeWebView?.visibility = android.view.View.GONE
exoPlayer?.apply {
volume = if (muted) 0f else 1f
volume = if (muted || wallMute) 0f else 1f
setMediaItem(MediaItem.fromUri(Uri.parse(url)))
prepare()
playWhenReady = true
@ -132,7 +138,7 @@ class MediaPlayerManager(
youtubeWebView?.visibility = android.view.View.GONE
exoPlayer?.apply {
volume = if (muted) 0f else 1f
volume = if (muted || wallMute) 0f else 1f
setMediaItem(MediaItem.fromUri(Uri.fromFile(file)))
prepare()
playWhenReady = true
@ -177,4 +183,55 @@ class MediaPlayerManager(
}
fun isPlayingVideo(): Boolean = currentType == MediaType.VIDEO && (exoPlayer?.isPlaying == true)
// ---- Video-wall (wall:sync) accessors. All must be called on the main thread. ----
/** Current video position in ms (0 when no video). */
fun currentPositionMs(): Long = exoPlayer?.currentPosition ?: 0L
/** Video duration in ms, or -1 when unknown/unprepared. */
fun durationMs(): Long {
val d = exoPlayer?.duration ?: C.TIME_UNSET
return if (d == C.TIME_UNSET) -1L else d
}
/** Exact (frame-accurate) seek for the follower drift controller's hard-seek path. */
fun seekExact(positionMs: Long) {
exoPlayer?.apply {
setSeekParameters(SeekParameters.EXACT)
seekTo(positionMs)
}
}
/** Playback rate — followers nudge ±3% to converge on the leader's clock. */
fun setSpeed(rate: Float) { exoPlayer?.setPlaybackSpeed(rate) }
/**
* Wall follower mute. Persists across item switches (the leader's sync can move a
* follower to an unmuted item, and N copies of the same audio out of phase flange),
* and enforces the mute on whatever is playing right now.
*/
fun setWallMute(mute: Boolean) {
wallMute = mute
if (mute) exoPlayer?.volume = 0f
}
/**
* Loop the current video for wall followers so they never freeze on the last frame
* if the leader's next index sync is slightly late; the leader plays through normally.
*/
fun setVideoLooping(loop: Boolean) {
exoPlayer?.repeatMode = if (loop) Player.REPEAT_MODE_ONE else Player.REPEAT_MODE_OFF
}
/**
* In wall mode the content fills its slice (object-fit:fill parity with the web/Tizen
* players); restore the default fit on exit.
*/
fun setWallMode(enabled: Boolean) {
playerView.resizeMode =
if (enabled) AspectRatioFrameLayout.RESIZE_MODE_FILL else AspectRatioFrameLayout.RESIZE_MODE_FIT
imageView.scaleType =
if (enabled) ImageView.ScaleType.FIT_XY else ImageView.ScaleType.FIT_CENTER
}
}

View file

@ -42,8 +42,32 @@ class PlaylistController(
@Volatile private var effectiveTimezone: String? = null
private var retryRunnable: Runnable? = null
// Video wall: followers don't self-advance — the leader's wall:sync drives the index.
private var wallFollower = false
// Wall-clock at which the current item started playing, for non-video sync position.
private var itemStartedAt = 0L
val isPlaying: Boolean get() = isRunning && currentIndex >= 0
/** Video wall: true on followers (suppress auto-advance; leader drives the index). */
fun setWallFollower(b: Boolean) { wallFollower = b }
/** Current playlist index (-1 if nothing is playing). */
fun getIndex(): Int = currentIndex
/** Wall-clock ms when the current item started; used for non-video sync position. */
fun itemStartedAtMs(): Long = itemStartedAt
/** Jump to an absolute index (wraps); used by wall followers tracking the leader. */
fun gotoIndex(idx: Int) {
if (items.isEmpty()) return
val n = items.size
val target = ((idx % n) + n) % n
if (target == currentIndex) return
currentIndex = target
playCurrentItem()
}
/** #74/#75: device-effective IANA timezone for per-item schedule evaluation. */
fun setTimezone(tz: String?) { effectiveTimezone = tz }
@ -187,7 +211,9 @@ class PlaylistController(
}
fun onVideoComplete() {
// Called when a video finishes naturally
// Called when a video finishes naturally. Wall followers don't self-advance —
// they hold (and loop) the leader's item until a wall:sync changes the index.
if (wallFollower) return
next()
}
@ -195,12 +221,14 @@ class PlaylistController(
cancelAdvance()
cancelRetry()
val item = currentItem ?: return
itemStartedAt = System.currentTimeMillis()
Log.i("PlaylistController", "Playing: ${item.filename} (index $currentIndex)")
onItemChanged(item)
// For images and widgets, auto-advance after duration. For videos, wait
// for the completion callback.
if (item.mimeType.startsWith("image/") || item.isWidget) {
// for the completion callback. Wall followers never auto-advance — the
// leader's wall:sync index drives every switch.
if (!wallFollower && (item.mimeType.startsWith("image/") || item.isWidget)) {
scheduleAdvance(item.durationSec * 1000L)
}
}

View file

@ -0,0 +1,143 @@
package com.remotedisplay.player.player
import android.os.Handler
import android.os.Looper
import android.util.Log
import org.json.JSONObject
import kotlin.math.abs
/**
* Video-wall (`wall:sync`) controller. Native Kotlin/ExoPlayer port of the web player
* (`server/player/index.html`) and the Tizen `WallController` same protocol, gates, and
* drift maths.
*
* - Leader: plays normally and broadcasts `wall:sync` at 4Hz (plus an immediate align on
* entry and on every `wall:sync-request`).
* - Follower: never self-advances; switches item only when a `wall:sync` carries a new
* `current_index`, and for video runs a latency-compensated drift controller that hard-
* seeks on large drift and nudges playbackRate on small drift. Followers are muted.
*
* The slice view transform (sizing/translating the root view to this screen's tile) lives in
* MainActivity and is invoked through [applyTransform]. Per-tile `rotation` is intentionally
* not applied (web/Tizen parity left as a TODO).
*
* All ExoPlayer/View touches happen on the main thread; the 4Hz timer runs on the main looper
* and `onSync` is delivered on the main thread by WebSocketService.
*/
class WallController(
private val media: MediaPlayerManager,
private val playlist: PlaylistController,
private val deviceId: () -> String,
private val emitSync: (wallId: String, idx: Int, contentId: String?, posSec: Float) -> Unit,
private val emitSyncRequest: (wallId: String) -> Unit,
private val applyTransform: (WallConfig?) -> Unit
) {
data class Rect(val x: Float, val y: Float, val w: Float, val h: Float)
data class WallConfig(
val wallId: String,
val screen: Rect,
val player: Rect,
val isLeader: Boolean,
val rotation: Int
)
private val handler = Handler(Looper.getMainLooper())
private var config: WallConfig? = null
private var tick: Runnable? = null
val isActive: Boolean get() = config != null
/** Enter/refresh wall mode for the given config (idempotent; handles role flips). */
fun apply(cfg: WallConfig) {
config = cfg
Log.i("WallController", "apply wall=${cfg.wallId} isLeader=${cfg.isLeader}")
applyTransform(cfg) // size/translate the root view to our slice
media.setWallMode(true) // object-fit:fill parity
playlist.setWallFollower(!cfg.isLeader) // followers don't self-advance
media.setWallMute(!cfg.isLeader) // followers muted (avoid flange)
media.setVideoLooping(!cfg.isLeader) // followers loop so they never freeze
stopTimer()
if (cfg.isLeader) {
tick = object : Runnable {
override fun run() { emitNow(); handler.postDelayed(this, 250) }
}
handler.postDelayed(tick!!, 250)
handler.postDelayed({ emitNow() }, 100) // immediate first align
} else {
emitSyncRequest(cfg.wallId) // align now, don't wait a tick
}
}
/** Leave wall mode and restore full-screen playback. */
fun exit() {
stopTimer()
val had = config != null
config = null
if (!had) return
Log.i("WallController", "exit wall mode")
playlist.setWallFollower(false)
media.setWallMute(false)
media.setVideoLooping(false)
media.setWallMode(false)
applyTransform(null)
}
private fun emitNow() {
val c = config ?: return
if (!c.isLeader) return
val item = playlist.currentItem ?: return
val pos = if (media.isPlayingVideo()) {
media.currentPositionMs() / 1000f
} else {
((System.currentTimeMillis() - playlist.itemStartedAtMs()) / 1000f).coerceAtLeast(0f)
}
emitSync(c.wallId, playlist.getIndex(), item.contentId.ifEmpty { null }, pos)
}
/** Handle an incoming `wall:sync` (followers only). */
fun onSync(data: JSONObject) {
val c = config ?: return
if (c.isLeader) return
if (data.optString("wall_id") != c.wallId) return
val leaderIdx = data.optInt("current_index", -1)
if (leaderIdx >= 0 && leaderIdx != playlist.getIndex()) playlist.gotoIndex(leaderIdx)
if (!media.isPlayingVideo()) return // images/widgets: index match is enough
val sentAt = data.optLong("sent_at", 0L)
val latency = if (sentAt > 0) ((System.currentTimeMillis() - sentAt) / 1000f).coerceAtLeast(0f) else 0f
val target = data.optDouble("position_sec", 0.0).toFloat() + latency
val curSec = media.currentPositionMs() / 1000f
val durMs = media.durationMs()
val durSec = if (durMs < 0) Float.NaN else durMs / 1000f
val drift = curSec - target
val ad = abs(drift)
when {
// Large drift: hard-seek (only when the target is within a known duration so we
// don't seek past the end). Don't seek every tick — exact seeks are expensive.
ad > 0.3f && !durSec.isNaN() && target < durSec -> {
media.seekExact((target * 1000).toLong())
media.setSpeed(1.0f)
}
// Small drift: gentle ±3% playbackRate nudge to converge.
ad > 0.05f -> media.setSpeed(if (drift > 0) 0.97f else 1.03f)
else -> media.setSpeed(1.0f)
}
}
/** Handle a follower's `wall:sync-request` (leader only): broadcast position now. */
fun onSyncRequest(data: JSONObject) {
val c = config ?: return
if (!c.isLeader) return
if (data.has("wall_id") && data.optString("wall_id") != c.wallId) return
emitNow()
}
private fun stopTimer() {
tick?.let { handler.removeCallbacks(it) }
tick = null
}
}

View file

@ -40,6 +40,8 @@ class WebSocketService : Service() {
var onRemoteTouch: ((Float, Float, String) -> Unit)? = null
var onRemoteKey: ((String) -> Unit)? = null
var onCommand: ((String, JSONObject?) -> Unit)? = null
var onWallSync: ((JSONObject) -> Unit)? = null
var onWallSyncRequest: ((JSONObject) -> Unit)? = null
inner class LocalBinder : Binder() {
fun getService(): WebSocketService = this@WebSocketService
@ -222,6 +224,18 @@ class WebSocketService : Service() {
handler.post { try { onRemoteKey?.invoke(keycode) } catch (e: Throwable) { Log.e("WebSocketService", "onRemoteKey cb: ${e.message}") } }
}
// Video wall. Post to the main thread: the handlers drive ExoPlayer
// (seek/speed/position), which is main-thread-only.
safeOn("wall:sync") { args ->
val data = args.firstOrNull() as? JSONObject ?: return@safeOn
handler.post { try { onWallSync?.invoke(data) } catch (e: Throwable) { Log.e("WebSocketService", "onWallSync cb: ${e.message}") } }
}
safeOn("wall:sync-request") { args ->
val data = args.firstOrNull() as? JSONObject ?: return@safeOn
handler.post { try { onWallSyncRequest?.invoke(data) } catch (e: Throwable) { Log.e("WebSocketService", "onWallSyncRequest cb: ${e.message}") } }
}
safeOn("device:command") { args ->
val data = args.firstOrNull() as? JSONObject ?: return@safeOn
val type = data.optString("type", "")
@ -525,6 +539,29 @@ class WebSocketService : Service() {
} catch (e: Throwable) { Log.w("WebSocketService", "sendPlaybackState: ${e.message}") }
}
// Video-wall senders. Guarded on socket.connected() like sendPlaybackState, so a
// pre-register tick is a no-op (the server would reject it as unauthenticated).
fun emitWallSync(wallId: String, currentIndex: Int, contentId: String?, positionSec: Float) {
if (socket?.connected() != true) return
try {
socket?.emit("wall:sync", JSONObject().apply {
put("wall_id", wallId)
put("device_id", config.deviceId)
put("current_index", currentIndex)
put("content_id", contentId ?: JSONObject.NULL)
put("position_sec", positionSec.toDouble())
put("sent_at", System.currentTimeMillis())
})
} catch (e: Throwable) { Log.w("WebSocketService", "emitWallSync: ${e.message}") }
}
fun emitWallSyncRequest(wallId: String) {
if (socket?.connected() != true) return
try {
socket?.emit("wall:sync-request", JSONObject().apply { put("wall_id", wallId) })
} catch (e: Throwable) { Log.w("WebSocketService", "emitWallSyncRequest: ${e.message}") }
}
fun disconnect() {
stopHeartbeat()
try { socket?.disconnect() } catch (e: Throwable) { Log.w("WebSocketService", "disconnect: ${e.message}") }