From 03c8753b0e3b2ce08a901b3d092c72778847e395 Mon Sep 17 00:00:00 2001 From: Alex Andres Date: Sat, 31 Oct 2026 09:39:41 +0100 Subject: [PATCH 1/2] feat: seek to the keyframe before a position with fastSeek MediaPlayer.fastSeek and MediaFileSource.fastSeek play from the keyframe at or before the position, without decoding up to it, so they take no longer however far the keyframes are apart, as a slider being dragged needs. --- docs/guide/media/media-files.md | 3 +++ .../src/main/cpp/include/JNI_MediaPlayer.h | 4 +-- .../src/main/cpp/include/media/MediaPlayer.h | 10 ++++--- .../src/main/cpp/src/JNI_MediaPlayer.cpp | 4 +-- .../src/main/cpp/src/media/MediaPlayer.cpp | 23 ++++++++++------ .../webrtc/media/player/MediaFileSource.java | 14 ++++++++++ .../webrtc/media/player/MediaPlayer.java | 27 ++++++++++++++++--- .../webrtc/media/player/MediaPlayerTest.java | 22 +++++++++++++++ 8 files changed, 88 insertions(+), 19 deletions(-) diff --git a/docs/guide/media/media-files.md b/docs/guide/media/media-files.md index 573510842..a1a3bf1e3 100644 --- a/docs/guide/media/media-files.md +++ b/docs/guide/media/media-files.md @@ -137,6 +137,7 @@ Duration is `0` for a source whose container does not say how long it runs, whic source.play(); // start, or resume after a pause source.pause(); // hold playback where it is source.seek(30_000_000); // move to 30 seconds +source.fastSeek(30_000_000); // move to the keyframe at or before 30 seconds source.setLooping(true); // start over instead of ending source.getPositionUs(); // where playback has got to source.getState(); // IDLE, PLAYING, PAUSED, ENDED or CLOSED @@ -146,6 +147,8 @@ Looping keeps the timing across the seam, so a looping file works as a stand-in A seek starts playback at the position asked for. The decoder has to go back to the keyframe at or before it to produce a picture at all, so the frames from there up to the position are decoded and dropped, and the audio is cut at it. How long a seek takes therefore depends on how often the media was encoded with keyframes. +`fastSeek` moves to that keyframe and plays from there, without decoding up to the position, so it takes no longer however far the keyframes are apart. That suits a slider being dragged, as browsers' `fastSeek()` does: fast seeks while it moves, and a `seek` once it is let go. In a container without an index, such as MPEG-TS, the keyframe can be the one after the position. + ## Following Playback ```java diff --git a/webrtc-media/src/main/cpp/include/JNI_MediaPlayer.h b/webrtc-media/src/main/cpp/include/JNI_MediaPlayer.h index 325cd184e..54f5d3271 100644 --- a/webrtc-media/src/main/cpp/include/JNI_MediaPlayer.h +++ b/webrtc-media/src/main/cpp/include/JNI_MediaPlayer.h @@ -49,10 +49,10 @@ extern "C" { /* * Class: dev_onvoid_webrtc_media_player_MediaPlayer * Method: seek - * Signature: (JJ)V + * Signature: (JJZ)V */ JNIEXPORT void JNICALL Java_dev_onvoid_webrtc_media_player_MediaPlayer_seek - (JNIEnv *, jclass, jlong, jlong); + (JNIEnv *, jclass, jlong, jlong, jboolean); /* * Class: dev_onvoid_webrtc_media_player_MediaPlayer diff --git a/webrtc-media/src/main/cpp/include/media/MediaPlayer.h b/webrtc-media/src/main/cpp/include/media/MediaPlayer.h index 3c6d29e14..f64b8f15c 100644 --- a/webrtc-media/src/main/cpp/include/media/MediaPlayer.h +++ b/webrtc-media/src/main/cpp/include/media/MediaPlayer.h @@ -90,9 +90,10 @@ namespace ffmpeg void Pause(); // Moves playback to the given position, in microseconds from the - // start. What is already queued is dropped, and what is decoded - // between the keyframe the seek lands on and the position as well. - void Seek(int64_t position_us); + // start. What is already queued is dropped. A precise seek drops + // what is decoded between the keyframe it lands on and the + // position as well; one that is not plays from that keyframe. + void Seek(int64_t position_us, bool precise); void SetLooping(bool looping); bool IsLooping() const; @@ -128,7 +129,7 @@ namespace ffmpeg int DecodeVideo(); int DecodeAudio(); - void PerformSeek(int64_t position_us); + void PerformSeek(int64_t position_us, bool precise); void SetState(int state); // Records the state, with the lock held. Returns whether it @@ -164,6 +165,7 @@ namespace ffmpeg bool seek_pending_ = false; int64_t seek_position_us_ = 0; + bool seek_precise_ = true; // Set once the source has run out and everything queued has been // delivered, so that the thread stops pumping until something diff --git a/webrtc-media/src/main/cpp/src/JNI_MediaPlayer.cpp b/webrtc-media/src/main/cpp/src/JNI_MediaPlayer.cpp index 133567661..f3ff87d1c 100644 --- a/webrtc-media/src/main/cpp/src/JNI_MediaPlayer.cpp +++ b/webrtc-media/src/main/cpp/src/JNI_MediaPlayer.cpp @@ -125,11 +125,11 @@ JNIEXPORT void JNICALL Java_dev_onvoid_webrtc_media_player_MediaPlayer_suspend } JNIEXPORT void JNICALL Java_dev_onvoid_webrtc_media_player_MediaPlayer_seek -(JNIEnv * env, jclass caller, jlong handle, jlong positionUs) +(JNIEnv * env, jclass caller, jlong handle, jlong positionUs, jboolean precise) { ffmpeg::CallFromJava(env, [&] { if (handle != 0) { - PlayerOf(handle)->Seek(positionUs); + PlayerOf(handle)->Seek(positionUs, precise == JNI_TRUE); } }); } diff --git a/webrtc-media/src/main/cpp/src/media/MediaPlayer.cpp b/webrtc-media/src/main/cpp/src/media/MediaPlayer.cpp index b90be950a..a7ad20fb9 100644 --- a/webrtc-media/src/main/cpp/src/media/MediaPlayer.cpp +++ b/webrtc-media/src/main/cpp/src/media/MediaPlayer.cpp @@ -120,6 +120,7 @@ namespace ffmpeg ended_ = false; seek_pending_ = true; seek_position_us_ = 0; + seek_precise_ = true; } // Under the lock, together with the flag: the decode thread stops @@ -157,7 +158,7 @@ namespace ffmpeg } } - void MediaPlayer::Seek(int64_t position_us) + void MediaPlayer::Seek(int64_t position_us, bool precise) { std::lock_guard lock(mutex_); @@ -167,6 +168,7 @@ namespace ffmpeg seek_pending_ = true; seek_position_us_ = position_us; + seek_precise_ = precise; ended_ = false; // The decode thread may be held up by a full queue, and it cannot act @@ -272,6 +274,7 @@ namespace ffmpeg for (;;) { int64_t seek_to = 0; + bool seek_precise = true; bool do_seek = false; { @@ -288,12 +291,13 @@ namespace ffmpeg if (seek_pending_) { seek_pending_ = false; seek_to = seek_position_us_; + seek_precise = seek_precise_; do_seek = true; } } if (do_seek) { - PerformSeek(seek_to); + PerformSeek(seek_to, seek_precise); continue; } @@ -500,7 +504,7 @@ namespace ffmpeg } } - void MediaPlayer::PerformSeek(int64_t position_us) + void MediaPlayer::PerformSeek(int64_t position_us, bool precise) { int result = reader_->Seek(position_us); @@ -514,11 +518,14 @@ namespace ffmpeg audio_decoder_.Flush(); // The reader lands on the keyframe at or before the position, which - // is where decoding has to begin. Playback begins at the position - // itself: what comes before it is decoded and dropped, and the audio - // frame it falls in is cut there. - video_decoder_.DiscardBefore(position_us); - audio_decoder_.DiscardBefore(position_us); + // is where decoding has to begin. A precise seek begins playback at + // the position itself: what comes before it is decoded and dropped, + // and the audio frame it falls in is cut there. Otherwise it begins + // at that keyframe, video and audio alike. + if (precise) { + video_decoder_.DiscardBefore(position_us); + audio_decoder_.DiscardBefore(position_us); + } // A seek starts a timeline of its own: the pacer maps the first item // that arrives onto the clock afresh, so the offset that kept looping diff --git a/webrtc-media/src/main/java/dev/onvoid/webrtc/media/player/MediaFileSource.java b/webrtc-media/src/main/java/dev/onvoid/webrtc/media/player/MediaFileSource.java index 4953cb0ec..83e4cca4a 100644 --- a/webrtc-media/src/main/java/dev/onvoid/webrtc/media/player/MediaFileSource.java +++ b/webrtc-media/src/main/java/dev/onvoid/webrtc/media/player/MediaFileSource.java @@ -244,11 +244,25 @@ public void pause() { * Moves playback to the given position. * * @param positionUs The position in microseconds from the start. + * + * @see MediaPlayer#seek(long) */ public void seek(long positionUs) { player.seek(positionUs); } + /** + * Moves playback to the keyframe at or before the given position, which + * takes no longer however far the keyframes are apart. + * + * @param positionUs The position in microseconds from the start. + * + * @see MediaPlayer#fastSeek(long) + */ + public void fastSeek(long positionUs) { + player.fastSeek(positionUs); + } + /** * Sets whether the source starts again when it runs out. * diff --git a/webrtc-media/src/main/java/dev/onvoid/webrtc/media/player/MediaPlayer.java b/webrtc-media/src/main/java/dev/onvoid/webrtc/media/player/MediaPlayer.java index 152681fe0..375aa6e97 100644 --- a/webrtc-media/src/main/java/dev/onvoid/webrtc/media/player/MediaPlayer.java +++ b/webrtc-media/src/main/java/dev/onvoid/webrtc/media/player/MediaPlayer.java @@ -186,13 +186,34 @@ public void pause() { /** * Moves playback to the given position. Whatever has been decoded but not - * yet delivered is dropped. + * yet delivered is dropped. Video is decoded from the keyframe at or + * before the position, and what comes before the position is dropped, so + * the further the keyframes are apart, the longer this takes. * * @param positionUs The position in microseconds from the start. + * + * @see #fastSeek(long) */ public void seek(long positionUs) { synchronized (lock) { - seek(handle, positionUs); + seek(handle, positionUs, true); + } + } + + /** + * Moves playback to the keyframe at or before the given position, and + * plays from there. Nothing before the position is dropped, so this takes + * no longer however far the keyframes are apart, which is what a slider + * being dragged needs; a {@link #seek(long)} once it is let go lands on + * the position itself. Whatever has been decoded but not yet delivered is + * dropped. In a container without an index, such as MPEG-TS, the keyframe + * can be the one after the position. + * + * @param positionUs The position in microseconds from the start. + */ + public void fastSeek(long positionUs) { + synchronized (lock) { + seek(handle, positionUs, false); } } @@ -350,7 +371,7 @@ private native long create(long readerHandle, long tableAddress, private static native void suspend(long handle); - private static native void seek(long handle, long positionUs); + private static native void seek(long handle, long positionUs, boolean precise); private static native void setLooping(long handle, boolean looping); diff --git a/webrtc-media/src/test/java/dev/onvoid/webrtc/media/player/MediaPlayerTest.java b/webrtc-media/src/test/java/dev/onvoid/webrtc/media/player/MediaPlayerTest.java index 7419abf8b..edaa64823 100644 --- a/webrtc-media/src/test/java/dev/onvoid/webrtc/media/player/MediaPlayerTest.java +++ b/webrtc-media/src/test/java/dev/onvoid/webrtc/media/player/MediaPlayerTest.java @@ -306,6 +306,28 @@ void seekStartsAtThePositionAskedFor() throws Exception { } } + @Test + void fastSeekPlaysFromTheKeyframeBefore() throws Exception { + try (Playback playback = new Playback()) { + // Between the keyframes at 1 s and 2 s: playback starts at the one + // at 1 s, with nothing dropped. + playback.player.fastSeek(1_500_000); + playback.player.play(); + + assertTrue(playback.ended.await(15, TimeUnit.SECONDS), "no end of stream"); + + assertEquals(1_000_000, playback.firstFramePositionUs.get()); + + // 1 s to 2.933 s, and the audio from the first packet the reader + // lands on, a few milliseconds after 1 s, uncut: about half a + // second more of each than seek gives. + assertTrue(Math.abs(playback.frames.get() - 30) <= 1, + "video frames: " + playback.frames.get()); + assertTrue(Math.abs(playback.chunks.get() - 198) <= 3, + "audio chunks: " + playback.chunks.get()); + } + } + @Test void seekCutsTheAudioFrameItFallsIn() throws Exception { // Two seconds in frames of 100 ms, at the rate WebRTC takes, so that From d1d43a695f19b0c80096a6506ef9514dd167cf74 Mon Sep 17 00:00:00 2001 From: Alex Andres Date: Sat, 31 Oct 2026 12:47:34 +0100 Subject: [PATCH 2/2] fix: forget a late seek landing when the decoder is flushed A seek whose first picture came only when the decoder was drained at the end left its late landing behind, and a looping source took it on the next pass, sending the reader back to the end again. --- .../src/main/cpp/include/media/VideoDecoder.h | 3 +- .../src/main/cpp/src/media/VideoDecoder.cpp | 6 ++ .../webrtc/media/player/MediaPlayerTest.java | 88 ++++++++++++++++++- 3 files changed, 95 insertions(+), 2 deletions(-) diff --git a/webrtc-media/src/main/cpp/include/media/VideoDecoder.h b/webrtc-media/src/main/cpp/include/media/VideoDecoder.h index d6142a003..f2f98b967 100644 --- a/webrtc-media/src/main/cpp/include/media/VideoDecoder.h +++ b/webrtc-media/src/main/cpp/include/media/VideoDecoder.h @@ -85,7 +85,8 @@ namespace ffmpeg // keyframe before the position, and the decoder had nothing to // show until the next one. A container without an index, such as // MPEG-TS, is searched by timestamp and can land that way. - // Reported once, by the call after that frame was taken. + // Reported once, by the call after that frame was taken, and + // forgotten by a flush. bool LandedPastPosition(); // Hands a packet to the decoder, or null to start draining at the diff --git a/webrtc-media/src/main/cpp/src/media/VideoDecoder.cpp b/webrtc-media/src/main/cpp/src/media/VideoDecoder.cpp index e42898745..0508b3e29 100644 --- a/webrtc-media/src/main/cpp/src/media/VideoDecoder.cpp +++ b/webrtc-media/src/main/cpp/src/media/VideoDecoder.cpp @@ -360,6 +360,12 @@ namespace ffmpeg awaiting_key_frame_ = false; skipped_ = 0; discarding_ = false; + + // A late landing not asked about before the flush, as at the end of + // the stream, belongs to the position that was left too: taken after + // it, it would send the reader back from wherever decoding goes on. + dropped_before_ = false; + landed_past_ = false; } void VideoDecoder::DiscardBefore(int64_t timestamp_us) diff --git a/webrtc-media/src/test/java/dev/onvoid/webrtc/media/player/MediaPlayerTest.java b/webrtc-media/src/test/java/dev/onvoid/webrtc/media/player/MediaPlayerTest.java index 53532f21d..db4ba08c4 100644 --- a/webrtc-media/src/test/java/dev/onvoid/webrtc/media/player/MediaPlayerTest.java +++ b/webrtc-media/src/test/java/dev/onvoid/webrtc/media/player/MediaPlayerTest.java @@ -24,9 +24,11 @@ import java.io.IOException; import java.net.URL; +import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.Paths; import java.util.ArrayList; +import java.util.Arrays; import java.util.Collections; import java.util.List; import java.util.concurrent.CountDownLatch; @@ -398,6 +400,46 @@ void seekWithoutIndexToItsStartPlaysItAll() throws Exception { } } + @Test + void loopAfterASeekWithoutIndexThatLandedLateStartsOver() throws Exception { + // The asset cut off after its last keyframe, at 8.133 s. A search for + // 7.8 s lands past the one at 4.133 s, and the decoder, holding + // pictures back for its B-frames, gives the one at 8.133 s only once + // it is drained at the end, where a seek that landed late is not + // retried. + Path ts = cutAfterLastKeyFrame( + Paths.get(MediaPlayerTest.class.getResource(TS_ASSET).toURI()), + tempDir.resolve("cut.ts")); + + try (Playback playback = new Playback(new MediaReader(ts))) { + playback.player.setLooping(true); + playback.player.seek(7_800_000); + playback.player.play(); + + // Past the seam and well into the next pass. + assertFalse(playback.ended.await(2000, TimeUnit.MILLISECONDS), + "a looping player reported an end of stream"); + + List positions = new ArrayList<>(playback.framePositionsUs); + + assertTrue(positions.size() > 10, "frames: " + positions); + + // The picture at 8.133 s, and then the next pass from the start. + // The late landing, left over from the seek once the source had + // started over, used to send the reader back before 7.8 s, so that + // the last pictures played again before the next pass. What the + // first frame reports depends on whether the decoding thread has + // already started over, which it does long before the frame is + // due; the frames after it are all from the next pass. + assertTrue(positions.get(1) < 1_000_000, "next pass not from the start: " + positions); + + for (int i = 2; i < positions.size(); i++) { + assertTrue(positions.get(i) > positions.get(i - 1), + "position went back at frame " + i + ": " + positions); + } + } + } + @Test void seekNearTheEndIsNotLost() throws Exception { // Two seconds of video only. All of it is decoded and queued within @@ -850,6 +892,45 @@ private static boolean channelsEqual(byte[] data) { return true; } + /** + * Copies an MPEG-TS file up to the next packet of its video stream after + * its last keyframe, so that the keyframe is the last picture. The muxer + * marks the first packet of a keyframe with the random access indicator + * of its adaptation field. + */ + private static Path cutAfterLastKeyFrame(Path source, Path target) throws IOException { + final int size = 188; + byte[] data = Files.readAllBytes(source); + + int keyFrame = -1; + int keyPid = -1; + + for (int i = 0; i + size <= data.length; i += size) { + boolean adaptation = (data[i + 3] & 0x20) != 0 && (data[i + 4] & 0xff) > 0; + + if (adaptation && (data[i + 5] & 0x40) != 0) { + keyFrame = i; + keyPid = ((data[i + 1] & 0x1f) << 8) | (data[i + 2] & 0xff); + } + } + + assertTrue(keyFrame >= 0, "no keyframe marked in " + source); + + int end = data.length; + + for (int i = keyFrame + size; i + size <= data.length; i += size) { + int pid = ((data[i + 1] & 0x1f) << 8) | (data[i + 2] & 0xff); + boolean start = (data[i + 1] & 0x40) != 0; + + if (pid == keyPid && start) { + end = i; + break; + } + } + + return Files.write(target, Arrays.copyOf(data, end)); + } + private static Path asset() throws Exception { URL url = MediaPlayerTest.class.getResource(ASSET); @@ -898,6 +979,7 @@ private final class Playback implements AutoCloseable { final AtomicReference error = new AtomicReference<>(); final List frameTimes = Collections.synchronizedList(new ArrayList<>()); final AtomicLong firstFramePositionUs = new AtomicLong(-1); + final List framePositionsUs = Collections.synchronizedList(new ArrayList<>()); final AtomicLong firstChunkNs = new AtomicLong(); final AtomicInteger unevenChunks = new AtomicInteger(); final AtomicLong nonZeroSamples = new AtomicLong(); @@ -906,10 +988,14 @@ private final class Playback implements AutoCloseable { private final VideoTrackSink videoSink = frame -> { // The pacer delivers on its own thread, having just moved the // position to the frame's presentation time. + long positionUs = Playback.this.player.getPositionUs(); + if (firstFramePositionUs.get() < 0) { - firstFramePositionUs.set(Playback.this.player.getPositionUs()); + firstFramePositionUs.set(positionUs); } + framePositionsUs.add(positionUs); + frameTimes.add(System.nanoTime()); frames.incrementAndGet(); width.set(frame.buffer.getWidth());