diff --git a/docs/guide/media/media-files.md b/docs/guide/media/media-files.md index 891ca90c2..48e127348 100644 --- a/docs/guide/media/media-files.md +++ b/docs/guide/media/media-files.md @@ -137,6 +137,7 @@ Duration is `0` for a source whose container does not say how long it runs, whic source.play(); // start, or resume after a pause source.pause(); // hold playback where it is source.seek(30_000_000); // move to 30 seconds +source.fastSeek(30_000_000); // move to the keyframe at or before 30 seconds source.setLooping(true); // start over instead of ending source.getPositionUs(); // where playback has got to source.getState(); // IDLE, PLAYING, PAUSED, ENDED or CLOSED @@ -146,6 +147,8 @@ Looping keeps the timing across the seam, so a looping file works as a stand-in A seek starts playback at the position asked for. The decoder has to go back to the keyframe at or before it to produce a picture at all, so the frames from there up to the position are decoded and dropped, and the audio is cut at it. How long a seek takes therefore depends on how often the media was encoded with keyframes, not on how long the media is: MKV and MP4 have an index that takes the reader straight to that keyframe. MPEG-TS has none, and a search for the position by its timestamps can land past the keyframe before it. The player then searches again further back, a second before the position, then two, four and so on, so that the seek still starts at the position. +`fastSeek` moves to that keyframe and plays from there, without decoding up to the position, so it takes no longer however far the keyframes are apart. That suits a slider being dragged, as browsers' `fastSeek()` does: fast seeks while it moves, and a `seek` once it is let go. In a container without an index, such as MPEG-TS, the keyframe can be the one after the position. + ## Following Playback ```java diff --git a/webrtc-media/src/main/cpp/include/JNI_MediaPlayer.h b/webrtc-media/src/main/cpp/include/JNI_MediaPlayer.h index 325cd184e..54f5d3271 100644 --- a/webrtc-media/src/main/cpp/include/JNI_MediaPlayer.h +++ b/webrtc-media/src/main/cpp/include/JNI_MediaPlayer.h @@ -49,10 +49,10 @@ extern "C" { /* * Class: dev_onvoid_webrtc_media_player_MediaPlayer * Method: seek - * Signature: (JJ)V + * Signature: (JJZ)V */ JNIEXPORT void JNICALL Java_dev_onvoid_webrtc_media_player_MediaPlayer_seek - (JNIEnv *, jclass, jlong, jlong); + (JNIEnv *, jclass, jlong, jlong, jboolean); /* * Class: dev_onvoid_webrtc_media_player_MediaPlayer diff --git a/webrtc-media/src/main/cpp/include/media/MediaPlayer.h b/webrtc-media/src/main/cpp/include/media/MediaPlayer.h index ea0128b0b..a06a4fd94 100644 --- a/webrtc-media/src/main/cpp/include/media/MediaPlayer.h +++ b/webrtc-media/src/main/cpp/include/media/MediaPlayer.h @@ -90,9 +90,10 @@ namespace ffmpeg void Pause(); // Moves playback to the given position, in microseconds from the - // start. What is already queued is dropped, and what is decoded - // between the keyframe the seek lands on and the position as well. - void Seek(int64_t position_us); + // start. What is already queued is dropped. A precise seek drops + // what is decoded between the keyframe it lands on and the + // position as well; one that is not plays from that keyframe. + void Seek(int64_t position_us, bool precise); void SetLooping(bool looping); bool IsLooping() const; @@ -131,7 +132,7 @@ namespace ffmpeg int DecodeVideo(bool at_end); int DecodeAudio(); - void PerformSeek(int64_t position_us); + void PerformSeek(int64_t position_us, bool precise); // Moves the reader to the given time and starts decoding from // there towards the position of the seek being carried out. @@ -140,9 +141,10 @@ namespace ffmpeg // Seeks again further back, after the video decoded since a seek // started past its position: the reader landed past the keyframe - // before it. Returns whether it did; there is nothing further back - // to try once the reader would land where it did already, or the - // distance has grown past kMaxSeekBackUs. + // before it. Only a precise seek does: a fast one plays from the + // keyframe it lands on. Returns whether it did; there is nothing + // further back to try once the reader would land where it did + // already, or the distance has grown past kMaxSeekBackUs. bool SeekFurtherBack(); void SetState(int state); @@ -179,6 +181,7 @@ namespace ffmpeg bool seek_pending_ = false; int64_t seek_position_us_ = 0; + bool seek_precise_ = true; // How far before the position a seek that landed too late looks // next, doubling each time, and how far at most: past that, a @@ -188,8 +191,10 @@ namespace ffmpeg static constexpr int64_t kMaxSeekBackUs = 64000000; // The seek being carried out, on the decode thread: its position, - // how far before it the reader was last sent, and where to. + // whether it is precise, how far before it the reader was last + // sent, and where to. int64_t seek_target_us_ = 0; + bool seek_target_precise_ = true; int64_t seek_back_us_ = 0; int64_t seek_from_us_ = 0; diff --git a/webrtc-media/src/main/cpp/include/media/VideoDecoder.h b/webrtc-media/src/main/cpp/include/media/VideoDecoder.h index d6142a003..f2f98b967 100644 --- a/webrtc-media/src/main/cpp/include/media/VideoDecoder.h +++ b/webrtc-media/src/main/cpp/include/media/VideoDecoder.h @@ -85,7 +85,8 @@ namespace ffmpeg // keyframe before the position, and the decoder had nothing to // show until the next one. A container without an index, such as // MPEG-TS, is searched by timestamp and can land that way. - // Reported once, by the call after that frame was taken. + // Reported once, by the call after that frame was taken, and + // forgotten by a flush. bool LandedPastPosition(); // Hands a packet to the decoder, or null to start draining at the diff --git a/webrtc-media/src/main/cpp/src/JNI_MediaPlayer.cpp b/webrtc-media/src/main/cpp/src/JNI_MediaPlayer.cpp index 133567661..f3ff87d1c 100644 --- a/webrtc-media/src/main/cpp/src/JNI_MediaPlayer.cpp +++ b/webrtc-media/src/main/cpp/src/JNI_MediaPlayer.cpp @@ -125,11 +125,11 @@ JNIEXPORT void JNICALL Java_dev_onvoid_webrtc_media_player_MediaPlayer_suspend } JNIEXPORT void JNICALL Java_dev_onvoid_webrtc_media_player_MediaPlayer_seek -(JNIEnv * env, jclass caller, jlong handle, jlong positionUs) +(JNIEnv * env, jclass caller, jlong handle, jlong positionUs, jboolean precise) { ffmpeg::CallFromJava(env, [&] { if (handle != 0) { - PlayerOf(handle)->Seek(positionUs); + PlayerOf(handle)->Seek(positionUs, precise == JNI_TRUE); } }); } diff --git a/webrtc-media/src/main/cpp/src/media/MediaPlayer.cpp b/webrtc-media/src/main/cpp/src/media/MediaPlayer.cpp index 1df87f794..bc5d3eab1 100644 --- a/webrtc-media/src/main/cpp/src/media/MediaPlayer.cpp +++ b/webrtc-media/src/main/cpp/src/media/MediaPlayer.cpp @@ -121,6 +121,7 @@ namespace ffmpeg ended_ = false; seek_pending_ = true; seek_position_us_ = 0; + seek_precise_ = true; } // Under the lock, together with the flag: the decode thread stops @@ -158,7 +159,7 @@ namespace ffmpeg } } - void MediaPlayer::Seek(int64_t position_us) + void MediaPlayer::Seek(int64_t position_us, bool precise) { std::lock_guard lock(mutex_); @@ -168,6 +169,7 @@ namespace ffmpeg seek_pending_ = true; seek_position_us_ = position_us; + seek_precise_ = precise; ended_ = false; // The decode thread may be held up by a full queue, and it cannot act @@ -273,6 +275,7 @@ namespace ffmpeg for (;;) { int64_t seek_to = 0; + bool seek_precise = true; bool do_seek = false; { @@ -289,12 +292,13 @@ namespace ffmpeg if (seek_pending_) { seek_pending_ = false; seek_to = seek_position_us_; + seek_precise = seek_precise_; do_seek = true; } } if (do_seek) { - PerformSeek(seek_to); + PerformSeek(seek_to, seek_precise); continue; } @@ -509,9 +513,10 @@ namespace ffmpeg } } - void MediaPlayer::PerformSeek(int64_t position_us) + void MediaPlayer::PerformSeek(int64_t position_us, bool precise) { seek_target_us_ = position_us; + seek_target_precise_ = precise; seek_back_us_ = 0; int result = SeekReader(position_us); @@ -523,6 +528,10 @@ namespace ffmpeg bool MediaPlayer::SeekFurtherBack() { + if (!seek_target_precise_) { + return false; + } + const int64_t back = seek_back_us_ == 0 ? kFirstSeekBackUs : seek_back_us_ * 2; const int64_t from = std::max(seek_target_us_ - back, 0); @@ -555,11 +564,14 @@ namespace ffmpeg audio_decoder_.Flush(); // The reader lands on the keyframe at or before the position, which - // is where decoding has to begin. Playback begins at the position - // itself: what comes before it is decoded and dropped, and the audio - // frame it falls in is cut there. - video_decoder_.DiscardBefore(position_us); - audio_decoder_.DiscardBefore(position_us); + // is where decoding has to begin. A precise seek begins playback at + // the position itself: what comes before it is decoded and dropped, + // and the audio frame it falls in is cut there. Otherwise it begins + // at that keyframe, video and audio alike. + if (seek_target_precise_) { + video_decoder_.DiscardBefore(position_us); + audio_decoder_.DiscardBefore(position_us); + } // A seek starts a timeline of its own: the pacer maps the first item // that arrives onto the clock afresh, so the offset that kept looping diff --git a/webrtc-media/src/main/cpp/src/media/VideoDecoder.cpp b/webrtc-media/src/main/cpp/src/media/VideoDecoder.cpp index e42898745..0508b3e29 100644 --- a/webrtc-media/src/main/cpp/src/media/VideoDecoder.cpp +++ b/webrtc-media/src/main/cpp/src/media/VideoDecoder.cpp @@ -360,6 +360,12 @@ namespace ffmpeg awaiting_key_frame_ = false; skipped_ = 0; discarding_ = false; + + // A late landing not asked about before the flush, as at the end of + // the stream, belongs to the position that was left too: taken after + // it, it would send the reader back from wherever decoding goes on. + dropped_before_ = false; + landed_past_ = false; } void VideoDecoder::DiscardBefore(int64_t timestamp_us) diff --git a/webrtc-media/src/main/java/dev/onvoid/webrtc/media/player/MediaFileSource.java b/webrtc-media/src/main/java/dev/onvoid/webrtc/media/player/MediaFileSource.java index 4953cb0ec..83e4cca4a 100644 --- a/webrtc-media/src/main/java/dev/onvoid/webrtc/media/player/MediaFileSource.java +++ b/webrtc-media/src/main/java/dev/onvoid/webrtc/media/player/MediaFileSource.java @@ -244,11 +244,25 @@ public void pause() { * Moves playback to the given position. * * @param positionUs The position in microseconds from the start. + * + * @see MediaPlayer#seek(long) */ public void seek(long positionUs) { player.seek(positionUs); } + /** + * Moves playback to the keyframe at or before the given position, which + * takes no longer however far the keyframes are apart. + * + * @param positionUs The position in microseconds from the start. + * + * @see MediaPlayer#fastSeek(long) + */ + public void fastSeek(long positionUs) { + player.fastSeek(positionUs); + } + /** * Sets whether the source starts again when it runs out. * diff --git a/webrtc-media/src/main/java/dev/onvoid/webrtc/media/player/MediaPlayer.java b/webrtc-media/src/main/java/dev/onvoid/webrtc/media/player/MediaPlayer.java index 152681fe0..375aa6e97 100644 --- a/webrtc-media/src/main/java/dev/onvoid/webrtc/media/player/MediaPlayer.java +++ b/webrtc-media/src/main/java/dev/onvoid/webrtc/media/player/MediaPlayer.java @@ -186,13 +186,34 @@ public void pause() { /** * Moves playback to the given position. Whatever has been decoded but not - * yet delivered is dropped. + * yet delivered is dropped. Video is decoded from the keyframe at or + * before the position, and what comes before the position is dropped, so + * the further the keyframes are apart, the longer this takes. * * @param positionUs The position in microseconds from the start. + * + * @see #fastSeek(long) */ public void seek(long positionUs) { synchronized (lock) { - seek(handle, positionUs); + seek(handle, positionUs, true); + } + } + + /** + * Moves playback to the keyframe at or before the given position, and + * plays from there. Nothing before the position is dropped, so this takes + * no longer however far the keyframes are apart, which is what a slider + * being dragged needs; a {@link #seek(long)} once it is let go lands on + * the position itself. Whatever has been decoded but not yet delivered is + * dropped. In a container without an index, such as MPEG-TS, the keyframe + * can be the one after the position. + * + * @param positionUs The position in microseconds from the start. + */ + public void fastSeek(long positionUs) { + synchronized (lock) { + seek(handle, positionUs, false); } } @@ -350,7 +371,7 @@ private native long create(long readerHandle, long tableAddress, private static native void suspend(long handle); - private static native void seek(long handle, long positionUs); + private static native void seek(long handle, long positionUs, boolean precise); private static native void setLooping(long handle, boolean looping); diff --git a/webrtc-media/src/test/java/dev/onvoid/webrtc/media/player/MediaPlayerTest.java b/webrtc-media/src/test/java/dev/onvoid/webrtc/media/player/MediaPlayerTest.java index cddbef903..db4ba08c4 100644 --- a/webrtc-media/src/test/java/dev/onvoid/webrtc/media/player/MediaPlayerTest.java +++ b/webrtc-media/src/test/java/dev/onvoid/webrtc/media/player/MediaPlayerTest.java @@ -24,9 +24,11 @@ import java.io.IOException; import java.net.URL; +import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.Paths; import java.util.ArrayList; +import java.util.Arrays; import java.util.Collections; import java.util.List; import java.util.concurrent.CountDownLatch; @@ -316,6 +318,28 @@ void seekStartsAtThePositionAskedFor() throws Exception { } } + @Test + void fastSeekPlaysFromTheKeyframeBefore() throws Exception { + try (Playback playback = new Playback()) { + // Between the keyframes at 1 s and 2 s: playback starts at the one + // at 1 s, with nothing dropped. + playback.player.fastSeek(1_500_000); + playback.player.play(); + + assertTrue(playback.ended.await(15, TimeUnit.SECONDS), "no end of stream"); + + assertEquals(1_000_000, playback.firstFramePositionUs.get()); + + // 1 s to 2.933 s, and the audio from the first packet the reader + // lands on, a few milliseconds after 1 s, uncut: about half a + // second more of each than seek gives. + assertTrue(Math.abs(playback.frames.get() - 30) <= 1, + "video frames: " + playback.frames.get()); + assertTrue(Math.abs(playback.chunks.get() - 198) <= 3, + "audio chunks: " + playback.chunks.get()); + } + } + @Test void seekCutsTheAudioFrameItFallsIn() throws Exception { // Two seconds in frames of 100 ms, at the rate WebRTC takes, so that @@ -376,6 +400,46 @@ void seekWithoutIndexToItsStartPlaysItAll() throws Exception { } } + @Test + void loopAfterASeekWithoutIndexThatLandedLateStartsOver() throws Exception { + // The asset cut off after its last keyframe, at 8.133 s. A search for + // 7.8 s lands past the one at 4.133 s, and the decoder, holding + // pictures back for its B-frames, gives the one at 8.133 s only once + // it is drained at the end, where a seek that landed late is not + // retried. + Path ts = cutAfterLastKeyFrame( + Paths.get(MediaPlayerTest.class.getResource(TS_ASSET).toURI()), + tempDir.resolve("cut.ts")); + + try (Playback playback = new Playback(new MediaReader(ts))) { + playback.player.setLooping(true); + playback.player.seek(7_800_000); + playback.player.play(); + + // Past the seam and well into the next pass. + assertFalse(playback.ended.await(2000, TimeUnit.MILLISECONDS), + "a looping player reported an end of stream"); + + List positions = new ArrayList<>(playback.framePositionsUs); + + assertTrue(positions.size() > 10, "frames: " + positions); + + // The picture at 8.133 s, and then the next pass from the start. + // The late landing, left over from the seek once the source had + // started over, used to send the reader back before 7.8 s, so that + // the last pictures played again before the next pass. What the + // first frame reports depends on whether the decoding thread has + // already started over, which it does long before the frame is + // due; the frames after it are all from the next pass. + assertTrue(positions.get(1) < 1_000_000, "next pass not from the start: " + positions); + + for (int i = 2; i < positions.size(); i++) { + assertTrue(positions.get(i) > positions.get(i - 1), + "position went back at frame " + i + ": " + positions); + } + } + } + @Test void seekNearTheEndIsNotLost() throws Exception { // Two seconds of video only. All of it is decoded and queued within @@ -828,6 +892,45 @@ private static boolean channelsEqual(byte[] data) { return true; } + /** + * Copies an MPEG-TS file up to the next packet of its video stream after + * its last keyframe, so that the keyframe is the last picture. The muxer + * marks the first packet of a keyframe with the random access indicator + * of its adaptation field. + */ + private static Path cutAfterLastKeyFrame(Path source, Path target) throws IOException { + final int size = 188; + byte[] data = Files.readAllBytes(source); + + int keyFrame = -1; + int keyPid = -1; + + for (int i = 0; i + size <= data.length; i += size) { + boolean adaptation = (data[i + 3] & 0x20) != 0 && (data[i + 4] & 0xff) > 0; + + if (adaptation && (data[i + 5] & 0x40) != 0) { + keyFrame = i; + keyPid = ((data[i + 1] & 0x1f) << 8) | (data[i + 2] & 0xff); + } + } + + assertTrue(keyFrame >= 0, "no keyframe marked in " + source); + + int end = data.length; + + for (int i = keyFrame + size; i + size <= data.length; i += size) { + int pid = ((data[i + 1] & 0x1f) << 8) | (data[i + 2] & 0xff); + boolean start = (data[i + 1] & 0x40) != 0; + + if (pid == keyPid && start) { + end = i; + break; + } + } + + return Files.write(target, Arrays.copyOf(data, end)); + } + private static Path asset() throws Exception { URL url = MediaPlayerTest.class.getResource(ASSET); @@ -876,6 +979,7 @@ private final class Playback implements AutoCloseable { final AtomicReference error = new AtomicReference<>(); final List frameTimes = Collections.synchronizedList(new ArrayList<>()); final AtomicLong firstFramePositionUs = new AtomicLong(-1); + final List framePositionsUs = Collections.synchronizedList(new ArrayList<>()); final AtomicLong firstChunkNs = new AtomicLong(); final AtomicInteger unevenChunks = new AtomicInteger(); final AtomicLong nonZeroSamples = new AtomicLong(); @@ -884,10 +988,14 @@ private final class Playback implements AutoCloseable { private final VideoTrackSink videoSink = frame -> { // The pacer delivers on its own thread, having just moved the // position to the frame's presentation time. + long positionUs = Playback.this.player.getPositionUs(); + if (firstFramePositionUs.get() < 0) { - firstFramePositionUs.set(Playback.this.player.getPositionUs()); + firstFramePositionUs.set(positionUs); } + framePositionsUs.add(positionUs); + frameTimes.add(System.nanoTime()); frames.incrementAndGet(); width.set(frame.buffer.getWidth());