fix: wide planar audio, audio timestamp gaps and vanished audio devices

docs/guide/client-api.md

@@ -416,7 +416,7 @@ devices.useMicrophone(devices.microphones().get(1));
 devices.useSpeaker(devices.speakers().get(0));
 ```
 
-`useMicrophone` and `useSpeaker` switch devices also for tracks that are sent or played now.
+`useMicrophone` and `useSpeaker` switch devices also for tracks that are sent or played now. A device that was disconnected since the list was read fails the returned future, and the device in use stays as it was. On Linux, where WebRTC's audio device module cannot tell devices apart, the default device is used whichever is given.
 
 ## Receiving Media
 

webrtc-java-media/src/main/cpp/include/media/AudioDecoder.h

@@ -46,6 +46,18 @@ namespace ffmpeg
 			// WebRTC takes 10 ms of audio at a time and nothing else.
 			static constexpr int kFramesPerChunk = kSampleRate / 100;
 
+			// How far a frame's timestamp may stray from where the audio
+			// before it ends. Containers round their timestamps, so a frame
+			// is followed by its sample count as long as it stays this close,
+			// which is also below the point where audio running ahead of
+			// video starts to show.
+			static constexpr int64_t kTimestampToleranceUs = 40000;
+
+			// The longest stretch of missing audio filled with silence, or of
+			// repeated audio skipped. Beyond it the source's clock is taken
+			// to have jumped, as ffplay does past the same ten seconds.
+			static constexpr int64_t kMaxGapUs = 10000000;
+
 			AudioDecoder() = default;
 			~AudioDecoder();
 
@@ -82,6 +94,17 @@ namespace ffmpeg
 			// Resamples one decoded frame into the pending buffer.
 			int Resample(const AVFrame * frame);
 
+			// Holds the audio to the frame's timestamp where it strays from
+			// where the audio before it ends: missing audio becomes silence,
+			// and a frame that repeats audio already given out skips what it
+			// repeats, which it reports as the number of input samples to
+			// skip.
+			int FollowTimestamp(int64_t timestamp_us, int * skip);
+
+			// Where the audio taken in so far ends, in microseconds on the
+			// source's timeline.
+			int64_t GetEndTimestampUs() const;
+
 			// Moves whatever swresample still holds into the pending buffer,
 			// which it only hands out when told that no more input follows.
 			int DrainResampler();
@@ -113,7 +136,9 @@ namespace ffmpeg
 			// When the next chunk starts, in microseconds on the source's own
 			// timeline. Chunks advance it in exact 10 ms steps, so the audio
 			// this produces is perfectly regular whatever the container's
-			// packet timing looks like.
+			// packet timing looks like. Gaps in the source's timestamps are
+			// filled with silence rather than closed up, so audio stays where
+			// video, which follows its own timestamps, expects it.
 			int64_t next_timestamp_us_ = 0;
 			bool have_timestamp_ = false;
 

webrtc-java-media/src/main/cpp/src/media/AudioDecoder.cpp

@@ -16,6 +16,10 @@
 
 #include "media/AudioDecoder.h"
 
+#include <algorithm>
+#include <climits>
+#include <cstdlib>
+
 extern "C" {
 #include <libavutil/channel_layout.h>
 #include <libavutil/opt.h>
@@ -205,15 +209,6 @@ namespace ffmpeg
 
 	int AudioDecoder::Resample(const AVFrame * frame)
 	{
-		if (!have_timestamp_) {
-			int64_t pts = frame->best_effort_timestamp != AV_NOPTS_VALUE
-					? frame->best_effort_timestamp : frame->pts;
-
-			next_timestamp_us_ = pts != AV_NOPTS_VALUE
-					? av_rescale_q(pts, time_base_, AV_TIME_BASE_Q) : 0;
-			have_timestamp_ = true;
-		}
-
 		if (frame->format != in_format_ || frame->sample_rate != in_rate_
 				|| av_channel_layout_compare(&frame->ch_layout, &in_layout_) != 0) {
 			// The stream changed its format. What swresample still holds of
@@ -232,24 +227,63 @@ namespace ffmpeg
 			}
 		}
 
+		int64_t pts = frame->best_effort_timestamp != AV_NOPTS_VALUE
+				? frame->best_effort_timestamp : frame->pts;
+		int skip = 0;
+
+		if (!have_timestamp_) {
+			next_timestamp_us_ = pts != AV_NOPTS_VALUE
+					? av_rescale_q(pts, time_base_, AV_TIME_BASE_Q) : 0;
+			have_timestamp_ = true;
+		}
+		else if (pts != AV_NOPTS_VALUE) {
+			int result = FollowTimestamp(av_rescale_q(pts, time_base_, AV_TIME_BASE_Q),
+					&skip);
+
+			if (result < 0) {
+				return result;
+			}
+		}
+
+		const int samples = frame->nb_samples - skip;
+
+		if (samples <= 0) {
+			return 0;
+		}
+
 		// swresample holds samples back when rates differ, and those come out
 		// of a later call, so the room needed is the delay plus this frame.
 		int64_t delay = swr_get_delay(swr_context_, in_rate_);
-		int64_t capacity = av_rescale_rnd(delay + frame->nb_samples, kSampleRate,
+		int64_t capacity = av_rescale_rnd(delay + samples, kSampleRate,
 				in_rate_, AV_ROUND_UP);
 
 		if (capacity <= 0) {
 			return 0;
 		}
 
+		// swresample reads a plane for every input channel, and data has room
+		// for eight: planar audio of more channels has the rest only in
+		// extended_data.
+		const AVSampleFormat format = static_cast<AVSampleFormat>(frame->format);
+		const bool planar = av_sample_fmt_is_planar(format) != 0;
+		const int planes = planar ? frame->ch_layout.nb_channels : 1;
+		const int stride = av_get_bytes_per_sample(format)
+				* (planar ? 1 : frame->ch_layout.nb_channels);
+
+		std::vector<const uint8_t *> input(static_cast<size_t>(planes));
+
+		for (int i = 0; i < planes; ++i) {
+			input[i] = frame->extended_data[i] + static_cast<size_t>(skip) * stride;
+		}
+
 		const size_t offset = pending_.size();
 
 		pending_.resize(offset + static_cast<size_t>(capacity) * channels_);
 
 		uint8_t * output = reinterpret_cast<uint8_t *>(pending_.data() + offset);
 
 		int converted = swr_convert(swr_context_, &output, static_cast<int>(capacity),
-				const_cast<const uint8_t **>(frame->data), frame->nb_samples);
+				input.data(), samples);
 
 		if (converted < 0) {
 			pending_.resize(offset);
@@ -263,6 +297,63 @@ namespace ffmpeg
 		return 0;
 	}
 
+	int AudioDecoder::FollowTimestamp(int64_t timestamp_us, int * skip)
+	{
+		const int64_t drift = timestamp_us - GetEndTimestampUs();
+
+		if (std::abs(drift) <= kTimestampToleranceUs) {
+			return 0;
+		}
+
+		if (std::abs(drift) > kMaxGapUs) {
+			// Too far to be audio that went missing: the source's clock
+			// jumped, as a live stream's does when it restarts it. The audio
+			// goes on without a break, on the new timeline. What is pending
+			// is less than a chunk, so it lands just before the jump.
+			next_timestamp_us_ += drift;
+
+			return 0;
+		}
+
+		if (drift < 0) {
+			// The frame starts where audio already given out has been, so
+			// that much of its start goes.
+			*skip = static_cast<int>(std::min<int64_t>(
+					av_rescale(-drift, in_rate_, AV_TIME_BASE), INT_MAX));
+
+			return 0;
+		}
+
+		// Audio is missing before this frame, and silence takes its place, so
+		// that what follows plays at its own time and not early. What
+		// swresample holds back belongs before the gap.
+		int result = DrainResampler();
+
+		if (result < 0) {
+			return result;
+		}
+
+		result = swr_init(swr_context_);
+
+		if (result < 0) {
+			return result;
+		}
+
+		const int64_t silence = av_rescale(drift, kSampleRate, AV_TIME_BASE);
+
+		pending_.resize(pending_.size() + static_cast<size_t>(silence) * channels_, 0);
+
+		return 0;
+	}
+
+	int64_t AudioDecoder::GetEndTimestampUs() const
+	{
+		const int64_t frames = static_cast<int64_t>(pending_.size() / channels_)
+				+ swr_get_delay(swr_context_, kSampleRate);
+
+		return next_timestamp_us_ + av_rescale(frames, AV_TIME_BASE, kSampleRate);
+	}
+
 	int AudioDecoder::DrainResampler()
 	{
 		if (swr_context_ == nullptr) {

webrtc-java-media/src/test/java/dev/onvoid/webrtc/media/player/MediaPlayerTest.java

@@ -34,6 +34,9 @@
 import java.util.concurrent.atomic.AtomicInteger;
 import java.util.concurrent.atomic.AtomicLong;
 import java.util.concurrent.atomic.AtomicReference;
+import java.util.stream.Collectors;
+import java.util.stream.IntStream;
+import java.util.stream.LongStream;
 
 import dev.onvoid.webrtc.PeerConnectionFactory;
 import dev.onvoid.webrtc.media.audio.AudioDeviceModule;
@@ -534,6 +537,74 @@ void deliversResamplerTail() throws Exception {
 		}
 	}
 
+	@Test
+	void downmixesMoreThanEightChannels() throws Exception {
+		// One second of 14 planar channels. A frame has the planes from the
+		// ninth on only in its extended data, and reading them from its data
+		// pointers instead crashes the process.
+		Path file = TestMedia.wideOpusMatroska(tempDir, 14, 50);
+
+		try (Playback playback = new Playback(new MediaReader(file))) {
+			playback.player.play();
+
+			assertTrue(playback.ended.await(10, TimeUnit.SECONDS), "no end of stream");
+
+			assertEquals(2, playback.channels.get());
+			assertTrue(Math.abs(playback.chunks.get() - 100) <= 2,
+					"audio chunks: " + playback.chunks.get());
+		}
+	}
+
+	@Test
+	void fillsAudioGapWithSilence() throws Exception {
+		// A second of audio, a second missing, and a second more. Closing the
+		// gap up would play the last second a second early, ahead of any
+		// video that follows its own timestamps.
+		Path file = TestMedia.pcmMatroska(tempDir, (short) 1000,
+				blockTimes(0, 10, 2000, 10));
+
+		try (Playback playback = new Playback(new MediaReader(file))) {
+			playback.player.play();
+
+			assertTrue(playback.ended.await(10, TimeUnit.SECONDS), "no end of stream");
+
+			assertEquals(300, playback.chunks.get());
+			assertEquals(96000, playback.nonZeroSamples.get());
+			assertEquals(IntStream.range(100, 200).boxed().collect(Collectors.toList()),
+					playback.silentChunks);
+		}
+	}
+
+	@Test
+	void skipsRepeatedAudio() throws Exception {
+		// The second second starts 50 ms before the first one ends, so 50 ms
+		// of it repeat what has played already.
+		Path file = TestMedia.pcmMatroska(tempDir, (short) 1000,
+				blockTimes(0, 10, 950, 10));
+
+		try (Playback playback = new Playback(new MediaReader(file))) {
+			playback.player.play();
+
+			assertTrue(playback.ended.await(10, TimeUnit.SECONDS), "no end of stream");
+
+			assertEquals(195, playback.chunks.get());
+			assertEquals(195 * 480, playback.nonZeroSamples.get());
+			assertTrue(playback.silentChunks.isEmpty(),
+					"silent chunks: " + playback.silentChunks);
+		}
+	}
+
+	/**
+	 * The times of two runs of 100 ms blocks, in milliseconds.
+	 */
+	private static long[] blockTimes(long firstStart, int firstBlocks,
+			long secondStart, int secondBlocks) {
+		return LongStream.concat(
+				LongStream.range(0, firstBlocks).map(i -> firstStart + i * 100),
+				LongStream.range(0, secondBlocks).map(i -> secondStart + i * 100))
+				.toArray();
+	}
+
 	private static int countNonZero(byte[] data) {
 		int count = 0;
 
@@ -608,6 +679,7 @@ private final class Playback implements AutoCloseable {
 		final AtomicLong firstChunkNs = new AtomicLong();
 		final AtomicInteger unevenChunks = new AtomicInteger();
 		final AtomicLong nonZeroSamples = new AtomicLong();
+		final List<Integer> silentChunks = Collections.synchronizedList(new ArrayList<>());
 
 		private final VideoTrackSink videoSink = frame -> {
 			frameTimes.add(System.nanoTime());
@@ -618,7 +690,7 @@ private final class Playback implements AutoCloseable {
 
 		private final AudioTrackSink audioSink = (data, bits, rate, ch, count) -> {
 			firstChunkNs.compareAndSet(0, System.nanoTime());
-			chunks.incrementAndGet();
+			int index = chunks.getAndIncrement();
 			sampleRate.set(rate);
 			channels.set(ch);
 			framesPerChunk.set(count);
@@ -627,7 +699,13 @@ private final class Playback implements AutoCloseable {
 				unevenChunks.incrementAndGet();
 			}
 
-			nonZeroSamples.addAndGet(countNonZero(data));
+			int nonZero = countNonZero(data);
+
+			if (nonZero == 0) {
+				silentChunks.add(index);
+			}
+
+			nonZeroSamples.addAndGet(nonZero);
 		};
 
 		private boolean closed;