From 97cf21949575254265429f0ab14b5267514b63ee Mon Sep 17 00:00:00 2001 From: Alex Andres Date: Sat, 31 Oct 2026 23:34:43 +0100 Subject: [PATCH 1/2] Fix voice activity hangover rounding at duration boundaries --- .../src/main/cpp/include/JNI_AudioTrack.h | 4 +- .../cpp/include/JNI_PeerConnectionFactory.h | 4 +- .../media/audio/VoiceActivityListener.h | 3 +- .../src/main/cpp/src/JNI_AudioTrack.cpp | 4 +- .../cpp/src/JNI_PeerConnectionFactory.cpp | 4 +- .../src/media/audio/VoiceActivityListener.cpp | 4 +- .../onvoid/webrtc/PeerConnectionFactory.java | 4 +- .../onvoid/webrtc/media/audio/AudioTrack.java | 4 +- .../audio/VoiceActivityListenerTest.java | 63 +++++++++++++++++++ 9 files changed, 79 insertions(+), 15 deletions(-) diff --git a/webrtc-jni/src/main/cpp/include/JNI_AudioTrack.h b/webrtc-jni/src/main/cpp/include/JNI_AudioTrack.h index 616ed1461..cc342ebe0 100644 --- a/webrtc-jni/src/main/cpp/include/JNI_AudioTrack.h +++ b/webrtc-jni/src/main/cpp/include/JNI_AudioTrack.h @@ -34,10 +34,10 @@ extern "C" { /* * Class: dev_onvoid_webrtc_media_audio_AudioTrack * Method: addVoiceActivityListenerInternal - * Signature: (Ldev/onvoid/webrtc/media/audio/VoiceActivityListener;FI)J + * Signature: (Ldev/onvoid/webrtc/media/audio/VoiceActivityListener;FJ)J */ JNIEXPORT jlong JNICALL Java_dev_onvoid_webrtc_media_audio_AudioTrack_addVoiceActivityListenerInternal - (JNIEnv *, jobject, jobject, jfloat, jint); + (JNIEnv *, jobject, jobject, jfloat, jlong); /* * Class: dev_onvoid_webrtc_media_audio_AudioTrack diff --git a/webrtc-jni/src/main/cpp/include/JNI_PeerConnectionFactory.h b/webrtc-jni/src/main/cpp/include/JNI_PeerConnectionFactory.h index 8995ce152..66beade22 100644 --- a/webrtc-jni/src/main/cpp/include/JNI_PeerConnectionFactory.h +++ b/webrtc-jni/src/main/cpp/include/JNI_PeerConnectionFactory.h @@ -26,10 +26,10 @@ extern "C" { /* * Class: dev_onvoid_webrtc_PeerConnectionFactory * Method: addVoiceActivityListenerInternal - * Signature: (Ldev/onvoid/webrtc/media/audio/VoiceActivityListener;FI)J + * Signature: (Ldev/onvoid/webrtc/media/audio/VoiceActivityListener;FJ)J */ JNIEXPORT jlong JNICALL Java_dev_onvoid_webrtc_PeerConnectionFactory_addVoiceActivityListenerInternal - (JNIEnv *, jobject, jobject, jfloat, jint); + (JNIEnv *, jobject, jobject, jfloat, jlong); /* * Class: dev_onvoid_webrtc_PeerConnectionFactory diff --git a/webrtc-jni/src/main/cpp/include/media/audio/VoiceActivityListener.h b/webrtc-jni/src/main/cpp/include/media/audio/VoiceActivityListener.h index b29a23c91..17d740aec 100644 --- a/webrtc-jni/src/main/cpp/include/media/audio/VoiceActivityListener.h +++ b/webrtc-jni/src/main/cpp/include/media/audio/VoiceActivityListener.h @@ -20,6 +20,7 @@ #include "JavaClass.h" #include "JavaRef.h" +#include #include #include @@ -34,7 +35,7 @@ namespace jni { public: VoiceActivityListener(JNIEnv * env, const JavaGlobalRef & listener, float threshold, - int hangoverMs); + int64_t hangoverNanos); ~VoiceActivityListener() = default; // Called on the thread that delivers the audio. diff --git a/webrtc-jni/src/main/cpp/src/JNI_AudioTrack.cpp b/webrtc-jni/src/main/cpp/src/JNI_AudioTrack.cpp index ac18b51b5..8c5ed1c35 100644 --- a/webrtc-jni/src/main/cpp/src/JNI_AudioTrack.cpp +++ b/webrtc-jni/src/main/cpp/src/JNI_AudioTrack.cpp @@ -62,14 +62,14 @@ JNIEXPORT void JNICALL Java_dev_onvoid_webrtc_media_audio_AudioTrack_removeSinkI } JNIEXPORT jlong JNICALL Java_dev_onvoid_webrtc_media_audio_AudioTrack_addVoiceActivityListenerInternal -(JNIEnv * env, jobject caller, jobject jListener, jfloat threshold, jint hangoverMs) +(JNIEnv * env, jobject caller, jobject jListener, jfloat threshold, jlong hangoverNanos) { try { webrtc::AudioTrackInterface * track = GetHandle(env, caller); CHECK_HANDLEV(track, 0); auto sink = new jni::VoiceActivitySink(std::make_unique(env, - jni::JavaGlobalRef(env, jListener), threshold, hangoverMs)); + jni::JavaGlobalRef(env, jListener), threshold, hangoverNanos)); track->AddSink(sink); diff --git a/webrtc-jni/src/main/cpp/src/JNI_PeerConnectionFactory.cpp b/webrtc-jni/src/main/cpp/src/JNI_PeerConnectionFactory.cpp index 122f3c824..dccb30544 100644 --- a/webrtc-jni/src/main/cpp/src/JNI_PeerConnectionFactory.cpp +++ b/webrtc-jni/src/main/cpp/src/JNI_PeerConnectionFactory.cpp @@ -257,14 +257,14 @@ JNIEXPORT void JNICALL Java_dev_onvoid_webrtc_PeerConnectionFactory_disposeInter } JNIEXPORT jlong JNICALL Java_dev_onvoid_webrtc_PeerConnectionFactory_addVoiceActivityListenerInternal -(JNIEnv * env, jobject caller, jobject jListener, jfloat threshold, jint hangoverMs) +(JNIEnv * env, jobject caller, jobject jListener, jfloat threshold, jlong hangoverNanos) { try { auto tap = GetHandle>(env, caller, "captureTapHandle"); CHECK_HANDLEV(tap, 0); auto listener = std::make_unique(env, jni::JavaGlobalRef(env, jListener), - threshold, hangoverMs); + threshold, hangoverNanos); return reinterpret_cast((*tap)->AddListener(std::move(listener))); } diff --git a/webrtc-jni/src/main/cpp/src/media/audio/VoiceActivityListener.cpp b/webrtc-jni/src/main/cpp/src/media/audio/VoiceActivityListener.cpp index 2480084ad..4a6b63bc3 100644 --- a/webrtc-jni/src/main/cpp/src/media/audio/VoiceActivityListener.cpp +++ b/webrtc-jni/src/main/cpp/src/media/audio/VoiceActivityListener.cpp @@ -23,11 +23,11 @@ namespace jni { VoiceActivityListener::VoiceActivityListener(JNIEnv * env, const JavaGlobalRef & listener, - float threshold, int hangoverMs) : + float threshold, int64_t hangoverNanos) : listener(listener), javaClass(JavaClasses::get(env)), threshold(threshold), - hangoverChunks((hangoverMs + 9) / 10), + hangoverChunks(static_cast(hangoverNanos / 10000000 + (hangoverNanos % 10000000 != 0))), quietChunks(0), speaking(false) { diff --git a/webrtc/src/main/java/dev/onvoid/webrtc/PeerConnectionFactory.java b/webrtc/src/main/java/dev/onvoid/webrtc/PeerConnectionFactory.java index 7869b7026..4f8652aab 100644 --- a/webrtc/src/main/java/dev/onvoid/webrtc/PeerConnectionFactory.java +++ b/webrtc/src/main/java/dev/onvoid/webrtc/PeerConnectionFactory.java @@ -447,7 +447,7 @@ public void addVoiceActivityListener(VoiceActivityConfig config, VoiceActivityLi } long nativeListener = addVoiceActivityListenerInternal(listener, (float) config.getThreshold(), - (int) config.getHangover().toMillis()); + config.getHangover().toNanos()); voiceActivityListeners.put(listener, nativeListener); } @@ -489,7 +489,7 @@ public void dispose() { private native void disposeInternal(); private native long addVoiceActivityListenerInternal(VoiceActivityListener listener, float threshold, - int hangoverMs); + long hangoverNanos); private native void removeVoiceActivityListenerInternal(long listenerHandle); diff --git a/webrtc/src/main/java/dev/onvoid/webrtc/media/audio/AudioTrack.java b/webrtc/src/main/java/dev/onvoid/webrtc/media/audio/AudioTrack.java index 7859915ae..a0750a101 100644 --- a/webrtc/src/main/java/dev/onvoid/webrtc/media/audio/AudioTrack.java +++ b/webrtc/src/main/java/dev/onvoid/webrtc/media/audio/AudioTrack.java @@ -122,7 +122,7 @@ public void addVoiceActivityListener(VoiceActivityConfig config, VoiceActivityLi } final long nativeSink = addVoiceActivityListenerInternal(listener, (float) config.getThreshold(), - (int) config.getHangover().toMillis()); + config.getHangover().toNanos()); voiceActivityListeners.put(listener, nativeSink); } @@ -177,7 +177,7 @@ public void setVolume(double volume) { private native void removeSinkInternal(long sinkHandle); private native long addVoiceActivityListenerInternal(VoiceActivityListener listener, float threshold, - int hangoverMs); + long hangoverNanos); private native void removeVoiceActivityListenerInternal(long sinkHandle); diff --git a/webrtc/src/test/java/dev/onvoid/webrtc/media/audio/VoiceActivityListenerTest.java b/webrtc/src/test/java/dev/onvoid/webrtc/media/audio/VoiceActivityListenerTest.java index 03b65370c..678f4d66f 100644 --- a/webrtc/src/test/java/dev/onvoid/webrtc/media/audio/VoiceActivityListenerTest.java +++ b/webrtc/src/test/java/dev/onvoid/webrtc/media/audio/VoiceActivityListenerTest.java @@ -26,6 +26,8 @@ import java.util.Arrays; import java.util.List; import java.util.concurrent.CopyOnWriteArrayList; +import java.util.function.BiConsumer; +import java.util.function.Consumer; import dev.onvoid.webrtc.PeerConnectionFactory; import dev.onvoid.webrtc.TestBase; @@ -112,6 +114,65 @@ void hangoverHoldsSpeech() { + " chunks"); } + @Test + void trackHangoverBoundaries() { + assertHangoverBoundaries(track::addVoiceActivityListener, pcm -> push(pcm, 16000, 1)); + } + + @Test + void factoryHangoverBoundaries() { + AudioProcessing processing = new AudioProcessing(); + HeadlessAudioDeviceModule module = new HeadlessAudioDeviceModule(); + PeerConnectionFactory processed = new PeerConnectionFactory(module, processing); + + try { + assertHangoverBoundaries(processed::addVoiceActivityListener, + pcm -> process(processing, pcm, 16000)); + } + finally { + processed.dispose(); + module.dispose(); + processing.dispose(); + } + } + + private void assertHangoverBoundaries( + BiConsumer addListener, + Consumer deliver) { + Duration[] hangovers = { Duration.ZERO, Duration.ofNanos(1), Duration.ofMillis(10), + Duration.ofMillis(10).plusNanos(1), Duration.ofNanos(10500000), Duration.ofMillis(20) }; + int[] chunk = { 0 }; + int[] lastEnd = new int[hangovers.length]; + Arrays.fill(lastEnd, -1); + + for (int i = 0; i < hangovers.length; i++) { + final int index = i; + addListener.accept(new VoiceActivityConfig(0.5, hangovers[i]), speaking -> { + if (!speaking) { + lastEnd[index] = chunk[0]; + } + }); + } + + List maximumChanges = new CopyOnWriteArrayList<>(); + addListener.accept(new VoiceActivityConfig(0.5, Duration.ofMillis(Integer.MAX_VALUE)), + maximumChanges::add); + + byte[] pcm = SpeechFixture.concat(speech(16000, 1), silence(2000, 16000, 1)); + for (int offset = 0; offset + 320 <= pcm.length; offset += 320) { + deliver.accept(Arrays.copyOfRange(pcm, offset, offset + 320)); + chunk[0]++; + } + + assertTrue(lastEnd[0] >= 0, "Speech must end for the rounding comparisons"); + assertEquals(lastEnd[0] + 1, lastEnd[1], "1 ns rounds up to 10 ms"); + assertEquals(lastEnd[1], lastEnd[2], "Exactly 10 ms stays one chunk"); + assertEquals(lastEnd[2] + 1, lastEnd[3], "10 ms + 1 ns rounds up to 20 ms"); + assertEquals(lastEnd[3], lastEnd[4], "10.5 ms rounds up to 20 ms"); + assertEquals(lastEnd[4], lastEnd[5], "Exactly 20 ms stays two chunks"); + assertEquals(List.of(true), maximumChanges, "The maximum hangover must keep speech active"); + } + @Test void shortHangoverSplitsSpeech() { // Without a hangover, the pauses between words end speech. @@ -278,6 +339,8 @@ void configRejectsOutOfRange() { assertThrows(IllegalArgumentException.class, () -> new VoiceActivityConfig(0.5, Duration.ofMillis(-10))); assertThrows(IllegalArgumentException.class, () -> new VoiceActivityConfig(0.5, Duration.ofDays(30))); + assertThrows(IllegalArgumentException.class, () -> new VoiceActivityConfig(0.5, + Duration.ofMillis(Integer.MAX_VALUE).plusNanos(1))); assertThrows(NullPointerException.class, () -> new VoiceActivityConfig(0.5, null)); } From d3fa74067c3b72b81c4033e8bc3c0c7c1a15e3b0 Mon Sep 17 00:00:00 2001 From: Alex Andres Date: Sun, 1 Nov 2026 00:04:25 +0100 Subject: [PATCH 2/2] Keep voice activity hangover in rounded milliseconds --- webrtc-jni/src/main/cpp/include/JNI_AudioTrack.h | 4 ++-- .../src/main/cpp/include/JNI_PeerConnectionFactory.h | 4 ++-- .../main/cpp/include/media/audio/VoiceActivityListener.h | 3 +-- webrtc-jni/src/main/cpp/src/JNI_AudioTrack.cpp | 4 ++-- webrtc-jni/src/main/cpp/src/JNI_PeerConnectionFactory.cpp | 4 ++-- .../main/cpp/src/media/audio/VoiceActivityListener.cpp | 4 ++-- .../java/dev/onvoid/webrtc/PeerConnectionFactory.java | 8 ++++++-- .../java/dev/onvoid/webrtc/media/audio/AudioTrack.java | 8 ++++++-- 8 files changed, 23 insertions(+), 16 deletions(-) diff --git a/webrtc-jni/src/main/cpp/include/JNI_AudioTrack.h b/webrtc-jni/src/main/cpp/include/JNI_AudioTrack.h index cc342ebe0..616ed1461 100644 --- a/webrtc-jni/src/main/cpp/include/JNI_AudioTrack.h +++ b/webrtc-jni/src/main/cpp/include/JNI_AudioTrack.h @@ -34,10 +34,10 @@ extern "C" { /* * Class: dev_onvoid_webrtc_media_audio_AudioTrack * Method: addVoiceActivityListenerInternal - * Signature: (Ldev/onvoid/webrtc/media/audio/VoiceActivityListener;FJ)J + * Signature: (Ldev/onvoid/webrtc/media/audio/VoiceActivityListener;FI)J */ JNIEXPORT jlong JNICALL Java_dev_onvoid_webrtc_media_audio_AudioTrack_addVoiceActivityListenerInternal - (JNIEnv *, jobject, jobject, jfloat, jlong); + (JNIEnv *, jobject, jobject, jfloat, jint); /* * Class: dev_onvoid_webrtc_media_audio_AudioTrack diff --git a/webrtc-jni/src/main/cpp/include/JNI_PeerConnectionFactory.h b/webrtc-jni/src/main/cpp/include/JNI_PeerConnectionFactory.h index 66beade22..8995ce152 100644 --- a/webrtc-jni/src/main/cpp/include/JNI_PeerConnectionFactory.h +++ b/webrtc-jni/src/main/cpp/include/JNI_PeerConnectionFactory.h @@ -26,10 +26,10 @@ extern "C" { /* * Class: dev_onvoid_webrtc_PeerConnectionFactory * Method: addVoiceActivityListenerInternal - * Signature: (Ldev/onvoid/webrtc/media/audio/VoiceActivityListener;FJ)J + * Signature: (Ldev/onvoid/webrtc/media/audio/VoiceActivityListener;FI)J */ JNIEXPORT jlong JNICALL Java_dev_onvoid_webrtc_PeerConnectionFactory_addVoiceActivityListenerInternal - (JNIEnv *, jobject, jobject, jfloat, jlong); + (JNIEnv *, jobject, jobject, jfloat, jint); /* * Class: dev_onvoid_webrtc_PeerConnectionFactory diff --git a/webrtc-jni/src/main/cpp/include/media/audio/VoiceActivityListener.h b/webrtc-jni/src/main/cpp/include/media/audio/VoiceActivityListener.h index 17d740aec..b29a23c91 100644 --- a/webrtc-jni/src/main/cpp/include/media/audio/VoiceActivityListener.h +++ b/webrtc-jni/src/main/cpp/include/media/audio/VoiceActivityListener.h @@ -20,7 +20,6 @@ #include "JavaClass.h" #include "JavaRef.h" -#include #include #include @@ -35,7 +34,7 @@ namespace jni { public: VoiceActivityListener(JNIEnv * env, const JavaGlobalRef & listener, float threshold, - int64_t hangoverNanos); + int hangoverMs); ~VoiceActivityListener() = default; // Called on the thread that delivers the audio. diff --git a/webrtc-jni/src/main/cpp/src/JNI_AudioTrack.cpp b/webrtc-jni/src/main/cpp/src/JNI_AudioTrack.cpp index 8c5ed1c35..ac18b51b5 100644 --- a/webrtc-jni/src/main/cpp/src/JNI_AudioTrack.cpp +++ b/webrtc-jni/src/main/cpp/src/JNI_AudioTrack.cpp @@ -62,14 +62,14 @@ JNIEXPORT void JNICALL Java_dev_onvoid_webrtc_media_audio_AudioTrack_removeSinkI } JNIEXPORT jlong JNICALL Java_dev_onvoid_webrtc_media_audio_AudioTrack_addVoiceActivityListenerInternal -(JNIEnv * env, jobject caller, jobject jListener, jfloat threshold, jlong hangoverNanos) +(JNIEnv * env, jobject caller, jobject jListener, jfloat threshold, jint hangoverMs) { try { webrtc::AudioTrackInterface * track = GetHandle(env, caller); CHECK_HANDLEV(track, 0); auto sink = new jni::VoiceActivitySink(std::make_unique(env, - jni::JavaGlobalRef(env, jListener), threshold, hangoverNanos)); + jni::JavaGlobalRef(env, jListener), threshold, hangoverMs)); track->AddSink(sink); diff --git a/webrtc-jni/src/main/cpp/src/JNI_PeerConnectionFactory.cpp b/webrtc-jni/src/main/cpp/src/JNI_PeerConnectionFactory.cpp index dccb30544..122f3c824 100644 --- a/webrtc-jni/src/main/cpp/src/JNI_PeerConnectionFactory.cpp +++ b/webrtc-jni/src/main/cpp/src/JNI_PeerConnectionFactory.cpp @@ -257,14 +257,14 @@ JNIEXPORT void JNICALL Java_dev_onvoid_webrtc_PeerConnectionFactory_disposeInter } JNIEXPORT jlong JNICALL Java_dev_onvoid_webrtc_PeerConnectionFactory_addVoiceActivityListenerInternal -(JNIEnv * env, jobject caller, jobject jListener, jfloat threshold, jlong hangoverNanos) +(JNIEnv * env, jobject caller, jobject jListener, jfloat threshold, jint hangoverMs) { try { auto tap = GetHandle>(env, caller, "captureTapHandle"); CHECK_HANDLEV(tap, 0); auto listener = std::make_unique(env, jni::JavaGlobalRef(env, jListener), - threshold, hangoverNanos); + threshold, hangoverMs); return reinterpret_cast((*tap)->AddListener(std::move(listener))); } diff --git a/webrtc-jni/src/main/cpp/src/media/audio/VoiceActivityListener.cpp b/webrtc-jni/src/main/cpp/src/media/audio/VoiceActivityListener.cpp index 4a6b63bc3..11bf13fbf 100644 --- a/webrtc-jni/src/main/cpp/src/media/audio/VoiceActivityListener.cpp +++ b/webrtc-jni/src/main/cpp/src/media/audio/VoiceActivityListener.cpp @@ -23,11 +23,11 @@ namespace jni { VoiceActivityListener::VoiceActivityListener(JNIEnv * env, const JavaGlobalRef & listener, - float threshold, int64_t hangoverNanos) : + float threshold, int hangoverMs) : listener(listener), javaClass(JavaClasses::get(env)), threshold(threshold), - hangoverChunks(static_cast(hangoverNanos / 10000000 + (hangoverNanos % 10000000 != 0))), + hangoverChunks(hangoverMs / 10 + (hangoverMs % 10 != 0)), quietChunks(0), speaking(false) { diff --git a/webrtc/src/main/java/dev/onvoid/webrtc/PeerConnectionFactory.java b/webrtc/src/main/java/dev/onvoid/webrtc/PeerConnectionFactory.java index 4f8652aab..7a46158c0 100644 --- a/webrtc/src/main/java/dev/onvoid/webrtc/PeerConnectionFactory.java +++ b/webrtc/src/main/java/dev/onvoid/webrtc/PeerConnectionFactory.java @@ -446,8 +446,12 @@ public void addVoiceActivityListener(VoiceActivityConfig config, VoiceActivityLi return; } + // Preserve partial milliseconds before native code rounds to 10 ms chunks. + int hangoverMs = (int) config.getHangover().toMillis() + + (config.getHangover().getNano() % 1_000_000 == 0 ? 0 : 1); + long nativeListener = addVoiceActivityListenerInternal(listener, (float) config.getThreshold(), - config.getHangover().toNanos()); + hangoverMs); voiceActivityListeners.put(listener, nativeListener); } @@ -489,7 +493,7 @@ public void dispose() { private native void disposeInternal(); private native long addVoiceActivityListenerInternal(VoiceActivityListener listener, float threshold, - long hangoverNanos); + int hangoverMs); private native void removeVoiceActivityListenerInternal(long listenerHandle); diff --git a/webrtc/src/main/java/dev/onvoid/webrtc/media/audio/AudioTrack.java b/webrtc/src/main/java/dev/onvoid/webrtc/media/audio/AudioTrack.java index a0750a101..47b8f1428 100644 --- a/webrtc/src/main/java/dev/onvoid/webrtc/media/audio/AudioTrack.java +++ b/webrtc/src/main/java/dev/onvoid/webrtc/media/audio/AudioTrack.java @@ -121,8 +121,12 @@ public void addVoiceActivityListener(VoiceActivityConfig config, VoiceActivityLi return; } + // Preserve partial milliseconds before native code rounds to 10 ms chunks. + int hangoverMs = (int) config.getHangover().toMillis() + + (config.getHangover().getNano() % 1_000_000 == 0 ? 0 : 1); + final long nativeSink = addVoiceActivityListenerInternal(listener, (float) config.getThreshold(), - config.getHangover().toNanos()); + hangoverMs); voiceActivityListeners.put(listener, nativeSink); } @@ -177,7 +181,7 @@ public void setVolume(double volume) { private native void removeSinkInternal(long sinkHandle); private native long addVoiceActivityListenerInternal(VoiceActivityListener listener, float threshold, - long hangoverNanos); + int hangoverMs); private native void removeVoiceActivityListenerInternal(long sinkHandle);