Skip to content
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion .github/workflows/android-apk.yml
Original file line number Diff line number Diff line change
Expand Up @@ -494,7 +494,7 @@ jobs:
working-directory: openless-all/app
env:
OPENLESS_UPDATE_APK_DIR: ${{ steps.stage.outputs.out_dir }}
OPENLESS_UPDATE_REPO: Open-Less/openless
OPENLESS_UPDATE_REPO: ${{ github.repository }}
OPENLESS_UPDATE_MIRROR_BASE_URL: https://fastgit.cc/https://github.com
OPENLESS_RELEASE_CHANNEL: ${{ env.OPENLESS_RELEASE_CHANNEL }}
OPENLESS_RELEASE_TAG: ${{ github.ref_name }}
Expand Down
2 changes: 1 addition & 1 deletion .github/workflows/release-tauri.yml
Original file line number Diff line number Diff line change
Expand Up @@ -498,7 +498,7 @@ jobs:
TAURI_SIGNING_PRIVATE_KEY: ${{ secrets.TAURI_SIGNING_PRIVATE_KEY }}
OPENLESS_UPDATE_TARGET: ${{ matrix.updater-target }}
OPENLESS_UPDATE_ARCH: ${{ matrix.updater-arch }}
OPENLESS_UPDATE_REPO: Open-Less/openless
OPENLESS_UPDATE_REPO: ${{ github.repository }}
OPENLESS_UPDATE_MIRROR_BASE_URL: https://fastgit.cc/https://github.com
# On the beta channel output latest-{tgt}-{arch}-beta.json; stable keeps the old filename.
OPENLESS_RELEASE_CHANNEL: ${{ env.OPENLESS_RELEASE_CHANNEL }}
Expand Down
969 changes: 886 additions & 83 deletions openless-all/app/crates/openless-core/src/api.rs

Large diffs are not rendered by default.

Original file line number Diff line number Diff line change
Expand Up @@ -172,6 +172,55 @@ pub const PREFERENCE_FIELDS: &[PreferenceField] = &[
shape: PreferenceShape::Boolean,
reason: "local_only_microphone_capture",
},
PreferenceField {
rust_name: "voice_activation_enabled",
key: "voiceActivationEnabled",
class: PreferenceClass::Excluded,
shape: PreferenceShape::Boolean,
reason: "local_only_microphone_capture",
},
PreferenceField {
rust_name: "voice_activation_wake_enabled",
key: "voiceActivationWakeEnabled",
class: PreferenceClass::Excluded,
shape: PreferenceShape::Boolean,
reason: "local_only_microphone_capture",
},
PreferenceField {
rust_name: "voice_activation_finish_enabled",
key: "voiceActivationFinishEnabled",
class: PreferenceClass::Excluded,
shape: PreferenceShape::Boolean,
reason: "local_only_microphone_capture",
},
PreferenceField {
rust_name: "voice_activation_wake_phrase",
key: "voiceActivationWakePhrase",
class: PreferenceClass::Excluded,
shape: PreferenceShape::Text,
reason: "local_only_microphone_capture",
},
PreferenceField {
rust_name: "voice_activation_finish_phrase",
key: "voiceActivationFinishPhrase",
class: PreferenceClass::Excluded,
shape: PreferenceShape::Text,
reason: "local_only_microphone_capture",
},
PreferenceField {
rust_name: "voice_activation_timeout_enabled",
key: "voiceActivationTimeoutEnabled",
class: PreferenceClass::Excluded,
shape: PreferenceShape::Boolean,
reason: "local_only_microphone_capture",
},
PreferenceField {
rust_name: "voice_activation_timeout_seconds",
key: "voiceActivationTimeoutSeconds",
class: PreferenceClass::Excluded,
shape: PreferenceShape::Unsigned32,
reason: "local_only_microphone_capture",
},
PreferenceField {
rust_name: "active_asr_provider",
key: "activeAsrProvider",
Expand Down
2 changes: 2 additions & 0 deletions openless-all/app/crates/openless-core/src/config.rs
Original file line number Diff line number Diff line change
Expand Up @@ -72,6 +72,7 @@ pub struct BackendDependencies {
pub dictation_engine: Arc<dyn DictationEngine>,
pub task_spawner: Arc<dyn TaskSpawner>,
pub credential_store: Arc<dyn CredentialStore>,
pub voice_activation_runtime: Option<Arc<dyn crate::ports::VoiceActivationRuntime>>,
pub services: BackendServices,
pub local_asr_runtime: Option<Arc<dyn crate::local_asr_service::ModelRuntimeAdapter>>,
pub marketplace_config: Option<crate::marketplace::MarketplaceConfig>,
Expand All @@ -91,6 +92,7 @@ impl BackendDependencies {
dictation_engine: Arc::new(UnsupportedDictationEngine),
task_spawner: Arc::new(TokioTaskSpawner),
credential_store: Arc::new(crate::credentials::UnsupportedCredentialStore),
voice_activation_runtime: None,
services: BackendServices::unsupported(),
local_asr_runtime: None,
marketplace_config: None,
Expand Down
1 change: 1 addition & 0 deletions openless-all/app/crates/openless-core/src/events.rs
Original file line number Diff line number Diff line change
Expand Up @@ -407,6 +407,7 @@ pub enum BackendEventKind {
BackendStarted,
BackendStopping,
DictationStateChanged(DictationStateSnapshot),
VoiceActivationStateChanged(crate::voice_activation::VoiceActivationStateSnapshot),
TranscriptDelta(TranscriptDelta),
PolishDelta(PolishDelta),
DictationCompleted(DictationResult),
Expand Down
13 changes: 12 additions & 1 deletion openless-all/app/crates/openless-core/src/lib.rs
Original file line number Diff line number Diff line change
Expand Up @@ -74,6 +74,7 @@ pub mod style_packs;
pub mod testing;
pub mod types;
pub mod vocabulary;
pub mod voice_activation;
pub mod voice_edit_session;
mod voice_session;

Expand Down Expand Up @@ -301,7 +302,9 @@ pub use ports::{
PendingAudioCapture, PolishOutput, RecordingArchive, RecordingControlSink, RecordingEvent,
RecordingProgressSink, ResourceResolver, TextInserter, TextInsertionSession, TextPolisher,
TextStreamChunk, TextStreamSink, TranscriptOutput, TranscriptionEngine, TranscriptionSession,
UnsupportedTextInserter, VoiceCapture,
UnsupportedTextInserter, UnsupportedVoiceActivationRuntime, VoiceActivationError,
VoiceActivationHandle, VoiceActivationRuntime, VoiceActivationSink, VoiceCapture,
VoiceDictationHandoff, VoiceDictationHandoffRequest,
};
pub use preferences::PreferencesStore;
pub use prompt_compose::{
Expand Down Expand Up @@ -375,6 +378,14 @@ pub use vocabulary::{
builtin_vocab_presets, list_vocab_presets, resolve_vocab_presets, save_vocab_presets,
DictionaryStore,
};
pub use voice_activation::{
normalize_phrase, validate_phrase, PhraseValidationError, VoiceActivationCommand,
VoiceActivationConfig, VoiceActivationDecision, VoiceActivationGeneration,
VoiceActivationPhase, VoiceActivationSession, VoiceActivationSignal,
VoiceActivationStateSnapshot, VoiceActivationStopReason, VoiceCommandMatch,
VoiceCommandRecognizer, VoicePhraseMatcher, DEFAULT_FINISH_PHRASE, DEFAULT_TIMEOUT_SECONDS,
DEFAULT_WAKE_PHRASE, MAX_PHRASE_CHARS, MAX_TIMEOUT_SECONDS, MIN_TIMEOUT_SECONDS,
};
pub use voice_edit_session::{
TextSelection, VoiceEditCommit, VoiceEditContext, VoiceEditError, VoiceEditPhase,
VoiceEditSession, VoiceEditSnapshot, VoiceEditTarget, VoiceEditTurn,
Expand Down
80 changes: 80 additions & 0 deletions openless-all/app/crates/openless-core/src/ports.rs
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,7 @@ use crate::audio::AudioProcessingConfig;
use crate::dictation_context::DictationContext;
use crate::errors::{BackendError, BackendErrorCode};
use crate::types::{InsertStatus, PolishDelta, SessionId, TranscriptDelta};
use crate::voice_activation::{VoiceActivationConfig, VoiceActivationSignal};

#[derive(Debug, Clone, PartialEq, Eq)]
pub enum HostAction {
Expand Down Expand Up @@ -73,6 +74,85 @@ impl HostActions for NoopHostActions {
}
}

/// Host-owned sink for semantic voice activation signals. Standby recognition
/// must only publish wake/finish commands and runtime state; it must never
/// publish the recognized standby text as a normal transcript delta.
pub trait VoiceActivationSink: Send + Sync {
fn publish(&self, signal: VoiceActivationSignal) -> Result<(), BackendError>;
}

#[derive(Debug, Clone, PartialEq, Eq)]
pub enum VoiceActivationError {
Unsupported,
ModelMissing,
PermissionDenied,
MicBusy,
InvalidConfig(String),
Runtime(String),
}

impl std::fmt::Display for VoiceActivationError {
fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
match self {
Self::Unsupported => formatter.write_str("voice activation is unsupported"),
Self::ModelMissing => formatter.write_str("voice activation model is not installed"),
Self::PermissionDenied => formatter.write_str("microphone permission was denied"),
Self::MicBusy => formatter.write_str("microphone is already in use"),
Self::InvalidConfig(message) | Self::Runtime(message) => formatter.write_str(message),
}
}
}

impl std::error::Error for VoiceActivationError {}

#[derive(Debug, Clone, PartialEq, Eq)]
pub struct VoiceDictationHandoffRequest {
pub activation_generation: u64,
}

pub struct VoiceDictationHandoff {
pub capture: Box<dyn PendingAudioCapture>,
}

/// A single host-owned microphone/runtime seam for voice activation. The Core
/// decides lifecycle and stale-event policy; the host decides how local ASR and
/// the native capture stream are implemented.
pub trait VoiceActivationRuntime: Send + Sync + 'static {
fn configure(
&self,
config: VoiceActivationConfig,
sink: Arc<dyn VoiceActivationSink>,
) -> BoxFuture<'static, Result<Box<dyn VoiceActivationHandle>, VoiceActivationError>>;
}

pub trait VoiceActivationHandle: Send + Sync {
/// Bind the Core-owned dictation session so finish signals can carry the
/// same session identity even though the host performs recognition.
fn bind_dictation_session(&self, _session_id: SessionId) {}

fn handoff_to_dictation(
&self,
request: VoiceDictationHandoffRequest,
) -> BoxFuture<'static, Result<VoiceDictationHandoff, VoiceActivationError>>;

fn stop(
&self,
reason: crate::voice_activation::VoiceActivationStopReason,
) -> BoxFuture<'static, Result<(), VoiceActivationError>>;
}

pub struct UnsupportedVoiceActivationRuntime;

impl VoiceActivationRuntime for UnsupportedVoiceActivationRuntime {
fn configure(
&self,
_config: VoiceActivationConfig,
_sink: Arc<dyn VoiceActivationSink>,
) -> BoxFuture<'static, Result<Box<dyn VoiceActivationHandle>, VoiceActivationError>> {
Box::pin(async { Err(VoiceActivationError::Unsupported) })
}
}

/// Resolve a packaged resource without exposing a framework-specific resource
/// directory object to the core or UI.
pub trait ResourceResolver: Send + Sync {
Expand Down
65 changes: 65 additions & 0 deletions openless-all/app/crates/openless-core/src/shared_types.rs
Original file line number Diff line number Diff line change
Expand Up @@ -360,6 +360,18 @@ fn default_silence_auto_stop_seconds() -> f32 {
3.0
}

fn default_voice_activation_wake_phrase() -> String {
crate::voice_activation::DEFAULT_WAKE_PHRASE.into()
}

fn default_voice_activation_finish_phrase() -> String {
crate::voice_activation::DEFAULT_FINISH_PHRASE.into()
}

fn default_voice_activation_timeout_seconds() -> u32 {
crate::voice_activation::DEFAULT_TIMEOUT_SECONDS
}

fn resolve_windows_insertion_mode(
mode: WindowsInsertionMode,
legacy_sendinput_only: bool,
Expand Down Expand Up @@ -436,6 +448,21 @@ pub struct UserPreferences {
/// Recording input device name. Empty string = system default microphone.
#[serde(default)]
pub microphone_device_name: String,
/// Opt-in local voice command listener. Standby audio is never sent to a cloud provider.
#[serde(default)]
pub voice_activation_enabled: bool,
#[serde(default = "default_true")]
pub voice_activation_wake_enabled: bool,
#[serde(default = "default_true")]
pub voice_activation_finish_enabled: bool,
#[serde(default = "default_voice_activation_wake_phrase")]
pub voice_activation_wake_phrase: String,
#[serde(default = "default_voice_activation_finish_phrase")]
pub voice_activation_finish_phrase: String,
#[serde(default = "default_true")]
pub voice_activation_timeout_enabled: bool,
#[serde(default = "default_voice_activation_timeout_seconds")]
pub voice_activation_timeout_seconds: u32,
pub active_asr_provider: String, // "volcengine" | "apple-speech" | ...
pub active_llm_provider: String, // "ark" | "openai" | ...
/// Recognition pipeline mode (experimental, issue #902). With
Expand Down Expand Up @@ -985,6 +1012,20 @@ struct UserPreferencesWire {
silence_auto_stop_seconds: f32,
#[serde(default)]
microphone_device_name: String,
#[serde(default)]
voice_activation_enabled: bool,
#[serde(default = "default_true")]
voice_activation_wake_enabled: bool,
#[serde(default = "default_true")]
voice_activation_finish_enabled: bool,
#[serde(default = "default_voice_activation_wake_phrase")]
voice_activation_wake_phrase: String,
#[serde(default = "default_voice_activation_finish_phrase")]
voice_activation_finish_phrase: String,
#[serde(default = "default_true")]
voice_activation_timeout_enabled: bool,
#[serde(default = "default_voice_activation_timeout_seconds")]
voice_activation_timeout_seconds: u32,
active_asr_provider: String,
active_llm_provider: String,
#[serde(default = "default_pipeline_mode")]
Expand Down Expand Up @@ -1240,6 +1281,13 @@ impl Default for UserPreferencesWire {
silence_auto_stop_enabled: prefs.silence_auto_stop_enabled,
silence_auto_stop_seconds: prefs.silence_auto_stop_seconds,
microphone_device_name: prefs.microphone_device_name,
voice_activation_enabled: prefs.voice_activation_enabled,
voice_activation_wake_enabled: prefs.voice_activation_wake_enabled,
voice_activation_finish_enabled: prefs.voice_activation_finish_enabled,
voice_activation_wake_phrase: prefs.voice_activation_wake_phrase,
voice_activation_finish_phrase: prefs.voice_activation_finish_phrase,
voice_activation_timeout_enabled: prefs.voice_activation_timeout_enabled,
voice_activation_timeout_seconds: prefs.voice_activation_timeout_seconds,
active_asr_provider: prefs.active_asr_provider,
active_llm_provider: prefs.active_llm_provider,
pipeline_mode: prefs.pipeline_mode,
Expand Down Expand Up @@ -1443,6 +1491,16 @@ impl<'de> Deserialize<'de> for UserPreferences {
silence_auto_stop_enabled: wire.silence_auto_stop_enabled,
silence_auto_stop_seconds: wire.silence_auto_stop_seconds,
microphone_device_name: wire.microphone_device_name,
voice_activation_enabled: wire.voice_activation_enabled,
voice_activation_wake_enabled: wire.voice_activation_wake_enabled,
voice_activation_finish_enabled: wire.voice_activation_finish_enabled,
voice_activation_wake_phrase: wire.voice_activation_wake_phrase,
voice_activation_finish_phrase: wire.voice_activation_finish_phrase,
voice_activation_timeout_enabled: wire.voice_activation_timeout_enabled,
voice_activation_timeout_seconds: wire.voice_activation_timeout_seconds.clamp(
crate::voice_activation::MIN_TIMEOUT_SECONDS,
crate::voice_activation::MAX_TIMEOUT_SECONDS,
),
active_asr_provider: wire.active_asr_provider,
active_llm_provider: wire.active_llm_provider,
pipeline_mode: wire.pipeline_mode,
Expand Down Expand Up @@ -1823,6 +1881,13 @@ impl Default for UserPreferences {
silence_auto_stop_enabled: false,
silence_auto_stop_seconds: default_silence_auto_stop_seconds(),
microphone_device_name: String::new(),
voice_activation_enabled: false,
voice_activation_wake_enabled: true,
voice_activation_finish_enabled: true,
voice_activation_wake_phrase: default_voice_activation_wake_phrase(),
voice_activation_finish_phrase: default_voice_activation_finish_phrase(),
voice_activation_timeout_enabled: true,
voice_activation_timeout_seconds: default_voice_activation_timeout_seconds(),
active_asr_provider: default_active_asr_provider(),
active_llm_provider: "ark".into(),
pipeline_mode: PipelineMode::Traditional,
Expand Down
Loading
Loading