diff --git a/androidApp/src/androidTest/kotlin/dev/obiente/nextcloudnative/SessionCipherInstrumentedTest.kt b/androidApp/src/androidTest/kotlin/dev/obiente/nextcloudnative/SessionCipherInstrumentedTest.kt new file mode 100644 index 000000000..fd798bffc --- /dev/null +++ b/androidApp/src/androidTest/kotlin/dev/obiente/nextcloudnative/SessionCipherInstrumentedTest.kt @@ -0,0 +1,40 @@ +package dev.obiente.nextcloudnative + +import android.util.Base64 +import androidx.test.ext.junit.runners.AndroidJUnit4 +import javax.crypto.AEADBadTagException +import org.junit.Assert.assertTrue +import org.junit.Test +import org.junit.runner.RunWith + +@RunWith(AndroidJUnit4::class) +class SessionCipherInstrumentedTest { + @Test + fun invalidBase64EnvelopeIsDefinitivelyRejected() { + val failure = invalidCiphertextFailure("not-base64.invalid") + + assertTrue(failure.cause is IllegalArgumentException) + } + + @Test + fun authenticatedCiphertextCorruptionIsDefinitivelyRejected() { + val cipher = SessionCipher() + val encrypted = cipher.encrypt("private upload capability") + val parts = encrypted.split('.', limit = 2) + val payload = Base64.decode(parts[1], Base64.NO_WRAP).also { bytes -> + bytes[0] = (bytes[0].toInt() xor 1).toByte() + } + val corrupted = parts[0] + "." + Base64.encodeToString(payload, Base64.NO_WRAP) + + val failure = invalidCiphertextFailure(corrupted) + + assertTrue(failure.cause is AEADBadTagException) + } + + private fun invalidCiphertextFailure(value: String): InvalidSessionCiphertextException = try { + SessionCipher().decrypt(value) + error("Corrupt ciphertext was accepted.") + } catch (failure: InvalidSessionCiphertextException) { + failure + } +} diff --git a/androidApp/src/main/kotlin/dev/obiente/nextcloudnative/AndroidDurableMultipartUploads.kt b/androidApp/src/main/kotlin/dev/obiente/nextcloudnative/AndroidDurableMultipartUploads.kt index 18308c0cc..d34d79ab8 100644 --- a/androidApp/src/main/kotlin/dev/obiente/nextcloudnative/AndroidDurableMultipartUploads.kt +++ b/androidApp/src/main/kotlin/dev/obiente/nextcloudnative/AndroidDurableMultipartUploads.kt @@ -16,20 +16,26 @@ import dev.obiente.nextcloudnative.app.DurableUploadStatus import dev.obiente.nextcloudnative.app.LocalUploadFile import dev.obiente.nextcloudnative.app.MAX_DURABLE_UPLOAD_MESSAGE_CHARACTERS import dev.obiente.nextcloudnative.app.MultipartTextField -import dev.obiente.nextcloudnative.app.NextcloudAccountId -import dev.obiente.nextcloudnative.app.NextcloudAccountRecord import dev.obiente.nextcloudnative.app.NextcloudApiMethod import dev.obiente.nextcloudnative.app.NextcloudMultipartUploadRequest import dev.obiente.nextcloudnative.app.NextcloudSession import dev.obiente.nextcloudnative.app.localUploadFile import java.util.UUID import kotlinx.coroutines.CancellationException +import kotlinx.coroutines.Dispatchers +import kotlinx.coroutines.flow.first +import kotlinx.coroutines.withContext import org.json.JSONArray import org.json.JSONObject -internal class AndroidDurableMultipartUploads(context: Context) { +internal class AndroidDurableMultipartUploads( + context: Context, + localUploadPicker: AndroidLocalUploadPicker? = null, +) { private val appContext = context.applicationContext + private val picker = localUploadPicker ?: AndroidLocalUploadPicker(appContext) private val store = AndroidDurableMultipartUploadStore(appContext) + private val workManager = WorkManager.getInstance(appContext) suspend fun enqueue( session: NextcloudSession, @@ -37,9 +43,7 @@ internal class AndroidDurableMultipartUploads(context: Context) { request: NextcloudMultipartUploadRequest, ): DurableUploadEnqueueResult { val accountId = NextcloudDocumentIds.accountKey(session) - val picker = AndroidLocalUploadPicker(appContext) - var storedJob: AndroidDurableMultipartUploadJob? = null - return runCatching { + return try { val safeRequest = request.requireSafe() picker.requirePersisted(safeRequest.file) val job = AndroidDurableMultipartUploadJob( @@ -51,17 +55,15 @@ internal class AndroidDurableMultipartUploads(context: Context) { state = DurableUploadState.Queued, message = null, ) - store.add(job) - storedJob = job - schedule(job).await() - DurableUploadEnqueueResult.Queued(job.status()) - }.getOrElse { error -> - storedJob?.let { job -> - runCatching { store.remove(job.id) } - } - if (!store.hasActiveSelection(request.file.selectionId)) { - picker.release(request.file) - } + persistAndScheduleDurableUpload( + job = job, + persist = store::add, + schedule = { queued -> schedule(queued).await() }, + ) + } catch (cancelled: CancellationException) { + throw cancelled + } catch (error: Exception) { + releaseIfUnowned(request.file) DurableUploadEnqueueResult.Rejected( error.message?.take(MAX_DURABLE_UPLOAD_MESSAGE_CHARACTERS) ?: "The background upload could not be scheduled.", @@ -69,35 +71,84 @@ internal class AndroidDurableMultipartUploads(context: Context) { } } - fun statuses(session: NextcloudSession, scope: DurableUploadScope): List = - store.list(NextcloudDocumentIds.accountKey(session), scope) - .asSequence() - .onEach { job -> - if (job.state == DurableUploadState.Queued) { - runCatching { schedule(job) } - } - } + fun releaseIfUnowned(file: LocalUploadFile): Boolean = releaseUnownedDurableUploadSelection( + selectionId = file.selectionId, + hasActiveSelection = store::hasActiveSelection, + releaseSelection = { picker.release(file) }, + markOwnershipCheckPending = { picker.markOwnershipCheckPending(file) }, + ) + + suspend fun runEnqueueWithCancellationCleanup( + file: LocalUploadFile, + enqueue: suspend () -> Result, + ): Result = runDurableUploadEnqueueWithCancellationCleanup( + enqueue = { withContext(Dispatchers.IO) { enqueue() } }, + releaseUnownedSelection = { releaseIfUnowned(file) }, + ) + + fun statuses(session: NextcloudSession, scope: DurableUploadScope): List { + val jobs = store.list(NextcloudDocumentIds.accountKey(session), scope) + requestDurableUploadSchedulingRecoveryForQueuedStatuses(jobs) + return jobs.asSequence() .sortedByDescending(AndroidDurableMultipartUploadJob::updatedAtEpochMillis) .take(MAX_VISIBLE_UPLOADS_PER_RESOURCE) .map(AndroidDurableMultipartUploadJob::status) .toList() + } suspend fun resumeQueuedForAccount(accountId: String) { queuedDurableUploadsForAccount(store.list(), accountId).forEach { job -> try { - schedule(job, ExistingWorkPolicy.APPEND_OR_REPLACE).await() + replaceDeferredDurableUploadWork( + expected = job, + load = store::find, + replace = { queued -> + schedule(queued, DURABLE_UPLOAD_ACCOUNT_RECOVERY_WORK_POLICY).await() + }, + ) } catch (cancelled: CancellationException) { + runCatching { requestQueuedDurableUploadSchedulingRecovery() } throw cancelled } catch (_: Exception) { - // The queue stays authoritative; status refresh or a later activation can retry. + requestQueuedDurableUploadSchedulingRecovery() } } } - suspend fun reconcileQueuedUploads(): Boolean = reconcileQueuedDurableUploads( - jobs = store.list(), - schedule = { job -> schedule(job).await() }, - ) + suspend fun reconcileQueuedUploads( + allowQueuedScheduling: Boolean = true, + schedulingRecoverySignal: AndroidDurableUploadSchedulingRecoverySignal = + ANDROID_DURABLE_UPLOAD_SCHEDULING_RECOVERY_SIGNAL, + ): Boolean { + val (jobs, capabilitiesRecovered) = synchronized(AndroidDurableMultipartUploadStore.LOCK) { + val snapshot = store.list() + val retainedSelectionIds = durableUploadCapabilityRetainedSelectionIds(snapshot) + snapshot to picker.reconcileCapabilities(retainedSelectionIds) + } + val uploadsRecovered = reconcileQueuedDurableUploads( + jobs = jobs, + allowQueuedScheduling = allowQueuedScheduling, + schedulerOwns = { job -> + workManager.getWorkInfosForUniqueWorkFlow(durableUploadWorkName(job.id)) + .first() + .any { work -> !work.state.isFinished } + }, + cleanupCapability = { job -> + check( + reconcileTerminalDurableUploadCapabilityCleanup( + release = { onQuarantined -> picker.release(job.request.file, onQuarantined) }, + complete = { store.completeCapabilityCleanup(job.id) }, + ), + ) { + "The durable upload capability cleanup remains pending." + } + }, + schedule = { job -> + schedulingRecoverySignal.scheduleUnlessBackedOff(job.id) { schedule(job) }?.await() + }, + ) + return capabilitiesRecovered && uploadsRecovered + } fun dismiss(session: NextcloudSession, scope: DurableUploadScope, uploadId: String): Boolean { val job = store.find(uploadId) ?: return false @@ -108,23 +159,24 @@ internal class AndroidDurableMultipartUploads(context: Context) { ) { return false } - if (!AndroidLocalUploadPicker(appContext).release(job.request.file)) return false - store.remove(uploadId) - return true + return dismissTerminalDurableUploadStatus( + release = { onQuarantined -> picker.release(job.request.file, onQuarantined) }, + removeStatus = { store.remove(uploadId) }, + ) } private fun schedule( job: AndroidDurableMultipartUploadJob, policy: ExistingWorkPolicy = ExistingWorkPolicy.KEEP, ): Operation = - WorkManager.getInstance(appContext).enqueueUniqueWork( + workManager.enqueueUniqueWork( durableUploadWorkName(job.id), policy, OneTimeWorkRequestBuilder() .setInputData(Data.Builder().putString(DeckAttachmentUploadWorker.KEY_JOB_ID, job.id).build()) .setConstraints( Constraints.Builder() - .setRequiredNetworkType(NetworkType.CONNECTED) + .setRequiredNetworkType(networkTypeForDurableUploadWork(job)) .build(), ) .build(), @@ -135,150 +187,89 @@ internal class AndroidDurableMultipartUploads(context: Context) { } } -internal fun durableUploadWorkName(jobId: String) = "deck-attachment-$jobId" - -internal suspend fun reconcileQueuedDurableUploads( - jobs: List, - schedule: suspend (AndroidDurableMultipartUploadJob) -> Unit, -): Boolean { - var allScheduled = true - jobs.filter { job -> job.state == DurableUploadState.Queued }.forEach { job -> +internal fun releaseUnownedDurableUploadSelection( + selectionId: String, + hasActiveSelection: (String) -> Boolean, + releaseSelection: () -> Boolean, + markOwnershipCheckPending: () -> Boolean = { false }, +): Boolean = synchronized(AndroidDurableMultipartUploadStore.LOCK) { + val active = try { + hasActiveSelection(selectionId) + } catch (cancelled: CancellationException) { + throw cancelled + } catch (_: Exception) { try { - schedule(job) + markOwnershipCheckPending() } catch (cancelled: CancellationException) { throw cancelled } catch (_: Exception) { - allScheduled = false + // The capability remains in its previous fail-closed state. } + return@synchronized false } - return allScheduled -} - -internal suspend fun constructAndReconcileQueuedDurableUploads( - createReconciler: () -> suspend () -> Boolean, -): Boolean { - val reconcile = try { - createReconciler() + if (active) return@synchronized false + try { + releaseSelection() } catch (cancelled: CancellationException) { throw cancelled - } catch (failure: Exception) { - throw AndroidDurableMultipartUploadRecoveryException(failure) + } catch (_: Exception) { + false } - return reconcile() } -internal suspend fun retryQueuedDurableUploadScheduling( - retryDelaysMillis: List = listOf(1_000L, 5_000L), - reconcile: suspend () -> Boolean, - wait: suspend (Long) -> Unit, -): Boolean { - if (reconcile()) return true - retryDelaysMillis.forEach { delayMillis -> - require(delayMillis >= 0L) - wait(delayMillis) - if (reconcile()) return true - } - return false +internal suspend fun runDurableUploadEnqueueWithCancellationCleanup( + enqueue: suspend () -> Result, + releaseUnownedSelection: () -> Unit, +): Result = try { + enqueue() +} catch (cancelled: CancellationException) { + runCatching(releaseUnownedSelection) + throw cancelled } -internal suspend fun keepRetryingQueuedDurableUploadScheduling( - retryDelaysMillis: List = listOf(1_000L, 5_000L), - followUpDelayMillis: Long = 60_000L, - reconcile: suspend () -> Boolean, - wait: suspend (Long) -> Unit, - recordRecoveryFailure: () -> Unit = {}, -) { - require(followUpDelayMillis > 0L) - var recoveryFailureReported = false - while (true) { - val recovered = try { - retryQueuedDurableUploadScheduling(retryDelaysMillis, reconcile, wait) - } catch (cancelled: CancellationException) { - throw cancelled - } catch (failure: AndroidDurableMultipartUploadRecoveryException) { - if (failure.disposition == DurableUploadQueueRecoveryDisposition.Quarantine) { - if (!recoveryFailureReported) runCatching(recordRecoveryFailure) - return - } - false - } - if (recovered) return - if (!recoveryFailureReported) { - runCatching(recordRecoveryFailure) - recoveryFailureReported = true - } - wait(followUpDelayMillis) - } -} +internal val DURABLE_UPLOAD_ACCOUNT_RECOVERY_WORK_POLICY = ExistingWorkPolicy.REPLACE -internal sealed interface DurableUploadAccountResolution { - data class Available(val session: NextcloudSession) : DurableUploadAccountResolution - data object RegistryUnavailable : DurableUploadAccountResolution - data object CredentialUnavailable : DurableUploadAccountResolution - data object DeferAccountActivation : DurableUploadAccountResolution - data object AccountUnavailable : DurableUploadAccountResolution -} - -internal sealed interface DurableUploadAccountRegistry { - data class Available( - val accounts: List, - val activeAccountId: NextcloudAccountId? = null, - ) : DurableUploadAccountRegistry +internal fun durableUploadWorkName(jobId: String) = "deck-attachment-$jobId" - data object Unavailable : DurableUploadAccountRegistry +internal fun networkTypeForDurableUploadWork(job: AndroidDurableMultipartUploadJob): NetworkType { + require(job.state == DurableUploadState.Queued && !job.capabilityCleanupPending) { + "Only a queued durable upload can use network-constrained upload work." + } + return NetworkType.CONNECTED } -internal fun queuedDurableUploadsForAccount( +internal fun requestDurableUploadSchedulingRecoveryForQueuedStatuses( jobs: List, - accountId: String, -): List = jobs.filter { job -> - job.accountId == accountId && job.state == DurableUploadState.Queued + requestRecovery: () -> Unit = ::requestQueuedDurableUploadSchedulingRecovery, +) { + if (jobs.any { job -> job.state == DurableUploadState.Queued }) requestRecovery() } -internal fun resolveDurableUploadSession( - expectedAccountId: String, - registry: DurableUploadAccountRegistry, - loadSession: (NextcloudAccountId) -> NextcloudSession?, -): DurableUploadAccountResolution { - val availableRegistry = when (registry) { - is DurableUploadAccountRegistry.Available -> registry - DurableUploadAccountRegistry.Unavailable -> return DurableUploadAccountResolution.RegistryUnavailable - } - val account = availableRegistry.accounts.singleOrNull { record -> - NextcloudDocumentIds.accountKey(record.serverUrl, record.loginName) == expectedAccountId - } ?: return DurableUploadAccountResolution.AccountUnavailable - val session = loadSession(account.id) - ?.takeIf { loaded -> NextcloudDocumentIds.accountKey(loaded) == expectedAccountId } - ?: return if (account.id == availableRegistry.activeAccountId) { - DurableUploadAccountResolution.CredentialUnavailable - } else { - DurableUploadAccountResolution.DeferAccountActivation - } - return DurableUploadAccountResolution.Available(session) +internal fun reconcileTerminalDurableUploadCapabilityCleanup( + release: (onQuarantined: () -> Unit) -> Boolean, + complete: () -> Unit, +): Boolean { + if (releaseOrQuarantineDurableUploadCapability(release)) { + complete() + return true + } + return false } -internal fun resolveDurableUploadSessionWithRegistryRecovery( - expectedAccountId: String, - readRegistry: () -> DurableUploadAccountRegistry, - recoverRegistry: () -> NextcloudSession?, - loadSession: (NextcloudAccountId) -> NextcloudSession?, -): DurableUploadAccountResolution { - val initial = readRegistry() - val recoveryRequired = when (initial) { - DurableUploadAccountRegistry.Unavailable -> true - is DurableUploadAccountRegistry.Available -> initial.accounts.none { account -> - NextcloudDocumentIds.accountKey(account.serverUrl, account.loginName) == expectedAccountId - } - } - if (!recoveryRequired) return resolveDurableUploadSession(expectedAccountId, initial, loadSession) - val recoveredSession = recoverRegistry() - if ( - recoveredSession != null && - NextcloudDocumentIds.accountKey(recoveredSession) == expectedAccountId - ) { - return DurableUploadAccountResolution.Available(recoveredSession) - } - return resolveDurableUploadSession(expectedAccountId, readRegistry(), loadSession) +internal fun releaseOrQuarantineDurableUploadCapability( + release: (onQuarantined: () -> Unit) -> Boolean, +): Boolean { + var quarantined = false + return release { quarantined = true } || quarantined +} + +internal fun dismissTerminalDurableUploadStatus( + release: (onQuarantined: () -> Unit) -> Boolean, + removeStatus: () -> Unit, +): Boolean { + if (!releaseOrQuarantineDurableUploadCapability(release)) return false + removeStatus() + return true } internal data class AndroidDurableMultipartUploadJob( @@ -289,6 +280,7 @@ internal data class AndroidDurableMultipartUploadJob( val request: NextcloudMultipartUploadRequest, val state: DurableUploadState, val message: String?, + val capabilityCleanupPending: Boolean = false, val updatedAtEpochMillis: Long = System.currentTimeMillis(), ) { init { @@ -304,6 +296,9 @@ internal data class AndroidDurableMultipartUploadJob( require(message == null || message.length <= MAX_DURABLE_UPLOAD_MESSAGE_CHARACTERS) { "The durable upload message is too long." } + require(!capabilityCleanupPending || state.isTerminal()) { + "Only a terminal durable upload can have pending capability cleanup." + } } fun status(): DurableUploadStatus = DurableUploadStatus( @@ -315,6 +310,15 @@ internal data class AndroidDurableMultipartUploadJob( ) } +internal fun durableUploadCapabilityRetainedSelectionIds( + jobs: Iterable, +): Set = jobs.asSequence() + .filter { job -> + job.state == DurableUploadState.Queued || job.state == DurableUploadState.Uploading + } + .map { job -> job.request.file.selectionId } + .toSet() + internal data class AndroidDurableUploadResource( val feature: String, val boardId: String?, @@ -370,7 +374,7 @@ internal class AndroidDurableMultipartUploadStore( fun hasActiveSelection(selectionId: String): Boolean = synchronized(LOCK) { readAll().any { it.request.file.selectionId == selectionId && - !it.state.isTerminal() + it.mustRetain() } } @@ -385,6 +389,14 @@ internal class AndroidDurableMultipartUploadStore( removed } + fun completeCapabilityCleanup(id: String) = synchronized(LOCK) { + val current = readAll().toMutableList() + val index = current.indexOfFirst { job -> job.id == id } + if (index < 0 || !current[index].capabilityCleanupPending) return@synchronized + current[index] = current[index].copy(capabilityCleanupPending = false) + writeAll(pruneDurableUploadJobs(current)) + } + fun transition( id: String, expected: DurableUploadState, @@ -400,6 +412,7 @@ internal class AndroidDurableMultipartUploadStore( val updated = current[index].copy( state = target, message = message?.take(MAX_DURABLE_UPLOAD_MESSAGE_CHARACTERS), + capabilityCleanupPending = target.isTerminal(), updatedAtEpochMillis = System.currentTimeMillis(), ) current[index] = updated @@ -535,6 +548,12 @@ internal fun requireCanAddDurableUpload( require(current.none { it.id == job.id }) { "The attachment upload id is already in use." } + require( + current.count(AndroidDurableMultipartUploadJob::mustRetain) < + AndroidDurableMultipartUploadStore.MAX_STORED_UPLOADS, + ) { + "Background upload cleanup must finish before another upload can be queued." + } require(active.size < AndroidDurableMultipartUploadStore.MAX_ACTIVE_UPLOADS) { "Too many attachment uploads are already pending." } @@ -562,13 +581,16 @@ internal fun requireCanAddDurableUpload( internal fun pruneDurableUploadJobs( jobs: List, ): List { - val active = jobs.filterNot { it.state.isTerminal() } - val terminal = jobs.filter { it.state.isTerminal() } + val retained = jobs.filter(AndroidDurableMultipartUploadJob::mustRetain) + val terminal = jobs.filterNot(AndroidDurableMultipartUploadJob::mustRetain) .sortedByDescending(AndroidDurableMultipartUploadJob::updatedAtEpochMillis) - .take((AndroidDurableMultipartUploadStore.MAX_STORED_UPLOADS - active.size).coerceAtLeast(0)) - return (active + terminal).sortedBy(AndroidDurableMultipartUploadJob::updatedAtEpochMillis) + .take((AndroidDurableMultipartUploadStore.MAX_STORED_UPLOADS - retained.size).coerceAtLeast(0)) + return (retained + terminal).sortedBy(AndroidDurableMultipartUploadJob::updatedAtEpochMillis) } +private fun AndroidDurableMultipartUploadJob.mustRetain(): Boolean = + !state.isTerminal() || capabilityCleanupPending + private fun DurableUploadState.isTerminal(): Boolean = this == DurableUploadState.Completed || this == DurableUploadState.Failed || @@ -611,6 +633,7 @@ private fun AndroidDurableMultipartUploadJob.toJson(): JSONObject = JSONObject() .put("itemId", resource.itemId) .put("state", state.name) .put("message", message) + .put("capabilityCleanupPending", capabilityCleanupPending) .put("updatedAt", updatedAtEpochMillis) .put("method", request.method.name) .put("relativePath", request.relativePath) @@ -685,10 +708,20 @@ private fun JSONObject.toJob(): AndroidDurableMultipartUploadJob { request = request, state = DurableUploadState.valueOf(getString("state")), message = if (isNull("message")) null else getString("message"), + capabilityCleanupPending = readCapabilityCleanupPending(), updatedAtEpochMillis = getLong("updatedAt"), ) } +private fun JSONObject.readCapabilityCleanupPending(): Boolean { + if (!has("capabilityCleanupPending")) return false + val persisted = get("capabilityCleanupPending") + check(persisted is Boolean) { + "The persisted capability cleanup marker is not a boolean." + } + return persisted +} + internal fun resolveDurableUploadResource( scope: DurableUploadScope, request: NextcloudMultipartUploadRequest, diff --git a/androidApp/src/main/kotlin/dev/obiente/nextcloudnative/AndroidDurableUploadAccountCleanup.kt b/androidApp/src/main/kotlin/dev/obiente/nextcloudnative/AndroidDurableUploadAccountCleanup.kt index 926b6c9ba..d12aa609c 100644 --- a/androidApp/src/main/kotlin/dev/obiente/nextcloudnative/AndroidDurableUploadAccountCleanup.kt +++ b/androidApp/src/main/kotlin/dev/obiente/nextcloudnative/AndroidDurableUploadAccountCleanup.kt @@ -15,12 +15,23 @@ internal class AndroidDurableUploadAccountCleanup(context: Context) { cancelWork = { job -> WorkManager.getInstance(appContext).cancelUniqueWork(durableUploadWorkName(job.id)).await() }, - releaseCapability = { job -> picker.release(job.request.file) }, + releaseCapability = { job -> + releaseAndroidDurableUploadCapabilityForAccountRemoval { onQuarantined -> + picker.release(job.request.file, onQuarantined) + } + }, removeJob = store::remove, ) } } +internal fun releaseAndroidDurableUploadCapabilityForAccountRemoval( + release: (onQuarantined: () -> Unit) -> Boolean, +): Boolean { + var quarantined = false + return release { quarantined = true } || quarantined +} + internal suspend fun removeAndroidDurableUploadJobs( jobs: List, cancelWork: suspend (AndroidDurableMultipartUploadJob) -> Unit, diff --git a/androidApp/src/main/kotlin/dev/obiente/nextcloudnative/AndroidDurableUploadAccountResolution.kt b/androidApp/src/main/kotlin/dev/obiente/nextcloudnative/AndroidDurableUploadAccountResolution.kt new file mode 100644 index 000000000..89b3ab8c5 --- /dev/null +++ b/androidApp/src/main/kotlin/dev/obiente/nextcloudnative/AndroidDurableUploadAccountResolution.kt @@ -0,0 +1,76 @@ +package dev.obiente.nextcloudnative + +import dev.obiente.nextcloudnative.app.DurableUploadState +import dev.obiente.nextcloudnative.app.NextcloudAccountId +import dev.obiente.nextcloudnative.app.NextcloudAccountRecord +import dev.obiente.nextcloudnative.app.NextcloudSession + +internal sealed interface DurableUploadAccountResolution { + data class Available(val session: NextcloudSession) : DurableUploadAccountResolution + data object RegistryUnavailable : DurableUploadAccountResolution + data object CredentialUnavailable : DurableUploadAccountResolution + data object DeferAccountActivation : DurableUploadAccountResolution + data object AccountUnavailable : DurableUploadAccountResolution +} + +internal sealed interface DurableUploadAccountRegistry { + data class Available( + val accounts: List, + val activeAccountId: NextcloudAccountId? = null, + ) : DurableUploadAccountRegistry + + data object Unavailable : DurableUploadAccountRegistry +} + +internal fun queuedDurableUploadsForAccount( + jobs: List, + accountId: String, +): List = jobs.filter { job -> + job.accountId == accountId && job.state == DurableUploadState.Queued +} + +internal fun resolveDurableUploadSession( + expectedAccountId: String, + registry: DurableUploadAccountRegistry, + loadSession: (NextcloudAccountId) -> NextcloudSession?, +): DurableUploadAccountResolution { + val availableRegistry = when (registry) { + is DurableUploadAccountRegistry.Available -> registry + DurableUploadAccountRegistry.Unavailable -> return DurableUploadAccountResolution.RegistryUnavailable + } + val account = availableRegistry.accounts.singleOrNull { record -> + NextcloudDocumentIds.accountKey(record.serverUrl, record.loginName) == expectedAccountId + } ?: return DurableUploadAccountResolution.AccountUnavailable + val session = loadSession(account.id) + ?.takeIf { loaded -> NextcloudDocumentIds.accountKey(loaded) == expectedAccountId } + ?: return if (account.id == availableRegistry.activeAccountId) { + DurableUploadAccountResolution.CredentialUnavailable + } else { + DurableUploadAccountResolution.DeferAccountActivation + } + return DurableUploadAccountResolution.Available(session) +} + +internal fun resolveDurableUploadSessionWithRegistryRecovery( + expectedAccountId: String, + readRegistry: () -> DurableUploadAccountRegistry, + recoverRegistry: () -> NextcloudSession?, + loadSession: (NextcloudAccountId) -> NextcloudSession?, +): DurableUploadAccountResolution { + val initial = readRegistry() + val recoveryRequired = when (initial) { + DurableUploadAccountRegistry.Unavailable -> true + is DurableUploadAccountRegistry.Available -> initial.accounts.none { account -> + NextcloudDocumentIds.accountKey(account.serverUrl, account.loginName) == expectedAccountId + } + } + if (!recoveryRequired) return resolveDurableUploadSession(expectedAccountId, initial, loadSession) + val recoveredSession = recoverRegistry() + if ( + recoveredSession != null && + NextcloudDocumentIds.accountKey(recoveredSession) == expectedAccountId + ) { + return DurableUploadAccountResolution.Available(recoveredSession) + } + return resolveDurableUploadSession(expectedAccountId, readRegistry(), loadSession) +} diff --git a/androidApp/src/main/kotlin/dev/obiente/nextcloudnative/AndroidDurableUploadScheduling.kt b/androidApp/src/main/kotlin/dev/obiente/nextcloudnative/AndroidDurableUploadScheduling.kt new file mode 100644 index 000000000..3974984ac --- /dev/null +++ b/androidApp/src/main/kotlin/dev/obiente/nextcloudnative/AndroidDurableUploadScheduling.kt @@ -0,0 +1,420 @@ +package dev.obiente.nextcloudnative + +import dev.obiente.nextcloudnative.app.DurableUploadEnqueueResult +import dev.obiente.nextcloudnative.app.DurableUploadState +import java.util.UUID +import kotlinx.coroutines.CancellationException +import kotlinx.coroutines.CoroutineStart +import kotlinx.coroutines.async +import kotlinx.coroutines.channels.Channel +import kotlinx.coroutines.coroutineScope +import kotlinx.coroutines.selects.select +import kotlinx.coroutines.sync.Mutex +import kotlinx.coroutines.sync.withLock + +internal class AndroidDurableUploadStartCoordinator { + private val monitor = Any() + private val jobLeases = mutableMapOf() + + suspend fun withJob(jobId: String, action: suspend () -> Result): Result { + require(jobId.isNotBlank()) + val lease = synchronized(monitor) { + jobLeases.getOrPut(jobId) { JobLease() }.also { it.references += 1 } + } + return try { + lease.mutex.withLock { action() } + } finally { + synchronized(monitor) { + lease.references -= 1 + if (lease.references == 0) jobLeases.remove(jobId, lease) + } + } + } + + private class JobLease( + val mutex: Mutex = Mutex(), + var references: Int = 0, + ) +} + +private val ANDROID_DURABLE_UPLOAD_START_COORDINATOR = AndroidDurableUploadStartCoordinator() + +internal const val ANDROID_DURABLE_UPLOAD_SCHEDULING_FOLLOW_UP_DELAY_MILLIS = 60_000L + +internal data class AndroidDurableUploadSchedulingRecoveryBatch( + val immediate: Boolean, + val workIdsToAwait: Map, +) + +internal sealed interface AndroidDurableUploadSchedulingRecoveryStep { + data object Completed : AndroidDurableUploadSchedulingRecoveryStep + + data class Interrupted( + val batch: AndroidDurableUploadSchedulingRecoveryBatch, + ) : AndroidDurableUploadSchedulingRecoveryStep +} + +internal class AndroidDurableUploadSchedulingRecoverySignal( + private val beforeBatchClaim: suspend () -> Unit = {}, +) { + private val monitor = Any() + private val wakeups = Channel(Channel.CONFLATED) + private var immediatePending = false + private val workIdsToAwait = linkedMapOf() + private val backedOffWorkIds = mutableMapOf() + + fun request() { + synchronized(monitor) { + immediatePending = true + wakeups.trySend(Unit) + } + } + + fun requestAfterWorkStopsRunning(jobId: String, workId: UUID) { + require(jobId.isNotBlank()) + synchronized(monitor) { + workIdsToAwait[jobId] = workId + backedOffWorkIds[jobId] = workId + wakeups.trySend(Unit) + } + } + + fun scheduleUnlessBackedOff(jobId: String, schedule: () -> Result): Result? { + require(jobId.isNotBlank()) + return synchronized(monitor) { + if (jobId in backedOffWorkIds) null else schedule() + } + } + + fun retireBackoff(jobId: String, workId: UUID): Boolean = synchronized(monitor) { + if (jobId in workIdsToAwait) false else backedOffWorkIds.remove(jobId, workId) + } + + suspend fun await(): AndroidDurableUploadSchedulingRecoveryBatch { + wakeups.receive() + beforeBatchClaim() + return takeBatch() + } + + fun tryTakePending(): AndroidDurableUploadSchedulingRecoveryBatch? = synchronized(monitor) { + if (!immediatePending && workIdsToAwait.isEmpty()) null else takeBatchLocked() + } + + suspend fun runUntilRequested( + action: suspend () -> Unit, + ): AndroidDurableUploadSchedulingRecoveryStep = coroutineScope { + val running = async(start = CoroutineStart.UNDISPATCHED) { action() } + try { + select { + running.onAwait { AndroidDurableUploadSchedulingRecoveryStep.Completed } + wakeups.onReceive { + beforeBatchClaim() + AndroidDurableUploadSchedulingRecoveryStep.Interrupted(takeBatch()) + } + } + } finally { + running.cancel() + } + } + + private fun takeBatch(): AndroidDurableUploadSchedulingRecoveryBatch = synchronized(monitor) { + takeBatchLocked() + } + + private fun takeBatchLocked(): AndroidDurableUploadSchedulingRecoveryBatch { + while (wakeups.tryReceive().isSuccess) { + // Every request represented by a drained token is included in the pending state below. + } + return AndroidDurableUploadSchedulingRecoveryBatch( + immediate = immediatePending, + workIdsToAwait = workIdsToAwait.toMap(), + ).also { + immediatePending = false + workIdsToAwait.clear() + } + } +} + +internal val ANDROID_DURABLE_UPLOAD_SCHEDULING_RECOVERY_SIGNAL = + AndroidDurableUploadSchedulingRecoverySignal() + +internal fun requestQueuedDurableUploadSchedulingRecovery() { + ANDROID_DURABLE_UPLOAD_SCHEDULING_RECOVERY_SIGNAL.request() +} + +internal fun requestQueuedDurableUploadSchedulingRecoveryAfterWorkStopsRunning(jobId: String, workId: UUID) { + ANDROID_DURABLE_UPLOAD_SCHEDULING_RECOVERY_SIGNAL.requestAfterWorkStopsRunning(jobId, workId) +} + +internal suspend fun monitorQueuedDurableUploadScheduling( + recover: suspend () -> Boolean, + awaitWorkStopsRunning: suspend (UUID) -> Unit = {}, + wait: suspend (Long) -> Unit, + workerFailureFollowUpDelayMillis: Long = + ANDROID_DURABLE_UPLOAD_SCHEDULING_FOLLOW_UP_DELAY_MILLIS, + monotonicTimeMillis: () -> Long = { System.nanoTime() / 1_000_000L }, + afterEmptyPendingBatchClaim: () -> Unit = {}, + recoverySignal: AndroidDurableUploadSchedulingRecoverySignal = + ANDROID_DURABLE_UPLOAD_SCHEDULING_RECOVERY_SIGNAL, +) { + require(workerFailureFollowUpDelayMillis > 0L) + var immediatePending = false + val workIdsToAwait = linkedMapOf() + val followUpDeadlinesMillis = mutableMapOf() + val stoppedWorkIds = mutableMapOf() + var recoveryRetryDeadlineMillis: Long? = null + + fun addRequests(batch: AndroidDurableUploadSchedulingRecoveryBatch) { + immediatePending = immediatePending || batch.immediate + batch.workIdsToAwait.forEach { (jobId, workId) -> + if (workIdsToAwait.put(jobId, workId) != workId) { + followUpDeadlinesMillis[jobId] = + monotonicTimeMillis() + workerFailureFollowUpDelayMillis + stoppedWorkIds.remove(jobId) + } + } + } + + suspend fun recoverOnce() { + val recovered = recover() + // Cleanup can request recovery itself. Coalesce signals raised during this pass into + // a timed retry instead of letting the same failure bypass every worker deadline. + val pending = recoverySignal.tryTakePending() + if (pending != null) addRequests(pending.copy(immediate = false)) + recoveryRetryDeadlineMillis = if (recovered && pending?.immediate != true) { + null + } else { + monotonicTimeMillis() + workerFailureFollowUpDelayMillis + } + } + + recoverySignal.tryTakePending()?.let(::addRequests) + recoverOnce() + + while (true) { + if (!immediatePending && workIdsToAwait.isEmpty()) { + val retryDeadline = recoveryRetryDeadlineMillis + if (retryDeadline == null) { + addRequests(recoverySignal.await()) + } else { + val retryDelay = (retryDeadline - monotonicTimeMillis()).coerceAtLeast(0L) + val step = recoverySignal.runUntilRequested { if (retryDelay > 0L) wait(retryDelay) } + if (step is AndroidDurableUploadSchedulingRecoveryStep.Interrupted) { + addRequests(step.batch) + continue + } + recoverOnce() + continue + } + } + if (!immediatePending && workIdsToAwait.isEmpty()) continue + if (immediatePending) { + immediatePending = false + recoverOnce() + continue + } + + val (jobId, workId) = workIdsToAwait.entries.first() + if (stoppedWorkIds[jobId] != workId) { + when (val step = recoverySignal.runUntilRequested { awaitWorkStopsRunning(workId) }) { + AndroidDurableUploadSchedulingRecoveryStep.Completed -> { + if (workIdsToAwait[jobId] != workId) continue + stoppedWorkIds[jobId] = workId + } + is AndroidDurableUploadSchedulingRecoveryStep.Interrupted -> { + addRequests(step.batch) + continue + } + } + } + + val remainingDelayMillis = + (followUpDeadlinesMillis.getValue(jobId) - monotonicTimeMillis()).coerceAtLeast(0L) + if (remainingDelayMillis > 0L) { + val recoveryRetryDelayMillis = recoveryRetryDeadlineMillis + ?.let { deadline -> (deadline - monotonicTimeMillis()).coerceAtLeast(0L) } + if (recoveryRetryDelayMillis == 0L) { + recoverOnce() + continue + } + val recoveryRetryFirst = + recoveryRetryDelayMillis != null && recoveryRetryDelayMillis < remainingDelayMillis + when ( + val step = recoverySignal.runUntilRequested { + wait(if (recoveryRetryFirst) requireNotNull(recoveryRetryDelayMillis) else remainingDelayMillis) + } + ) { + AndroidDurableUploadSchedulingRecoveryStep.Completed -> if (recoveryRetryFirst) { + recoverOnce() + continue + } + is AndroidDurableUploadSchedulingRecoveryStep.Interrupted -> { + addRequests(step.batch) + continue + } + } + } + val pendingBatch = recoverySignal.tryTakePending() + if (pendingBatch != null) { + addRequests(pendingBatch) + continue + } + afterEmptyPendingBatchClaim() + followUpDeadlinesMillis.remove(jobId) + stoppedWorkIds.remove(jobId) + workIdsToAwait.remove(jobId, workId) + recoverySignal.retireBackoff(jobId, workId) + recoverOnce() + } +} + +internal suspend fun awaitDurableUploadWorkToStopRunning( + workId: UUID, + retryDelayMillis: Long = 1_000L, + awaitWorkStopsRunning: suspend (UUID) -> Unit, + wait: suspend (Long) -> Unit, +) { + require(retryDelayMillis > 0L) + while (true) { + try { + awaitWorkStopsRunning(workId) + return + } catch (cancelled: CancellationException) { + throw cancelled + } catch (_: Exception) { + wait(retryDelayMillis) + } + } +} + +internal suspend fun claimQueuedDurableUploadForExecution( + jobId: String, + coordinator: AndroidDurableUploadStartCoordinator = ANDROID_DURABLE_UPLOAD_START_COORDINATOR, + claim: suspend () -> AndroidDurableMultipartUploadJob?, +): AndroidDurableMultipartUploadJob? = coordinator.withJob(jobId, claim) + +internal suspend fun replaceDeferredDurableUploadWork( + expected: AndroidDurableMultipartUploadJob, + load: (String) -> AndroidDurableMultipartUploadJob?, + replace: suspend (AndroidDurableMultipartUploadJob) -> Unit, + coordinator: AndroidDurableUploadStartCoordinator = ANDROID_DURABLE_UPLOAD_START_COORDINATOR, +): Boolean = coordinator.withJob(expected.id) { + val current = load(expected.id) + if ( + current == null || + current.accountId != expected.accountId || + current.state != DurableUploadState.Queued + ) { + return@withJob false + } + replace(current) + true +} + +internal suspend fun constructAndReconcileQueuedDurableUploads( + createReconciler: () -> suspend () -> Boolean, +): Boolean { + val reconcile = try { + createReconciler() + } catch (cancelled: CancellationException) { + throw cancelled + } catch (failure: Exception) { + throw AndroidDurableMultipartUploadRecoveryException(failure) + } + return reconcile() +} + +internal suspend fun reconcileQueuedDurableUploads( + jobs: List, + allowQueuedScheduling: Boolean = true, + schedulerOwns: suspend (AndroidDurableMultipartUploadJob) -> Boolean = { false }, + cleanupCapability: suspend (AndroidDurableMultipartUploadJob) -> Unit, + schedule: suspend (AndroidDurableMultipartUploadJob) -> Unit, +): Boolean { + var allScheduled = true + jobs.filter { job -> job.requiresSchedulingRecovery(allowQueuedScheduling) }.forEach { job -> + try { + if (job.capabilityCleanupPending) { + cleanupCapability(job) + } else if (!schedulerOwns(job)) { + schedule(job) + } + } catch (cancelled: CancellationException) { + throw cancelled + } catch (_: Exception) { + allScheduled = false + } + } + return allScheduled +} + +private fun AndroidDurableMultipartUploadJob.requiresSchedulingRecovery( + allowQueuedScheduling: Boolean, +): Boolean = capabilityCleanupPending || (allowQueuedScheduling && state == DurableUploadState.Queued) + +internal suspend fun retryQueuedDurableUploadScheduling( + retryDelaysMillis: List = listOf(1_000L, 5_000L), + reconcile: suspend () -> Boolean, + wait: suspend (Long) -> Unit, +): Boolean { + if (reconcile()) return true + retryDelaysMillis.forEach { delayMillis -> + require(delayMillis >= 0L) + wait(delayMillis) + if (reconcile()) return true + } + return false +} + +internal suspend fun keepRetryingQueuedDurableUploadScheduling( + retryDelaysMillis: List = listOf(1_000L, 5_000L), + followUpDelayMillis: Long = ANDROID_DURABLE_UPLOAD_SCHEDULING_FOLLOW_UP_DELAY_MILLIS, + reconcile: suspend () -> Boolean, + wait: suspend (Long) -> Unit, + recordRecoveryFailure: () -> Unit = {}, +) { + require(followUpDelayMillis > 0L) + var recoveryFailureReported = false + while (true) { + val recovered = try { + retryQueuedDurableUploadScheduling(retryDelaysMillis, reconcile, wait) + } catch (cancelled: CancellationException) { + throw cancelled + } catch (failure: AndroidDurableMultipartUploadRecoveryException) { + if (failure.disposition == DurableUploadQueueRecoveryDisposition.Quarantine) { + if (!recoveryFailureReported) runCatching(recordRecoveryFailure) + return + } + false + } + if (recovered) return + if (!recoveryFailureReported) { + runCatching(recordRecoveryFailure) + recoveryFailureReported = true + } + wait(followUpDelayMillis) + } +} + +/** + * Persists the upload before asking WorkManager to schedule it. WorkManager acceptance and its + * completion signal are not atomic, so a scheduling failure after persistence is ambiguous: the + * durable queued job must remain authoritative and can be scheduled again after process restart. + */ +internal suspend fun persistAndScheduleDurableUpload( + job: AndroidDurableMultipartUploadJob, + persist: (AndroidDurableMultipartUploadJob) -> Unit, + schedule: suspend (AndroidDurableMultipartUploadJob) -> Unit, + requestRecovery: () -> Unit = ::requestQueuedDurableUploadSchedulingRecovery, +): DurableUploadEnqueueResult.Queued { + persist(job) + try { + schedule(job) + } catch (cancelled: CancellationException) { + runCatching(requestRecovery) + throw cancelled + } catch (_: Exception) { + runCatching(requestRecovery) + } + return DurableUploadEnqueueResult.Queued(job.status()) +} diff --git a/androidApp/src/main/kotlin/dev/obiente/nextcloudnative/AndroidDurableUploadWorker.kt b/androidApp/src/main/kotlin/dev/obiente/nextcloudnative/AndroidDurableUploadWorker.kt index 825a9cc0d..dd36560a5 100644 --- a/androidApp/src/main/kotlin/dev/obiente/nextcloudnative/AndroidDurableUploadWorker.kt +++ b/androidApp/src/main/kotlin/dev/obiente/nextcloudnative/AndroidDurableUploadWorker.kt @@ -4,6 +4,7 @@ import android.content.Context import androidx.work.CoroutineWorker import androidx.work.WorkerParameters import dev.obiente.nextcloudnative.app.DurableUploadState +import dev.obiente.nextcloudnative.app.NextcloudSession import dev.obiente.nextcloudnative.app.SupportDiagnosticComponent import dev.obiente.nextcloudnative.app.SupportDiagnosticEventDraft import dev.obiente.nextcloudnative.app.SupportDiagnosticFieldDraft @@ -11,6 +12,7 @@ import dev.obiente.nextcloudnative.app.SupportDiagnosticSeverity import dev.obiente.nextcloudnative.app.SupportDiagnosticValuePrivacy import dev.obiente.nextcloudnative.app.afterProcessRecovery import dev.obiente.nextcloudnative.app.toSupportDiagnosticExceptionDraft +import java.io.FileNotFoundException import kotlinx.coroutines.CancellationException import kotlinx.coroutines.Dispatchers import kotlinx.coroutines.withContext @@ -19,11 +21,26 @@ internal class DeckAttachmentUploadWorker( appContext: Context, params: WorkerParameters, ) : CoroutineWorker(appContext, params) { - override suspend fun doWork(): Result = withContext(Dispatchers.IO) { + override suspend fun doWork(): Result = runDurableUploadWorkerWithRecoverySignal( + requestRecovery = { + val jobId = inputData.getString(KEY_JOB_ID)?.takeIf(String::isNotBlank) + if (jobId == null) { + requestQueuedDurableUploadSchedulingRecovery() + } else { + requestQueuedDurableUploadSchedulingRecoveryAfterWorkStopsRunning(jobId, id) + } + }, + ) { + withContext(Dispatchers.IO) { + executeDurableUploadWork() + } + } + + private suspend fun executeDurableUploadWork(): Result { val jobId = inputData.getString(KEY_JOB_ID)?.takeIf(String::isNotBlank) - ?: return@withContext Result.failure() + ?: return Result.failure() val store = AndroidDurableMultipartUploadStore(applicationContext) - val initial = store.find(jobId) ?: return@withContext Result.success() + val initial = store.find(jobId) ?: return Result.success() val picker = AndroidLocalUploadPicker(applicationContext) if (initial.state.afterProcessRecovery() != initial.state) { store.transition( @@ -32,18 +49,31 @@ internal class DeckAttachmentUploadWorker( target = DurableUploadState.OutcomeUnknown, message = "The app restarted while this upload was in progress. Check the card before uploading again.", ) - picker.release(initial.request.file) recordUploadDiagnostic( severity = SupportDiagnosticSeverity.Warning, outcome = "process-recovery", accountId = initial.accountId, jobId = jobId, ) - return@withContext Result.success() + return resultAfterDurableUploadCapabilityReleaseOrQuarantine( + releaseCapability = { onQuarantined -> picker.release(initial.request.file, onQuarantined) }, + completeCapabilityCleanup = { store.completeCapabilityCleanup(jobId) }, + onCleanupRetained = ::requestQueuedDurableUploadSchedulingRecovery, + releasedResult = Result.success(), + retainedResult = Result.retry(), + ) + } + if (initial.state != DurableUploadState.Queued) { + return resultAfterDurableUploadCapabilityReleaseOrQuarantine( + releaseCapability = { onQuarantined -> picker.release(initial.request.file, onQuarantined) }, + completeCapabilityCleanup = { store.completeCapabilityCleanup(jobId) }, + onCleanupRetained = ::requestQueuedDurableUploadSchedulingRecovery, + releasedResult = Result.success(), + retainedResult = Result.retry(), + ) } - if (initial.state != DurableUploadState.Queued) return@withContext Result.success() - return@withContext uploadQueuedJob(store, initial, picker, jobId) + return uploadQueuedJob(store, initial, picker, jobId) } private suspend fun uploadQueuedJob( @@ -71,15 +101,10 @@ internal class DeckAttachmentUploadWorker( ) val session = when (accountResolution) { is DurableUploadAccountResolution.Available -> accountResolution.session - DurableUploadAccountResolution.RegistryUnavailable, - DurableUploadAccountResolution.CredentialUnavailable, - -> { + DurableUploadAccountResolution.RegistryUnavailable -> { recordUploadDiagnostic( severity = SupportDiagnosticSeverity.Warning, - outcome = when (accountResolution) { - DurableUploadAccountResolution.RegistryUnavailable -> "account-registry-unavailable" - else -> "account-resolution-deferred" - }, + outcome = "account-registry-unavailable", accountId = initial.accountId, jobId = jobId, ) @@ -94,6 +119,15 @@ internal class DeckAttachmentUploadWorker( ) return Result.success() } + DurableUploadAccountResolution.CredentialUnavailable -> { + recordUploadDiagnostic( + severity = SupportDiagnosticSeverity.Warning, + outcome = "account-resolution-deferred", + accountId = initial.accountId, + jobId = jobId, + ) + return Result.retry() + } DurableUploadAccountResolution.AccountUnavailable -> { return failQueuedDurableUploadForUnavailableAccount( transitionToFailed = { @@ -104,7 +138,9 @@ internal class DeckAttachmentUploadWorker( message = "The account used for this upload is no longer available.", ) }, - releaseSelection = { picker.release(initial.request.file) }, + releaseSelection = { onQuarantined -> picker.release(initial.request.file, onQuarantined) }, + completeCapabilityCleanup = { store.completeCapabilityCleanup(jobId) }, + onCleanupRetained = ::requestQueuedDurableUploadSchedulingRecovery, recordFailure = { recordUploadDiagnostic( severity = SupportDiagnosticSeverity.Warning, @@ -114,35 +150,65 @@ internal class DeckAttachmentUploadWorker( ) }, failureResult = Result.failure(), + retryResult = Result.retry(), ) } } - val capabilityReady = runCatching { - picker.requirePersisted(initial.request.file) - picker.open(initial.request.file).use { } - }.isSuccess - if (!capabilityReady) { + return processQueuedDurableUploadSource( + requireCapability = { picker.requirePersisted(initial.request.file) }, + openSource = { picker.open(initial.request.file).use { } }, + onCapabilityUnavailable = { + store.transition( + jobId, + expected = DurableUploadState.Queued, + target = DurableUploadState.Failed, + message = "The selected file is no longer available. Select it again to retry.", + ) + recordUploadDiagnostic( + severity = SupportDiagnosticSeverity.Warning, + outcome = "source-unavailable", + accountId = initial.accountId, + jobId = jobId, + ) + resultAfterDurableUploadCapabilityReleaseOrQuarantine( + releaseCapability = { onQuarantined -> picker.release(initial.request.file, onQuarantined) }, + completeCapabilityCleanup = { store.completeCapabilityCleanup(jobId) }, + onCleanupRetained = ::requestQueuedDurableUploadSchedulingRecovery, + releasedResult = Result.failure(), + retainedResult = Result.retry(), + ) + }, + onProviderUnavailable = { failure -> + recordUploadDiagnostic( + severity = SupportDiagnosticSeverity.Warning, + outcome = "source-open-deferred", + accountId = initial.accountId, + jobId = jobId, + failure = failure, + ) + Result.retry() + }, + onReady = { + uploadReadyQueuedJob(store, initial, picker, jobId, session) + }, + ) + } + + private suspend fun uploadReadyQueuedJob( + store: AndroidDurableMultipartUploadStore, + initial: AndroidDurableMultipartUploadJob, + picker: AndroidLocalUploadPicker, + jobId: String, + session: NextcloudSession, + ): Result { + val started = claimQueuedDurableUploadForExecution(jobId) { store.transition( jobId, expected = DurableUploadState.Queued, - target = DurableUploadState.Failed, - message = "The selected file is no longer available. Select it again to retry.", + target = DurableUploadState.Uploading, + message = null, ) - picker.release(initial.request.file) - recordUploadDiagnostic( - severity = SupportDiagnosticSeverity.Warning, - outcome = "source-unavailable", - accountId = initial.accountId, - jobId = jobId, - ) - return Result.failure() - } - val started = store.transition( - jobId, - expected = DurableUploadState.Queued, - target = DurableUploadState.Uploading, - message = null, - ) ?: return Result.success() + } ?: return Result.success() val uploadServices = AndroidNextcloudServices( applicationContext, localUploadPicker = picker, @@ -186,7 +252,6 @@ internal class DeckAttachmentUploadWorker( code = "HTTP:${response.status}", ) } - picker.release(started.request.file) }.onFailure { failure -> // Once the request body starts, a transport exception cannot prove whether the server // created the attachment. Never replay it automatically and risk a duplicate. @@ -203,9 +268,14 @@ internal class DeckAttachmentUploadWorker( jobId = jobId, failure = failure, ) - picker.release(started.request.file) } - return Result.success() + return resultAfterDurableUploadCapabilityReleaseOrQuarantine( + releaseCapability = { onQuarantined -> picker.release(started.request.file, onQuarantined) }, + completeCapabilityCleanup = { store.completeCapabilityCleanup(jobId) }, + onCleanupRetained = ::requestQueuedDurableUploadSchedulingRecovery, + releasedResult = Result.success(), + retainedResult = Result.retry(), + ) } private fun recordUploadDiagnostic( @@ -239,14 +309,99 @@ internal class DeckAttachmentUploadWorker( internal fun failQueuedDurableUploadForUnavailableAccount( transitionToFailed: () -> Unit, - releaseSelection: () -> Unit, + releaseSelection: (onQuarantined: () -> Unit) -> Boolean, + completeCapabilityCleanup: () -> Unit = {}, + onCleanupRetained: () -> Unit = {}, recordFailure: () -> Unit, failureResult: Result, + retryResult: Result, ): Result { transitionToFailed() - releaseSelection() + val result = resultAfterDurableUploadCapabilityReleaseOrQuarantine( + releaseCapability = releaseSelection, + completeCapabilityCleanup = completeCapabilityCleanup, + onCleanupRetained = onCleanupRetained, + releasedResult = failureResult, + retainedResult = retryResult, + ) recordFailure() - return failureResult + return result +} + +internal fun resultAfterDurableUploadCapabilityRelease( + releaseCapability: () -> Boolean, + completeCapabilityCleanup: () -> Unit = {}, + onCleanupRetained: () -> Unit = {}, + releasedResult: Result, + retainedResult: Result, +): Result = try { + if (releaseCapability()) { + completeCapabilityCleanup() + releasedResult + } else { + runCatching(onCleanupRetained) + retainedResult + } +} catch (cancelled: CancellationException) { + throw cancelled +} catch (_: Exception) { + runCatching(onCleanupRetained) + retainedResult +} + +internal fun resultAfterDurableUploadCapabilityReleaseOrQuarantine( + releaseCapability: (onQuarantined: () -> Unit) -> Boolean, + completeCapabilityCleanup: () -> Unit = {}, + onCleanupRetained: () -> Unit = {}, + releasedResult: Result, + retainedResult: Result, +): Result = resultAfterDurableUploadCapabilityRelease( + releaseCapability = { releaseOrQuarantineDurableUploadCapability(releaseCapability) }, + completeCapabilityCleanup = completeCapabilityCleanup, + onCleanupRetained = onCleanupRetained, + releasedResult = releasedResult, + retainedResult = retainedResult, +) + +internal suspend fun processQueuedDurableUploadSource( + requireCapability: () -> Unit, + openSource: () -> Unit, + onCapabilityUnavailable: suspend () -> Result, + onProviderUnavailable: suspend (Exception) -> Result, + onReady: suspend () -> Result, +): Result { + try { + requireCapability() + openSource() + } catch (cancelled: CancellationException) { + throw cancelled + } catch (_: DurableUploadCapabilityOverflowException) { + return onCapabilityUnavailable() + } catch (failure: AndroidLocalUploadCapabilityReadException) { + return onProviderUnavailable(failure) + } catch (_: AndroidLocalUploadCapabilityUnavailableException) { + return onCapabilityUnavailable() + } catch (_: FileNotFoundException) { + return onCapabilityUnavailable() + } catch (_: SecurityException) { + return onCapabilityUnavailable() + } catch (failure: Exception) { + return onProviderUnavailable(failure) + } + return onReady() +} + +internal suspend fun runDurableUploadWorkerWithRecoverySignal( + requestRecovery: () -> Unit = ::requestQueuedDurableUploadSchedulingRecovery, + work: suspend () -> WorkResult, +): WorkResult = try { + work() +} catch (cancelled: CancellationException) { + runCatching(requestRecovery) + throw cancelled +} catch (failure: Exception) { + runCatching(requestRecovery) + throw failure } internal suspend fun captureDurableUploadRequestOutcome( diff --git a/androidApp/src/main/kotlin/dev/obiente/nextcloudnative/AndroidLocalUploadCapabilityPreferences.kt b/androidApp/src/main/kotlin/dev/obiente/nextcloudnative/AndroidLocalUploadCapabilityPreferences.kt new file mode 100644 index 000000000..e8e894875 --- /dev/null +++ b/androidApp/src/main/kotlin/dev/obiente/nextcloudnative/AndroidLocalUploadCapabilityPreferences.kt @@ -0,0 +1,43 @@ +package dev.obiente.nextcloudnative + +import android.content.Context +import java.io.File + +internal class AndroidLocalUploadCapabilityPreferences( + context: Context, + private val preferenceName: String, + private val preferencePrefix: String, + private val maximumFileBytes: Long, +) { + private val preferenceFile = File(context.dataDir, "shared_prefs/$preferenceName.xml") + private val preferenceBackupFile = File("${preferenceFile.path}.bak") + private val preferences by lazy { + requireBoundedStorage() + context.getSharedPreferences(preferenceName, Context.MODE_PRIVATE) + } + + fun requireBoundedStorage() { + if ( + durableUploadCapabilityPreferenceStorageIsOversized( + primaryFileBytes = preferenceFile.length(), + backupFileBytes = preferenceBackupFile.length(), + maximumFileBytes = maximumFileBytes, + ) + ) throw DurableUploadCapabilityOverflowException() + } + + fun selectionIds(maximumRows: Int?): List = boundedDurableUploadCapabilitySelectionIds( + primaryFileBytes = preferenceFile.length(), + backupFileBytes = preferenceBackupFile.length(), + maximumFileBytes = maximumFileBytes, + maximumRows = maximumRows, + preferencePrefix = preferencePrefix, + preferenceKeys = { preferences.all.keys }, + ) + + fun getString(key: String): String? = preferences.getString(key, null) + + fun putString(key: String, value: String): Boolean = preferences.edit().putString(key, value).commit() + + fun remove(key: String): Boolean = preferences.edit().remove(key).commit() +} diff --git a/androidApp/src/main/kotlin/dev/obiente/nextcloudnative/AndroidLocalUploadCapabilityRead.kt b/androidApp/src/main/kotlin/dev/obiente/nextcloudnative/AndroidLocalUploadCapabilityRead.kt new file mode 100644 index 000000000..c9ec73ae0 --- /dev/null +++ b/androidApp/src/main/kotlin/dev/obiente/nextcloudnative/AndroidLocalUploadCapabilityRead.kt @@ -0,0 +1,93 @@ +package dev.obiente.nextcloudnative + +import kotlinx.coroutines.CancellationException +import org.json.JSONObject + +internal class AndroidLocalUploadCapabilityUnavailableException( + message: String, + cause: Throwable? = null, +) : IllegalStateException(message, cause) + +internal class AndroidLocalUploadCapabilityReadException( + message: String, + cause: Throwable? = null, +) : IllegalStateException(message, cause) + +internal class AndroidLocalUploadCapabilityMalformedException( + message: String, + cause: Throwable? = null, + val cleanupPermissionIdentity: String? = null, + val grantPreExisting: Boolean? = null, +) : IllegalStateException(message, cause) + +internal fun requireDurableUploadCapabilityReady(phase: CapabilityPhase) { + if (phase == CapabilityPhase.OwnershipCheckPending) { + throw AndroidLocalUploadCapabilityReadException( + "The local file selection ownership check is still pending.", + ) + } + if (!isDurableUploadCapabilityReady(phase)) { + throw AndroidLocalUploadCapabilityUnavailableException( + "The local file selection is pending capability cleanup.", + ) + } +} + +internal inline fun readAndroidLocalUploadCapabilityPreference(read: () -> String?): String? = try { + read() +} catch (failure: ClassCastException) { + throw AndroidLocalUploadCapabilityMalformedException( + "The local file selection metadata has an invalid stored type.", + failure, + ) +} + +internal inline fun decryptAndroidLocalUploadCapability(decrypt: () -> String): String = try { + decrypt() +} catch (cancelled: CancellationException) { + throw cancelled +} catch (failure: InvalidSessionCiphertextException) { + throw AndroidLocalUploadCapabilityMalformedException( + "The encrypted local file selection metadata is invalid.", + failure, + ) +} + +internal inline fun readAndroidLocalUploadCapability(load: () -> Result): Result = try { + load() +} catch (cancelled: CancellationException) { + throw cancelled +} catch (overflow: DurableUploadCapabilityOverflowException) { + throw overflow +} catch (failure: AndroidLocalUploadCapabilityMalformedException) { + throw AndroidLocalUploadCapabilityUnavailableException( + "The local file selection metadata is invalid.", + failure, + ) +} catch (failure: Exception) { + throw AndroidLocalUploadCapabilityReadException( + "The local file selection metadata could not be read.", + failure, + ) +} + +internal fun JSONObject.optionalStrictString(key: String): String? { + if (!has(key) || isNull(key)) return null + return requireStrictString(key) +} + +internal fun JSONObject.requireStrictString(key: String): String = get(key).let { value -> + require(value is String) { "The $key value changed type." } + value +} + +internal fun JSONObject.optionalStrictBoolean(key: String): Boolean? { + if (!has(key) || isNull(key)) return null + return get(key).let { value -> + require(value is Boolean) { "The $key value changed type." } + value + } +} + +internal fun persistedDurableUploadGrantPreExisting(payload: JSONObject): Boolean = + payload.optionalStrictBoolean("grantPreExisting") ?: false diff --git a/androidApp/src/main/kotlin/dev/obiente/nextcloudnative/AndroidLocalUploadCapabilityRecovery.kt b/androidApp/src/main/kotlin/dev/obiente/nextcloudnative/AndroidLocalUploadCapabilityRecovery.kt new file mode 100644 index 000000000..74bb39ba4 --- /dev/null +++ b/androidApp/src/main/kotlin/dev/obiente/nextcloudnative/AndroidLocalUploadCapabilityRecovery.kt @@ -0,0 +1,521 @@ +package dev.obiente.nextcloudnative + +import kotlinx.coroutines.CancellationException + +/** + * Revokes the URI grant before synchronously deleting capability metadata. Android may throw when + * the grant is already absent, so an exception is accepted only after absence is verified. + */ +internal fun releaseDurableUploadCapability( + releasePermission: () -> Unit, + removeMetadata: () -> Boolean, + isPermissionAbsent: () -> Boolean = { false }, +): Boolean { + if (!releaseDurableUploadPermission(releasePermission, isPermissionAbsent)) return false + return durableUploadCleanupStep(removeMetadata) +} + +internal fun releaseDurableUploadPermission( + releasePermission: () -> Unit, + isPermissionAbsent: () -> Boolean, +): Boolean = try { + releasePermission() + true +} catch (cancelled: CancellationException) { + throw cancelled +} catch (_: Exception) { + try { + isPermissionAbsent() + } catch (cancelled: CancellationException) { + throw cancelled + } catch (_: Exception) { + false + } +} + +internal fun releaseStoredDurableUploadCapability( + cachedCapability: Capability?, + loadCapability: () -> Capability?, + releasePermission: (Capability) -> Unit, + removeMetadata: () -> Boolean, + otherCapabilityOwnsPermission: (Capability) -> Boolean = { false }, + isPermissionAbsent: (Capability) -> Boolean = { false }, +): Boolean { + val capability = cachedCapability ?: try { + loadCapability() + } catch (cancelled: CancellationException) { + throw cancelled + } catch (_: Exception) { + return false + } + return releaseDurableUploadCapability( + releasePermission = { + capability?.let { stored -> + if (!otherCapabilityOwnsPermission(stored)) releasePermission(stored) + } + }, + removeMetadata = removeMetadata, + isPermissionAbsent = { capability == null || isPermissionAbsent(capability) }, + ) +} + +internal fun mergeDurableUploadCapabilities( + cachedCapabilities: Map, + storedSelectionIds: Iterable, + loadStoredCapability: (String) -> Capability?, +): Map = buildMap { + putAll(cachedCapabilities) + storedSelectionIds.forEach { selectionId -> + if (selectionId !in this) { + put( + selectionId, + checkNotNull(loadStoredCapability(selectionId)) { + "The picker capability disappeared during recovery." + }, + ) + } + } +} + +internal data class MalformedDurableUploadCapability( + val selectionId: String, + val cleanupPermissionIdentity: String?, + val grantPreExisting: Boolean?, +) + +internal data class DurableUploadCapabilitySnapshot( + val capabilities: Map, + val malformedCapabilities: Map, + private val storedCapabilityCount: Int? = null, + val scanComplete: Boolean = true, + val recoveryQuarantined: Boolean = false, +) { + val trackedCapabilityCount: Int + get() = storedCapabilityCount ?: (capabilities.keys + malformedCapabilities.keys).size +} + +internal class DurableUploadCapabilityOverflowException : IllegalStateException( + "Too many picker capabilities are pending bounded recovery.", +) + +internal fun malformedDurableUploadCapabilityCanBecomeActionable( + capability: MalformedDurableUploadCapability, +): Boolean = capability.cleanupPermissionIdentity != null + +internal enum class DurableUploadMalformedRecoveryDisposition { + Recover, + Retry, + Quarantine, +} + +internal enum class DurableUploadMalformedReleaseResult { + Released, + Retry, + Quarantine, +} + +internal fun releaseMalformedDurableUploadCapability( + disposition: DurableUploadMalformedRecoveryDisposition, + recover: () -> Boolean, +): DurableUploadMalformedReleaseResult = when (disposition) { + DurableUploadMalformedRecoveryDisposition.Recover -> if (recover()) { + DurableUploadMalformedReleaseResult.Released + } else { + DurableUploadMalformedReleaseResult.Retry + } + DurableUploadMalformedRecoveryDisposition.Retry -> DurableUploadMalformedReleaseResult.Retry + DurableUploadMalformedRecoveryDisposition.Quarantine -> DurableUploadMalformedReleaseResult.Quarantine +} + +internal fun durableUploadMalformedRecoveryDisposition( + grantPreExisting: Boolean?, + peerProtection: DurableUploadPermissionPeerProtection, + permissionAbsent: Boolean?, +): DurableUploadMalformedRecoveryDisposition { + val cleanupPlan = durableUploadPermissionCleanupPlan( + grantPreExisting = grantPreExisting, + peerProtection = peerProtection, + permissionAbsent = permissionAbsent == true, + ) + return when { + cleanupPlan != DurableUploadPermissionCleanupPlan.Retain -> + DurableUploadMalformedRecoveryDisposition.Recover + permissionAbsent == null -> DurableUploadMalformedRecoveryDisposition.Retry + grantPreExisting == null && peerProtection == DurableUploadPermissionPeerProtection.None -> + DurableUploadMalformedRecoveryDisposition.Quarantine + else -> DurableUploadMalformedRecoveryDisposition.Quarantine + } +} + +internal fun loadDurableUploadCapabilitySnapshot( + cachedCapabilities: Map, + storedSelectionIds: Iterable, + maximumRecoverableCapabilities: Int = Int.MAX_VALUE, + loadStoredCapability: (String) -> Capability?, +): DurableUploadCapabilitySnapshot { + require(maximumRecoverableCapabilities > 0) + val trackedIds = cachedCapabilities.keys.toMutableSet() + if (trackedIds.size > maximumRecoverableCapabilities) throw DurableUploadCapabilityOverflowException() + val storedIds = linkedSetOf() + storedSelectionIds.forEach { selectionId -> + storedIds += selectionId + if (trackedIds.add(selectionId) && trackedIds.size > maximumRecoverableCapabilities) { + throw DurableUploadCapabilityOverflowException() + } + } + val capabilities = cachedCapabilities.toMutableMap() + val malformed = linkedMapOf() + storedIds.forEach { selectionId -> + if (selectionId in capabilities) return@forEach + val stored = try { + checkNotNull(loadStoredCapability(selectionId)) { + "The picker capability disappeared during recovery." + } + } catch (cancelled: CancellationException) { + throw cancelled + } catch (failure: AndroidLocalUploadCapabilityMalformedException) { + malformed[selectionId] = MalformedDurableUploadCapability( + selectionId, + failure.cleanupPermissionIdentity, + failure.grantPreExisting, + ) + return@forEach + } + capabilities[selectionId] = stored + } + return DurableUploadCapabilitySnapshot(capabilities.toMap(), malformed.toMap()) +} + +internal fun malformedDurableUploadCapabilitiesForRecovery( + capabilities: Map, + ownedSelectionIds: Set, +): List = capabilities.values + .filterNot { capability -> capability.selectionId in ownedSelectionIds } + .sortedWith( + compareBy { capability -> + when (capability.grantPreExisting) { + true -> 0 + null -> 1 + false -> 2 + } + }.thenBy(MalformedDurableUploadCapability::selectionId), + ) + +internal enum class DurableUploadPermissionPeerProtection { + None, + RetainedAppOwnedGrant, + Ambiguous, +} + +internal data class DurableUploadPermissionPeer( + val selectionId: String, + val permission: Permission?, + val grantPreExisting: Boolean?, +) + +internal fun durableUploadPermissionPeerProtection( + peers: Iterable>, + targetSelectionId: String, + targetPermission: Permission, + samePermission: (Permission, Permission) -> Boolean, +): DurableUploadPermissionPeerProtection { + var ambiguous = false + peers.forEach { peer -> + if (peer.selectionId == targetSelectionId) return@forEach + val permission = peer.permission + if (permission == null) { + ambiguous = true + } else if (samePermission(targetPermission, permission)) { + if (peer.grantPreExisting == false) { + return DurableUploadPermissionPeerProtection.RetainedAppOwnedGrant + } + ambiguous = true + } + } + return if (ambiguous) { + DurableUploadPermissionPeerProtection.Ambiguous + } else { + DurableUploadPermissionPeerProtection.None + } +} + +internal enum class DurableUploadMalformedPeerCleanupDisposition { + Proceed, + Retry, + Quarantine, +} + +internal fun durableUploadMalformedPeerCleanupDisposition( + malformedPeers: Iterable>, + targetSelectionId: String, + targetPermission: Permission, + samePermission: (Permission, Permission) -> Boolean, + targetGrantPreExisting: Boolean = false, +): DurableUploadMalformedPeerCleanupDisposition { + if (targetGrantPreExisting) return DurableUploadMalformedPeerCleanupDisposition.Proceed + val peers = malformedPeers.toList() + if (peers.any { peer -> peer.selectionId != targetSelectionId && peer.permission == null }) { + return DurableUploadMalformedPeerCleanupDisposition.Quarantine + } + return when (durableUploadPermissionPeerProtection(peers, targetSelectionId, targetPermission, samePermission)) { + DurableUploadPermissionPeerProtection.Ambiguous -> DurableUploadMalformedPeerCleanupDisposition.Quarantine + DurableUploadPermissionPeerProtection.RetainedAppOwnedGrant -> DurableUploadMalformedPeerCleanupDisposition.Retry + DurableUploadPermissionPeerProtection.None -> DurableUploadMalformedPeerCleanupDisposition.Proceed + } +} + +internal enum class DurableUploadPermissionCleanupPlan { + ReleaseThenRemove, + RemoveWithoutRelease, + Retain, +} + +internal fun durableUploadPermissionCleanupPlan( + grantPreExisting: Boolean?, + peerProtection: DurableUploadPermissionPeerProtection, + permissionAbsent: Boolean = false, +): DurableUploadPermissionCleanupPlan = when { + grantPreExisting == true -> DurableUploadPermissionCleanupPlan.RemoveWithoutRelease + peerProtection == DurableUploadPermissionPeerProtection.RetainedAppOwnedGrant -> + DurableUploadPermissionCleanupPlan.RemoveWithoutRelease + peerProtection == DurableUploadPermissionPeerProtection.Ambiguous -> + DurableUploadPermissionCleanupPlan.Retain + grantPreExisting == false -> DurableUploadPermissionCleanupPlan.ReleaseThenRemove + permissionAbsent -> DurableUploadPermissionCleanupPlan.RemoveWithoutRelease + else -> DurableUploadPermissionCleanupPlan.Retain +} + +internal fun recoverMalformedDurableUploadCapability( + capability: MalformedDurableUploadCapability, + permission: Permission?, + peerProtection: DurableUploadPermissionPeerProtection, + releasePermission: (Permission) -> Unit, + isPermissionAbsent: (Permission) -> Boolean, + removeMetadata: (String) -> Boolean, +): Boolean { + permission ?: return false + val grantPreExisting = capability.grantPreExisting + val permissionAbsent = if ( + grantPreExisting == null && + peerProtection == DurableUploadPermissionPeerProtection.None + ) { + try { + isPermissionAbsent(permission) + } catch (cancelled: CancellationException) { + throw cancelled + } catch (_: Exception) { + false + } + } else { + false + } + val cleanupPlan = durableUploadPermissionCleanupPlan( + grantPreExisting = grantPreExisting, + peerProtection = peerProtection, + permissionAbsent = permissionAbsent, + ) + if (cleanupPlan == DurableUploadPermissionCleanupPlan.Retain) return false + return releaseDurableUploadCapability( + releasePermission = { + if (cleanupPlan == DurableUploadPermissionCleanupPlan.ReleaseThenRemove) { + releasePermission(permission) + } + }, + isPermissionAbsent = { + cleanupPlan == DurableUploadPermissionCleanupPlan.RemoveWithoutRelease || + isPermissionAbsent(permission) + }, + removeMetadata = { removeMetadata(capability.selectionId) }, + ) +} + +internal fun durableUploadCapabilityPermissionOwnedByAnother( + capabilities: Map, + targetSelectionId: String, + targetPermission: Permission, + permissionOf: (Capability) -> Permission, + samePermission: (Permission, Permission) -> Boolean, +): Boolean = capabilities.any { (selectionId, capability) -> + selectionId != targetSelectionId && samePermission(targetPermission, permissionOf(capability)) +} + +internal fun malformedDurableUploadCapabilityPermissionOwnedByAnother( + capabilities: Map, + targetSelectionId: String, + targetPermission: Permission, + permissionOf: (MalformedDurableUploadCapability) -> Permission?, + samePermission: (Permission, Permission) -> Boolean, +): Boolean = capabilities.any { (selectionId, capability) -> + val permission = permissionOf(capability) + selectionId != targetSelectionId && permission != null && samePermission(targetPermission, permission) +} + +internal fun DurableUploadCapabilitySnapshot<*>.malformedCapabilityOwnsPermission( + targetSelectionId: String, + targetPermission: String, +): Boolean = malformedDurableUploadCapabilityPermissionOwnedByAnother( + capabilities = malformedCapabilities, + targetSelectionId = targetSelectionId, + targetPermission = targetPermission, + permissionOf = MalformedDurableUploadCapability::cleanupPermissionIdentity, + samePermission = String::equals, +) + +internal fun durableUploadCapabilityPreferenceStorageIsOversized( + primaryFileBytes: Long, + backupFileBytes: Long, + maximumFileBytes: Long, +): Boolean { + require(primaryFileBytes >= 0L && backupFileBytes >= 0L && maximumFileBytes > 0L) + return primaryFileBytes > maximumFileBytes || backupFileBytes > maximumFileBytes +} + +internal fun boundedDurableUploadCapabilitySelectionIds( + primaryFileBytes: Long, + backupFileBytes: Long, + maximumFileBytes: Long, + maximumRows: Int?, + preferencePrefix: String, + preferenceKeys: () -> Set, +): List { + require(maximumRows == null || maximumRows >= 0) + if ( + durableUploadCapabilityPreferenceStorageIsOversized( + primaryFileBytes, + backupFileBytes, + maximumFileBytes, + ) + ) throw DurableUploadCapabilityOverflowException() + val selectionIds = preferenceKeys().asSequence() + .filter { key -> key.startsWith(preferencePrefix) } + .map { key -> key.removePrefix(preferencePrefix) } + return maximumRows?.let { limit -> selectionIds.take(limit + 1).toList() } ?: selectionIds.toList() +} + +internal fun shouldReleaseDurableUploadPermission( + grantPreExisting: Boolean, + ownedByAnotherCapability: Boolean, +): Boolean = !grantPreExisting && !ownedByAnotherCapability + +internal fun retainDurableUploadCapabilityCleanup(onCleanupRetained: () -> Unit): Boolean { + runCatching(onCleanupRetained) + return false +} + +internal fun durableUploadCleanupStep(action: () -> Boolean): Boolean = try { + action() +} catch (cancelled: CancellationException) { + throw cancelled +} catch (_: Exception) { + false +} + +/** + * Acquires a durable picker capability without exposing an interval where a successful selection + * can be reported before its metadata reaches app-private storage. + */ +internal fun acquireDurableUploadCapability( + takePermission: () -> Unit, + persistMetadata: () -> Boolean, + releasePermission: () -> Unit, + persistAcquiring: () -> Boolean = { true }, + markCleanupPending: () -> Boolean = { true }, + isPermissionAbsent: () -> Boolean = { false }, + removeCapability: () -> Boolean = { true }, + onRollbackRetained: () -> Unit = {}, +) { + val acquiringPersisted = try { + persistAcquiring() + } catch (cancelled: CancellationException) { + durableUploadCleanupStep(removeCapability) + runCatching(onRollbackRetained) + throw cancelled + } catch (failure: Exception) { + durableUploadCleanupStep(removeCapability) + runCatching(onRollbackRetained) + throw failure + } + if (!acquiringPersisted) { + durableUploadCleanupStep(removeCapability) + runCatching(onRollbackRetained) + error("The picker capability rollback could not be saved.") + } + try { + takePermission() + } catch (cancelled: CancellationException) { + runCatching(onRollbackRetained) + throw cancelled + } catch (failure: Exception) { + runCatching(onRollbackRetained) + throw failure + } + val persisted = runCatching { persistMetadata() } + if (persisted.getOrNull() == true) return + if (!durableUploadCleanupStep(markCleanupPending)) runCatching(onRollbackRetained) + val released = try { + releaseDurableUploadPermission(releasePermission, isPermissionAbsent) + } catch (cancelled: CancellationException) { + runCatching(onRollbackRetained) + throw cancelled + } + if (released) { + if (!durableUploadCleanupStep(removeCapability)) runCatching(onRollbackRetained) + } else { + runCatching(onRollbackRetained) + } + persisted.exceptionOrNull()?.let { throw it } + error("The durable upload capability could not be saved.") +} + +internal enum class CapabilityPhase(val persistedValue: String) { + Acquiring("acquiring"), + Ready("ready"), + OwnershipCheckPending("ownership-check-pending"), + CleanupPending("cleanup-pending"); + + companion object { + fun fromPersistedValue(value: String): CapabilityPhase = entries.singleOrNull { + phase -> phase.persistedValue == value + } ?: error("The picker capability phase is invalid.") + } +} + +internal fun shouldRecoverDurableUploadCapability( + phase: CapabilityPhase, + processGeneration: String?, + currentProcessGeneration: String, + ownedByDurableJob: Boolean, + cleanupExplicitlyPending: Boolean, +): Boolean = !ownedByDurableJob && ( + cleanupExplicitlyPending || + phase != CapabilityPhase.Ready || + processGeneration != currentProcessGeneration +) + +internal fun shouldRestoreDurableUploadCapabilityAfterOwnershipCheck( + phase: CapabilityPhase, + ownedByDurableJob: Boolean, +): Boolean = phase == CapabilityPhase.OwnershipCheckPending && ownedByDurableJob + +internal fun isDurableUploadCapabilityReady(phase: CapabilityPhase): Boolean = + phase == CapabilityPhase.Ready + +internal fun durableUploadCapabilityHasCapacity( + trackedCapabilityCount: Int, + maximumTrackedCapabilities: Int, +): Boolean { + require(trackedCapabilityCount >= 0) + require(maximumTrackedCapabilities > 0) + return trackedCapabilityCount < maximumTrackedCapabilities +} + +internal fun finalizeDurableUploadCapabilityDelivery( + publishReady: () -> Unit, + continuationIsActive: () -> Boolean, + cleanupUndelivered: () -> Unit, +): Boolean { + publishReady() + if (continuationIsActive()) return true + runCatching(cleanupUndelivered) + return false +} diff --git a/androidApp/src/main/kotlin/dev/obiente/nextcloudnative/AndroidLocalUploadCapabilityRecoveryScan.kt b/androidApp/src/main/kotlin/dev/obiente/nextcloudnative/AndroidLocalUploadCapabilityRecoveryScan.kt new file mode 100644 index 000000000..a27e65e9e --- /dev/null +++ b/androidApp/src/main/kotlin/dev/obiente/nextcloudnative/AndroidLocalUploadCapabilityRecoveryScan.kt @@ -0,0 +1,68 @@ +package dev.obiente.nextcloudnative + +import kotlinx.coroutines.CancellationException + +internal class DurableUploadCapabilityRecoveryScan { + private val capabilities = linkedMapOf() + private val malformed = linkedMapOf() + + fun loadPage( + cachedCapabilities: Map, + storedSelectionIds: Iterable, + maximumRows: Int, + loadStoredCapability: (String) -> Capability?, + ): DurableUploadCapabilitySnapshot { + require(maximumRows > 0) + val storedIds = linkedSetOf() + storedSelectionIds.forEach { selectionId -> + if (selectionId !in storedIds && storedIds.size == maximumRows) { + capabilities.clear() + malformed.clear() + return DurableUploadCapabilitySnapshot( + capabilities = emptyMap(), + malformedCapabilities = emptyMap(), + storedCapabilityCount = maximumRows + 1, + scanComplete = false, + recoveryQuarantined = true, + ) + } + storedIds += selectionId + } + capabilities.keys.retainAll(storedIds) + malformed.keys.retainAll(storedIds) + cachedCapabilities.forEach { (selectionId, capability) -> + if (selectionId in storedIds) { + capabilities[selectionId] = capability + malformed.remove(selectionId) + } + } + storedIds.asSequence() + .filterNot { selectionId -> selectionId in capabilities || selectionId in malformed } + .sorted() + .take(maximumRows) + .forEach { selectionId -> + val stored = try { + checkNotNull(loadStoredCapability(selectionId)) { + "The picker capability disappeared during recovery." + } + } catch (cancelled: CancellationException) { + throw cancelled + } catch (failure: AndroidLocalUploadCapabilityMalformedException) { + malformed[selectionId] = MalformedDurableUploadCapability( + selectionId, + failure.cleanupPermissionIdentity, + failure.grantPreExisting, + ) + return@forEach + } + capabilities[selectionId] = stored + } + val scannedIds = capabilities.keys + malformed.keys + return DurableUploadCapabilitySnapshot( + capabilities = capabilities.toMap(), + malformedCapabilities = malformed.toMap(), + storedCapabilityCount = storedIds.size, + scanComplete = scannedIds.containsAll(storedIds), + ) + } +} diff --git a/androidApp/src/main/kotlin/dev/obiente/nextcloudnative/AndroidLocalUploadPicker.kt b/androidApp/src/main/kotlin/dev/obiente/nextcloudnative/AndroidLocalUploadPicker.kt index b7dc27a62..9d5436bc2 100644 --- a/androidApp/src/main/kotlin/dev/obiente/nextcloudnative/AndroidLocalUploadPicker.kt +++ b/androidApp/src/main/kotlin/dev/obiente/nextcloudnative/AndroidLocalUploadPicker.kt @@ -15,6 +15,7 @@ import java.io.InputStream import java.util.UUID import java.util.concurrent.ConcurrentHashMap import kotlinx.coroutines.CancellableContinuation +import kotlinx.coroutines.CancellationException import kotlinx.coroutines.suspendCancellableCoroutine import org.json.JSONObject import kotlin.coroutines.resume @@ -29,9 +30,11 @@ import kotlin.coroutines.resume internal class AndroidLocalUploadPicker(context: Context) { private val appContext = context.applicationContext private val resolver = context.applicationContext.contentResolver - private val preferences = appContext.getSharedPreferences(PREFERENCES, Context.MODE_PRIVATE) + private val preferences = AndroidLocalUploadCapabilityPreferences( + appContext, PREFERENCES, PREFERENCE_PREFIX, MAX_CAPABILITY_PREFERENCE_FILE_BYTES, + ) private val cipher = SessionCipher() - private val selections = ConcurrentHashMap() + private val selections = PROCESS_SELECTIONS private var launcher: ActivityResultLauncher>? = null private var pending: PendingSelection? = null @@ -51,6 +54,7 @@ internal class AndroidLocalUploadPicker(context: Context) { pending = selection continuation.invokeOnCancellation { if (pending === selection) pending = null + runCatching { selection.readyFile?.let(::release) } } activeLauncher.launch(accepted.toTypedArray()) } @@ -63,16 +67,16 @@ internal class AndroidLocalUploadPicker(context: Context) { selection.continuation.resume(LocalUploadSelectionResult.Cancelled) return } - val result = runCatching { + val result = runCatching selectionResult@{ val metadata = resolver.queryUploadMetadata(uri) val mimeType = resolver.getType(uri)?.trim()?.lowercase()?.takeIf(String::isNotBlank) if (!isAcceptedUploadMimeType(mimeType, selection.acceptedMimeTypes)) { - return@runCatching LocalUploadSelectionResult.Rejected( + return@selectionResult LocalUploadSelectionResult.Rejected( "The selected file type is not accepted.", ) } if (metadata.sizeBytes != null && metadata.sizeBytes > selection.maximumBytes) { - return@runCatching LocalUploadSelectionResult.Rejected( + return@selectionResult LocalUploadSelectionResult.Rejected( "The selected file is larger than the allowed upload limit.", ) } @@ -83,36 +87,91 @@ internal class AndroidLocalUploadPicker(context: Context) { mimeType = mimeType, sizeBytes = metadata.sizeBytes, ) - runCatching { - acquireDurableUploadCapability( - takePermission = { - resolver.takePersistableUriPermission( - uri, - Intent.FLAG_GRANT_READ_URI_PERMISSION, - ) - }, - persistMetadata = { persist(source = SelectedSource(uri, file)) }, - releasePermission = { - resolver.releasePersistableUriPermission( - uri, - Intent.FLAG_GRANT_READ_URI_PERMISSION, - ) - }, - ) - }.getOrElse { - return@runCatching LocalUploadSelectionResult.Rejected( + val source = SelectedSource(uri, file) + var cancelledAfterAcquire = false + val acquisitionFailure = runCatching { + synchronized(CAPABILITY_LOCK) { + val snapshot = loadCapabilitySnapshot() + val existing = snapshot.capabilities + check( + durableUploadCapabilityHasCapacity( + trackedCapabilityCount = snapshot.trackedCapabilityCount, + maximumTrackedCapabilities = MAX_TRACKED_CAPABILITIES, + ), + ) { + "Too many picker capabilities are tracked." + } + check( + !durableUploadCapabilityPermissionOwnedByAnother( + capabilities = existing, + targetSelectionId = token, + targetPermission = uri, + permissionOf = SelectedSource::uri, + samePermission = { first, second -> first == second }, + ), + ) { + "The selected file already has an active picker capability." + } + check(!snapshot.malformedCapabilityOwnsPermission(token, uri.toString())) { + "The selected file already has a quarantined picker capability." + } + val grantPreExisting = !exactReadPermissionIsAbsent(uri) + val acquiring = source.copy( + phase = CapabilityPhase.Acquiring, + processGeneration = PROCESS_GENERATION, + grantPreExisting = grantPreExisting, + ) + val ready = acquiring.copy(phase = CapabilityPhase.Ready) + val cleanupPending = acquiring.copy(phase = CapabilityPhase.CleanupPending) + acquireDurableUploadCapability( + persistAcquiring = { persist(acquiring) }, + takePermission = { + resolver.takePersistableUriPermission( + uri, + Intent.FLAG_GRANT_READ_URI_PERMISSION, + ) + }, + persistMetadata = { persist(ready) }, + markCleanupPending = { + PENDING_CLEANUP_SELECTIONS += token + persist(cleanupPending) + }, + releasePermission = { + if (shouldReleaseDurableUploadPermission(grantPreExisting, false)) { + releasePermission(uri) + } + }, + isPermissionAbsent = { exactReadPermissionIsAbsent(uri) }, + removeCapability = { removeMetadata(token) }, + onRollbackRetained = ::requestQueuedDurableUploadSchedulingRecovery, + ) + cancelledAfterAcquire = !finalizeDurableUploadCapabilityDelivery( + publishReady = { + selections[token] = ready + selection.readyFile = file + }, + continuationIsActive = { selection.continuation.isActive }, + cleanupUndelivered = { release(file) }, + ) + } + }.exceptionOrNull() + if (acquisitionFailure != null) { + return@selectionResult LocalUploadSelectionResult.Rejected( "The selected file provider cannot keep access for a background upload.", ) } - val source = SelectedSource(uri, file) - selections[token] = source + if (cancelledAfterAcquire) return@selectionResult LocalUploadSelectionResult.Cancelled LocalUploadSelectionResult.Selected(file) }.getOrElse { LocalUploadSelectionResult.Rejected( "The selected file could not be opened.", ) } - selection.continuation.resume(result) + resumeLocalUploadSelectionResult( + continuation = selection.continuation, + result = result, + releaseSelected = { file -> release(file) }, + ) } fun open(file: LocalUploadFile): InputStream { @@ -125,32 +184,245 @@ internal class AndroidLocalUploadPicker(context: Context) { } fun requirePersisted(file: LocalUploadFile) { - val source = load(file.selectionId) - ?: error("The local file selection was not durably saved.") - require(source.file == file) { "The persisted local file metadata changed." } + requiredSource(file, useCachedSource = false) } - fun release(file: LocalUploadFile): Boolean { - val source = selections[file.selectionId] ?: load(file.selectionId) - return releaseDurableUploadCapability( + fun release(file: LocalUploadFile, onQuarantined: () -> Unit = {}): Boolean = synchronized(CAPABILITY_LOCK) { + val source = try { + preferences.requireBoundedStorage() + selections[file.selectionId] ?: load(file.selectionId) + } catch (cancelled: CancellationException) { + throw cancelled + } catch (_: DurableUploadCapabilityOverflowException) { + return@synchronized quarantineCapabilityCleanup(file.selectionId, onQuarantined) + } catch (malformed: AndroidLocalUploadCapabilityMalformedException) { + return@synchronized releaseMalformedCapability(file.selectionId, malformed, onQuarantined) + } catch (_: Exception) { + return@synchronized retainCapabilityCleanup(file.selectionId) + } + if (source == null) { + val removed = durableUploadCleanupStep { + removeMetadata(file.selectionId) + } + return@synchronized if (removed) true else retainCapabilityCleanup(file.selectionId) + } + val cleanupPending = source.copy(phase = CapabilityPhase.CleanupPending) + PENDING_CLEANUP_SELECTIONS += file.selectionId + selections[file.selectionId] = cleanupPending + if (!durableUploadCleanupStep { persist(cleanupPending) }) { + requestQueuedDurableUploadSchedulingRecovery() + return@synchronized false + } + val snapshot = try { + loadCapabilitySnapshot() + } catch (cancelled: CancellationException) { + throw cancelled + } catch (_: DurableUploadCapabilityOverflowException) { + return@synchronized quarantineCapabilityCleanup(file.selectionId, onQuarantined) + } catch (_: Exception) { + return@synchronized retainCapabilityCleanup(file.selectionId) + } + when (malformedPeerCleanupDisposition( + snapshot.malformedCapabilities, file.selectionId, source.uri.toString(), source.grantPreExisting, + )) { + DurableUploadMalformedPeerCleanupDisposition.Quarantine -> + return@synchronized quarantineCapabilityCleanup(file.selectionId, onQuarantined) + DurableUploadMalformedPeerCleanupDisposition.Retry -> + return@synchronized retainCapabilityCleanup(file.selectionId) + DurableUploadMalformedPeerCleanupDisposition.Proceed -> Unit + } + val cleanupPlan = durableUploadPermissionCleanupPlan( + grantPreExisting = source.grantPreExisting, + peerProtection = permissionPeerProtection( + capabilities = snapshot.capabilities, + malformedCapabilities = emptyMap(), + targetSelectionId = file.selectionId, + targetPermissionIdentity = source.uri.toString(), + ), + ) + if (cleanupPlan == DurableUploadPermissionCleanupPlan.Retain) { + return@synchronized retainCapabilityCleanup(file.selectionId) + } + releaseDurableUploadCapability( releasePermission = { - source?.let { - resolver.releasePersistableUriPermission( - it.uri, - Intent.FLAG_GRANT_READ_URI_PERMISSION, - ) + if (cleanupPlan == DurableUploadPermissionCleanupPlan.ReleaseThenRemove) { + releasePermission(source.uri) } }, - removeMetadata = { - preferences.edit() - .remove(preferenceKey(file.selectionId)) - .commit() + isPermissionAbsent = { + cleanupPlan == DurableUploadPermissionCleanupPlan.RemoveWithoutRelease || + exactReadPermissionIsAbsent(source.uri) }, + removeMetadata = { removeMetadata(file.selectionId) }, ).also { released -> - if (released) selections.remove(file.selectionId) + if (released) { + selections.remove(file.selectionId) + } else { + requestQueuedDurableUploadSchedulingRecovery() + } } } + fun markOwnershipCheckPending(file: LocalUploadFile): Boolean = synchronized(CAPABILITY_LOCK) { + val source = try { + selections[file.selectionId] ?: load(file.selectionId) + } catch (cancelled: CancellationException) { + throw cancelled + } catch (_: Exception) { + return@synchronized retainCapabilityCleanup(file.selectionId) + } + if (source == null || source.file != file) { + return@synchronized retainCapabilityCleanup(file.selectionId) + } + if (source.phase != CapabilityPhase.Ready) { + return@synchronized retainCapabilityCleanup(file.selectionId) + } + val ownershipCheckPending = source.copy(phase = CapabilityPhase.OwnershipCheckPending) + selections[file.selectionId] = ownershipCheckPending + val persisted = durableUploadCleanupStep { persist(ownershipCheckPending) } + requestQueuedDurableUploadSchedulingRecovery() + persisted + } + + fun reconcileCapabilities(ownedSelectionIds: Set): Boolean = synchronized(CAPABILITY_LOCK) { + val snapshot = try { + loadCapabilityRecoverySnapshot() + } catch (cancelled: CancellationException) { + throw cancelled + } catch (_: DurableUploadCapabilityOverflowException) { + return@synchronized true + } catch (_: Exception) { + return@synchronized false + } + if (snapshot.malformedCapabilities.isNotEmpty()) { + PENDING_CLEANUP_SELECTIONS += snapshot.malformedCapabilities.keys + } + if (snapshot.recoveryQuarantined) return@synchronized true + if (!snapshot.scanComplete) { + requestQueuedDurableUploadSchedulingRecovery() + return@synchronized false + } + val capabilities = snapshot.capabilities.toMutableMap() + val malformedCapabilities = snapshot.malformedCapabilities.toMutableMap() + var allRecovered = true + var remainingRecoveryActions = MAX_RECOVERY_ROWS_PER_PASS + val malformedRecovery = malformedDurableUploadCapabilitiesForRecovery( + malformedCapabilities, + ownedSelectionIds, + ) + malformedRecovery.forEach { malformed -> + val disposition = malformedRecoveryDisposition(malformed, capabilities, malformedCapabilities) + if (disposition != DurableUploadMalformedRecoveryDisposition.Recover) { + if (disposition == DurableUploadMalformedRecoveryDisposition.Retry) allRecovered = false + else PENDING_CLEANUP_SELECTIONS.remove(malformed.selectionId) + return@forEach + } + if (remainingRecoveryActions == 0) { + allRecovered = false + return@forEach + } + remainingRecoveryActions -= 1 + val recovered = recoverMalformedCapability( + malformed, + capabilities, + malformedCapabilities, + ) + if (recovered) { + malformedCapabilities.remove(malformed.selectionId) + selections.remove(malformed.selectionId) + PENDING_CLEANUP_SELECTIONS.remove(malformed.selectionId) + } else { + allRecovered = false + } + } + capabilities.values + .sortedWith( + compareByDescending { capability -> capability.grantPreExisting } + .thenBy { capability -> capability.file.selectionId }, + ) + .forEach { capability -> + val selectionId = capability.file.selectionId + val ownedByDurableJob = selectionId in ownedSelectionIds + if ( + shouldRestoreDurableUploadCapabilityAfterOwnershipCheck( + phase = capability.phase, + ownedByDurableJob = ownedByDurableJob, + ) + ) { + if (remainingRecoveryActions == 0) { + allRecovered = false + return@forEach + } + remainingRecoveryActions -= 1 + val ready = capability.copy( + phase = CapabilityPhase.Ready, + processGeneration = PROCESS_GENERATION, + ) + if (durableUploadCleanupStep { persist(ready) }) { + capabilities[selectionId] = ready + selections[selectionId] = ready + } else { + allRecovered = false + } + return@forEach + } + if (!shouldRecoverDurableUploadCapability( + phase = capability.phase, + processGeneration = capability.processGeneration, + currentProcessGeneration = PROCESS_GENERATION, + ownedByDurableJob = ownedByDurableJob, + cleanupExplicitlyPending = selectionId in PENDING_CLEANUP_SELECTIONS, + )) return@forEach + if ( + malformedPeerCleanupDisposition( + malformedCapabilities, + selectionId, + capability.uri.toString(), + ) == DurableUploadMalformedPeerCleanupDisposition.Quarantine + ) { + PENDING_CLEANUP_SELECTIONS.remove(selectionId) + return@forEach + } + val cleanupPlan = durableUploadPermissionCleanupPlan( + grantPreExisting = capability.grantPreExisting, + peerProtection = permissionPeerProtection( + capabilities = capabilities, + malformedCapabilities = malformedCapabilities, + targetSelectionId = selectionId, + targetPermissionIdentity = capability.uri.toString(), + ), + ) + if (cleanupPlan == DurableUploadPermissionCleanupPlan.Retain) { + allRecovered = false + return@forEach + } + if (remainingRecoveryActions == 0) { + allRecovered = false + return@forEach + } + remainingRecoveryActions -= 1 + val released = releaseDurableUploadCapability( + releasePermission = { + if (cleanupPlan == DurableUploadPermissionCleanupPlan.ReleaseThenRemove) { + releasePermission(capability.uri) + } + }, + isPermissionAbsent = { + cleanupPlan == DurableUploadPermissionCleanupPlan.RemoveWithoutRelease || + exactReadPermissionIsAbsent(capability.uri) + }, + removeMetadata = { removeMetadata(selectionId) }, + ) + if (released) { + capabilities.remove(selectionId) + selections.remove(selectionId) + } else { + allRecovered = false + } + } + allRecovered + } + private fun persist(source: SelectedSource): Boolean { val payload = JSONObject() .put("uri", source.uri.toString()) @@ -158,23 +430,256 @@ internal class AndroidLocalUploadPicker(context: Context) { .put("displayName", source.file.displayName) .put("mimeType", source.file.mimeType) .put("sizeBytes", source.file.sizeBytes) - .toString() - return preferences.edit() - .putString(preferenceKey(source.file.selectionId), cipher.encrypt(payload)) - .commit() + .put("phase", source.phase.persistedValue) + .put("grantPreExisting", source.grantPreExisting) + source.processGeneration?.let { generation -> payload.put("processGeneration", generation) } + val encrypted = cipher.encrypt(payload.toString()) + return preferences.putString(preferenceKey(source.file.selectionId), encrypted) + } + + private fun removeMetadata(selectionId: String): Boolean = preferences.remove(preferenceKey(selectionId)) + .also { removed -> if (removed) PENDING_CLEANUP_SELECTIONS.remove(selectionId) } + + private fun retainCapabilityCleanup(selectionId: String): Boolean { + PENDING_CLEANUP_SELECTIONS += selectionId + return retainDurableUploadCapabilityCleanup(::requestQueuedDurableUploadSchedulingRecovery) + } + + private fun quarantineCapabilityCleanup(selectionId: String, onQuarantined: () -> Unit): Boolean { + PENDING_CLEANUP_SELECTIONS.remove(selectionId) + onQuarantined() + return false + } + + private fun releasePermission(uri: Uri) { + resolver.releasePersistableUriPermission(uri, Intent.FLAG_GRANT_READ_URI_PERMISSION) + } + + private fun exactReadPermissionIsAbsent(uri: Uri): Boolean = resolver.persistedUriPermissions.none { + it.uri == uri && it.isReadPermission } + private fun loadCapabilitySnapshot(): DurableUploadCapabilitySnapshot { + val storedSelectionIds = storedCapabilitySelectionIds(MAX_RECOVERABLE_CAPABILITIES) + val snapshot = loadDurableUploadCapabilitySnapshot( + cachedCapabilities = selections.toMap(), + storedSelectionIds = storedSelectionIds, + maximumRecoverableCapabilities = MAX_RECOVERABLE_CAPABILITIES, + loadStoredCapability = ::load, + ) + if (snapshot.malformedCapabilities.isNotEmpty()) { + PENDING_CLEANUP_SELECTIONS += snapshot.malformedCapabilities.keys + if (snapshot.malformedCapabilities.values.any(::malformedDurableUploadCapabilityCanBecomeActionable)) { + requestQueuedDurableUploadSchedulingRecovery() + } + } + return snapshot + } + + private fun loadCapabilityRecoverySnapshot(): DurableUploadCapabilitySnapshot = + RECOVERY_SCAN.loadPage( + cachedCapabilities = selections.toMap(), + storedSelectionIds = storedCapabilitySelectionIds(MAX_RECOVERY_ROWS_PER_PASS), + maximumRows = MAX_RECOVERY_ROWS_PER_PASS, + loadStoredCapability = ::load, + ) + + private fun storedCapabilitySelectionIds(maximumRows: Int? = null): List = + preferences.selectionIds(maximumRows) + + private fun releaseMalformedCapability( + selectionId: String, + malformed: AndroidLocalUploadCapabilityMalformedException, + onQuarantined: () -> Unit, + ): Boolean { + PENDING_CLEANUP_SELECTIONS += selectionId + val snapshot = try { + loadCapabilitySnapshot() + } catch (cancelled: CancellationException) { + throw cancelled + } catch (_: DurableUploadCapabilityOverflowException) { + return quarantineCapabilityCleanup(selectionId, onQuarantined) + } catch (_: Exception) { + return retainCapabilityCleanup(selectionId) + } + val isolated = snapshot.malformedCapabilities[selectionId] + ?: MalformedDurableUploadCapability( + selectionId, + malformed.cleanupPermissionIdentity, + malformed.grantPreExisting, + ) + return when (releaseMalformedDurableUploadCapability( + disposition = malformedRecoveryDisposition( + isolated, + snapshot.capabilities, + snapshot.malformedCapabilities, + ), + recover = { + recoverMalformedCapability( + isolated, + snapshot.capabilities, + snapshot.malformedCapabilities, + ) + }, + )) { + DurableUploadMalformedReleaseResult.Released -> true.also { + selections.remove(selectionId) + PENDING_CLEANUP_SELECTIONS.remove(selectionId) + } + DurableUploadMalformedReleaseResult.Retry -> retainCapabilityCleanup(selectionId) + DurableUploadMalformedReleaseResult.Quarantine -> + quarantineCapabilityCleanup(selectionId, onQuarantined) + } + } + + private fun recoverMalformedCapability( + malformed: MalformedDurableUploadCapability, + capabilities: Map, + malformedCapabilities: Map, + ): Boolean { + val permission = malformed.cleanupPermissionIdentity?.let(Uri::parse) + val peerProtection = permission?.let { target -> + permissionPeerProtection( + capabilities = capabilities, + malformedCapabilities = malformedCapabilities, + targetSelectionId = malformed.selectionId, + targetPermissionIdentity = target.toString(), + ) + } ?: DurableUploadPermissionPeerProtection.Ambiguous + return recoverMalformedDurableUploadCapability( + capability = malformed, + permission = permission, + peerProtection = peerProtection, + releasePermission = ::releasePermission, + isPermissionAbsent = ::exactReadPermissionIsAbsent, + removeMetadata = ::removeMetadata, + ) + } + + private fun malformedRecoveryDisposition( + malformed: MalformedDurableUploadCapability, + capabilities: Map, + malformedCapabilities: Map, + ): DurableUploadMalformedRecoveryDisposition { + val permission = malformed.cleanupPermissionIdentity?.let(Uri::parse) + ?: return DurableUploadMalformedRecoveryDisposition.Quarantine + if (malformedPeerCleanupDisposition(malformedCapabilities, malformed.selectionId, permission.toString()) == + DurableUploadMalformedPeerCleanupDisposition.Quarantine + ) return DurableUploadMalformedRecoveryDisposition.Quarantine + val peerProtection = permissionPeerProtection( + capabilities = capabilities, + malformedCapabilities = malformedCapabilities, + targetSelectionId = malformed.selectionId, + targetPermissionIdentity = permission.toString(), + ) + val permissionAbsent: Boolean? = if ( + malformed.grantPreExisting == null && + peerProtection == DurableUploadPermissionPeerProtection.None + ) { + try { + exactReadPermissionIsAbsent(permission) + } catch (cancelled: CancellationException) { + throw cancelled + } catch (_: Exception) { + null + } + } else { + false + } + return durableUploadMalformedRecoveryDisposition(malformed.grantPreExisting, peerProtection, permissionAbsent) + } + + private fun permissionPeerProtection( + capabilities: Map, + malformedCapabilities: Map, + targetSelectionId: String, + targetPermissionIdentity: String, + ): DurableUploadPermissionPeerProtection = durableUploadPermissionPeerProtection( + peers = ( + capabilities.asSequence().map { (selectionId, capability) -> + DurableUploadPermissionPeer( + selectionId, + capability.uri.toString(), + capability.grantPreExisting, + ) + } + malformedCapabilities.asSequence().map { (_, capability) -> + DurableUploadPermissionPeer( + capability.selectionId, + capability.cleanupPermissionIdentity, + capability.grantPreExisting, + ) + } + ).asIterable(), + targetSelectionId = targetSelectionId, + targetPermission = targetPermissionIdentity, + samePermission = String::equals, + ) + + private fun malformedPeerCleanupDisposition( + malformedCapabilities: Map, + targetSelectionId: String, + targetPermissionIdentity: String, + targetGrantPreExisting: Boolean = false, + ): DurableUploadMalformedPeerCleanupDisposition = durableUploadMalformedPeerCleanupDisposition( + malformedPeers = malformedCapabilities.values.asSequence().map { capability -> + DurableUploadPermissionPeer( + capability.selectionId, + capability.cleanupPermissionIdentity, + capability.grantPreExisting, + ) + }.asIterable(), + targetSelectionId = targetSelectionId, + targetPermission = targetPermissionIdentity, + samePermission = String::equals, + targetGrantPreExisting = targetGrantPreExisting, + ) + private fun persistedSource(file: LocalUploadFile): SelectedSource { - val source = selections[file.selectionId] ?: load(file.selectionId) - ?: error("The local file selection has expired.") - require(source.file == file) { "The local file selection metadata changed." } + return requiredSource(file, useCachedSource = true) + } + + private fun requiredSource( + file: LocalUploadFile, + useCachedSource: Boolean, + ): SelectedSource { + val source = readAndroidLocalUploadCapability { + selections[file.selectionId].takeIf { useCachedSource } ?: load(file.selectionId) + } ?: throw AndroidLocalUploadCapabilityUnavailableException( + "The local file selection was not durably saved.", + ) + if (source.file != file) { + throw AndroidLocalUploadCapabilityUnavailableException( + "The persisted local file metadata changed.", + ) + } + requireDurableUploadCapabilityReady(source.phase) return source } private fun load(selectionId: String): SelectedSource? { - val encrypted = preferences.getString(preferenceKey(selectionId), null) ?: return null - return runCatching { - val payload = JSONObject(cipher.decrypt(encrypted)) + val encrypted = readAndroidLocalUploadCapabilityPreference { + preferences.getString(preferenceKey(selectionId)) + } ?: return null + val decrypted = decryptAndroidLocalUploadCapability { cipher.decrypt(encrypted) } + val payload = try { + JSONObject(decrypted) + } catch (failure: Exception) { + throw AndroidLocalUploadCapabilityMalformedException( + "The local file selection metadata is invalid.", + failure, + ) + } + val cleanupPermissionIdentity = try { + payload.requireStrictString("uri") + } catch (_: Exception) { + null + } + val cleanupGrantPreExisting = try { + persistedDurableUploadGrantPreExisting(payload) + } catch (_: Exception) { + null + } + return try { val file = localUploadFile( selectionId = payload.getString("selectionId"), displayName = payload.getString("displayName"), @@ -182,57 +687,83 @@ internal class AndroidLocalUploadPicker(context: Context) { sizeBytes = if (payload.isNull("sizeBytes")) null else payload.getLong("sizeBytes"), ) require(file.selectionId == selectionId) { "The persisted upload capability changed." } - SelectedSource(Uri.parse(payload.getString("uri")), file) - }.getOrNull() + val phase = if (payload.has("phase")) { + CapabilityPhase.fromPersistedValue(payload.requireStrictString("phase")) + } else { + CapabilityPhase.Ready + } + val processGeneration = payload.optionalStrictString("processGeneration") + ?.also(::requireSafeProcessGeneration) + val grantPreExisting = persistedDurableUploadGrantPreExisting(payload) + SelectedSource( + uri = Uri.parse(payload.getString("uri")), + file = file, + phase = phase, + processGeneration = processGeneration, + grantPreExisting = grantPreExisting, + ) + } catch (cancelled: CancellationException) { + throw cancelled + } catch (failure: Exception) { + throw AndroidLocalUploadCapabilityMalformedException( + "The local file selection metadata is invalid.", + failure, + cleanupPermissionIdentity, + cleanupGrantPreExisting, + ) + } } private fun preferenceKey(selectionId: String): String = "$PREFERENCE_PREFIX$selectionId" - private data class PendingSelection( + private class PendingSelection( val continuation: CancellableContinuation, val acceptedMimeTypes: List, val maximumBytes: Long, - ) + ) { + @Volatile + var readyFile: LocalUploadFile? = null + } private data class SelectedSource( val uri: Uri, val file: LocalUploadFile, + val phase: CapabilityPhase = CapabilityPhase.Ready, + val processGeneration: String? = PROCESS_GENERATION, + val grantPreExisting: Boolean = false, ) private companion object { const val PREFERENCES = "nextcloud_native_upload_capabilities" const val PREFERENCE_PREFIX = "upload_" + const val MAX_TRACKED_CAPABILITIES = 64 + const val MAX_RECOVERABLE_CAPABILITIES = 1_024 + const val MAX_RECOVERY_ROWS_PER_PASS = 1_024 + const val MAX_CAPABILITY_PREFERENCE_FILE_BYTES = 8L * 1024L * 1024L + val PROCESS_GENERATION = UUID.randomUUID().toString() + val PROCESS_SELECTIONS = ConcurrentHashMap() + val PENDING_CLEANUP_SELECTIONS = ConcurrentHashMap.newKeySet() + val CAPABILITY_LOCK = Any() + val RECOVERY_SCAN = DurableUploadCapabilityRecoveryScan() } } -/** - * Acquires a durable picker capability without exposing an interval where a successful selection - * can be reported before its metadata reaches app-private storage. - */ -internal fun acquireDurableUploadCapability( - takePermission: () -> Unit, - persistMetadata: () -> Boolean, - releasePermission: () -> Unit, -) { - takePermission() - val persisted = runCatching { persistMetadata() } - if (persisted.getOrNull() == true) return - runCatching(releasePermission) - persisted.exceptionOrNull()?.let { throw it } - error("The durable upload capability could not be saved.") +private fun requireSafeProcessGeneration(value: String) { + require(value.length in 16..96 && value.all { it.isLetterOrDigit() || it == '-' }) { + "The picker capability process generation is invalid." + } } -/** - * Revokes the URI grant before synchronously deleting capability metadata. A failed grant release - * is still followed by metadata deletion because Android also throws when the grant was already - * absent; in either case the app must not retain an indefinitely reusable picker capability. - */ -internal fun releaseDurableUploadCapability( - releasePermission: () -> Unit, - removeMetadata: () -> Boolean, -): Boolean { - runCatching(releasePermission) - return runCatching(removeMetadata).getOrDefault(false) +internal fun resumeLocalUploadSelectionResult( + continuation: CancellableContinuation, + result: LocalUploadSelectionResult, + releaseSelected: (LocalUploadFile) -> Unit, +) { + continuation.resume(result) { _, undeliveredResult, _ -> + if (undeliveredResult is LocalUploadSelectionResult.Selected) { + runCatching { releaseSelected(undeliveredResult.file) } + } + } } private data class AndroidUploadMetadata( diff --git a/androidApp/src/main/kotlin/dev/obiente/nextcloudnative/AndroidNextcloudServices.kt b/androidApp/src/main/kotlin/dev/obiente/nextcloudnative/AndroidNextcloudServices.kt index 1121b7bff..261e9b25a 100644 --- a/androidApp/src/main/kotlin/dev/obiente/nextcloudnative/AndroidNextcloudServices.kt +++ b/androidApp/src/main/kotlin/dev/obiente/nextcloudnative/AndroidNextcloudServices.kt @@ -451,7 +451,7 @@ internal class AndroidNextcloudServices( requestPermissions = requestPlatformPermissions, ) private val projectContent = AndroidProjectContentClient(appContext, activity) - private val durableMultipartUploads = AndroidDurableMultipartUploads(appContext) + private val durableMultipartUploads = AndroidDurableMultipartUploads(appContext, localUploadPicker) private val deckCardDrafts = AndroidDeckCardDraftStore(appContext) private val supportDiagnostics = AndroidSupportDiagnostics.get(appContext) private val supportBundleExporter = AndroidSupportBundleExporter( @@ -2937,7 +2937,7 @@ internal class AndroidNextcloudServices( session: NextcloudSession, scope: DurableUploadScope, request: NextcloudMultipartUploadRequest, - ): DurableUploadEnqueueResult = withContext(Dispatchers.IO) { + ): DurableUploadEnqueueResult = durableMultipartUploads.runEnqueueWithCancellationCleanup(request.file) { ANDROID_ACCOUNT_OPERATION_GUARD.withExactAccountSession( expectedSession = session, resolveSession = ::loadSession, diff --git a/androidApp/src/main/kotlin/dev/obiente/nextcloudnative/NextcloudNativeApplication.kt b/androidApp/src/main/kotlin/dev/obiente/nextcloudnative/NextcloudNativeApplication.kt index 37310ca46..d2091db13 100644 --- a/androidApp/src/main/kotlin/dev/obiente/nextcloudnative/NextcloudNativeApplication.kt +++ b/androidApp/src/main/kotlin/dev/obiente/nextcloudnative/NextcloudNativeApplication.kt @@ -3,6 +3,8 @@ package dev.obiente.nextcloudnative import android.app.Application import android.content.Context import android.content.SharedPreferences +import androidx.work.WorkInfo +import androidx.work.WorkManager import dev.obiente.nextcloudnative.app.SupportDiagnosticComponent import dev.obiente.nextcloudnative.app.SupportDiagnosticEventDraft import dev.obiente.nextcloudnative.app.SupportDiagnosticSeverity @@ -11,6 +13,7 @@ import kotlinx.coroutines.CoroutineScope import kotlinx.coroutines.Dispatchers import kotlinx.coroutines.SupervisorJob import kotlinx.coroutines.delay +import kotlinx.coroutines.flow.first import kotlinx.coroutines.launch class NextcloudNativeApplication : Application() { @@ -40,25 +43,63 @@ class NextcloudNativeApplication : Application() { runAndroidDurableUploadStartupRecovery( recover = { var uploads: AndroidDurableMultipartUploads? = null - keepRetryingQueuedDurableUploadScheduling( - reconcile = { - constructAndReconcileQueuedDurableUploads { - val accountPreferences = getSharedPreferences( - ANDROID_ACCOUNT_PREFERENCES_NAME, - Context.MODE_PRIVATE, + var recoveryFailureReported = false + monitorQueuedDurableUploadScheduling( + recover = { + val recovered = try { + retryQueuedDurableUploadScheduling( + reconcile = { + constructAndReconcileQueuedDurableUploads { + val accountPreferences = getSharedPreferences( + ANDROID_ACCOUNT_PREFERENCES_NAME, + Context.MODE_PRIVATE, + ) + val accountResolutionAvailable = + accountPreferences.durableUploadAccountResolutionAvailable() + val available = uploads ?: AndroidDurableMultipartUploads( + this@NextcloudNativeApplication, + ).also { uploads = it } + suspend { + available.reconcileQueuedUploads( + allowQueuedScheduling = accountResolutionAvailable, + ) + } + } + }, + wait = { delayMillis -> delay(delayMillis) }, ) - if (accountPreferences.durableUploadAccountResolutionAvailable()) { - val available = uploads ?: AndroidDurableMultipartUploads( - this@NextcloudNativeApplication, - ).also { uploads = it } - available::reconcileQueuedUploads - } else { - suspend { true } + } catch (cancelled: CancellationException) { + throw cancelled + } catch (failure: AndroidDurableMultipartUploadRecoveryException) { + if (failure.disposition == DurableUploadQueueRecoveryDisposition.Quarantine) { + if (!recoveryFailureReported) { + runCatching(recordRecoveryFailure) + recoveryFailureReported = true + } + return@monitorQueuedDurableUploadScheduling true } + false } + if (recovered) { + recoveryFailureReported = false + } else if (!recoveryFailureReported) { + runCatching(recordRecoveryFailure) + recoveryFailureReported = true + } + recovered + }, + awaitWorkStopsRunning = { workId -> + awaitDurableUploadWorkToStopRunning( + workId = workId, + awaitWorkStopsRunning = { requestedWorkId -> + WorkManager.getInstance(this@NextcloudNativeApplication) + .getWorkInfoByIdFlow(requestedWorkId) + .first { work -> work == null || work.state != WorkInfo.State.RUNNING } + }, + wait = { retryDelayMillis -> delay(retryDelayMillis) }, + ) }, - wait = { delayMillis -> delay(delayMillis) }, - recordRecoveryFailure = recordRecoveryFailure, + wait = { retryDelayMillis -> delay(retryDelayMillis) }, ) }, recordRecoveryFailure = recordRecoveryFailure, diff --git a/androidApp/src/main/kotlin/dev/obiente/nextcloudnative/SessionCipher.kt b/androidApp/src/main/kotlin/dev/obiente/nextcloudnative/SessionCipher.kt index a9e327750..88babecd3 100644 --- a/androidApp/src/main/kotlin/dev/obiente/nextcloudnative/SessionCipher.kt +++ b/androidApp/src/main/kotlin/dev/obiente/nextcloudnative/SessionCipher.kt @@ -1,14 +1,23 @@ package dev.obiente.nextcloudnative import android.security.keystore.KeyGenParameterSpec +import android.security.keystore.KeyPermanentlyInvalidatedException import android.security.keystore.KeyProperties import android.util.Base64 +import java.security.GeneralSecurityException +import java.security.InvalidAlgorithmParameterException import java.security.KeyStore +import javax.crypto.AEADBadTagException import javax.crypto.Cipher import javax.crypto.KeyGenerator import javax.crypto.SecretKey import javax.crypto.spec.GCMParameterSpec +internal class InvalidSessionCiphertextException( + message: String, + cause: Throwable? = null, +) : GeneralSecurityException(message, cause) + class SessionCipher { private val keyStore = KeyStore.getInstance(KEYSTORE_PROVIDER).apply { load(null) } @@ -22,13 +31,30 @@ class SessionCipher { } fun decrypt(value: String): String { + val (iv, encrypted) = decodeEnvelope(value) + val cipher = Cipher.getInstance(TRANSFORMATION) + try { + cipher.init(Cipher.DECRYPT_MODE, getOrCreateKey(), GCMParameterSpec(GCM_TAG_BITS, iv)) + return cipher.doFinal(encrypted).toString(Charsets.UTF_8) + } catch (failure: InvalidAlgorithmParameterException) { + throw InvalidSessionCiphertextException("Invalid encrypted session envelope.", failure) + } catch (failure: KeyPermanentlyInvalidatedException) { + throw InvalidSessionCiphertextException("Encrypted session key is no longer valid.", failure) + } catch (failure: AEADBadTagException) { + throw InvalidSessionCiphertextException("Encrypted session authentication failed.", failure) + } + } + + private fun decodeEnvelope(value: String): Pair = try { val parts = value.split(SEPARATOR, limit = 2) require(parts.size == 2) { "Invalid encrypted session." } val iv = Base64.decode(parts[0], Base64.NO_WRAP) val encrypted = Base64.decode(parts[1], Base64.NO_WRAP) - val cipher = Cipher.getInstance(TRANSFORMATION) - cipher.init(Cipher.DECRYPT_MODE, getOrCreateKey(), GCMParameterSpec(128, iv)) - return cipher.doFinal(encrypted).toString(Charsets.UTF_8) + require(iv.size == GCM_IV_BYTES) { "Invalid encrypted session IV." } + require(encrypted.size >= GCM_TAG_BYTES) { "Invalid encrypted session payload." } + iv to encrypted + } catch (failure: IllegalArgumentException) { + throw InvalidSessionCiphertextException("Invalid encrypted session envelope.", failure) } private fun getOrCreateKey(): SecretKey { @@ -52,5 +78,8 @@ class SessionCipher { const val KEY_ALIAS = "dev.obiente.nextcloudnative.session" const val TRANSFORMATION = "AES/GCM/NoPadding" const val SEPARATOR = "." + const val GCM_IV_BYTES = 12 + const val GCM_TAG_BITS = 128 + const val GCM_TAG_BYTES = GCM_TAG_BITS / Byte.SIZE_BITS } } diff --git a/androidApp/src/test/kotlin/dev/obiente/nextcloudnative/AndroidDurableMultipartUploadPolicyTest.kt b/androidApp/src/test/kotlin/dev/obiente/nextcloudnative/AndroidDurableMultipartUploadPolicyTest.kt index 2f07b1280..340c17422 100644 --- a/androidApp/src/test/kotlin/dev/obiente/nextcloudnative/AndroidDurableMultipartUploadPolicyTest.kt +++ b/androidApp/src/test/kotlin/dev/obiente/nextcloudnative/AndroidDurableMultipartUploadPolicyTest.kt @@ -1,5 +1,7 @@ package dev.obiente.nextcloudnative +import androidx.work.ExistingWorkPolicy +import dev.obiente.nextcloudnative.app.DurableUploadEnqueueResult import dev.obiente.nextcloudnative.app.DurableUploadScope import dev.obiente.nextcloudnative.app.DurableUploadState import dev.obiente.nextcloudnative.app.NextcloudApiMethod @@ -9,16 +11,66 @@ import dev.obiente.nextcloudnative.app.accountRecord import dev.obiente.nextcloudnative.app.afterProcessRecovery import dev.obiente.nextcloudnative.app.localUploadFile import java.io.IOException +import java.util.UUID import kotlinx.coroutines.CancellationException +import kotlinx.coroutines.CompletableDeferred +import kotlinx.coroutines.async import kotlinx.coroutines.runBlocking +import kotlinx.coroutines.yield import kotlin.test.Test import kotlin.test.assertEquals import kotlin.test.assertFailsWith import kotlin.test.assertFalse +import kotlin.test.assertIs import kotlin.test.assertTrue import org.json.JSONArray class AndroidDurableMultipartUploadPolicyTest { + @Test + fun `successful worker execution does not request scheduling recovery`() = runBlocking { + var recoveryRequests = 0 + + val result = runDurableUploadWorkerWithRecoverySignal( + requestRecovery = { recoveryRequests += 1 }, + work = { "completed" }, + ) + + assertEquals("completed", result) + assertEquals(0, recoveryRequests) + } + + @Test + fun `worker failure requests scheduling recovery before preserving the failure`() = runBlocking { + var recoveryRequests = 0 + val expected = IOException("journal read failed") + + val actual = assertFailsWith { + runDurableUploadWorkerWithRecoverySignal( + requestRecovery = { recoveryRequests += 1 }, + work = { throw expected }, + ) + } + + assertTrue(actual === expected) + assertEquals(1, recoveryRequests) + } + + @Test + fun `worker cancellation requests scheduling recovery before preserving cancellation`() = runBlocking { + var recoveryRequests = 0 + val expected = CancellationException("worker stopped") + + val actual = assertFailsWith { + runDurableUploadWorkerWithRecoverySignal( + requestRecovery = { recoveryRequests += 1 }, + work = { throw expected }, + ) + } + + assertTrue(actual === expected) + assertEquals(1, recoveryRequests) + } + @Test fun `worker cancellation does not become a terminal upload outcome`() = runBlocking { assertFailsWith { @@ -339,57 +391,93 @@ class AndroidDurableMultipartUploadPolicyTest { } @Test - fun `inactive retained account defers when its credential is temporarily unavailable`() { - val retainedSession = NextcloudSession( - serverUrl = "https://cloud.example.test/nextcloud", - loginName = "alice", - appPassword = "fixture-password", - ) - val accountId = NextcloudDocumentIds.accountKey(retainedSession) + fun `removed account terminally fails and releases its queued upload exactly once`() { + val events = mutableListOf() - val resolution = resolveDurableUploadSession( - expectedAccountId = accountId, - registry = DurableUploadAccountRegistry.Available(listOf(retainedSession.accountRecord())), - loadSession = { null }, + val result = failQueuedDurableUploadForUnavailableAccount( + transitionToFailed = { events += "fail" }, + releaseSelection = { _ -> + events += "release" + true + }, + recordFailure = { events += "diagnose" }, + failureResult = "worker-failure", + retryResult = "worker-retry", ) - assertEquals(DurableUploadAccountResolution.DeferAccountActivation, resolution) + assertEquals("worker-failure", result) + assertEquals(listOf("fail", "release", "diagnose"), events) } @Test - fun `active retained account retries when its credential is temporarily unavailable`() { - val retainedSession = NextcloudSession( - serverUrl = "https://cloud.example.test/nextcloud", - loginName = "alice", - appPassword = "fixture-password", - ) - val accountId = NextcloudDocumentIds.accountKey(retainedSession) + fun `account recovery uses replacement only for deferred worker backoff`() { + assertEquals(ExistingWorkPolicy.REPLACE, DURABLE_UPLOAD_ACCOUNT_RECOVERY_WORK_POLICY) + } - val resolution = resolveDurableUploadSession( - expectedAccountId = accountId, - registry = DurableUploadAccountRegistry.Available( - accounts = listOf(retainedSession.accountRecord()), - activeAccountId = retainedSession.accountId, - ), - loadSession = { null }, - ) + @Test + fun `account recovery never replaces a worker after it starts its upload`() = runBlocking { + val queued = fixtureJob(index = 1, account = ACCOUNT_A, cardId = 42) + var current = queued + val claimEntered = CompletableDeferred() + val allowClaim = CompletableDeferred() + var replacementScheduled = false + + val claim = async { + claimQueuedDurableUploadForExecution(queued.id) { + claimEntered.complete(Unit) + allowClaim.await() + current = queued.copy(state = DurableUploadState.Uploading) + current + } + } + claimEntered.await() + val recovery = async { + replaceDeferredDurableUploadWork( + expected = queued, + load = { current }, + replace = { replacementScheduled = true }, + ) + } + yield() - assertEquals(DurableUploadAccountResolution.CredentialUnavailable, resolution) + assertFalse(recovery.isCompleted) + allowClaim.complete(Unit) + assertEquals(DurableUploadState.Uploading, claim.await()?.state) + assertFalse(recovery.await()) + assertFalse(replacementScheduled) } @Test - fun `removed account terminally fails and releases its queued upload exactly once`() { - val events = mutableListOf() + fun `slow account recovery does not block an unrelated upload claim`() = runBlocking { + val recovering = fixtureJob(index = 1, account = ACCOUNT_A, cardId = 42) + val unrelated = fixtureJob(index = 2, account = ACCOUNT_B, cardId = 43) + val coordinator = AndroidDurableUploadStartCoordinator() + val replacementEntered = CompletableDeferred() + val releaseReplacement = CompletableDeferred() + val recovery = async { + replaceDeferredDurableUploadWork( + expected = recovering, + load = { recovering }, + replace = { + replacementEntered.complete(Unit) + releaseReplacement.await() + }, + coordinator = coordinator, + ) + } + replacementEntered.await() - val result = failQueuedDurableUploadForUnavailableAccount( - transitionToFailed = { events += "fail" }, - releaseSelection = { events += "release" }, - recordFailure = { events += "diagnose" }, - failureResult = "worker-failure", - ) + val claimed = claimQueuedDurableUploadForExecution( + jobId = unrelated.id, + coordinator = coordinator, + ) { + unrelated.copy(state = DurableUploadState.Uploading) + } - assertEquals("worker-failure", result) - assertEquals(listOf("fail", "release", "diagnose"), events) + assertEquals(DurableUploadState.Uploading, claimed?.state) + assertFalse(recovery.isCompleted) + releaseReplacement.complete(Unit) + assertTrue(recovery.await()) } @Test @@ -409,6 +497,245 @@ class AndroidDurableMultipartUploadPolicyTest { ) } + @Test + fun `ambiguous scheduling keeps the durable job queued across restart`() = runBlocking { + val job = fixtureJob(index = 1, account = ACCOUNT_A, cardId = 42) + val persisted = mutableListOf() + val acceptedWork = mutableSetOf() + var recoveryRequests = 0 + + val result = persistAndScheduleDurableUpload( + job = job, + persist = persisted::add, + schedule = { queued -> + acceptedWork += queued.id + throw IOException("The scheduler completion signal was lost") + }, + requestRecovery = { recoveryRequests += 1 }, + ) + + assertIs(result) + assertEquals(listOf(job), persisted) + assertEquals(setOf(job.id), acceptedWork) + assertEquals(1, recoveryRequests) + + val workRecoveredAfterRestart = persisted + .filter { queued -> queued.state == DurableUploadState.Queued } + .map(AndroidDurableMultipartUploadJob::id) + assertEquals(listOf(job.id), workRecoveredAfterRestart) + } + + @Test + fun `startup reconciliation skips queued uploads already owned by WorkManager`() = runBlocking { + val owned = fixtureJob(index = 1, account = ACCOUNT_A, cardId = 42) + val missing = fixtureJob(index = 2, account = ACCOUNT_B, cardId = 43) + val attempted = mutableListOf() + + val allScheduled = reconcileQueuedDurableUploads( + jobs = listOf(owned, missing), + schedulerOwns = { job -> job == owned }, + cleanupCapability = { error("Queued uploads must not enter local cleanup.") }, + schedule = { job -> attempted += job.id }, + ) + + assertTrue(allScheduled) + assertEquals(listOf(missing.id), attempted) + } + + @Test + fun `recovery signal conflates immediate requests and replacement workers per durable job`() = runBlocking { + val recoverySignal = AndroidDurableUploadSchedulingRecoverySignal() + val replacedWorkId = UUID.randomUUID() + val replacementWorkId = UUID.randomUUID() + val otherWorkId = UUID.randomUUID() + + recoverySignal.request() + recoverySignal.request() + recoverySignal.requestAfterWorkStopsRunning("job-1", replacedWorkId) + repeat(100) { recoverySignal.requestAfterWorkStopsRunning("job-1", replacementWorkId) } + recoverySignal.requestAfterWorkStopsRunning("job-2", otherWorkId) + + assertEquals( + AndroidDurableUploadSchedulingRecoveryBatch( + immediate = true, + workIdsToAwait = mapOf("job-1" to replacementWorkId, "job-2" to otherWorkId), + ), + recoverySignal.await(), + ) + } + + @Test + fun `worker readiness retries a transient state query for the same work id`() = runBlocking { + val workId = UUID.randomUUID() + val requestedWorkIds = mutableListOf() + val waits = mutableListOf() + + awaitDurableUploadWorkToStopRunning( + workId = workId, + retryDelayMillis = 25L, + awaitWorkStopsRunning = { requestedWorkId -> + requestedWorkIds += requestedWorkId + if (requestedWorkIds.size == 1) { + throw IOException("Synthetic WorkManager database failure") + } + }, + wait = waits::add, + ) + + assertEquals(listOf(workId, workId), requestedWorkIds) + assertEquals(listOf(25L), waits) + } + + @Test + fun `worker readiness cancellation propagates without retrying`() = runBlocking { + val workId = UUID.randomUUID() + val expected = CancellationException("Recovery owner stopped") + val waits = mutableListOf() + + val actual = assertFailsWith { + awaitDurableUploadWorkToStopRunning( + workId = workId, + awaitWorkStopsRunning = { requestedWorkId -> + assertEquals(workId, requestedWorkId) + throw expected + }, + wait = waits::add, + ) + } + + assertTrue(actual === expected) + assertTrue(waits.isEmpty()) + } + + @Test + fun `queued status snapshot requests one scheduling recovery`() { + val first = fixtureJob(index = 1, account = ACCOUNT_A, cardId = 42) + val second = fixtureJob(index = 2, account = ACCOUNT_A, cardId = 43) + var recoveryRequests = 0 + + requestDurableUploadSchedulingRecoveryForQueuedStatuses( + jobs = listOf(first, second), + requestRecovery = { recoveryRequests += 1 }, + ) + + assertEquals(1, recoveryRequests) + } + + @Test + fun `terminal status snapshot does not request scheduling recovery`() { + val completed = fixtureJob( + index = 1, + account = ACCOUNT_A, + cardId = 42, + state = DurableUploadState.Completed, + ) + val failed = fixtureJob( + index = 2, + account = ACCOUNT_A, + cardId = 43, + state = DurableUploadState.Failed, + ) + var recoveryRequests = 0 + + requestDurableUploadSchedulingRecoveryForQueuedStatuses( + jobs = listOf(completed, failed), + requestRecovery = { recoveryRequests += 1 }, + ) + + assertEquals(0, recoveryRequests) + } + + @Test + fun `startup scheduling retries until a transient journal read failure clears`() = runBlocking { + var attempts = 0 + val waits = mutableListOf() + var diagnostics = 0 + + keepRetryingQueuedDurableUploadScheduling( + followUpDelayMillis = 100L, + reconcile = { + attempts += 1 + when (attempts) { + 1, 2 -> throw AndroidDurableMultipartUploadRecoveryException( + IOException("Synthetic unreadable journal"), + ) + else -> true + } + }, + wait = waits::add, + recordRecoveryFailure = { diagnostics += 1 }, + ) + + assertEquals(3, attempts) + assertEquals(listOf(100L, 100L), waits) + assertEquals(1, diagnostics) + } + + @Test + fun `startup scheduling retries when uploader construction is temporarily unavailable`() = runBlocking { + var constructions = 0 + val waits = mutableListOf() + var diagnostics = 0 + + keepRetryingQueuedDurableUploadScheduling( + followUpDelayMillis = 100L, + reconcile = { + constructAndReconcileQueuedDurableUploads { + constructions += 1 + when (constructions) { + 1 -> throw IOException("Synthetic keystore initialization failure") + else -> suspend { true } + } + } + }, + wait = waits::add, + recordRecoveryFailure = { diagnostics += 1 }, + ) + + assertEquals(2, constructions) + assertEquals(listOf(100L), waits) + assertEquals(1, diagnostics) + } + + @Test + fun `cancellation after persistence propagates without discarding restart state`() { + val job = fixtureJob(index = 1, account = ACCOUNT_A, cardId = 42) + val persisted = mutableListOf() + var recoveryRequests = 0 + + assertFailsWith { + runBlocking { + persistAndScheduleDurableUpload( + job = job, + persist = persisted::add, + schedule = { throw CancellationException("Owner stopped") }, + requestRecovery = { recoveryRequests += 1 }, + ) + } + } + + assertEquals(listOf(job), persisted) + assertEquals(1, recoveryRequests) + } + + @Test + fun `persistence failure never reaches the scheduler`() { + val job = fixtureJob(index = 1, account = ACCOUNT_A, cardId = 42) + var scheduled = false + + assertFailsWith { + runBlocking { + persistAndScheduleDurableUpload( + job = job, + persist = { throw IOException("Queue storage is unavailable") }, + schedule = { scheduled = true }, + ) + } + } + + assertFalse(scheduled) + } + @Test fun `background upload resolves the queued account instead of the active account`() { val queuedSession = fixtureSession("alice") @@ -530,10 +857,14 @@ class AndroidDurableMultipartUploadPolicyTest { ) val attempted = mutableListOf() - val allScheduled = reconcileQueuedDurableUploads(listOf(first, completed, second)) { job -> - attempted += job.id - if (job == first) throw IOException("Synthetic scheduler rejection") - } + val allScheduled = reconcileQueuedDurableUploads( + jobs = listOf(first, completed, second), + cleanupCapability = { error("Completed history must not enter cleanup.") }, + schedule = { job -> + attempted += job.id + if (job == first) throw IOException("Synthetic scheduler rejection") + }, + ) assertEquals(listOf(first.id, second.id), attempted) assertFalse(allScheduled) @@ -674,6 +1005,29 @@ class AndroidDurableMultipartUploadPolicyTest { assertTrue(events.isEmpty()) } + @Test + fun `unsupported registry remains unavailable after credential recovery attempt`() { + var recoveryAttempts = 0 + var credentialReads = 0 + + val resolved = resolveDurableUploadSessionWithRegistryRecovery( + expectedAccountId = NextcloudDocumentIds.accountKey(fixtureSession("alice")), + readRegistry = { DurableUploadAccountRegistry.Unavailable }, + recoverRegistry = { + recoveryAttempts += 1 + null + }, + loadSession = { + credentialReads += 1 + fixtureSession("alice") + }, + ) + + assertEquals(DurableUploadAccountResolution.RegistryUnavailable, resolved) + assertEquals(1, recoveryAttempts) + assertEquals(0, credentialReads) + } + @Test fun `background upload never substitutes another account on the same server path`() { val queuedSession = fixtureSession("alice") diff --git a/androidApp/src/test/kotlin/dev/obiente/nextcloudnative/AndroidDurableUploadAccountResolutionTest.kt b/androidApp/src/test/kotlin/dev/obiente/nextcloudnative/AndroidDurableUploadAccountResolutionTest.kt new file mode 100644 index 000000000..ec1106e6d --- /dev/null +++ b/androidApp/src/test/kotlin/dev/obiente/nextcloudnative/AndroidDurableUploadAccountResolutionTest.kt @@ -0,0 +1,43 @@ +package dev.obiente.nextcloudnative + +import dev.obiente.nextcloudnative.app.NextcloudSession +import dev.obiente.nextcloudnative.app.accountRecord +import kotlin.test.Test +import kotlin.test.assertEquals + +class AndroidDurableUploadAccountResolutionTest { + @Test + fun `inactive retained account defers when its credential is temporarily unavailable`() { + val retainedSession = fixtureSession() + + val resolution = resolveDurableUploadSession( + expectedAccountId = NextcloudDocumentIds.accountKey(retainedSession), + registry = DurableUploadAccountRegistry.Available(listOf(retainedSession.accountRecord())), + loadSession = { null }, + ) + + assertEquals(DurableUploadAccountResolution.DeferAccountActivation, resolution) + } + + @Test + fun `active retained account retries when its credential is temporarily unavailable`() { + val retainedSession = fixtureSession() + + val resolution = resolveDurableUploadSession( + expectedAccountId = NextcloudDocumentIds.accountKey(retainedSession), + registry = DurableUploadAccountRegistry.Available( + accounts = listOf(retainedSession.accountRecord()), + activeAccountId = retainedSession.accountId, + ), + loadSession = { null }, + ) + + assertEquals(DurableUploadAccountResolution.CredentialUnavailable, resolution) + } + + private fun fixtureSession() = NextcloudSession( + serverUrl = "https://cloud.example.test/nextcloud", + loginName = "alice", + appPassword = "fixture-password", + ) +} diff --git a/androidApp/src/test/kotlin/dev/obiente/nextcloudnative/AndroidDurableUploadCleanupPruningTest.kt b/androidApp/src/test/kotlin/dev/obiente/nextcloudnative/AndroidDurableUploadCleanupPruningTest.kt new file mode 100644 index 000000000..a25e708f1 --- /dev/null +++ b/androidApp/src/test/kotlin/dev/obiente/nextcloudnative/AndroidDurableUploadCleanupPruningTest.kt @@ -0,0 +1,360 @@ +package dev.obiente.nextcloudnative + +import androidx.work.NetworkType +import dev.obiente.nextcloudnative.app.DurableUploadScope +import dev.obiente.nextcloudnative.app.DurableUploadState +import dev.obiente.nextcloudnative.app.NextcloudApiMethod +import dev.obiente.nextcloudnative.app.NextcloudMultipartUploadRequest +import dev.obiente.nextcloudnative.app.localUploadFile +import kotlinx.coroutines.CancellationException +import kotlinx.coroutines.runBlocking +import kotlin.test.Test +import kotlin.test.assertEquals +import kotlin.test.assertFailsWith +import kotlin.test.assertFalse +import kotlin.test.assertTrue +import org.json.JSONArray +import org.json.JSONObject + +class AndroidDurableUploadCleanupPruningTest { + @Test + fun `oversized terminal cleanup state is quarantined before capability loading`() { + val jobs = (1..1_025).map { index -> + fixtureJob( + index = index, + state = DurableUploadState.Failed, + cleanupPending = true, + ) + } + val retained = durableUploadCapabilityRetainedSelectionIds(jobs) + val storedIds = jobs.map { job -> job.request.file.selectionId } + val scan = DurableUploadCapabilityRecoveryScan() + + var loaded = 0 + val snapshot = scan.loadPage(emptyMap(), storedIds, maximumRows = 1_024) { + loaded += 1 + CapabilityPhase.CleanupPending + } + + assertTrue(retained.isEmpty()) + assertFalse(snapshot.scanComplete) + assertTrue(snapshot.recoveryQuarantined) + assertTrue(snapshot.capabilities.isEmpty()) + assertEquals(0, loaded) + } + + @Test + fun `reconciliation runs terminal cleanup without consulting upload work ownership`() = runBlocking { + val pending = fixtureJob(index = 1, cleanupPending = true) + val history = fixtureJob(index = 2) + val cleaned = mutableListOf() + + val allScheduled = reconcileQueuedDurableUploads( + jobs = listOf(pending, history), + schedulerOwns = { error("Local cleanup must not wait for upload work ownership.") }, + cleanupCapability = { job -> cleaned += job }, + schedule = { error("Terminal cleanup must not use network-constrained upload work.") }, + ) + + assertTrue(allScheduled) + assertEquals(listOf(pending), cleaned) + } + + @Test + fun `permanently malformed terminal cleanup preserves its terminal status`() { + val storage = MemoryStorage() + val store = AndroidDurableMultipartUploadStore(storage, PlaintextCipher) + val quarantined = fixtureJob(index = 1, cleanupPending = true) + val retained = fixtureJob(index = 2, cleanupPending = true) + store.add(quarantined) + store.add(retained) + + val reconciled = reconcileTerminalDurableUploadCapabilityCleanup( + release = { onQuarantined -> + onQuarantined() + false + }, + complete = { store.completeCapabilityCleanup(quarantined.id) }, + ) + + assertTrue(reconciled) + assertEquals( + listOf(quarantined.copy(capabilityCleanupPending = false), retained), + AndroidDurableMultipartUploadStore(storage, PlaintextCipher).list(), + ) + } + + @Test + fun `account cleanup accepts a quarantined unknowable capability`() { + var quarantined = false + + val ready = releaseAndroidDurableUploadCapabilityForAccountRemoval { onQuarantined -> + onQuarantined() + quarantined = true + false + } + + assertTrue(ready) + assertTrue(quarantined) + } + + @Test + fun `only queued uploads are eligible for connected upload work`() { + val queued = fixtureJob(index = 1, state = DurableUploadState.Queued) + val pending = fixtureJob(index = 2, cleanupPending = true) + + assertEquals(NetworkType.CONNECTED, networkTypeForDurableUploadWork(queued)) + assertFailsWith { + networkTypeForDurableUploadWork(pending) + } + } + + @Test + fun `terminal cleanup scheduling failure is aggregated without blocking queued work`() = runBlocking { + val pending = fixtureJob(index = 1, cleanupPending = true) + val queued = fixtureJob(index = 2, state = DurableUploadState.Queued) + val attempts = mutableListOf() + + val allScheduled = reconcileQueuedDurableUploads( + jobs = listOf(pending, queued), + cleanupCapability = { job -> + attempts += job.id + error("synthetic cleanup failure") + }, + schedule = { job -> attempts += job.id }, + ) + + assertFalse(allScheduled) + assertEquals(listOf(pending.id, queued.id), attempts) + } + + @Test + fun `unsupported account registry runs terminal cleanup but not queued uploads`() = runBlocking { + val pending = fixtureJob(index = 1, cleanupPending = true) + val queued = fixtureJob(index = 2, state = DurableUploadState.Queued) + val cleaned = mutableListOf() + val scheduled = mutableListOf() + val accountResolutionAvailable = androidCredentialFreeRegistryAllowsAccountResolution( + """{"version":99,"accounts":[]}""", + ) + + val allScheduled = reconcileQueuedDurableUploads( + jobs = listOf(pending, queued), + allowQueuedScheduling = accountResolutionAvailable, + cleanupCapability = cleaned::add, + schedule = scheduled::add, + ) + + assertFalse(accountResolutionAvailable) + assertTrue(allScheduled) + assertEquals(listOf(pending), cleaned) + assertTrue(scheduled.isEmpty()) + } + + @Test + fun `wrong typed account registry runs terminal cleanup but not queued uploads`() = runBlocking { + val pending = fixtureJob(index = 1, cleanupPending = true) + val queued = fixtureJob(index = 2, state = DurableUploadState.Queued) + val cleaned = mutableListOf() + val scheduled = mutableListOf() + val accountResolutionAvailable = durableUploadAccountResolutionAvailable { + throw ClassCastException("synthetic wrong-typed account registry") + } + + val allScheduled = reconcileQueuedDurableUploads( + jobs = listOf(pending, queued), + allowQueuedScheduling = accountResolutionAvailable, + cleanupCapability = cleaned::add, + schedule = scheduled::add, + ) + + assertFalse(accountResolutionAvailable) + assertTrue(allScheduled) + assertEquals(listOf(pending), cleaned) + assertTrue(scheduled.isEmpty()) + } + + @Test + fun `terminal cleanup reconciliation preserves cancellation`() = runBlocking { + val pending = fixtureJob(index = 1, cleanupPending = true) + + assertFailsWith { + reconcileQueuedDurableUploads( + jobs = listOf(pending), + cleanupCapability = { throw CancellationException("recovery stopped") }, + schedule = { error("Terminal cleanup must not schedule upload work.") }, + ) + } + Unit + } + + @Test + fun `pruning retains terminal rows until capability cleanup commits`() { + val pending = fixtureJob( + index = 1, + cleanupPending = true, + updatedAt = 0L, + ) + val history = (2..70).map { index -> fixtureJob(index = index, updatedAt = index.toLong()) } + + val pruned = pruneDurableUploadJobs(history + pending) + + assertEquals(AndroidDurableMultipartUploadStore.MAX_STORED_UPLOADS, pruned.size) + assertTrue(pending in pruned) + assertFalse(pruned.any { job -> job.id == fixtureId(2) }) + } + + @Test + fun `terminal transition persists cleanup until its commit`() { + val storage = MemoryStorage() + val store = AndroidDurableMultipartUploadStore(storage, PlaintextCipher) + val queued = fixtureJob(index = 1, state = DurableUploadState.Queued) + store.add(queued) + + store.transition(queued.id, DurableUploadState.Queued, DurableUploadState.Failed, "failed") + + assertTrue(AndroidDurableMultipartUploadStore(storage, PlaintextCipher).list().single().capabilityCleanupPending) + store.completeCapabilityCleanup(queued.id) + assertFalse(AndroidDurableMultipartUploadStore(storage, PlaintextCipher).list().single().capabilityCleanupPending) + } + + @Test + fun `legacy terminal rows default to completed cleanup`() { + val storage = MemoryStorage() + val store = AndroidDurableMultipartUploadStore(storage, PlaintextCipher) + val queued = fixtureJob(index = 1, state = DurableUploadState.Queued) + store.add(queued) + store.transition(queued.id, DurableUploadState.Queued, DurableUploadState.Failed, "failed") + val legacy = JSONArray(checkNotNull(storage.value)) + legacy.getJSONObject(0).remove("capabilityCleanupPending") + storage.value = legacy.toString() + + val restored = AndroidDurableMultipartUploadStore(storage, PlaintextCipher).list().single() + + assertFalse(restored.capabilityCleanupPending) + } + + @Test + fun `explicit cleanup marker booleans restore without coercion`() { + listOf(true, false).forEach { cleanupPending -> + val storage = MemoryStorage() + AndroidDurableMultipartUploadStore(storage, PlaintextCipher).add( + fixtureJob(index = 1, cleanupPending = cleanupPending), + ) + + val restored = AndroidDurableMultipartUploadStore(storage, PlaintextCipher).list().single() + + assertEquals(cleanupPending, restored.capabilityCleanupPending) + } + } + + @Test + fun `malformed cleanup markers leave the recovery queue unchanged`() { + val malformedValues = listOf( + "true", + "false", + 1, + JSONObject.NULL, + JSONObject().put("pending", true), + JSONArray().put(true), + ) + + malformedValues.forEach { malformedValue -> + val storage = MemoryStorage() + val store = AndroidDurableMultipartUploadStore(storage, PlaintextCipher) + store.add(fixtureJob(index = 1, cleanupPending = true)) + val malformedSnapshot = JSONArray(checkNotNull(storage.value)).also { array -> + array.getJSONObject(0).put("capabilityCleanupPending", malformedValue) + }.toString() + storage.value = malformedSnapshot + + assertFailsWith { store.list() } + assertFailsWith { + store.add(fixtureJob(index = 2, state = DurableUploadState.Queued)) + } + assertEquals(malformedSnapshot, storage.value) + } + } + + @Test + fun `pending cleanup consumes bounded queue capacity`() { + val pending = (1..AndroidDurableMultipartUploadStore.MAX_STORED_UPLOADS).map { index -> + fixtureJob(index = index, cleanupPending = true) + } + + assertFailsWith { + requireCanAddDurableUpload( + current = pending, + job = fixtureJob(index = 100, state = DurableUploadState.Queued), + ) + } + } + + @Test + fun `cleanup commit failure retries and preserves cancellation`() { + var recoveryRequests = 0 + assertEquals( + "retry", + resultAfterDurableUploadCapabilityRelease( + releaseCapability = { true }, + completeCapabilityCleanup = { error("queue unavailable") }, + onCleanupRetained = { recoveryRequests += 1 }, + releasedResult = "finished", + retainedResult = "retry", + ), + ) + assertEquals(1, recoveryRequests) + assertFailsWith { + resultAfterDurableUploadCapabilityRelease( + releaseCapability = { true }, + completeCapabilityCleanup = { throw CancellationException("worker stopped") }, + releasedResult = "finished", + retainedResult = "retry", + ) + } + } + + private fun fixtureJob( + index: Int, + state: DurableUploadState = DurableUploadState.Completed, + cleanupPending: Boolean = false, + updatedAt: Long = index.toLong(), + ): AndroidDurableMultipartUploadJob { + val cardId = index.toLong() + val scope = DurableUploadScope("deck-attachment", cardId.toString()) + val request = NextcloudMultipartUploadRequest( + method = NextcloudApiMethod.POST, + relativePath = "/index.php/apps/deck/api/v1.1/boards/7/stacks/11/cards/$cardId/attachments", + file = localUploadFile( + selectionId = "selection-${index.toString().padStart(16, '0')}", + displayName = "fixture-$index.txt", + mimeType = "text/plain", + sizeBytes = 16L, + ), + maximumFileBytes = 1024L, + ) + return AndroidDurableMultipartUploadJob( + id = fixtureId(index), + accountId = "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", + scope = scope, + resource = resolveDurableUploadResource(scope, request), + request = request, + state = state, + message = null, + capabilityCleanupPending = cleanupPending, + updatedAtEpochMillis = updatedAt, + ) + } + + private fun fixtureId(index: Int) = "upload-${index.toString().padStart(16, '0')}" + + private class MemoryStorage(var value: String? = null) : AndroidDurableMultipartUploadEncryptedStorage { + override fun read(): String? = value + override fun write(value: String): Boolean = true.also { this.value = value } + } + + private object PlaintextCipher : AndroidDurableMultipartUploadCipher { + override fun encrypt(value: String): String = value + override fun decrypt(value: String): String = value + } +} diff --git a/androidApp/src/test/kotlin/dev/obiente/nextcloudnative/AndroidDurableUploadEnqueueCancellationTest.kt b/androidApp/src/test/kotlin/dev/obiente/nextcloudnative/AndroidDurableUploadEnqueueCancellationTest.kt new file mode 100644 index 000000000..bc7050901 --- /dev/null +++ b/androidApp/src/test/kotlin/dev/obiente/nextcloudnative/AndroidDurableUploadEnqueueCancellationTest.kt @@ -0,0 +1,141 @@ +package dev.obiente.nextcloudnative + +import java.security.GeneralSecurityException +import kotlinx.coroutines.CancellationException +import kotlinx.coroutines.runBlocking +import kotlin.test.Test +import kotlin.test.assertEquals +import kotlin.test.assertFailsWith +import kotlin.test.assertFalse +import kotlin.test.assertTrue + +class AndroidDurableUploadEnqueueCancellationTest { + @Test + fun `cancellation releases a selection with no durable owner`() = runBlocking { + val expected = CancellationException("screen closed") + var releases = 0 + + val actual = assertFailsWith { + runDurableUploadEnqueueWithCancellationCleanup( + enqueue = { throw expected }, + releaseUnownedSelection = { + assertTrue( + releaseUnownedDurableUploadSelection( + selectionId = "selection-123456", + hasActiveSelection = { false }, + releaseSelection = { + releases += 1 + true + }, + ), + ) + }, + ) + } + + assertTrue(actual === expected) + assertEquals(1, releases) + } + + @Test + fun `cancellation retains a selection owned by a queued job`() = runBlocking { + var releases = 0 + + assertFailsWith { + runDurableUploadEnqueueWithCancellationCleanup( + enqueue = { throw CancellationException("scheduling cancelled") }, + releaseUnownedSelection = { + assertFalse( + releaseUnownedDurableUploadSelection( + selectionId = "selection-123456", + hasActiveSelection = { true }, + releaseSelection = { + releases += 1 + true + }, + ), + ) + }, + ) + } + + assertEquals(0, releases) + } + + @Test + fun `cancellation releases cached capability when encrypted storage is unreadable`() = runBlocking { + val expected = CancellationException("screen closed") + var cleanupResult: Boolean? = null + var encryptedMetadata: String? = "unreadable-encrypted-capability" + var loadAttempts = 0 + val events = mutableListOf() + + val actual = assertFailsWith { + runDurableUploadEnqueueWithCancellationCleanup( + enqueue = { throw expected }, + releaseUnownedSelection = { + cleanupResult = releaseUnownedDurableUploadSelection( + selectionId = "selection-123456", + hasActiveSelection = { false }, + releaseSelection = { + releaseStoredDurableUploadCapability( + cachedCapability = "content://cached/upload", + loadCapability = { + loadAttempts += 1 + throw GeneralSecurityException("synthetic decryption failure") + }, + releasePermission = { events += "permission:$it" }, + removeMetadata = { + events += "metadata" + true.also { encryptedMetadata = null } + }, + ) + }, + ) + }, + ) + } + + assertTrue(actual === expected) + assertTrue(cleanupResult == true) + assertEquals(0, loadAttempts) + assertEquals(listOf("permission:content://cached/upload", "metadata"), events) + assertEquals(null, encryptedMetadata) + } + + @Test + fun `unreadable ownership state persists cleanup intent without releasing`() { + var releases = 0 + var ownershipChecksPending = 0 + + val released = releaseUnownedDurableUploadSelection( + selectionId = "selection-123456", + hasActiveSelection = { error("journal unavailable") }, + releaseSelection = { + releases += 1 + true + }, + markOwnershipCheckPending = { + ownershipChecksPending += 1 + true + }, + ) + + assertFalse(released) + assertEquals(0, releases) + assertEquals(1, ownershipChecksPending) + } + + @Test + fun `successful enqueue does not run cancellation cleanup`() = runBlocking { + var cleanupCalls = 0 + + val result = runDurableUploadEnqueueWithCancellationCleanup( + enqueue = { "queued" }, + releaseUnownedSelection = { cleanupCalls += 1 }, + ) + + assertEquals("queued", result) + assertEquals(0, cleanupCalls) + } +} diff --git a/androidApp/src/test/kotlin/dev/obiente/nextcloudnative/AndroidDurableUploadSchedulingRecoveryTest.kt b/androidApp/src/test/kotlin/dev/obiente/nextcloudnative/AndroidDurableUploadSchedulingRecoveryTest.kt new file mode 100644 index 000000000..981a568e3 --- /dev/null +++ b/androidApp/src/test/kotlin/dev/obiente/nextcloudnative/AndroidDurableUploadSchedulingRecoveryTest.kt @@ -0,0 +1,599 @@ +package dev.obiente.nextcloudnative + +import dev.obiente.nextcloudnative.app.DurableUploadScope +import dev.obiente.nextcloudnative.app.DurableUploadState +import dev.obiente.nextcloudnative.app.NextcloudApiMethod +import dev.obiente.nextcloudnative.app.NextcloudMultipartUploadRequest +import dev.obiente.nextcloudnative.app.localUploadFile +import java.util.UUID +import kotlinx.coroutines.CancellationException +import kotlinx.coroutines.CompletableDeferred +import kotlinx.coroutines.async +import kotlinx.coroutines.cancelAndJoin +import kotlinx.coroutines.runBlocking +import kotlinx.coroutines.yield +import kotlin.test.Test +import kotlin.test.assertEquals +import kotlin.test.assertFailsWith +import kotlin.test.assertFalse +import kotlin.test.assertTrue + +class AndroidDurableUploadSchedulingRecoveryTest { + @Test + fun `a recovery request wakes the idle scheduling monitor without polling`() = runBlocking { + val recoverySignal = AndroidDurableUploadSchedulingRecoverySignal() + var recoveryRuns = 0 + recoverySignal.request() + + assertFailsWith { + monitorQueuedDurableUploadScheduling( + recover = { + recoveryRuns += 1 + if (recoveryRuns == 2) throw CancellationException("Lifecycle stopped") + true + }, + wait = { error("an immediate wake must not wait") }, + recoverySignal = recoverySignal, + ) + } + + assertEquals(2, recoveryRuns) + } + + @Test + fun `coalesced immediate recovery preempts worker ownership and follow up waits`() = runBlocking { + val recoverySignal = AndroidDurableUploadSchedulingRecoverySignal() + val jobId = "job-1" + val workId = UUID.randomUUID() + val expectedCancellation = CancellationException("recovery owner stopped") + var recoveryRuns = 0 + var ownershipWaits = 0 + var delayRuns = 0 + recoverySignal.request() + recoverySignal.requestAfterWorkStopsRunning(jobId, workId) + + val actual = assertFailsWith { + monitorQueuedDurableUploadScheduling( + recover = { + recoveryRuns += 1 + if (recoveryRuns == 2) throw expectedCancellation + true + }, + awaitWorkStopsRunning = { requestedWorkId -> + assertEquals(workId, requestedWorkId) + ownershipWaits += 1 + }, + wait = { delayMillis -> + assertEquals(ANDROID_DURABLE_UPLOAD_SCHEDULING_FOLLOW_UP_DELAY_MILLIS, delayMillis) + delayRuns += 1 + }, + recoverySignal = recoverySignal, + ) + } + + assertTrue(actual === expectedCancellation) + assertEquals(2, recoveryRuns) + assertEquals(0, ownershipWaits) + assertEquals(0, delayRuns) + } + + @Test + fun `failed idle reconciliation retries without a new signal and success stops polling`() = runBlocking { + val recoverySignal = AndroidDurableUploadSchedulingRecoverySignal() + val recovered = CompletableDeferred() + val waits = mutableListOf() + var recoveryRuns = 0 + var nowMillis = 1_000L + val monitor = async { + monitorQueuedDurableUploadScheduling( + recover = { + recoveryRuns += 1 + if (recoveryRuns == 2) recovered.complete(Unit) + recoveryRuns == 2 + }, + wait = { delayMillis -> + waits += delayMillis + nowMillis += delayMillis + }, + monotonicTimeMillis = { nowMillis }, + recoverySignal = recoverySignal, + ) + } + try { + recovered.await() + yield() + assertEquals(2, recoveryRuns) + assertEquals(listOf(60_000L), waits) + } finally { + monitor.cancelAndJoin() + } + } + + @Test + fun `self signaling cleanup cannot prevent a stopped worker backoff from expiring`() = runBlocking { + val recoverySignal = AndroidDurableUploadSchedulingRecoverySignal() + val workId = UUID.randomUUID() + val expected = CancellationException("stopped after the queued upload was scheduled") + val scheduled = mutableListOf() + val waits = mutableListOf() + var recoveryRuns = 0 + var ownershipWaits = 0 + var nowMillis = 1_000L + recoverySignal.requestAfterWorkStopsRunning("job-1", workId) + + val actual = assertFailsWith { + monitorQueuedDurableUploadScheduling( + recover = { + recoveryRuns += 1 + assertTrue(recoveryRuns <= 3, "Cleanup signals must not spin ahead of the worker deadline") + recoverySignal.request() + recoverySignal.scheduleUnlessBackedOff("job-1") { scheduled += "job-1" } + if (scheduled.isNotEmpty()) throw expected + false + }, + awaitWorkStopsRunning = { requestedWorkId -> + assertEquals(workId, requestedWorkId) + ownershipWaits += 1 + }, + wait = { delayMillis -> + waits += delayMillis + nowMillis += delayMillis + }, + monotonicTimeMillis = { nowMillis }, + recoverySignal = recoverySignal, + ) + } + + assertTrue(actual === expected) + assertEquals(2, recoveryRuns) + assertEquals(1, ownershipWaits) + assertEquals(listOf(60_000L), waits) + assertEquals(listOf("job-1"), scheduled) + } + + @Test + fun `idle self signaling cleanup waits for its retry deadline instead of spinning`() = runBlocking { + val recoverySignal = AndroidDurableUploadSchedulingRecoverySignal() + val expected = CancellationException("stopped after the bounded cleanup retry") + val waits = mutableListOf() + var recoveryRuns = 0 + var nowMillis = 1_000L + + val actual = assertFailsWith { + monitorQueuedDurableUploadScheduling( + recover = { + recoveryRuns += 1 + recoverySignal.request() + if (recoveryRuns == 2) { + assertEquals(listOf(60_000L), waits) + throw expected + } + false + }, + wait = { delayMillis -> + waits += delayMillis + nowMillis += delayMillis + }, + monotonicTimeMillis = { nowMillis }, + recoverySignal = recoverySignal, + ) + } + + assertTrue(actual === expected) + assertEquals(2, recoveryRuns) + assertEquals(listOf(60_000L), waits) + } + + @Test + fun `request crossing wakeup consumption is claimed without a stale token`() = runBlocking { + val wakeupConsumed = CompletableDeferred() + val releaseBatchClaim = CompletableDeferred() + val recoverySignal = AndroidDurableUploadSchedulingRecoverySignal { + wakeupConsumed.complete(Unit) + releaseBatchClaim.await() + } + val workId = UUID.randomUUID() + recoverySignal.requestAfterWorkStopsRunning("job-1", workId) + val firstBatch = async { recoverySignal.await() } + wakeupConsumed.await() + + recoverySignal.request() + releaseBatchClaim.complete(Unit) + + assertEquals( + AndroidDurableUploadSchedulingRecoveryBatch( + immediate = true, + workIdsToAwait = mapOf("job-1" to workId), + ), + firstBatch.await(), + ) + val nextBatch = async { recoverySignal.await() } + yield() + assertFalse(nextBatch.isCompleted) + nextBatch.cancel() + } + + @Test + fun `immediate recovery interrupts an unrelated worker follow up delay`() = runBlocking { + val recoverySignal = AndroidDurableUploadSchedulingRecoverySignal() + val workId = UUID.randomUUID() + val delayEntered = CompletableDeferred() + val expected = CancellationException("monitor stopped after immediate recovery") + var recoveryRuns = 0 + val scheduledJobIds = mutableListOf() + + recoverySignal.requestAfterWorkStopsRunning("job-1", workId) + val monitoring = async { + assertFailsWith { + monitorQueuedDurableUploadScheduling( + recover = { + recoveryRuns += 1 + recoverySignal.scheduleUnlessBackedOff("job-1") { + scheduledJobIds += "job-1" + } + recoverySignal.scheduleUnlessBackedOff("job-2") { + scheduledJobIds += "job-2" + } + if (recoveryRuns == 2) throw expected + true + }, + awaitWorkStopsRunning = { requestedWorkId -> + assertEquals(workId, requestedWorkId) + }, + wait = { + delayEntered.complete(Unit) + CompletableDeferred().await() + }, + recoverySignal = recoverySignal, + ) + } + } + + delayEntered.await() + recoverySignal.request() + + assertTrue(monitoring.await() === expected) + assertEquals(2, recoveryRuns) + assertEquals(listOf("job-2", "job-2"), scheduledJobIds) + } + + @Test + fun `persistent cleanup failure yields through immediate recovery to worker backoff expiry`() = runBlocking { + val recoverySignal = AndroidDurableUploadSchedulingRecoverySignal() + val workId = UUID.randomUUID() + val initialRetryWaitEntered = CompletableDeferred() + val backoffWaitEntered = CompletableDeferred() + val expected = CancellationException("monitor stopped after backed off upload recovered") + val scheduledJobIds = mutableListOf() + val waits = mutableListOf() + var recoveryRuns = 0 + var secondJobQueued = false + var nowMillis = 1_000L + + val monitoring = async { + assertFailsWith { + monitorQueuedDurableUploadScheduling( + recover = { + recoveryRuns += 1 + if (secondJobQueued) { + recoverySignal.scheduleUnlessBackedOff("job-2") { + scheduledJobIds += "job-2" + } + } + if (scheduledJobIds.isNotEmpty()) throw expected + false + }, + awaitWorkStopsRunning = { requestedWorkId -> assertEquals(workId, requestedWorkId) }, + wait = { delayMillis -> + waits += delayMillis + when (waits.size) { + 1 -> { + initialRetryWaitEntered.complete(Unit) + CompletableDeferred().await() + } + 2 -> { + backoffWaitEntered.complete(Unit) + CompletableDeferred().await() + } + else -> nowMillis += delayMillis + } + }, + workerFailureFollowUpDelayMillis = 60_000L, + monotonicTimeMillis = { nowMillis }, + recoverySignal = recoverySignal, + ) + } + } + + initialRetryWaitEntered.await() + secondJobQueued = true + recoverySignal.requestAfterWorkStopsRunning("job-2", workId) + backoffWaitEntered.await() + recoverySignal.request() + + assertTrue(monitoring.await() === expected) + assertEquals(3, recoveryRuns) + assertEquals(listOf(60_000L, 60_000L, 60_000L), waits) + assertEquals(listOf("job-2"), scheduledJobIds) + } + + @Test + fun `immediate recovery preserves the failed job deadline`() = runBlocking { + val recoverySignal = AndroidDurableUploadSchedulingRecoverySignal() + val workId = UUID.randomUUID() + val firstDelayEntered = CompletableDeferred() + val expected = CancellationException("monitor stopped after failed job recovery") + val waits = mutableListOf() + val scheduledJobIds = mutableListOf() + var recoveryRuns = 0 + var nowMillis = 1_000L + + recoverySignal.requestAfterWorkStopsRunning("job-1", workId) + val monitoring = async { + assertFailsWith { + monitorQueuedDurableUploadScheduling( + recover = { + recoveryRuns += 1 + recoverySignal.scheduleUnlessBackedOff("job-1") { + scheduledJobIds += "job-1" + } + if (recoveryRuns == 3) throw expected + true + }, + awaitWorkStopsRunning = { requestedWorkId -> + assertEquals(workId, requestedWorkId) + }, + wait = { delayMillis -> + waits += delayMillis + if (waits.size == 1) { + firstDelayEntered.complete(Unit) + CompletableDeferred().await() + } + }, + workerFailureFollowUpDelayMillis = 60_000L, + monotonicTimeMillis = { nowMillis }, + recoverySignal = recoverySignal, + ) + } + } + + firstDelayEntered.await() + nowMillis += 25_000L + recoverySignal.request() + + assertTrue(monitoring.await() === expected) + assertEquals(listOf(60_000L, 35_000L), waits) + assertEquals(listOf("job-1"), scheduledJobIds) + } + + @Test + fun `replacement work resets and coalesces the failed job deadline`() = runBlocking { + val recoverySignal = AndroidDurableUploadSchedulingRecoverySignal() + val replacedWorkId = UUID.randomUUID() + val supersededWorkId = UUID.randomUUID() + val replacementWorkId = UUID.randomUUID() + val expected = CancellationException("monitor stopped after replacement recovery") + val awaitedWorkIds = mutableListOf() + val waits = mutableListOf() + val scheduledJobIds = mutableListOf() + var recoveryRuns = 0 + + recoverySignal.requestAfterWorkStopsRunning("job-1", replacedWorkId) + val actual = assertFailsWith { + monitorQueuedDurableUploadScheduling( + recover = { + recoveryRuns += 1 + recoverySignal.scheduleUnlessBackedOff("job-1") { + scheduledJobIds += "job-1" + } + if (recoveryRuns == 2) throw expected + true + }, + awaitWorkStopsRunning = { workId -> awaitedWorkIds += workId }, + wait = { delayMillis -> + waits += delayMillis + if (waits.size == 1) { + recoverySignal.requestAfterWorkStopsRunning("job-1", supersededWorkId) + repeat(100) { + recoverySignal.requestAfterWorkStopsRunning("job-1", replacementWorkId) + } + } + }, + workerFailureFollowUpDelayMillis = 60_000L, + monotonicTimeMillis = { 1_000L }, + recoverySignal = recoverySignal, + ) + } + + assertTrue(actual === expected) + assertEquals(listOf(replacedWorkId, replacementWorkId), awaitedWorkIds) + assertEquals(listOf(60_000L, 60_000L), waits) + assertEquals(listOf("job-1"), scheduledJobIds) + } + + @Test + fun `immediate reconciliation skips backed off upload and schedules unrelated work`() = runBlocking { + val recoverySignal = AndroidDurableUploadSchedulingRecoverySignal() + val backedOff = fixtureQueuedJob(index = 1) + val unrelated = fixtureQueuedJob(index = 2) + val attempted = mutableListOf() + recoverySignal.requestAfterWorkStopsRunning(backedOff.id, UUID.randomUUID()) + + val allScheduled = reconcileQueuedDurableUploads( + jobs = listOf(backedOff, unrelated), + cleanupCapability = { error("Queued uploads must not enter local cleanup.") }, + schedule = { job -> + recoverySignal.scheduleUnlessBackedOff(job.id) { attempted += job.id } + }, + ) + + assertTrue(allScheduled) + assertEquals(listOf(unrelated.id), attempted) + } + + @Test + fun `backoff exclusion does not delay terminal capability cleanup`() = runBlocking { + val recoverySignal = AndroidDurableUploadSchedulingRecoverySignal() + val terminalCleanup = fixtureQueuedJob(index = 1).copy( + state = DurableUploadState.Failed, + capabilityCleanupPending = true, + ) + var cleaned = false + recoverySignal.requestAfterWorkStopsRunning(terminalCleanup.id, UUID.randomUUID()) + + val allScheduled = reconcileQueuedDurableUploads( + jobs = listOf(terminalCleanup), + cleanupCapability = { cleaned = true }, + schedule = { job -> + recoverySignal.scheduleUnlessBackedOff(job.id) { + error("Terminal cleanup must not schedule upload work.") + } + }, + ) + + assertTrue(allScheduled) + assertTrue(cleaned) + } + + @Test + fun `replacement request in the post drain gap remains excluded`() = runBlocking { + val recoverySignal = AndroidDurableUploadSchedulingRecoverySignal() + val replacedWorkId = UUID.randomUUID() + val replacementWorkId = UUID.randomUUID() + val expected = CancellationException("monitor stopped after replacement recovery") + val awaitedWorkIds = mutableListOf() + val scheduledJobIds = mutableListOf() + var recoveryRuns = 0 + var gapRuns = 0 + + recoverySignal.requestAfterWorkStopsRunning("job-1", replacedWorkId) + val actual = assertFailsWith { + monitorQueuedDurableUploadScheduling( + recover = { + recoveryRuns += 1 + recoverySignal.scheduleUnlessBackedOff("job-1") { + scheduledJobIds += "job-1" + } + if (recoveryRuns == 3) throw expected + true + }, + awaitWorkStopsRunning = { workId -> awaitedWorkIds += workId }, + wait = {}, + workerFailureFollowUpDelayMillis = 60_000L, + monotonicTimeMillis = { 1_000L }, + afterEmptyPendingBatchClaim = { + if (gapRuns++ == 0) { + recoverySignal.requestAfterWorkStopsRunning("job-1", replacementWorkId) + } + }, + recoverySignal = recoverySignal, + ) + } + + assertTrue(actual === expected) + assertEquals(listOf(replacedWorkId, replacementWorkId), awaitedWorkIds) + assertEquals(listOf("job-1"), scheduledJobIds) + } + + @Test + fun `same work request in the post drain gap starts a fresh backoff`() = runBlocking { + val recoverySignal = AndroidDurableUploadSchedulingRecoverySignal() + val workId = UUID.randomUUID() + val expected = CancellationException("monitor stopped after repeated work recovery") + val awaitedWorkIds = mutableListOf() + val waits = mutableListOf() + val scheduledJobIds = mutableListOf() + var recoveryRuns = 0 + var gapRuns = 0 + + recoverySignal.requestAfterWorkStopsRunning("job-1", workId) + val actual = assertFailsWith { + monitorQueuedDurableUploadScheduling( + recover = { + recoveryRuns += 1 + recoverySignal.scheduleUnlessBackedOff("job-1") { + scheduledJobIds += "job-1" + } + if (recoveryRuns == 3) throw expected + true + }, + awaitWorkStopsRunning = { requestedWorkId -> awaitedWorkIds += requestedWorkId }, + wait = { delayMillis -> waits += delayMillis }, + workerFailureFollowUpDelayMillis = 60_000L, + monotonicTimeMillis = { 1_000L }, + afterEmptyPendingBatchClaim = { + if (gapRuns++ == 0) { + recoverySignal.requestAfterWorkStopsRunning("job-1", workId) + } + }, + recoverySignal = recoverySignal, + ) + } + + assertTrue(actual === expected) + assertEquals(listOf(workId, workId), awaitedWorkIds) + assertEquals(listOf(60_000L, 60_000L), waits) + assertEquals(listOf("job-1"), scheduledJobIds) + } + + @Test + fun `coalesced failed jobs age through one follow up interval`() = runBlocking { + val recoverySignal = AndroidDurableUploadSchedulingRecoverySignal() + val firstWorkId = UUID.randomUUID() + val secondWorkId = UUID.randomUUID() + val expected = CancellationException("monitor stopped after both jobs recovered") + val awaitedWorkIds = mutableListOf() + val waits = mutableListOf() + var recoveryRuns = 0 + var nowMillis = 1_000L + + recoverySignal.requestAfterWorkStopsRunning("job-1", firstWorkId) + recoverySignal.requestAfterWorkStopsRunning("job-2", secondWorkId) + val actual = assertFailsWith { + monitorQueuedDurableUploadScheduling( + recover = { + recoveryRuns += 1 + if (recoveryRuns == 3) throw expected + true + }, + awaitWorkStopsRunning = { workId -> awaitedWorkIds += workId }, + wait = { delayMillis -> + waits += delayMillis + nowMillis += delayMillis + }, + workerFailureFollowUpDelayMillis = 60_000L, + monotonicTimeMillis = { nowMillis }, + recoverySignal = recoverySignal, + ) + } + + assertTrue(actual === expected) + assertEquals(listOf(firstWorkId, secondWorkId), awaitedWorkIds) + assertEquals(listOf(60_000L), waits) + } + + private fun fixtureQueuedJob(index: Int): AndroidDurableMultipartUploadJob { + val scope = DurableUploadScope("deck-attachment", index.toString()) + val request = NextcloudMultipartUploadRequest( + method = NextcloudApiMethod.POST, + relativePath = "/index.php/apps/deck/api/v1.1/boards/7/stacks/11/cards/$index/attachments", + file = localUploadFile( + selectionId = "selection-${index.toString().padStart(16, '0')}", + displayName = "fixture-$index.txt", + mimeType = "text/plain", + sizeBytes = 16L, + ), + maximumFileBytes = 1_024L, + ) + return AndroidDurableMultipartUploadJob( + id = "upload-${index.toString().padStart(16, '0')}", + accountId = index.toString(16).padStart(32, '0'), + scope = scope, + resource = resolveDurableUploadResource(scope, request), + request = request, + state = DurableUploadState.Queued, + message = null, + updatedAtEpochMillis = index.toLong(), + ) + } +} diff --git a/androidApp/src/test/kotlin/dev/obiente/nextcloudnative/AndroidDurableUploadSourcePreflightTest.kt b/androidApp/src/test/kotlin/dev/obiente/nextcloudnative/AndroidDurableUploadSourcePreflightTest.kt new file mode 100644 index 000000000..83e46126f --- /dev/null +++ b/androidApp/src/test/kotlin/dev/obiente/nextcloudnative/AndroidDurableUploadSourcePreflightTest.kt @@ -0,0 +1,313 @@ +package dev.obiente.nextcloudnative + +import java.io.FileNotFoundException +import java.io.IOException +import java.security.GeneralSecurityException +import javax.crypto.AEADBadTagException +import kotlinx.coroutines.CancellationException +import kotlinx.coroutines.runBlocking +import kotlin.test.Test +import kotlin.test.assertEquals +import kotlin.test.assertFailsWith +import kotlin.test.assertFalse +import kotlin.test.assertTrue + +class AndroidDurableUploadSourcePreflightTest { + @Test + fun `oversized capability storage terminally fails without opening the provider`() = runBlocking { + var providerOpened = false + var transientRetries = 0 + + val result = processQueuedDurableUploadSource( + requireCapability = { + readAndroidLocalUploadCapability { throw DurableUploadCapabilityOverflowException() } + }, + openSource = { providerOpened = true }, + onCapabilityUnavailable = { "failed" }, + onProviderUnavailable = { + transientRetries += 1 + "retried" + }, + onReady = { "started" }, + ) + + assertEquals("failed", result) + assertFalse(providerOpened) + assertEquals(0, transientRetries) + } + + @Test + fun `missing or mismatched private metadata terminally fails and releases`() = runBlocking { + listOf("missing", "mismatched").forEach { reason -> + var providerOpened = false + var queued = true + var retained = true + + val result = processQueuedDurableUploadSource( + requireCapability = { + throw AndroidLocalUploadCapabilityUnavailableException(reason) + }, + openSource = { providerOpened = true }, + onCapabilityUnavailable = { + queued = false + retained = false + "failed" + }, + onProviderUnavailable = { "retried" }, + onReady = { "started" }, + ) + + assertEquals("failed", result) + assertFalse(providerOpened) + assertFalse(queued) + assertFalse(retained) + } + } + + @Test + fun `permanently unavailable provider source terminally fails and releases`() = runBlocking { + listOf( + FileNotFoundException("document removed"), + SecurityException("grant revoked"), + ).forEach { failure -> + var queued = true + var retained = true + var starts = 0 + + val result = processQueuedDurableUploadSource( + requireCapability = { }, + openSource = { throw failure }, + onCapabilityUnavailable = { + queued = false + retained = false + "failed" + }, + onProviderUnavailable = { "retried" }, + onReady = { + starts += 1 + "started" + }, + ) + + assertEquals("failed", result) + assertFalse(queued) + assertFalse(retained) + assertEquals(0, starts) + } + } + + @Test + fun `transient provider failure leaves queued capability retained`() = runBlocking { + var queued = true + var retained = true + var starts = 0 + + val result = processQueuedDurableUploadSource( + requireCapability = { }, + openSource = { throw IOException("provider unavailable") }, + onCapabilityUnavailable = { + queued = false + retained = false + "failed" + }, + onProviderUnavailable = { "retried" }, + onReady = { + starts += 1 + "started" + }, + ) + + assertEquals("retried", result) + assertTrue(queued) + assertTrue(retained) + assertEquals(0, starts) + } + + @Test + fun `transient capability metadata failure leaves queued capability retained`() = runBlocking { + var queued = true + var retained = true + var starts = 0 + + val result = processQueuedDurableUploadSource( + requireCapability = { + throw AndroidLocalUploadCapabilityReadException( + "credential store unavailable", + IOException("keystore restarting"), + ) + }, + openSource = { starts += 1 }, + onCapabilityUnavailable = { + queued = false + retained = false + "failed" + }, + onProviderUnavailable = { "retried" }, + onReady = { + starts += 1 + "started" + }, + ) + + assertEquals("retried", result) + assertTrue(queued) + assertTrue(retained) + assertEquals(0, starts) + } + + @Test + fun `pending ownership check defers the worker without releasing its capability`() = runBlocking { + var terminalDispositions = 0 + var transientRetries = 0 + + val result = processQueuedDurableUploadSource( + requireCapability = { + requireDurableUploadCapabilityReady(CapabilityPhase.OwnershipCheckPending) + }, + openSource = { error("A pending capability must not open its provider.") }, + onCapabilityUnavailable = { + terminalDispositions += 1 + "failed" + }, + onProviderUnavailable = { + transientRetries += 1 + "retried" + }, + onReady = { error("A pending capability must not start an upload.") }, + ) + + assertEquals("retried", result) + assertEquals(0, terminalDispositions) + assertEquals(1, transientRetries) + } + + @Test + fun `malformed capability metadata is terminal while storage failures stay retryable`() { + assertFailsWith { + readAndroidLocalUploadCapability { + throw AndroidLocalUploadCapabilityMalformedException("invalid JSON") + } + } + assertFailsWith { + readAndroidLocalUploadCapability { + readAndroidLocalUploadCapabilityPreference { + throw ClassCastException("not a string") + } + } + } + assertFailsWith { + readAndroidLocalUploadCapability { + readAndroidLocalUploadCapabilityPreference { + throw IOException("preferences unavailable") + } + } + } + assertFailsWith { + readAndroidLocalUploadCapability { + decryptAndroidLocalUploadCapability { + throw GeneralSecurityException("keystore temporarily unavailable") + } + } + } + } + + @Test + fun `invalid encrypted capability envelope and authentication terminally fail the job`() = runBlocking { + listOf( + InvalidSessionCiphertextException( + "invalid base64 envelope", + IllegalArgumentException("bad base64"), + ), + InvalidSessionCiphertextException( + "authentication failed", + AEADBadTagException("bad tag"), + ), + ).forEach { failure -> + var queued = true + var terminalDispositions = 0 + var transientRetries = 0 + val result = processQueuedDurableUploadSource( + requireCapability = { + readAndroidLocalUploadCapability { + decryptAndroidLocalUploadCapability { throw failure } + } + }, + openSource = { error("Corrupt capability metadata must not open the provider.") }, + onCapabilityUnavailable = { + queued = false + terminalDispositions += 1 + "failed" + }, + onProviderUnavailable = { + transientRetries += 1 + "retried" + }, + onReady = { error("Corrupt capability metadata must not start an upload.") }, + ) + + assertEquals("failed", result) + assertFalse(queued) + assertEquals(1, terminalDispositions) + assertEquals(0, transientRetries) + } + } + + @Test + fun `later provider success starts exactly once`() = runBlocking { + var providerAttempts = 0 + var starts = 0 + + suspend fun attempt(): String = processQueuedDurableUploadSource( + requireCapability = { }, + openSource = { + providerAttempts += 1 + if (providerAttempts == 1) throw IOException("provider restarting") + }, + onCapabilityUnavailable = { "failed" }, + onProviderUnavailable = { "retried" }, + onReady = { + starts += 1 + "started" + }, + ) + + assertEquals("retried", attempt()) + assertEquals(0, starts) + assertEquals("started", attempt()) + assertEquals(1, starts) + } + + @Test + fun `cancellation is preserved without running a disposition`() = runBlocking { + listOf(true, false).forEach { cancelDuringCapabilityRead -> + var dispositions = 0 + val expected = CancellationException("worker stopped") + + val actual = assertFailsWith { + processQueuedDurableUploadSource( + requireCapability = { + if (cancelDuringCapabilityRead) throw expected + }, + openSource = { + if (!cancelDuringCapabilityRead) throw expected + }, + onCapabilityUnavailable = { + dispositions += 1 + Unit + }, + onProviderUnavailable = { + dispositions += 1 + Unit + }, + onReady = { + dispositions += 1 + Unit + }, + ) + } + + assertTrue(actual === expected) + assertEquals(0, dispositions) + } + } +} diff --git a/androidApp/src/test/kotlin/dev/obiente/nextcloudnative/AndroidDurableUploadTerminalCapabilityTest.kt b/androidApp/src/test/kotlin/dev/obiente/nextcloudnative/AndroidDurableUploadTerminalCapabilityTest.kt new file mode 100644 index 000000000..bb7ef5b98 --- /dev/null +++ b/androidApp/src/test/kotlin/dev/obiente/nextcloudnative/AndroidDurableUploadTerminalCapabilityTest.kt @@ -0,0 +1,105 @@ +package dev.obiente.nextcloudnative + +import kotlin.test.Test +import kotlin.test.assertEquals +import kotlin.test.assertFalse +import kotlin.test.assertTrue + +class AndroidDurableUploadTerminalCapabilityTest { + @Test + fun `terminal worker retries while capability cleanup remains uncommitted`() { + var releaseAttempts = 0 + var recoveryRequests = 0 + + val result = resultAfterDurableUploadCapabilityReleaseOrQuarantine( + releaseCapability = { + releaseAttempts += 1 + false + }, + onCleanupRetained = { recoveryRequests += 1 }, + releasedResult = "finished", + retainedResult = "retry", + ) + + assertEquals("retry", result) + assertEquals(1, releaseAttempts) + assertEquals(1, recoveryRequests) + } + + @Test + fun `terminal worker finishes after capability cleanup commits`() { + var releaseAttempts = 0 + + val result = resultAfterDurableUploadCapabilityRelease( + releaseCapability = { + releaseAttempts += 1 + true + }, + releasedResult = "finished", + retainedResult = "retry", + ) + + assertEquals("finished", result) + assertEquals(1, releaseAttempts) + } + + @Test + fun `terminal worker finishes and commits cleanup after capability quarantine`() { + val events = mutableListOf() + + val result = resultAfterDurableUploadCapabilityReleaseOrQuarantine( + releaseCapability = { onQuarantined -> + events += "quarantine" + onQuarantined() + false + }, + completeCapabilityCleanup = { events += "complete" }, + onCleanupRetained = { events += "retry" }, + releasedResult = "finished", + retainedResult = "retry", + ) + + assertEquals("finished", result) + assertEquals(listOf("quarantine", "complete"), events) + } + + @Test + fun `terminal status dismissal accepts quarantine but not transient cleanup failure`() { + val events = mutableListOf() + assertTrue( + dismissTerminalDurableUploadStatus( + release = { onQuarantined -> + onQuarantined() + false + }, + removeStatus = { events += "remove" }, + ), + ) + assertFalse( + dismissTerminalDurableUploadStatus( + release = { false }, + removeStatus = { events += "unexpected" }, + ), + ) + assertEquals(listOf("remove"), events) + } + + @Test + fun `removed account retries after terminal transition when release is retained`() { + val events = mutableListOf() + + val result = failQueuedDurableUploadForUnavailableAccount( + transitionToFailed = { events += "fail" }, + releaseSelection = { _ -> + events += "release" + false + }, + recordFailure = { events += "diagnose" }, + failureResult = "failed", + retryResult = "retry", + ) + + assertEquals("retry", result) + assertEquals(listOf("fail", "release", "diagnose"), events) + } +} diff --git a/androidApp/src/test/kotlin/dev/obiente/nextcloudnative/AndroidLocalUploadCapabilityLifecycleTest.kt b/androidApp/src/test/kotlin/dev/obiente/nextcloudnative/AndroidLocalUploadCapabilityLifecycleTest.kt index 47d08df13..cee9137e7 100644 --- a/androidApp/src/test/kotlin/dev/obiente/nextcloudnative/AndroidLocalUploadCapabilityLifecycleTest.kt +++ b/androidApp/src/test/kotlin/dev/obiente/nextcloudnative/AndroidLocalUploadCapabilityLifecycleTest.kt @@ -1,5 +1,18 @@ package dev.obiente.nextcloudnative +import dev.obiente.nextcloudnative.app.LocalUploadFile +import dev.obiente.nextcloudnative.app.LocalUploadSelectionResult +import dev.obiente.nextcloudnative.app.localUploadFile +import java.security.GeneralSecurityException +import kotlinx.coroutines.CancellationException +import kotlinx.coroutines.CoroutineDispatcher +import kotlinx.coroutines.CoroutineScope +import kotlinx.coroutines.CoroutineStart +import kotlinx.coroutines.Job +import kotlinx.coroutines.launch +import kotlinx.coroutines.suspendCancellableCoroutine +import org.json.JSONObject +import kotlin.coroutines.CoroutineContext import kotlin.test.Test import kotlin.test.assertEquals import kotlin.test.assertFailsWith @@ -7,6 +20,80 @@ import kotlin.test.assertFalse import kotlin.test.assertTrue class AndroidLocalUploadCapabilityLifecycleTest { + @Test + fun `selected capability is released when cancellation wins result delivery`() { + val file = localUploadFile( + selectionId = "selection-1234567890", + displayName = "cancelled.txt", + mimeType = "text/plain", + sizeBytes = 12L, + ) + val dispatcher = PausedDispatcher() + var resumeSelection: ((LocalUploadSelectionResult) -> Unit)? = null + var delivered = false + var persistedCapability: LocalUploadFile? = file + var cachedCapability: LocalUploadFile? = file + val scopeJob = Job() + val selectionJob = CoroutineScope(scopeJob + dispatcher).launch(start = CoroutineStart.UNDISPATCHED) { + suspendCancellableCoroutine { continuation -> + resumeSelection = { result -> + resumeLocalUploadSelectionResult( + continuation = continuation, + result = result, + releaseSelected = { cancelledFile -> + assertEquals(cachedCapability, cancelledFile) + persistedCapability = null + cachedCapability = null + }, + ) + } + } + delivered = true + } + + checkNotNull(resumeSelection)(LocalUploadSelectionResult.Selected(file)) + selectionJob.cancel() + dispatcher.runAll() + + assertTrue(selectionJob.isCancelled) + assertFalse(delivered) + assertEquals(null, persistedCapability) + assertEquals(null, cachedCapability) + scopeJob.cancel() + } + + @Test + fun `non-selected results do not request capability cleanup when delivery is cancelled`() { + listOf( + LocalUploadSelectionResult.Cancelled, + LocalUploadSelectionResult.Rejected("synthetic rejection"), + ).forEach { result -> + val dispatcher = PausedDispatcher() + var resumeSelection: (() -> Unit)? = null + var releases = 0 + val scopeJob = Job() + val selectionJob = CoroutineScope(scopeJob + dispatcher).launch(start = CoroutineStart.UNDISPATCHED) { + suspendCancellableCoroutine { continuation -> + resumeSelection = { + resumeLocalUploadSelectionResult( + continuation = continuation, + result = result, + releaseSelected = { releases += 1 }, + ) + } + } + } + + checkNotNull(resumeSelection).invoke() + selectionJob.cancel() + dispatcher.runAll() + + assertTrue(selectionJob.isCancelled) + assertEquals(0, releases) + scopeJob.cancel() + } + } + @Test fun `permission is taken before metadata commit and retained after success`() { val events = mutableListOf() @@ -69,4 +156,671 @@ class AndroidLocalUploadCapabilityLifecycleTest { assertFalse(released) assertTrue(permissionReleased) } + + @Test + fun `release exception with exact read grant present retains metadata`() { + var metadataPresent = true + + val released = releaseDurableUploadCapability( + releasePermission = { error("provider failure") }, + isPermissionAbsent = { false }, + removeMetadata = { true.also { metadataPresent = false } }, + ) + + assertFalse(released) + assertTrue(metadataPresent) + } + + @Test + fun `release exception with exact read grant absent deletes metadata`() { + var metadataPresent = true + + val released = releaseDurableUploadCapability( + releasePermission = { error("grant already absent") }, + isPermissionAbsent = { true }, + removeMetadata = { true.also { metadataPresent = false } }, + ) + + assertTrue(released) + assertFalse(metadataPresent) + } + + @Test + fun `release verification failure retains metadata`() { + var metadataPresent = true + + val released = releaseDurableUploadCapability( + releasePermission = { error("provider failure") }, + isPermissionAbsent = { error("permission query failed") }, + removeMetadata = { true.also { metadataPresent = false } }, + ) + + assertFalse(released) + assertTrue(metadataPresent) + } + + @Test + fun `uncached unreadable encrypted metadata is retained without claiming capability release`() { + var encryptedMetadata: String? = "unreadable-encrypted-capability" + var permissionReleased = false + var cleanupPending = true + + val result = resultAfterDurableUploadCapabilityRelease( + releaseCapability = { + releaseStoredDurableUploadCapability( + cachedCapability = null, + loadCapability = { throw GeneralSecurityException("synthetic decryption failure") }, + releasePermission = { permissionReleased = true }, + removeMetadata = { true.also { encryptedMetadata = null } }, + ) + }, + completeCapabilityCleanup = { cleanupPending = false }, + releasedResult = "finished", + retainedResult = "retry", + ) + + assertEquals("retry", result) + assertTrue(cleanupPending) + assertFalse(permissionReleased) + assertEquals("unreadable-encrypted-capability", encryptedMetadata) + } + + @Test + fun `restored capability release revokes permission before deleting metadata`() { + val events = mutableListOf() + + val released = releaseStoredDurableUploadCapability( + cachedCapability = null, + loadCapability = { + events += "restore" + "content://synthetic/upload" + }, + releasePermission = { events += "permission:$it" }, + removeMetadata = { + events += "metadata" + true + }, + ) + + assertTrue(released) + assertEquals( + listOf("restore", "permission:content://synthetic/upload", "metadata"), + events, + ) + } + + @Test + fun `missing capability metadata is an idempotent cleanup success`() { + var metadataRemovals = 0 + + val released = releaseStoredDurableUploadCapability( + cachedCapability = null, + loadCapability = { null }, + releasePermission = { error("Missing metadata has no URI grant to release.") }, + removeMetadata = { + metadataRemovals += 1 + true + }, + ) + + assertTrue(released) + assertEquals(1, metadataRemovals) + } + + @Test + fun `cached capability releases without reading redundant stored metadata`() { + val events = mutableListOf() + + val released = releaseStoredDurableUploadCapability( + cachedCapability = "content://cached/upload", + loadCapability = { error("Cached cleanup must not read stored metadata.") }, + releasePermission = { events += "permission:$it" }, + removeMetadata = { + events += "metadata" + true + }, + ) + + assertTrue(released) + assertEquals( + listOf("permission:content://cached/upload", "metadata"), + events, + ) + } + + @Test + fun `shared uri cleanup deletes only current capability metadata`() { + val events = mutableListOf() + + val released = releaseStoredDurableUploadCapability( + cachedCapability = "content://shared/upload", + loadCapability = { error("cache is authoritative") }, + otherCapabilityOwnsPermission = { true }, + releasePermission = { events += "permission" }, + isPermissionAbsent = { error("shared grant must remain") }, + removeMetadata = { + events += "metadata" + true + }, + ) + + assertTrue(released) + assertEquals(listOf("metadata"), events) + } + + @Test + fun `unreadable shared uri ownership retains current capability`() { + var metadataPresent = true + + val released = releaseStoredDurableUploadCapability( + cachedCapability = "content://shared/upload", + loadCapability = { error("cache is authoritative") }, + otherCapabilityOwnsPermission = { error("another capability is unreadable") }, + releasePermission = { error("ownership must be known before release") }, + isPermissionAbsent = { false }, + removeMetadata = { true.also { metadataPresent = false } }, + ) + + assertFalse(released) + assertTrue(metadataPresent) + } + + @Test + fun `cached duplicate selection owns shared uri without reading redundant storage`() { + var storedLoads = 0 + + val capabilities = mergeDurableUploadCapabilities( + cachedCapabilities = mapOf("selection-cached" to "content://shared/upload"), + storedSelectionIds = listOf("selection-cached"), + loadStoredCapability = { + storedLoads += 1 + error("cached capability must be authoritative") + }, + ) + + assertEquals("content://shared/upload", capabilities["selection-cached"]) + assertTrue( + durableUploadCapabilityPermissionOwnedByAnother( + capabilities = capabilities, + targetSelectionId = "selection-target", + targetPermission = "content://shared/upload", + permissionOf = { capability -> capability }, + samePermission = String::equals, + ), + ) + assertEquals(0, storedLoads) + } + + @Test + fun `persisted duplicate selection owns shared uri`() { + val capabilities = mergeDurableUploadCapabilities( + cachedCapabilities = emptyMap(), + storedSelectionIds = listOf("selection-persisted"), + loadStoredCapability = { "content://shared/upload" }, + ) + + assertEquals("content://shared/upload", capabilities["selection-persisted"]) + assertTrue( + durableUploadCapabilityPermissionOwnedByAnother( + capabilities = capabilities, + targetSelectionId = "selection-target", + targetPermission = "content://shared/upload", + permissionOf = { capability -> capability }, + samePermission = String::equals, + ), + ) + } + + @Test + fun `malformed persisted peer does not hide valid capabilities or block a new selection`() { + val snapshot = loadDurableUploadCapabilitySnapshot( + cachedCapabilities = emptyMap(), + storedSelectionIds = listOf("selection-malformed", "selection-valid"), + loadStoredCapability = { selectionId -> + if (selectionId == "selection-malformed") { + throw AndroidLocalUploadCapabilityMalformedException( + message = "invalid phase", + cleanupPermissionIdentity = "content://synthetic/malformed", + grantPreExisting = false, + ) + } + "content://synthetic/valid" + }, + ) + + assertEquals(mapOf("selection-valid" to "content://synthetic/valid"), snapshot.capabilities) + assertEquals(setOf("selection-malformed"), snapshot.malformedCapabilities.keys) + assertEquals(2, snapshot.trackedCapabilityCount) + assertTrue( + durableUploadCapabilityHasCapacity( + trackedCapabilityCount = snapshot.trackedCapabilityCount, + maximumTrackedCapabilities = 64, + ), + ) + assertEquals( + "content://synthetic/new", + (snapshot.capabilities + ("selection-new" to "content://synthetic/new"))["selection-new"], + ) + assertFalse( + malformedDurableUploadCapabilityPermissionOwnedByAnother( + capabilities = snapshot.malformedCapabilities, + targetSelectionId = "selection-new", + targetPermission = "content://synthetic/new", + permissionOf = MalformedDurableUploadCapability::cleanupPermissionIdentity, + samePermission = String::equals, + ), + ) + assertTrue( + malformedDurableUploadCapabilityPermissionOwnedByAnother( + capabilities = snapshot.malformedCapabilities, + targetSelectionId = "selection-new", + targetPermission = "content://synthetic/malformed", + permissionOf = MalformedDurableUploadCapability::cleanupPermissionIdentity, + samePermission = String::equals, + ), + ) + } + + @Test + fun `snapshot isolation preserves transient capability read failures`() { + assertFailsWith { + loadDurableUploadCapabilitySnapshot( + cachedCapabilities = emptyMap(), + storedSelectionIds = listOf("selection-unreadable"), + loadStoredCapability = { throw GeneralSecurityException("synthetic decryption failure") }, + ) + } + } + + @Test + fun `corrupt ciphertext remains durable when grant ownership cannot be reconstructed`() { + var encryptedMetadataPresent = true + val snapshot = loadDurableUploadCapabilitySnapshot( + cachedCapabilities = emptyMap(), + storedSelectionIds = listOf("selection-corrupt"), + loadStoredCapability = { + decryptAndroidLocalUploadCapability { + throw InvalidSessionCiphertextException("authentication failed") + } + }, + ) + val corrupt = snapshot.malformedCapabilities.getValue("selection-corrupt") + + val recovered = recoverMalformedDurableUploadCapability( + capability = corrupt, + permission = null, + peerProtection = DurableUploadPermissionPeerProtection.Ambiguous, + releasePermission = { error("Unknown permission must not be released.") }, + isPermissionAbsent = { error("Unknown permission cannot be queried.") }, + removeMetadata = { true.also { encryptedMetadataPresent = false } }, + ) + + assertEquals(null, corrupt.cleanupPermissionIdentity) + assertEquals(null, corrupt.grantPreExisting) + assertFalse(recovered) + assertTrue(encryptedMetadataPresent) + } + + @Test + fun `startup recovery releases a malformed app owned capability before deleting its row`() { + val events = mutableListOf() + val capability = MalformedDurableUploadCapability( + selectionId = "selection-malformed", + cleanupPermissionIdentity = "content://synthetic/malformed", + grantPreExisting = false, + ) + + val recovered = recoverMalformedDurableUploadCapability( + capability = capability, + permission = checkNotNull(capability.cleanupPermissionIdentity), + peerProtection = DurableUploadPermissionPeerProtection.None, + releasePermission = { events += "release:$it" }, + isPermissionAbsent = { false }, + removeMetadata = { + events += "remove:$it" + true + }, + ) + + assertTrue(recovered) + assertEquals( + listOf( + "release:content://synthetic/malformed", + "remove:selection-malformed", + ), + events, + ) + } + + @Test + fun `malformed capability with unknown grant ownership remains durable while permission exists`() { + var metadataPresent = true + + val recovered = recoverMalformedDurableUploadCapability( + capability = MalformedDurableUploadCapability( + selectionId = "selection-malformed", + cleanupPermissionIdentity = "content://synthetic/malformed", + grantPreExisting = null, + ), + permission = "content://synthetic/malformed", + peerProtection = DurableUploadPermissionPeerProtection.None, + releasePermission = { error("unknown ownership must not release") }, + isPermissionAbsent = { false }, + removeMetadata = { true.also { metadataPresent = false } }, + ) + + assertFalse(recovered) + assertTrue(metadataPresent) + } + + @Test + fun `unreadable persisted duplicate ownership fails closed`() { + assertFailsWith { + mergeDurableUploadCapabilities( + cachedCapabilities = emptyMap(), + storedSelectionIds = listOf("selection-unreadable"), + loadStoredCapability = { throw GeneralSecurityException("synthetic decryption failure") }, + ) + } + } + + @Test + fun `failed acquisition rollback retains capability record for recovery`() { + var acquiringTracked = false + var capabilityClears = 0 + var recoveryRequests = 0 + + assertFailsWith { + acquireDurableUploadCapability( + persistAcquiring = { true.also { acquiringTracked = true } }, + takePermission = {}, + persistMetadata = { false }, + releasePermission = { error("provider failure") }, + isPermissionAbsent = { false }, + removeCapability = { true.also { capabilityClears += 1 } }, + onRollbackRetained = { recoveryRequests += 1 }, + ) + } + + assertTrue(acquiringTracked) + assertEquals(0, capabilityClears) + assertEquals(1, recoveryRequests) + } + + @Test + fun `ambiguous permission acquisition retains capability record for recovery`() { + val expected = IllegalStateException("binder failure") + var capabilityClears = 0 + var recoveryRequests = 0 + + val actual = assertFailsWith { + acquireDurableUploadCapability( + persistAcquiring = { true }, + takePermission = { throw expected }, + persistMetadata = { error("metadata must not be written") }, + releasePermission = { error("ambiguous acquisition is reconciled later") }, + removeCapability = { true.also { capabilityClears += 1 } }, + onRollbackRetained = { recoveryRequests += 1 }, + ) + } + + assertTrue(actual === expected) + assertEquals(0, capabilityClears) + assertEquals(1, recoveryRequests) + } + + @Test + fun `failed ready persistence cleans possibly written capability after grant release`() { + val events = mutableListOf() + var persistedPhase: CapabilityPhase? = null + + assertFailsWith { + acquireDurableUploadCapability( + persistAcquiring = { + events += "acquiring" + true.also { persistedPhase = CapabilityPhase.Acquiring } + }, + takePermission = { events += "permission" }, + persistMetadata = { + events += "ready-false" + false.also { persistedPhase = CapabilityPhase.Ready } + }, + markCleanupPending = { + events += "cleanup-pending" + true.also { persistedPhase = CapabilityPhase.CleanupPending } + }, + releasePermission = { events += "release" }, + removeCapability = { + events += "metadata" + true.also { persistedPhase = null } + }, + ) + } + + assertEquals( + listOf("acquiring", "permission", "ready-false", "cleanup-pending", "release", "metadata"), + events, + ) + assertEquals(null, persistedPhase) + } + + @Test + fun `current ready record is retained unless cleanup was requested`() { + assertFalse( + shouldRecoverDurableUploadCapability( + phase = CapabilityPhase.Ready, + processGeneration = "current-generation", + currentProcessGeneration = "current-generation", + ownedByDurableJob = false, + cleanupExplicitlyPending = false, + ), + ) + assertTrue( + shouldRecoverDurableUploadCapability( + phase = CapabilityPhase.Ready, + processGeneration = "current-generation", + currentProcessGeneration = "current-generation", + ownedByDurableJob = false, + cleanupExplicitlyPending = true, + ), + ) + } + + @Test + fun `prior ready and cleanup phases recover unless durable job owns them`() { + assertTrue( + shouldRecoverDurableUploadCapability( + phase = CapabilityPhase.Ready, + processGeneration = "prior-generation", + currentProcessGeneration = "current-generation", + ownedByDurableJob = false, + cleanupExplicitlyPending = false, + ), + ) + assertTrue( + shouldRecoverDurableUploadCapability( + phase = CapabilityPhase.CleanupPending, + processGeneration = "current-generation", + currentProcessGeneration = "current-generation", + ownedByDurableJob = false, + cleanupExplicitlyPending = false, + ), + ) + assertFalse( + shouldRecoverDurableUploadCapability( + phase = CapabilityPhase.CleanupPending, + processGeneration = "prior-generation", + currentProcessGeneration = "current-generation", + ownedByDurableJob = true, + cleanupExplicitlyPending = false, + ), + ) + } + + @Test + fun `preexisting or shared grants are never revoked by capability cleanup`() { + assertFalse( + shouldReleaseDurableUploadPermission( + grantPreExisting = true, + ownedByAnotherCapability = false, + ), + ) + assertFalse( + shouldReleaseDurableUploadPermission( + grantPreExisting = false, + ownedByAnotherCapability = true, + ), + ) + assertTrue( + shouldReleaseDurableUploadPermission( + grantPreExisting = false, + ownedByAnotherCapability = false, + ), + ) + } + + @Test + fun `failed acquiring commit clears possible record before taking permission`() { + val events = mutableListOf() + var capabilityPresent = false + var recoveryRequests = 0 + + assertFailsWith { + acquireDurableUploadCapability( + persistAcquiring = { + events += "acquiring-false" + capabilityPresent = true + false + }, + takePermission = { events += "permission" }, + persistMetadata = { + events += "ready" + true + }, + releasePermission = { events += "release" }, + removeCapability = { + events += "metadata" + true.also { capabilityPresent = false } + }, + onRollbackRetained = { recoveryRequests += 1 }, + ) + } + + assertEquals(listOf("acquiring-false", "metadata"), events) + assertFalse(capabilityPresent) + assertEquals(1, recoveryRequests) + } + + @Test + fun `only ready capability phase may open or enqueue`() { + assertTrue(isDurableUploadCapabilityReady(CapabilityPhase.Ready)) + assertFalse(isDurableUploadCapabilityReady(CapabilityPhase.Acquiring)) + assertFalse(isDurableUploadCapabilityReady(CapabilityPhase.OwnershipCheckPending)) + assertFalse(isDurableUploadCapabilityReady(CapabilityPhase.CleanupPending)) + } + + @Test + fun `ownership check intent restores only after the durable job is found`() { + assertTrue( + shouldRestoreDurableUploadCapabilityAfterOwnershipCheck( + phase = CapabilityPhase.OwnershipCheckPending, + ownedByDurableJob = true, + ), + ) + assertFalse( + shouldRestoreDurableUploadCapabilityAfterOwnershipCheck( + phase = CapabilityPhase.OwnershipCheckPending, + ownedByDurableJob = false, + ), + ) + assertTrue( + shouldRecoverDurableUploadCapability( + phase = CapabilityPhase.OwnershipCheckPending, + processGeneration = "current-generation", + currentProcessGeneration = "current-generation", + ownedByDurableJob = false, + cleanupExplicitlyPending = false, + ), + ) + } + + @Test + fun `cancelled capability delivery publishes then cleans without leaking cleanup failure`() { + val events = mutableListOf() + + val delivered = finalizeDurableUploadCapabilityDelivery( + publishReady = { events += "ready" }, + continuationIsActive = { false }, + cleanupUndelivered = { + events += "cleanup" + error("synthetic retained cleanup") + }, + ) + + assertFalse(delivered) + assertEquals(listOf("ready", "cleanup"), events) + } + + @Test + fun `retained cleanup requests recovery and reports false`() { + var recoveryRequests = 0 + + val released = retainDurableUploadCapabilityCleanup { recoveryRequests += 1 } + + assertFalse(released) + assertEquals(1, recoveryRequests) + } + + @Test + fun `legacy nullable generation stays absent across cleanup serialization`() { + val payload = JSONObject().put("phase", CapabilityPhase.CleanupPending.persistedValue) + + assertEquals(null, payload.optionalStrictString("processGeneration")) + assertFalse(payload.has("processGeneration")) + } + + @Test + fun `legacy upload grants remain app owned while new provenance stays explicit`() { + val legacy = JSONObject() + val preExisting = JSONObject().put("grantPreExisting", true) + val appOwned = JSONObject().put("grantPreExisting", false) + val malformed = JSONObject().put("grantPreExisting", "false") + + assertFalse(persistedDurableUploadGrantPreExisting(legacy)) + assertTrue(persistedDurableUploadGrantPreExisting(preExisting)) + assertFalse(persistedDurableUploadGrantPreExisting(appOwned)) + assertFailsWith { + persistedDurableUploadGrantPreExisting(malformed) + } + } + + @Test + fun `capability acquisition reserves space before reaching the persisted limit`() { + assertTrue(durableUploadCapabilityHasCapacity(trackedCapabilityCount = 63, maximumTrackedCapabilities = 64)) + assertFalse(durableUploadCapabilityHasCapacity(trackedCapabilityCount = 64, maximumTrackedCapabilities = 64)) + } + + @Test + fun `capability restore preserves cancellation`() { + assertFailsWith { + releaseStoredDurableUploadCapability( + cachedCapability = null, + loadCapability = { throw CancellationException("cleanup stopped") }, + releasePermission = {}, + removeMetadata = { true }, + ) + } + } + + private class PausedDispatcher : CoroutineDispatcher() { + private val tasks = ArrayDeque() + + override fun dispatch(context: CoroutineContext, block: Runnable) { + tasks.addLast(block) + } + + fun runAll() { + while (tasks.isNotEmpty()) tasks.removeFirst().run() + } + } } diff --git a/androidApp/src/test/kotlin/dev/obiente/nextcloudnative/AndroidLocalUploadCapabilityOverflowRecoveryTest.kt b/androidApp/src/test/kotlin/dev/obiente/nextcloudnative/AndroidLocalUploadCapabilityOverflowRecoveryTest.kt new file mode 100644 index 000000000..11bb63946 --- /dev/null +++ b/androidApp/src/test/kotlin/dev/obiente/nextcloudnative/AndroidLocalUploadCapabilityOverflowRecoveryTest.kt @@ -0,0 +1,541 @@ +package dev.obiente.nextcloudnative + +import java.security.GeneralSecurityException +import kotlin.test.Test +import kotlin.test.assertEquals +import kotlin.test.assertFailsWith +import kotlin.test.assertFalse +import kotlin.test.assertTrue + +class AndroidLocalUploadCapabilityOverflowRecoveryTest { + @Test + fun `preference storage is bounded before capability values are enumerated`() { + val maximumBytes = 8L * 1024L * 1024L + var valuesEnumerated = false + + assertFalse( + durableUploadCapabilityPreferenceStorageIsOversized( + primaryFileBytes = maximumBytes, + backupFileBytes = 0L, + maximumFileBytes = maximumBytes, + ), + ) + assertTrue( + durableUploadCapabilityPreferenceStorageIsOversized( + primaryFileBytes = maximumBytes + 1L, + backupFileBytes = 0L, + maximumFileBytes = maximumBytes, + ), + ) + assertTrue( + durableUploadCapabilityPreferenceStorageIsOversized( + primaryFileBytes = 0L, + backupFileBytes = maximumBytes + 1L, + maximumFileBytes = maximumBytes, + ), + ) + assertFailsWith { + boundedDurableUploadCapabilitySelectionIds( + primaryFileBytes = maximumBytes + 1L, + backupFileBytes = 0L, + maximumFileBytes = maximumBytes, + maximumRows = 1_024, + preferencePrefix = "upload_", + preferenceKeys = { + valuesEnumerated = true + setOf("upload_selection") + }, + ) + } + assertFalse(valuesEnumerated) + } + + @Test + fun `over admission limit capability state remains recoverable`() { + val storedIds = (1..65).map { index -> "selection-$index" } + + val snapshot = loadDurableUploadCapabilitySnapshot( + cachedCapabilities = emptyMap(), + storedSelectionIds = storedIds, + maximumRecoverableCapabilities = 1_024, + loadStoredCapability = { selectionId -> "content://synthetic/$selectionId" }, + ) + + assertEquals(65, snapshot.trackedCapabilityCount) + assertEquals(65, snapshot.capabilities.size) + assertFalse( + durableUploadCapabilityHasCapacity( + trackedCapabilityCount = snapshot.trackedCapabilityCount, + maximumTrackedCapabilities = 64, + ), + ) + } + + @Test + fun `malformed capability owned by a durable job is excluded from recovery`() { + val queued = malformed("selection-queued") + val abandoned = malformed("selection-abandoned") + + val recoverable = malformedDurableUploadCapabilitiesForRecovery( + capabilities = mapOf( + queued.selectionId to queued, + abandoned.selectionId to abandoned, + ), + ownedSelectionIds = setOf(queued.selectionId), + ) + + assertEquals(listOf(abandoned), recoverable) + } + + @Test + fun `oversized state is quarantined without retaining or loading unbounded rows`() { + var rowLoads = 0 + val storedIds = (1..1_025).map { index -> "selection-$index" } + val scan = DurableUploadCapabilityRecoveryScan() + + val snapshot = scan.loadPage(emptyMap(), storedIds, maximumRows = 1_024) { + rowLoads += 1 + "content://synthetic/$it" + } + assertFalse(snapshot.scanComplete) + assertTrue(snapshot.recoveryQuarantined) + assertEquals(1_025, snapshot.trackedCapabilityCount) + assertTrue(snapshot.capabilities.isEmpty()) + assertTrue(snapshot.malformedCapabilities.isEmpty()) + assertEquals(0, rowLoads) + } + + @Test + fun `direct snapshot overflow is distinct from a transient row failure`() { + var rowLoads = 0 + + assertFailsWith { + loadDurableUploadCapabilitySnapshot( + cachedCapabilities = emptyMap(), + storedSelectionIds = listOf("selection-one", "selection-two"), + maximumRecoverableCapabilities = 1, + loadStoredCapability = { rowLoads += 1 }, + ) + } + assertEquals(0, rowLoads) + assertFailsWith { + loadDurableUploadCapabilitySnapshot( + cachedCapabilities = mapOf("selection-cached" to "content://synthetic/cached"), + storedSelectionIds = listOf("selection-cached", "selection-stored"), + maximumRecoverableCapabilities = 1, + loadStoredCapability = { rowLoads += 1 }, + ) + } + assertEquals(0, rowLoads) + } + + @Test + fun `malformed ciphertext without a permission identity remains quarantined without polling`() { + assertFalse( + malformedDurableUploadCapabilityCanBecomeActionable( + malformed("selection-corrupt").copy(cleanupPermissionIdentity = null), + ), + ) + assertTrue( + malformedDurableUploadCapabilityCanBecomeActionable( + malformed("selection-recoverable", "content://synthetic/recoverable"), + ), + ) + } + + @Test + fun `unknown grant provenance is quarantined while a transient permission read retries`() { + assertEquals( + DurableUploadMalformedRecoveryDisposition.Quarantine, + durableUploadMalformedRecoveryDisposition( + grantPreExisting = null, + peerProtection = DurableUploadPermissionPeerProtection.None, + permissionAbsent = false, + ), + ) + assertEquals( + DurableUploadMalformedRecoveryDisposition.Retry, + durableUploadMalformedRecoveryDisposition( + grantPreExisting = null, + peerProtection = DurableUploadPermissionPeerProtection.None, + permissionAbsent = null, + ), + ) + assertEquals( + DurableUploadMalformedRecoveryDisposition.Recover, + durableUploadMalformedRecoveryDisposition( + grantPreExisting = null, + peerProtection = DurableUploadPermissionPeerProtection.None, + permissionAbsent = true, + ), + ) + assertEquals( + DurableUploadMalformedReleaseResult.Quarantine, + releaseMalformedDurableUploadCapability(DurableUploadMalformedRecoveryDisposition.Quarantine) { + error("quarantined unknown provenance must not attempt recovery") + }, + ) + } + + @Test + fun `paged recovery isolates malformed rows without swallowing transient failures`() { + val scan = DurableUploadCapabilityRecoveryScan() + val malformed = AndroidLocalUploadCapabilityMalformedException("invalid metadata") + + assertFailsWith { + scan.loadPage( + cachedCapabilities = emptyMap(), + storedSelectionIds = listOf("selection-malformed", "selection-transient"), + maximumRows = 2, + ) { selectionId -> + when (selectionId) { + "selection-malformed" -> throw malformed + else -> throw GeneralSecurityException("synthetic decryption failure") + } + } + } + + val recovered = scan.loadPage( + cachedCapabilities = emptyMap(), + storedSelectionIds = listOf("selection-malformed", "selection-transient"), + maximumRows = 2, + loadStoredCapability = { selectionId -> "content://synthetic/$selectionId" }, + ) + + assertTrue(recovered.scanComplete) + assertEquals(setOf("selection-malformed"), recovered.malformedCapabilities.keys) + assertEquals("content://synthetic/selection-transient", recovered.capabilities["selection-transient"]) + } + + @Test + fun `owned malformed app grant permits abandoned valid metadata cleanup without revocation`() { + val sharedUri = "content://synthetic/shared" + val owned = malformed("selection-owned", sharedUri) + val protection = protection( + targetSelectionId = "selection-valid", + targetPermission = sharedUri, + peers = arrayOf(owned.peer()), + ) + val cleanupPlan = durableUploadPermissionCleanupPlan( + grantPreExisting = false, + peerProtection = protection, + ) + val events = mutableListOf() + + val recovered = releaseDurableUploadCapability( + releasePermission = { + if (cleanupPlan == DurableUploadPermissionCleanupPlan.ReleaseThenRemove) { + events += "release" + } + }, + isPermissionAbsent = { + cleanupPlan == DurableUploadPermissionCleanupPlan.RemoveWithoutRelease + }, + removeMetadata = { true.also { events += "remove-valid" } }, + ) + + assertEquals(DurableUploadPermissionPeerProtection.RetainedAppOwnedGrant, protection) + assertTrue(recovered) + assertEquals(listOf("remove-valid"), events) + } + + @Test + fun `owned malformed app grant permits abandoned malformed metadata cleanup without revocation`() { + val sharedUri = "content://synthetic/shared" + val owned = malformed("selection-owned", sharedUri) + val abandoned = malformed("selection-abandoned", sharedUri) + val protection = protection( + targetSelectionId = abandoned.selectionId, + targetPermission = sharedUri, + peers = arrayOf(owned.peer(), abandoned.peer()), + ) + val events = mutableListOf() + + val recovered = recoverMalformedDurableUploadCapability( + capability = abandoned, + permission = sharedUri, + peerProtection = protection, + releasePermission = { events += "release" }, + isPermissionAbsent = { false }, + removeMetadata = { true.also { events += "remove-malformed" } }, + ) + + assertEquals(DurableUploadPermissionPeerProtection.RetainedAppOwnedGrant, protection) + assertTrue(recovered) + assertEquals(listOf("remove-malformed"), events) + } + + @Test + fun `ambiguous malformed peers block direct valid and malformed cleanup`() { + val sharedUri = "content://synthetic/shared" + val appOwnedPeer = malformed("selection-app-owned", sharedUri).peer() + val exactPeer = malformed("selection-peer", sharedUri).copy(grantPreExisting = true).peer() + val unknownPeer = malformed("selection-unknown").copy(cleanupPermissionIdentity = null).peer() + + listOf(appOwnedPeer, exactPeer, unknownPeer).forEach { peer -> + assertTrue( + durableUploadMalformedPeerCleanupDisposition( + malformedPeers = listOf(peer), + targetSelectionId = "selection-valid", + targetPermission = sharedUri, + samePermission = String::equals, + ) != DurableUploadMalformedPeerCleanupDisposition.Proceed, + ) + assertTrue( + durableUploadMalformedPeerCleanupDisposition( + malformedPeers = listOf(peer), + targetSelectionId = "selection-malformed", + targetPermission = sharedUri, + samePermission = String::equals, + ) != DurableUploadMalformedPeerCleanupDisposition.Proceed, + ) + } + } + + @Test + fun `unknowable malformed peer quarantines blocked cleanup without polling`() { + val disposition = durableUploadMalformedPeerCleanupDisposition( + malformedPeers = listOf(peer("selection-unknown", permission = null, grantPreExisting = null)), + targetSelectionId = "selection-valid", + targetPermission = "content://synthetic/valid", + samePermission = String::equals, + ) + + assertEquals(DurableUploadMalformedPeerCleanupDisposition.Quarantine, disposition) + } + + @Test + fun `same permission malformed peer with stable provenance ambiguity quarantines valid cleanup`() { + val sharedUri = "content://synthetic/shared" + + listOf(null, true).forEach { grantPreExisting -> + val disposition = durableUploadMalformedPeerCleanupDisposition( + malformedPeers = listOf(peer("selection-malformed", sharedUri, grantPreExisting)), + targetSelectionId = "selection-valid", + targetPermission = sharedUri, + samePermission = String::equals, + ) + + assertEquals(DurableUploadMalformedPeerCleanupDisposition.Quarantine, disposition) + } + } + + @Test + fun `preexisting valid target removes only its metadata despite any malformed peer provenance`() { + val sharedUri = "content://synthetic/shared" + + listOf(null, false, true).forEach { peerGrantPreExisting -> + val peers = listOf(peer("selection-malformed", sharedUri, peerGrantPreExisting)) + val disposition = durableUploadMalformedPeerCleanupDisposition( + malformedPeers = peers, + targetSelectionId = "selection-valid-preexisting", + targetPermission = sharedUri, + samePermission = String::equals, + targetGrantPreExisting = true, + ) + val cleanupPlan = durableUploadPermissionCleanupPlan( + grantPreExisting = true, + peerProtection = protection("selection-valid-preexisting", sharedUri, peers.toTypedArray()), + ) + val events = mutableListOf() + + assertEquals(DurableUploadMalformedPeerCleanupDisposition.Proceed, disposition) + assertTrue(releaseDurableUploadCapability( + releasePermission = { + if (cleanupPlan == DurableUploadPermissionCleanupPlan.ReleaseThenRemove) events += "release" + }, + isPermissionAbsent = { cleanupPlan == DurableUploadPermissionCleanupPlan.RemoveWithoutRelease }, + removeMetadata = { events += "remove-target"; true }, + )) + assertEquals(listOf("remove-target"), events) + } + } + + @Test + fun `recoverable malformed owner still retries while unrelated malformed grants permit cleanup`() { + val sharedUri = "content://synthetic/shared" + val appOwnedPeer = peer("selection-malformed", sharedUri, grantPreExisting = false) + + assertEquals( + DurableUploadMalformedPeerCleanupDisposition.Retry, + durableUploadMalformedPeerCleanupDisposition( + listOf(appOwnedPeer), "selection-valid", sharedUri, String::equals, + ), + ) + assertEquals( + DurableUploadMalformedPeerCleanupDisposition.Proceed, + durableUploadMalformedPeerCleanupDisposition( + listOf(appOwnedPeer), "selection-valid", "content://synthetic/unrelated", String::equals, + ), + ) + } + + @Test + fun `peer protection keeps ambiguous provenance distinct from a retained app grant`() { + val sharedUri = "content://synthetic/shared" + val exactAppGrant = peer("selection-false", sharedUri, grantPreExisting = false) + val exactPreExisting = peer("selection-true", sharedUri, grantPreExisting = true) + val exactUnknownGrant = peer("selection-null", sharedUri, grantPreExisting = null) + val unknownPermission = peer("selection-unknown", permission = null, grantPreExisting = false) + + assertEquals( + DurableUploadPermissionPeerProtection.RetainedAppOwnedGrant, + protection("selection-target", sharedUri, arrayOf(exactAppGrant)), + ) + listOf(exactPreExisting, exactUnknownGrant, unknownPermission).forEach { peer -> + val protection = protection("selection-target", sharedUri, arrayOf(peer)) + assertEquals( + DurableUploadPermissionPeerProtection.Ambiguous, + protection, + ) + assertEquals( + DurableUploadPermissionCleanupPlan.Retain, + durableUploadPermissionCleanupPlan(false, protection), + ) + } + assertEquals( + DurableUploadMalformedRecoveryDisposition.Quarantine, + durableUploadMalformedRecoveryDisposition( + grantPreExisting = null, + peerProtection = protection("selection-target", sharedUri, arrayOf(exactUnknownGrant)), + permissionAbsent = false, + ), + ) + } + + @Test + fun `stable malformed peer ambiguity quarantines direct release without recovery`() { + val sharedUri = "content://synthetic/shared" + val disposition = durableUploadMalformedRecoveryDisposition( + grantPreExisting = null, + peerProtection = protection( + "selection-target", + sharedUri, + arrayOf(peer("selection-peer", sharedUri, grantPreExisting = null)), + ), + permissionAbsent = false, + ) + var recoveryAttempted = false + + val result = releaseMalformedDurableUploadCapability(disposition) { + recoveryAttempted = true + true + } + + assertEquals(DurableUploadMalformedReleaseResult.Quarantine, result) + assertFalse(recoveryAttempted) + } + + @Test + fun `valid false grant is retained while only peer claims preexisting ownership`() { + val protection = protection( + targetSelectionId = "selection-false", + targetPermission = "content://synthetic/shared", + peers = arrayOf( + peer("selection-false", "content://synthetic/shared", grantPreExisting = false), + peer("selection-true", "content://synthetic/shared", grantPreExisting = true), + ), + ) + + assertEquals(DurableUploadPermissionPeerProtection.Ambiguous, protection) + assertEquals( + DurableUploadPermissionCleanupPlan.Retain, + durableUploadPermissionCleanupPlan(false, protection), + ) + } + + @Test + fun `cleanup planning preserves unknown target provenance until absence is proven`() { + assertEquals( + DurableUploadPermissionCleanupPlan.RemoveWithoutRelease, + durableUploadPermissionCleanupPlan( + grantPreExisting = null, + peerProtection = DurableUploadPermissionPeerProtection.RetainedAppOwnedGrant, + ), + ) + assertEquals( + DurableUploadPermissionCleanupPlan.Retain, + durableUploadPermissionCleanupPlan( + grantPreExisting = null, + peerProtection = DurableUploadPermissionPeerProtection.Ambiguous, + ), + ) + assertEquals( + DurableUploadPermissionCleanupPlan.Retain, + durableUploadPermissionCleanupPlan( + grantPreExisting = null, + peerProtection = DurableUploadPermissionPeerProtection.None, + permissionAbsent = false, + ), + ) + assertEquals( + DurableUploadPermissionCleanupPlan.RemoveWithoutRelease, + durableUploadPermissionCleanupPlan( + grantPreExisting = null, + peerProtection = DurableUploadPermissionPeerProtection.None, + permissionAbsent = true, + ), + ) + assertEquals( + DurableUploadPermissionCleanupPlan.RemoveWithoutRelease, + durableUploadPermissionCleanupPlan( + grantPreExisting = true, + peerProtection = DurableUploadPermissionPeerProtection.Ambiguous, + ), + ) + } + + @Test + fun `mutable malformed ownership leaves the final duplicate to revoke the grant`() { + val sharedUri = "content://synthetic/shared" + val peers = linkedMapOf( + "selection-one" to peer("selection-one", sharedUri, grantPreExisting = false), + "selection-two" to peer("selection-two", sharedUri, grantPreExisting = false), + ) + + val firstPlan = durableUploadPermissionCleanupPlan( + grantPreExisting = false, + peerProtection = protection("selection-one", sharedUri, peers.values.toTypedArray()), + ) + peers.remove("selection-one") + val secondPlan = durableUploadPermissionCleanupPlan( + grantPreExisting = false, + peerProtection = protection("selection-two", sharedUri, peers.values.toTypedArray()), + ) + + assertEquals(DurableUploadPermissionCleanupPlan.RemoveWithoutRelease, firstPlan) + assertEquals(DurableUploadPermissionCleanupPlan.ReleaseThenRemove, secondPlan) + } + + private fun protection( + targetSelectionId: String, + targetPermission: String, + peers: Array>, + ): DurableUploadPermissionPeerProtection = durableUploadPermissionPeerProtection( + peers = peers.asIterable(), + targetSelectionId = targetSelectionId, + targetPermission = targetPermission, + samePermission = String::equals, + ) + + private fun peer( + selectionId: String, + permission: String?, + grantPreExisting: Boolean?, + ) = DurableUploadPermissionPeer(selectionId, permission, grantPreExisting) + + private fun MalformedDurableUploadCapability.peer() = DurableUploadPermissionPeer( + selectionId, + cleanupPermissionIdentity, + grantPreExisting, + ) + + private fun malformed( + selectionId: String, + permissionIdentity: String = "content://synthetic/$selectionId", + ) = MalformedDurableUploadCapability( + selectionId = selectionId, + cleanupPermissionIdentity = permissionIdentity, + grantPreExisting = false, + ) +} diff --git a/changes/unreleased/durable-multipart-scheduling-recovery.md b/changes/unreleased/durable-multipart-scheduling-recovery.md new file mode 100644 index 000000000..c406be0fb --- /dev/null +++ b/changes/unreleased/durable-multipart-scheduling-recovery.md @@ -0,0 +1,7 @@ +category: fix +issue: 52 +pull: 439 +platforms: android +user-facing: yes + +Restore queued attachment uploads after interrupted Android scheduling. Cleanup retries yield to failed-worker deadlines, and malformed grant ownership is quarantined without blocking other uploads or deleting retained grants. diff --git a/ui/src/commonMain/kotlin/dev/obiente/nextcloudnative/app/NextcloudPlatform.kt b/ui/src/commonMain/kotlin/dev/obiente/nextcloudnative/app/NextcloudPlatform.kt index ed004743e..27e346602 100644 --- a/ui/src/commonMain/kotlin/dev/obiente/nextcloudnative/app/NextcloudPlatform.kt +++ b/ui/src/commonMain/kotlin/dev/obiente/nextcloudnative/app/NextcloudPlatform.kt @@ -1346,8 +1346,7 @@ interface NextcloudPlatformServices : NextcloudAccountCredentialServices, DeckCa /** * Streams one picker-authorized file to a reviewed same-origin multipart endpoint. * - * Implementations use the supplied session's credentials, including for retained background - * work, reject redirects and arbitrary local paths, and enforce request and response limits. + * Use supplied-session credentials for retained work, reject redirects and arbitrary paths, and enforce limits. */ suspend fun executeNextcloudMultipartUpload( session: NextcloudSession, diff --git a/website/public/screenshots/capture-manifest.json b/website/public/screenshots/capture-manifest.json index 8c95d43d7..c0e48e17a 100644 --- a/website/public/screenshots/capture-manifest.json +++ b/website/public/screenshots/capture-manifest.json @@ -488,7 +488,7 @@ "ui/src/commonMain/kotlin/dev/obiente/nextcloudnative/app/DocumentPreview.kt": "a9a8743dd7a381504282cc6ddc68034425024ccc1ae51667da9734bbfc7b1a79", "ui/src/commonMain/kotlin/dev/obiente/nextcloudnative/app/DurableMutationRecoveryDialog.kt": "e720eadb477a347762cd1894285788ac9f6820972431fe0a1953d01955667bfe", "ui/src/commonMain/kotlin/dev/obiente/nextcloudnative/app/DynamicActionUi.kt": "2b7ef2d18b4a23615686ced0b7c9c621c58dc5edd0202104d0ca55b1ebf61d81", - "ui/src/commonMain/kotlin/dev/obiente/nextcloudnative/app/DynamicApiRequestCoalescer.kt": "ddf80ca67d954f6e063c9e88c75794fcb81cbd6d42887c45d1a04b1cefe4f2fd", + "ui/src/commonMain/kotlin/dev/obiente/nextcloudnative/app/DynamicApiRequestCoalescer.kt": "9aeb3dce3a1bd11651a7c84b5ab0e77e2d905055c68bc8f8cd02d412670c5ed5", "ui/src/commonMain/kotlin/dev/obiente/nextcloudnative/app/DynamicArtworkMemoryCache.kt": "c313daea9465087ab1862814bc5a772bdcc1f087bc673eb80f417db73668ea1c", "ui/src/commonMain/kotlin/dev/obiente/nextcloudnative/app/DynamicCollectionHeaderActions.kt": "d352d0a0fc28bdf5cfd3cf24b04dc7b23aa15de5ec29dbcf6e5c49f25599d1ab", "ui/src/commonMain/kotlin/dev/obiente/nextcloudnative/app/DynamicContractInfo.kt": "cff6ba11283705120375452d6d539c20581f4dd0115dd3d07049eb965242b019", @@ -645,7 +645,7 @@ "ui/src/commonMain/kotlin/dev/obiente/nextcloudnative/app/NextcloudNotesCache.kt": "aa9ec330c3a569e2bc9948858ddde1d939358069d38fbd7a0319b1ecc3baa363", "ui/src/commonMain/kotlin/dev/obiente/nextcloudnative/app/NextcloudPeople.kt": "cff910ea2cc77211ef81779c49ee0c957851f2b4a3ed32b857b12ded1cee643b", "ui/src/commonMain/kotlin/dev/obiente/nextcloudnative/app/NextcloudPhotoEditor.kt": "34d9b43cf3bbfc8342958dc40f2df7b4573f30a2ae1bd9bcb8bb470151313a3d", - "ui/src/commonMain/kotlin/dev/obiente/nextcloudnative/app/NextcloudPlatform.kt": "5b278014d8a2f6f98733095126ce7480a18618be82ec4a4aa0f252fa4dc54e50", + "ui/src/commonMain/kotlin/dev/obiente/nextcloudnative/app/NextcloudPlatform.kt": "5370183fb78190a6893e4a36638b0570de4d2d6b33acab18aef52f2b83d83589", "ui/src/commonMain/kotlin/dev/obiente/nextcloudnative/app/NextcloudResponseLimits.kt": "fead6cbf4f723ea46f99c9155b190967205d24f198687b7bfd5a213c474374b8", "ui/src/commonMain/kotlin/dev/obiente/nextcloudnative/app/NextcloudSession.kt": "792a381fd5eefc44e13eb73ee95a80f8d52e8dcec9d3876ba06ec5392b5c1f81", "ui/src/commonMain/kotlin/dev/obiente/nextcloudnative/app/NextcloudSessionLoading.kt": "c96941c7582218754243f780a0bb0954ba31484719d8045f1e609f5d8c04a7c5",