From d2813887da286b58c759f56e3953a54cb2ab4e2e Mon Sep 17 00:00:00 2001 From: Junwang Zhao Date: Sun, 2 Aug 2026 23:26:46 +0800 Subject: [PATCH 1/5] fix: enforce pending update lifecycle Pending updates need deterministic ownership and finalization across both standalone commits and explicit transactions. Register updates through shared ownership so transactions can retain and finalize them safely, and scope the temporary transaction binding used by standalone commits so it never escapes through TransactionContext. Finalize no-op commits and failed applies so every update reaches a terminal state and staged files are cleaned even when the caller never commits the transaction. During Transaction::Commit retries, defer eager finalization while updates are being reapplied; otherwise a retryable validation error would finalize the transaction and destroy staged state before RetryRunner can retry. Restore the retry lifecycle marker with RAII when commit exits or throws. Convert snapshot update factories and test helpers to shared_ptr to satisfy the ownership contract. Tests cover detached temporary transactions, cleared standalone bindings, rejection of unshared standalone updates, no-op and apply-failure finalization, staged-file cleanup, standalone retry reapplication, and lifecycle restoration after commit exceptions. --- src/iceberg/test/fast_append_test.cc | 20 +++ .../test/merging_snapshot_update_test.cc | 14 +- src/iceberg/test/transaction_test.cc | 131 ++++++++++++++++++ src/iceberg/transaction.cc | 79 +++++++---- src/iceberg/transaction.h | 13 +- src/iceberg/update/delete_files.cc | 4 +- src/iceberg/update/delete_files.h | 2 +- src/iceberg/update/fast_append.cc | 4 +- src/iceberg/update/fast_append.h | 2 +- src/iceberg/update/merge_append.cc | 4 +- src/iceberg/update/merge_append.h | 2 +- src/iceberg/update/pending_update.cc | 46 ++++-- src/iceberg/update/pending_update.h | 4 +- src/iceberg/update/replace_partitions.cc | 4 +- src/iceberg/update/replace_partitions.h | 2 +- src/iceberg/update/rewrite_files.cc | 4 +- src/iceberg/update/rewrite_files.h | 4 +- src/iceberg/update/row_delta.cc | 4 +- src/iceberg/update/row_delta.h | 2 +- 19 files changed, 281 insertions(+), 64 deletions(-) diff --git a/src/iceberg/test/fast_append_test.cc b/src/iceberg/test/fast_append_test.cc index f88d2e011..15cc78c40 100644 --- a/src/iceberg/test/fast_append_test.cc +++ b/src/iceberg/test/fast_append_test.cc @@ -315,6 +315,26 @@ TEST_F(FastAppendTest, FinalizeIgnoresCleanupDeleteFailure) { IsOk()); } +TEST_F(FastAppendTest, TransactionApplyFailureCleansUpStagedFiles) { + ICEBERG_UNWRAP_OR_FAIL(auto txn, table_->NewTransaction()); + ICEBERG_UNWRAP_OR_FAIL(auto fast_append, txn->NewFastAppend()); + std::vector deleted_paths; + fast_append->DeleteWith([&](const std::string& path) { + deleted_paths.push_back(path); + return file_io_->DeleteFile(path); + }); + fast_append->AppendFile(file_a_); + + EXPECT_THAT(static_cast(*fast_append).Apply(), IsOk()); + fast_append->AppendFile(nullptr); + + EXPECT_THAT(fast_append->Commit(), IsError(ErrorKind::kValidationFailed)); + EXPECT_THAT(deleted_paths, ::testing::SizeIs(2U)); + EXPECT_THAT(txn->Commit(), + ::testing::AllOf(IsError(ErrorKind::kValidationFailed), + HasErrorMessage("Transaction already finalized"))); +} + TEST_F(FastAppendTest, RetryCopiesAppendManifestAgain) { table_->metadata()->format_version = 1; const auto path = table_location_ + "/metadata/input.avro"; diff --git a/src/iceberg/test/merging_snapshot_update_test.cc b/src/iceberg/test/merging_snapshot_update_test.cc index 25907c445..f293b7923 100644 --- a/src/iceberg/test/merging_snapshot_update_test.cc +++ b/src/iceberg/test/merging_snapshot_update_test.cc @@ -81,11 +81,11 @@ class MergingSnapshotCapturingReporter final : public MetricsReporter { /// \brief Concrete subclass of MergingSnapshotUpdate for testing. class TestMergeAppend : public MergingSnapshotUpdate { public: - static Result> Make(std::string table_name, + static Result> Make(std::string table_name, std::shared_ptr table) { ICEBERG_ASSIGN_OR_RAISE( auto ctx, TransactionContext::Make(std::move(table), TransactionKind::kUpdate)); - return std::unique_ptr( + return std::shared_ptr( new TestMergeAppend(std::move(table_name), std::move(ctx))); } @@ -231,11 +231,11 @@ class TestMergeAppend : public MergingSnapshotUpdate { class TestOverwriteUpdate : public MergingSnapshotUpdate { public: - static Result> Make(std::string table_name, + static Result> Make(std::string table_name, std::shared_ptr
table) { ICEBERG_ASSIGN_OR_RAISE( auto ctx, TransactionContext::Make(std::move(table), TransactionKind::kUpdate)); - return std::unique_ptr( + return std::shared_ptr( new TestOverwriteUpdate(std::move(table_name), std::move(ctx))); } @@ -335,11 +335,11 @@ class MergingSnapshotUpdateTest : public MinimalUpdateTestBase { return f; } - Result> NewMergeAppend() { + Result> NewMergeAppend() { return TestMergeAppend::Make(TableName(), table_); } - Result> NewOverwriteUpdate() { + Result> NewOverwriteUpdate() { return TestOverwriteUpdate::Make(TableName(), table_); } @@ -1101,7 +1101,7 @@ class MergingSnapshotUpdateV1Test : public UpdateTestBase { return f; } - Result> NewMergeAppend() { + Result> NewMergeAppend() { return TestMergeAppend::Make(TableName(), table_); } diff --git a/src/iceberg/test/transaction_test.cc b/src/iceberg/test/transaction_test.cc index 3a13b7bc5..0998beacd 100644 --- a/src/iceberg/test/transaction_test.cc +++ b/src/iceberg/test/transaction_test.cc @@ -19,20 +19,38 @@ #include "iceberg/transaction.h" +#include +#include +#include +#include +#include + #include "iceberg/expression/expressions.h" #include "iceberg/expression/term.h" #include "iceberg/sort_order.h" +#include "iceberg/table_metadata.h" #include "iceberg/test/matchers.h" #include "iceberg/test/mock_catalog.h" #include "iceberg/test/update_test_base.h" #include "iceberg/transform.h" #include "iceberg/type.h" +#include "iceberg/update/fast_append.h" +#include "iceberg/update/set_snapshot.h" #include "iceberg/update/update_properties.h" #include "iceberg/update/update_schema.h" #include "iceberg/update/update_sort_order.h" namespace iceberg { +class TestPendingUpdate final : public PendingUpdate { + public: + explicit TestPendingUpdate(std::shared_ptr ctx) + : PendingUpdate(std::move(ctx)) {} + + Kind kind() const override { return Kind::kUpdateProperties; } + bool IsRetryable() const override { return true; } +}; + class TransactionTest : public UpdateTestBase {}; TEST_F(TransactionTest, CreateTransaction) { @@ -46,6 +64,58 @@ TEST_F(TransactionTest, CommitEmptyTransaction) { EXPECT_THAT(txn->Commit(), IsOk()); } +TEST_F(TransactionTest, TemporaryTransactionDoesNotAttachToContext) { + ICEBERG_UNWRAP_OR_FAIL(auto ctx, + TransactionContext::Make(table_, TransactionKind::kUpdate)); + ASSERT_FALSE(ctx->transaction.has_value()); + + ICEBERG_UNWRAP_OR_FAIL(auto txn, Transaction::Make(ctx)); + + EXPECT_NE(txn, nullptr); + EXPECT_FALSE(ctx->transaction.has_value()); +} + +TEST_F(TransactionTest, StandaloneCommitRequiresSharedOwnership) { + ICEBERG_UNWRAP_OR_FAIL(auto ctx, + TransactionContext::Make(table_, TransactionKind::kUpdate)); + auto update = std::make_unique(ctx); + + EXPECT_THAT(update->Commit(), + ::testing::AllOf( + IsError(ErrorKind::kInvalidArgument), + HasErrorMessage("PendingUpdate must be owned by std::shared_ptr"))); + EXPECT_FALSE(ctx->transaction.has_value()); +} + +TEST_F(TransactionTest, StandaloneCommitClearsTemporaryTransactionBinding) { + ICEBERG_UNWRAP_OR_FAIL(auto ctx, + TransactionContext::Make(table_, TransactionKind::kUpdate)); + ICEBERG_UNWRAP_OR_FAIL(auto update, UpdateProperties::Make(ctx)); + update->Set("standalone.property", "standalone.value"); + + EXPECT_THAT(update->Commit(), IsOk()); + EXPECT_FALSE(ctx->transaction.has_value()); +} + +TEST_F(TransactionTest, CommitNoOpUpdate) { + ICEBERG_UNWRAP_OR_FAIL(auto txn, table_->NewTransaction()); + ICEBERG_UNWRAP_OR_FAIL(auto update, txn->NewSetSnapshot()); + + EXPECT_THAT(update->Commit(), IsOk()); + EXPECT_THAT(txn->Commit(), IsOk()); +} + +TEST_F(TransactionTest, ApplyFailureFinalizesTransaction) { + ICEBERG_UNWRAP_OR_FAIL(auto txn, table_->NewTransaction()); + ICEBERG_UNWRAP_OR_FAIL(auto update, txn->NewFastAppend()); + update->AppendFile(nullptr); + + EXPECT_THAT(update->Commit(), IsError(ErrorKind::kValidationFailed)); + EXPECT_THAT(txn->Commit(), + ::testing::AllOf(IsError(ErrorKind::kValidationFailed), + HasErrorMessage("Transaction already finalized"))); +} + TEST_F(TransactionTest, CommitTransactionWithPropertyUpdate) { ICEBERG_UNWRAP_OR_FAIL(auto txn, table_->NewTransaction()); ICEBERG_UNWRAP_OR_FAIL(auto update, txn->NewUpdateProperties()); @@ -151,6 +221,39 @@ TEST_F(TransactionRetryTest, CommitRetrySucceedsAfterConflict) { EXPECT_EQ(update_call_count, 2); } +TEST_F(TransactionRetryTest, StandaloneCommitRetryReappliesUpdate) { + std::vector update_counts; + ON_CALL(*mock_catalog_, UpdateTable(::testing::_, ::testing::_, ::testing::_)) + .WillByDefault( + [this, &update_counts](const TableIdentifier&, + const std::vector>&, + const std::vector>& updates) + -> Result> { + update_counts.push_back(updates.size()); + if (update_counts.size() == 1) { + return CommitFailed("conflict on first attempt"); + } + return Table::Make(mock_table_->name(), mock_table_->metadata(), + std::string(mock_table_->metadata_file_location()), + mock_table_->io(), mock_catalog_); + }); + EXPECT_CALL(*mock_catalog_, LoadTable(::testing::_)) + .WillOnce([this](const TableIdentifier&) -> Result> { + auto builder = TableMetadataBuilder::BuildFrom(mock_table_->metadata().get()); + ICEBERG_ASSIGN_OR_RAISE(auto metadata, builder->Build()); + return Table::Make( + mock_table_->name(), std::shared_ptr(std::move(metadata)), + std::format("{}.refreshed", mock_table_->metadata_file_location()), + mock_table_->io(), mock_catalog_); + }); + + ICEBERG_UNWRAP_OR_FAIL(auto update, mock_table_->NewUpdateProperties()); + update->Set("retry.test", "value"); + + EXPECT_THAT(update->Commit(), IsOk()); + EXPECT_THAT(update_counts, ::testing::ElementsAre(1U, 1U)); +} + TEST_F(TransactionRetryTest, CommitRetryExhausted) { int update_call_count = 0; ON_CALL(*mock_catalog_, UpdateTable(::testing::_, ::testing::_, ::testing::_)) @@ -195,6 +298,34 @@ TEST_F(TransactionRetryTest, CommitNonRetryableErrorStopsImmediately) { EXPECT_EQ(update_call_count, 1); // Should not retry } +TEST_F(TransactionRetryTest, CommitExceptionRestoresLifecycleState) { + int update_call_count = 0; + ON_CALL(*mock_catalog_, UpdateTable(::testing::_, ::testing::_, ::testing::_)) + .WillByDefault( + [&update_call_count](const TableIdentifier&, + const std::vector>&, + const std::vector>&) + -> Result> { + ++update_call_count; + throw std::runtime_error("injected catalog failure"); + }); + + ICEBERG_UNWRAP_OR_FAIL(auto txn, mock_table_->NewTransaction()); + ICEBERG_UNWRAP_OR_FAIL(auto properties, txn->NewUpdateProperties()); + properties->Set("exception.test", "value"); + EXPECT_THAT(properties->Commit(), IsOk()); + + EXPECT_THROW(std::ignore = txn->Commit(), std::runtime_error); + + ICEBERG_UNWRAP_OR_FAIL(auto append, txn->NewFastAppend()); + append->AppendFile(nullptr); + EXPECT_THAT(append->Commit(), IsError(ErrorKind::kValidationFailed)); + EXPECT_THAT(txn->Commit(), + ::testing::AllOf(IsError(ErrorKind::kValidationFailed), + HasErrorMessage("Transaction already finalized"))); + EXPECT_EQ(update_call_count, 1); +} + TEST_F(TransactionRetryTest, CreateTransactionDoesNotRetry) { int update_call_count = 0; ON_CALL(*mock_catalog_, UpdateTable(::testing::_, ::testing::_, ::testing::_)) diff --git a/src/iceberg/transaction.cc b/src/iceberg/transaction.cc index bacd880de..b26232376 100644 --- a/src/iceberg/transaction.cc +++ b/src/iceberg/transaction.cc @@ -58,6 +58,21 @@ #include "iceberg/util/retry_util.h" namespace iceberg { +namespace { + +class ScopedTrue { + public: + explicit ScopedTrue(bool& value) : value_(value) { value_ = true; } + ~ScopedTrue() { value_ = false; } + + ScopedTrue(const ScopedTrue&) = delete; + ScopedTrue& operator=(const ScopedTrue&) = delete; + + private: + bool& value_; +}; + +} // namespace // --------------------------------------------------------------------------- // TransactionContext @@ -123,9 +138,7 @@ Result> Transaction::Make(std::shared_ptr
ta Result> Transaction::Make( std::shared_ptr ctx) { ICEBERG_PRECHECK(ctx != nullptr, "TransactionContext cannot be null"); - auto txn = std::shared_ptr(new Transaction(ctx)); - ctx->transaction = std::weak_ptr(txn); - return txn; + return std::shared_ptr(new Transaction(std::move(ctx))); } const std::shared_ptr
& Transaction::table() const { return ctx_->table; } @@ -139,6 +152,8 @@ std::string Transaction::MetadataFileLocation(std::string_view filename) const { } Status Transaction::AddUpdate(const std::shared_ptr& update) { + ICEBERG_CHECK(!committed_, "Cannot add update to a committed transaction"); + ICEBERG_CHECK(!finalized_, "Cannot add update to a finalized transaction"); ICEBERG_CHECK(last_update_committed_, "Cannot add update when previous update is not committed"); @@ -148,6 +163,9 @@ Status Transaction::AddUpdate(const std::shared_ptr& update) { } Status Transaction::Apply(PendingUpdate& update) { + ICEBERG_CHECK(!committed_, "Cannot apply update to a committed transaction"); + ICEBERG_CHECK(!finalized_, "Cannot apply update to a finalized transaction"); + switch (update.kind()) { case PendingUpdate::Kind::kExpireSnapshots: ICEBERG_RETURN_UNEXPECTED( @@ -359,40 +377,37 @@ Status Transaction::ApplyUpdatePartitionStatistics(UpdatePartitionStatistics& up Result> Transaction::Commit() { ICEBERG_CHECK(!committed_, "Transaction already committed"); + ICEBERG_CHECK(!finalized_, "Transaction already finalized"); ICEBERG_CHECK(last_update_committed_, "Cannot commit transaction when previous update is not committed"); const auto& updates = ctx_->metadata_builder->changes(); - if (updates.empty()) { - committed_ = true; - return ctx_->table; + Result> commit_result = ctx_->table; + if (!updates.empty()) { + const auto& props = ctx_->table->properties(); + int32_t num_retries = + CanRetry() ? static_cast(props.Get(TableProperties::kCommitNumRetries)) + : 0; + int32_t min_wait_ms = props.Get(TableProperties::kCommitMinRetryWaitMs); + int32_t max_wait_ms = props.Get(TableProperties::kCommitMaxRetryWaitMs); + int32_t total_timeout_ms = props.Get(TableProperties::kCommitTotalRetryTimeMs); + + bool is_first_attempt = true; + ScopedTrue committing(committing_); + commit_result = + MakeCommitRetryRunner(num_retries, min_wait_ms, max_wait_ms, total_timeout_ms) + .Run([this, &is_first_attempt]() -> Result> { + auto result = CommitOnce(is_first_attempt); + is_first_attempt = false; + return result; + }); } - const auto& props = ctx_->table->properties(); - int32_t num_retries = - CanRetry() ? static_cast(props.Get(TableProperties::kCommitNumRetries)) - : 0; - int32_t min_wait_ms = props.Get(TableProperties::kCommitMinRetryWaitMs); - int32_t max_wait_ms = props.Get(TableProperties::kCommitMaxRetryWaitMs); - int32_t total_timeout_ms = props.Get(TableProperties::kCommitTotalRetryTimeMs); - - bool is_first_attempt = true; - auto commit_result = - MakeCommitRetryRunner(num_retries, min_wait_ms, max_wait_ms, total_timeout_ms) - .Run([this, &is_first_attempt]() -> Result> { - auto result = CommitOnce(is_first_attempt); - is_first_attempt = false; - return result; - }); - Result finalize_result = commit_result.has_value() ? Result(commit_result.value()->metadata().get()) : std::unexpected(commit_result.error()); - - for (const auto& update : pending_updates_) { - std::ignore = update->Finalize(finalize_result); - } + FinalizeUpdates(finalize_result); ICEBERG_RETURN_UNEXPECTED(commit_result); @@ -403,6 +418,16 @@ Result> Transaction::Commit() { return ctx_->table; } +void Transaction::FinalizeUpdates(const Result& commit_result) { + if (finalized_) { + return; + } + finalized_ = true; + for (const auto& update : pending_updates_) { + std::ignore = update->Finalize(commit_result); + } +} + Result> Transaction::CommitOnce(bool is_first_attempt) { std::vector> requirements; diff --git a/src/iceberg/transaction.h b/src/iceberg/transaction.h index 3ee0372ad..4dd2b03bb 100644 --- a/src/iceberg/transaction.h +++ b/src/iceberg/transaction.h @@ -50,8 +50,8 @@ class ICEBERG_EXPORT Transaction : public std::enable_shared_from_this> Make( std::shared_ptr ctx); @@ -170,6 +170,9 @@ class ICEBERG_EXPORT Transaction : public std::enable_shared_from_this& commit_result); + private: friend class PendingUpdate; @@ -182,6 +185,12 @@ class ICEBERG_EXPORT Transaction : public std::enable_shared_from_this> DeleteFiles::Make( +Result> DeleteFiles::Make( std::string table_name, std::shared_ptr ctx) { ICEBERG_PRECHECK(!table_name.empty(), "Table name cannot be empty"); ICEBERG_PRECHECK(ctx != nullptr, "Cannot create DeleteFiles without a context"); - return std::unique_ptr( + return std::shared_ptr( new DeleteFiles(std::move(table_name), std::move(ctx))); } diff --git a/src/iceberg/update/delete_files.h b/src/iceberg/update/delete_files.h index 7e567830e..18756ff48 100644 --- a/src/iceberg/update/delete_files.h +++ b/src/iceberg/update/delete_files.h @@ -43,7 +43,7 @@ namespace iceberg { /// differently-normalized URIs are not considered matches. class ICEBERG_EXPORT DeleteFiles : public MergingSnapshotUpdate { public: - static Result> Make( + static Result> Make( std::string table_name, std::shared_ptr ctx); /// \brief Delete a file by path from the underlying table. diff --git a/src/iceberg/update/fast_append.cc b/src/iceberg/update/fast_append.cc index 4167387c7..3e21c67c1 100644 --- a/src/iceberg/update/fast_append.cc +++ b/src/iceberg/update/fast_append.cc @@ -35,11 +35,11 @@ namespace iceberg { -Result> FastAppend::Make( +Result> FastAppend::Make( std::string table_name, std::shared_ptr ctx) { ICEBERG_PRECHECK(!table_name.empty(), "Table name cannot be empty"); ICEBERG_PRECHECK(ctx != nullptr, "Cannot create FastAppend without a context"); - return std::unique_ptr( + return std::shared_ptr( new FastAppend(std::move(table_name), std::move(ctx))); } diff --git a/src/iceberg/update/fast_append.h b/src/iceberg/update/fast_append.h index f5e1ceaba..c28c61cdd 100644 --- a/src/iceberg/update/fast_append.h +++ b/src/iceberg/update/fast_append.h @@ -51,7 +51,7 @@ class ICEBERG_EXPORT FastAppend : public SnapshotUpdate { /// \param table_name The name of the table /// \param ctx The transaction context to use for this update /// \return A Result containing the FastAppend instance or an error - static Result> Make( + static Result> Make( std::string table_name, std::shared_ptr ctx); /// \brief Append a DataFile to the table. diff --git a/src/iceberg/update/merge_append.cc b/src/iceberg/update/merge_append.cc index 70cd7b8e7..fd6cc35ae 100644 --- a/src/iceberg/update/merge_append.cc +++ b/src/iceberg/update/merge_append.cc @@ -29,11 +29,11 @@ namespace iceberg { -Result> MergeAppend::Make( +Result> MergeAppend::Make( std::string table_name, std::shared_ptr ctx) { ICEBERG_PRECHECK(!table_name.empty(), "Table name cannot be empty"); ICEBERG_PRECHECK(ctx != nullptr, "Cannot create MergeAppend without a context"); - return std::unique_ptr( + return std::shared_ptr( new MergeAppend(std::move(table_name), std::move(ctx))); } diff --git a/src/iceberg/update/merge_append.h b/src/iceberg/update/merge_append.h index cd4f4acbb..18503b3c1 100644 --- a/src/iceberg/update/merge_append.h +++ b/src/iceberg/update/merge_append.h @@ -47,7 +47,7 @@ class ICEBERG_EXPORT MergeAppend : public MergingSnapshotUpdate { /// \param table_name The name of the table /// \param ctx The transaction context to use for this update /// \return A Result containing the MergeAppend instance or an error - static Result> Make( + static Result> Make( std::string table_name, std::shared_ptr ctx); /// \brief Append a DataFile to the table. diff --git a/src/iceberg/update/pending_update.cc b/src/iceberg/update/pending_update.cc index 4b3000652..70388e198 100644 --- a/src/iceberg/update/pending_update.cc +++ b/src/iceberg/update/pending_update.cc @@ -25,6 +25,23 @@ #include "iceberg/util/macros.h" namespace iceberg { +namespace { + +class ScopedTransactionBinding { + public: + ScopedTransactionBinding(TransactionContext& ctx, + const std::shared_ptr& txn) + : ctx_(ctx) { + ctx_.transaction = txn; + } + + ~ScopedTransactionBinding() { ctx_.transaction.reset(); } + + private: + TransactionContext& ctx_; +}; + +} // namespace PendingUpdate::PendingUpdate(std::shared_ptr ctx) : ctx_(std::move(ctx)) {} @@ -35,26 +52,39 @@ Status PendingUpdate::Commit() { if (!ctx_->transaction) { // Table-created path: no transaction exists yet, create a temporary one. ICEBERG_ASSIGN_OR_RAISE(auto txn, Transaction::Make(ctx_)); + auto self = weak_from_this().lock(); + ICEBERG_PRECHECK(self != nullptr, "PendingUpdate must be owned by std::shared_ptr"); + ICEBERG_RETURN_UNEXPECTED(txn->AddUpdate(self)); + // Keep Transaction::Make(ctx_) detached, but expose this live transaction while + // Commit() runs so an internal retry can reapply through update->Commit(). + ScopedTransactionBinding binding(*ctx_, txn); + auto apply_status = txn->Apply(*this); if (!apply_status.has_value()) { - std::ignore = Finalize(std::unexpected(apply_status.error())); + txn->FinalizeUpdates(std::unexpected(apply_status.error())); return apply_status; } auto commit_result = txn->Commit(); - if (!commit_result.has_value()) { - std::ignore = Finalize(std::unexpected(commit_result.error())); - return std::unexpected(commit_result.error()); - } - - std::ignore = Finalize(commit_result.value()->metadata().get()); + ICEBERG_RETURN_UNEXPECTED(commit_result); return {}; } + auto txn = ctx_->transaction->lock(); if (!txn) { return CommitFailed("Transaction has been destroyed"); } - return txn->Apply(*this); + + auto apply_status = txn->Apply(*this); + if (!apply_status.has_value() && !txn->committing_) { + // Finalize eagerly so a failed update cleans up its staged files even if the + // caller never commits the transaction. When the transaction is mid-commit, + // leave finalization to Transaction::Commit(): the failure may be retryable + // (e.g. RetryableValidationFailed from a stale sequence number), and + // finalizing here would destroy staged state before the retry runs. + txn->FinalizeUpdates(std::unexpected(apply_status.error())); + } + return apply_status; } Status PendingUpdate::Finalize( diff --git a/src/iceberg/update/pending_update.h b/src/iceberg/update/pending_update.h index 19998ddb3..dc9e705aa 100644 --- a/src/iceberg/update/pending_update.h +++ b/src/iceberg/update/pending_update.h @@ -38,7 +38,8 @@ namespace iceberg { /// /// \note Implementations are expected to use builder pattern and errors /// should be handled by the ErrorCollector base class. -class ICEBERG_EXPORT PendingUpdate : public ErrorCollector { +class ICEBERG_EXPORT PendingUpdate : public ErrorCollector, + public std::enable_shared_from_this { public: enum class Kind : uint8_t { kExpireSnapshots, @@ -66,6 +67,7 @@ class ICEBERG_EXPORT PendingUpdate : public ErrorCollector { /// - ValidationFailed: if it cannot be applied to the current table metadata. /// - CommitFailed: if it cannot be committed due to conflicts. /// - CommitStateUnknown: unknown status, no cleanup should be done. + /// \note The update must be owned by a `std::shared_ptr` before calling Commit(). virtual Status Commit(); /// \brief Finalize the pending update. diff --git a/src/iceberg/update/replace_partitions.cc b/src/iceberg/update/replace_partitions.cc index 3d224375c..4487751bf 100644 --- a/src/iceberg/update/replace_partitions.cc +++ b/src/iceberg/update/replace_partitions.cc @@ -32,11 +32,11 @@ namespace iceberg { -Result> ReplacePartitions::Make( +Result> ReplacePartitions::Make( std::string table_name, std::shared_ptr ctx) { ICEBERG_PRECHECK(!table_name.empty(), "Table name cannot be empty"); ICEBERG_PRECHECK(ctx != nullptr, "Cannot create ReplacePartitions without a context"); - return std::unique_ptr( + return std::shared_ptr( new ReplacePartitions(std::move(table_name), std::move(ctx))); } diff --git a/src/iceberg/update/replace_partitions.h b/src/iceberg/update/replace_partitions.h index 40048ba5c..ee57b6124 100644 --- a/src/iceberg/update/replace_partitions.h +++ b/src/iceberg/update/replace_partitions.h @@ -62,7 +62,7 @@ class ICEBERG_EXPORT ReplacePartitions : public MergingSnapshotUpdate { /// \param table_name The name of the table /// \param ctx The transaction context /// \return A Result containing the ReplacePartitions instance or an error - static Result> Make( + static Result> Make( std::string table_name, std::shared_ptr ctx); /// \brief Add a data file to the table. diff --git a/src/iceberg/update/rewrite_files.cc b/src/iceberg/update/rewrite_files.cc index b7fc048d3..3c743c01d 100644 --- a/src/iceberg/update/rewrite_files.cc +++ b/src/iceberg/update/rewrite_files.cc @@ -39,11 +39,11 @@ RewriteFiles::RewriteFiles(std::string table_name, FailMissingDeletePaths(); } -Result> RewriteFiles::Make( +Result> RewriteFiles::Make( std::string table_name, std::shared_ptr ctx) { ICEBERG_PRECHECK(!table_name.empty(), "Table name cannot be empty"); ICEBERG_PRECHECK(ctx != nullptr, "Cannot create RewriteFiles without a context"); - return std::unique_ptr( + return std::shared_ptr( new RewriteFiles(std::move(table_name), std::move(ctx))); } diff --git a/src/iceberg/update/rewrite_files.h b/src/iceberg/update/rewrite_files.h index ce219c3ae..283091c13 100644 --- a/src/iceberg/update/rewrite_files.h +++ b/src/iceberg/update/rewrite_files.h @@ -54,8 +54,8 @@ class ICEBERG_EXPORT RewriteFiles : public MergingSnapshotUpdate { /// /// \param table_name The name of the table /// \param ctx The transaction context - /// \return A unique pointer to the new RewriteFiles operation - static Result> Make( + /// \return A shared pointer to the new RewriteFiles operation + static Result> Make( std::string table_name, std::shared_ptr ctx); ~RewriteFiles() override = default; diff --git a/src/iceberg/update/row_delta.cc b/src/iceberg/update/row_delta.cc index dd3f50c58..f239ea495 100644 --- a/src/iceberg/update/row_delta.cc +++ b/src/iceberg/update/row_delta.cc @@ -38,11 +38,11 @@ namespace iceberg { -Result> RowDelta::Make( +Result> RowDelta::Make( std::string table_name, std::shared_ptr ctx) { ICEBERG_PRECHECK(!table_name.empty(), "Table name cannot be empty"); ICEBERG_PRECHECK(ctx != nullptr, "Cannot create RowDelta without a context"); - return std::unique_ptr(new RowDelta(std::move(table_name), std::move(ctx))); + return std::shared_ptr(new RowDelta(std::move(table_name), std::move(ctx))); } RowDelta::RowDelta(std::string table_name, std::shared_ptr ctx) diff --git a/src/iceberg/update/row_delta.h b/src/iceberg/update/row_delta.h index bf58a048c..f6faf8c92 100644 --- a/src/iceberg/update/row_delta.h +++ b/src/iceberg/update/row_delta.h @@ -47,7 +47,7 @@ namespace iceberg { class ICEBERG_EXPORT RowDelta : public MergingSnapshotUpdate { public: /// \brief Create a new RowDelta instance. - static Result> Make(std::string table_name, + static Result> Make(std::string table_name, std::shared_ptr ctx); /// \brief Add a DataFile to the table. From 5e90b57ba7df55e800702b0bc0832670cb596b92 Mon Sep 17 00:00:00 2001 From: Junwang Zhao Date: Mon, 31 Aug 2026 22:48:39 +0800 Subject: [PATCH 2/5] update misleading docs. Co-authored-by: Copilot Autofix powered by AI <175728472+Copilot@users.noreply.github.com> --- src/iceberg/update/pending_update.cc | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/iceberg/update/pending_update.cc b/src/iceberg/update/pending_update.cc index 70388e198..71e148b62 100644 --- a/src/iceberg/update/pending_update.cc +++ b/src/iceberg/update/pending_update.cc @@ -50,7 +50,8 @@ PendingUpdate::~PendingUpdate() = default; Status PendingUpdate::Commit() { if (!ctx_->transaction) { - // Table-created path: no transaction exists yet, create a temporary one. + // Standalone update path: no transaction is attached to the context, so create a + // temporary one for this Commit() call. ICEBERG_ASSIGN_OR_RAISE(auto txn, Transaction::Make(ctx_)); auto self = weak_from_this().lock(); ICEBERG_PRECHECK(self != nullptr, "PendingUpdate must be owned by std::shared_ptr"); From b508d03ae5389c03fe023752234c57e9b1c17710 Mon Sep 17 00:00:00 2001 From: Zhao Junwang Date: Wed, 9 Sep 2026 22:51:27 +0800 Subject: [PATCH 3/5] fix: make transaction updates replayable and cleanup-safe Enforce terminal transaction states, sequential updates, frozen intent, and explicit Abort cleanup. Replay registered updates internally and stop on Apply failures while preserving files for unknown catalog outcomes. Track and consume staged resources per generation, isolate terminal hooks, and suppress unsafe expiration cleanup after reference-adding updates. Add regression coverage for retries, no-ops, aliases, cleanup, and reentry. --- src/iceberg/test/expire_snapshots_test.cc | 148 ++++- src/iceberg/test/fast_append_test.cc | 581 ++++++++++++++++-- src/iceberg/test/merge_append_test.cc | 5 +- .../test/merging_snapshot_update_test.cc | 145 ++--- src/iceberg/test/transaction_test.cc | 140 ++++- src/iceberg/test/update_location_test.cc | 12 +- .../test/update_partition_spec_test.cc | 12 +- .../test/update_partition_statistics_test.cc | 16 +- src/iceberg/test/update_properties_test.cc | 30 +- src/iceberg/test/update_schema_test.cc | 392 +++++++----- src/iceberg/test/update_sort_order_test.cc | 12 +- src/iceberg/test/update_statistics_test.cc | 64 +- src/iceberg/test/update_test_base.h | 37 ++ src/iceberg/transaction.cc | 327 +++++++--- src/iceberg/transaction.h | 53 +- src/iceberg/update/delete_files.cc | 5 + src/iceberg/update/expire_snapshots.cc | 71 ++- src/iceberg/update/expire_snapshots.h | 29 +- src/iceberg/update/fast_append.cc | 38 +- src/iceberg/update/fast_append.h | 3 +- src/iceberg/update/merge_append.cc | 2 + src/iceberg/update/merging_snapshot_update.cc | 24 +- src/iceberg/update/overwrite_files.cc | 20 +- src/iceberg/update/pending_update.cc | 113 ++-- src/iceberg/update/pending_update.h | 82 ++- src/iceberg/update/replace_partitions.cc | 5 + src/iceberg/update/rewrite_files.cc | 13 +- src/iceberg/update/row_delta.cc | 18 +- src/iceberg/update/set_snapshot.cc | 5 +- src/iceberg/update/set_snapshot.h | 4 +- src/iceberg/update/snapshot_manager.cc | 24 + src/iceberg/update/snapshot_manager.h | 1 + src/iceberg/update/snapshot_update.cc | 147 +++-- src/iceberg/update/snapshot_update.h | 46 +- src/iceberg/update/update_location.cc | 3 +- src/iceberg/update/update_location.h | 6 +- src/iceberg/update/update_partition_spec.cc | 26 +- src/iceberg/update/update_partition_spec.h | 7 +- .../update/update_partition_statistics.cc | 17 +- .../update/update_partition_statistics.h | 5 +- src/iceberg/update/update_properties.cc | 15 +- src/iceberg/update/update_properties.h | 9 +- src/iceberg/update/update_schema.cc | 48 +- src/iceberg/update/update_schema.h | 12 +- .../update/update_snapshot_reference.cc | 26 +- .../update/update_snapshot_reference.h | 6 +- src/iceberg/update/update_sort_order.cc | 21 +- src/iceberg/update/update_sort_order.h | 8 +- src/iceberg/update/update_statistics.cc | 15 +- src/iceberg/update/update_statistics.h | 5 +- src/iceberg/update/update_util_internal.h | 191 ++++++ 51 files changed, 2314 insertions(+), 730 deletions(-) create mode 100644 src/iceberg/update/update_util_internal.h diff --git a/src/iceberg/test/expire_snapshots_test.cc b/src/iceberg/test/expire_snapshots_test.cc index b96754ec0..bf9b1e7e5 100644 --- a/src/iceberg/test/expire_snapshots_test.cc +++ b/src/iceberg/test/expire_snapshots_test.cc @@ -40,6 +40,9 @@ #include "iceberg/test/executor.h" #include "iceberg/test/matchers.h" #include "iceberg/test/update_test_base.h" +#include "iceberg/transaction.h" +#include "iceberg/update/fast_append.h" +#include "iceberg/update/set_snapshot.h" namespace iceberg { @@ -221,7 +224,7 @@ class ExpireSnapshotsCleanupTest : public UpdateTestBase { TEST_F(ExpireSnapshotsTest, DefaultExpireByAge) { ICEBERG_UNWRAP_OR_FAIL(auto update, table_->NewExpireSnapshots()); - ICEBERG_UNWRAP_OR_FAIL(auto result, update->Apply()); + ICEBERG_UNWRAP_OR_FAIL(auto result, update->Validate()); EXPECT_EQ(result.snapshot_ids_to_remove.size(), 1); EXPECT_EQ(result.snapshot_ids_to_remove.at(0), 3051729675574597004); } @@ -229,7 +232,7 @@ TEST_F(ExpireSnapshotsTest, DefaultExpireByAge) { TEST_F(ExpireSnapshotsTest, KeepAll) { ICEBERG_UNWRAP_OR_FAIL(auto update, table_->NewExpireSnapshots()); update->RetainLast(2); - ICEBERG_UNWRAP_OR_FAIL(auto result, update->Apply()); + ICEBERG_UNWRAP_OR_FAIL(auto result, update->Validate()); EXPECT_TRUE(result.snapshot_ids_to_remove.empty()); EXPECT_TRUE(result.refs_to_remove.empty()); } @@ -237,7 +240,7 @@ TEST_F(ExpireSnapshotsTest, KeepAll) { TEST_F(ExpireSnapshotsTest, ExpireById) { ICEBERG_UNWRAP_OR_FAIL(auto update, table_->NewExpireSnapshots()); update->ExpireSnapshotId(3051729675574597004); - ICEBERG_UNWRAP_OR_FAIL(auto result, update->Apply()); + ICEBERG_UNWRAP_OR_FAIL(auto result, update->Validate()); EXPECT_EQ(result.snapshot_ids_to_remove.size(), 1); EXPECT_EQ(result.snapshot_ids_to_remove.at(0), 3051729675574597004); } @@ -247,7 +250,7 @@ TEST_F(ExpireSnapshotsTest, ExpireByIdOverridesRetainLast) { update->RetainLast(2); update->ExpireSnapshotId(3051729675574597004); - ICEBERG_UNWRAP_OR_FAIL(auto result, update->Apply()); + ICEBERG_UNWRAP_OR_FAIL(auto result, update->Validate()); EXPECT_THAT(result.snapshot_ids_to_remove, testing::ElementsAre(3051729675574597004)); } @@ -262,7 +265,7 @@ TEST_F(ExpireSnapshotsTest, ExpireOlderThan) { for (const auto& test_case : test_cases) { ICEBERG_UNWRAP_OR_FAIL(auto update, table_->NewExpireSnapshots()); update->ExpireOlderThan(test_case.expire_older_than); - ICEBERG_UNWRAP_OR_FAIL(auto result, update->Apply()); + ICEBERG_UNWRAP_OR_FAIL(auto result, update->Validate()); EXPECT_EQ(result.snapshot_ids_to_remove.size(), test_case.expected_num_expired); } } @@ -286,26 +289,20 @@ TEST_F(ExpireSnapshotsCleanupTest, RetainsUnreferencedSnapshotAtExpireThreshold) ICEBERG_UNWRAP_OR_FAIL(auto update, table_->NewExpireSnapshots()); update->ExpireOlderThan(expire_at_ms); - ICEBERG_UNWRAP_OR_FAIL(auto result, update->Apply()); + ICEBERG_UNWRAP_OR_FAIL(auto result, update->Validate()); EXPECT_THAT(result.snapshot_ids_to_remove, testing::Not(testing::Contains(unreferenced_snapshot_id))); } -TEST_F(ExpireSnapshotsTest, FinalizeRequiresCommittedMetadata) { +TEST_F(ExpireSnapshotsTest, ValidateDoesNotScheduleCleanup) { std::vector deleted_files; - ICEBERG_UNWRAP_OR_FAIL(auto update, table_->NewExpireSnapshots()); + ICEBERG_UNWRAP_OR_FAIL(auto txn, table_->NewTransaction()); + ICEBERG_UNWRAP_OR_FAIL(auto update, txn->NewExpireSnapshots()); update->DeleteWith( [&deleted_files](const std::string& path) { deleted_files.push_back(path); }); - - // Apply first so apply_result_ is cached - ICEBERG_UNWRAP_OR_FAIL(auto result, update->Apply()); + ICEBERG_UNWRAP_OR_FAIL(auto result, update->Validate()); EXPECT_EQ(result.snapshot_ids_to_remove.size(), 1); - - // A successful finalize now requires the committed metadata from the catalog. - auto finalize_status = update->Finalize(static_cast(nullptr)); - EXPECT_THAT(finalize_status, IsError(ErrorKind::kInvalidArgument)); - EXPECT_THAT(finalize_status, - HasErrorMessage("Missing committed table metadata for cleanup")); + EXPECT_THAT(txn->Abort(), IsOk()); EXPECT_TRUE(deleted_files.empty()); } @@ -316,28 +313,24 @@ TEST_F(ExpireSnapshotsTest, CleanupNoneSkipsDeletion) { update->DeleteWith( [&deleted_files](const std::string& path) { deleted_files.push_back(path); }); - ICEBERG_UNWRAP_OR_FAIL(auto result, update->Apply()); + ICEBERG_UNWRAP_OR_FAIL(auto result, update->Validate()); EXPECT_EQ(result.snapshot_ids_to_remove.size(), 1); // With kNone cleanup level, Finalize should skip all file deletion - auto finalize_status = update->Finalize(static_cast(nullptr)); + auto finalize_status = update->Commit(); EXPECT_THAT(finalize_status, IsOk()); EXPECT_TRUE(deleted_files.empty()); } -TEST_F(ExpireSnapshotsTest, FinalizeSkippedOnCommitError) { +TEST_F(ExpireSnapshotsTest, AbortSkipsExpirationDeletion) { std::vector deleted_files; - ICEBERG_UNWRAP_OR_FAIL(auto update, table_->NewExpireSnapshots()); + ICEBERG_UNWRAP_OR_FAIL(auto txn, table_->NewTransaction()); + ICEBERG_UNWRAP_OR_FAIL(auto update, txn->NewExpireSnapshots()); update->DeleteWith( [&deleted_files](const std::string& path) { deleted_files.push_back(path); }); - - ICEBERG_UNWRAP_OR_FAIL(auto result, update->Apply()); - EXPECT_EQ(result.snapshot_ids_to_remove.size(), 1); - - // Simulate a commit failure - Finalize should not delete any files - auto finalize_status = update->Finalize(Result(std::unexpected( - Error{.kind = ErrorKind::kCommitFailed, .message = "simulated failure"}))); - EXPECT_THAT(finalize_status, IsOk()); + EXPECT_THAT(update->Commit(), IsOk()); + EXPECT_THAT(txn->Abort(), IsOk()); + EXPECT_THAT(txn->Abort(), IsOk()); EXPECT_TRUE(deleted_files.empty()); } @@ -348,11 +341,11 @@ TEST_F(ExpireSnapshotsTest, FinalizeSkipsWhenNothingExpired) { update->DeleteWith( [&deleted_files](const std::string& path) { deleted_files.push_back(path); }); - ICEBERG_UNWRAP_OR_FAIL(auto result, update->Apply()); + ICEBERG_UNWRAP_OR_FAIL(auto result, update->Validate()); EXPECT_TRUE(result.snapshot_ids_to_remove.empty()); // No snapshots expired, so Finalize should not delete any files - auto finalize_status = update->Finalize(static_cast(nullptr)); + auto finalize_status = update->Commit(); EXPECT_THAT(finalize_status, IsOk()); EXPECT_TRUE(deleted_files.empty()); } @@ -1016,4 +1009,97 @@ TEST_F(ExpireSnapshotsCleanupTest, CommitIgnoresMalformedSourceSnapshotIdCleanup EXPECT_EQ(committed_metadata->snapshots.at(0)->snapshot_id, kCurrentSnapshotId); } +class ExpirationLifecycleTest + : public ExpireSnapshotsCleanupTest, + public ::testing::WithParamInterface> {}; + +TEST_P(ExpirationLifecycleTest, LaterReferencesSuppressPhysicalDeletion) { + auto [cleanup_level, custom_delete, reattach, retry] = GetParam(); + const auto data_path = table_location_ + "/data/reattached.parquet"; + const auto manifest_path = table_location_ + "/metadata/expired.avro"; + const auto expired_list = table_location_ + "/metadata/expired-list.avro"; + const auto current_list = table_location_ + "/metadata/current-list.avro"; + auto data_file = MakeDataFile(data_path); + data_file->partition_spec_id = DefaultSpec()->spec_id(); + ASSERT_THAT(file_io_->WriteFile(data_path, "data"), IsOk()); + auto manifest = WriteDataManifest(manifest_path, kExpiredSnapshotId, + {MakeEntry(ManifestStatus::kAdded, kExpiredSnapshotId, + kExpiredSequenceNumber, data_file)}); + WriteManifestList(expired_list, kExpiredSnapshotId, 0, kExpiredSequenceNumber, + {manifest}); + WriteManifestList(current_list, kCurrentSnapshotId, kExpiredSnapshotId, + kCurrentSequenceNumber, {}); + RewriteTableWithManifestLists(expired_list, current_list); + if (retry) { + FailCommits(1); + } + ICEBERG_UNWRAP_OR_FAIL(auto txn, table_->NewTransaction()); + ICEBERG_UNWRAP_OR_FAIL(auto expire, txn->NewExpireSnapshots()); + expire->ExpireSnapshotId(kExpiredSnapshotId).CleanupLevel(cleanup_level); + int deletes = 0; + if (custom_delete) { + expire->DeleteWith([&](const std::string&) { ++deletes; }); + } + ASSERT_THAT(expire->Commit(), IsOk()); + if (reattach) { + // FastAppend inherits the conservative default capability from PendingUpdate. + ICEBERG_UNWRAP_OR_FAIL(auto append, txn->NewFastAppend()); + append->AppendFile(data_file); + ASSERT_THAT(append->Commit(), IsOk()); + } else { + ICEBERG_UNWRAP_OR_FAIL(auto noop, txn->NewSetSnapshot()); + noop->SetCurrentSnapshot(kCurrentSnapshotId); + ASSERT_THAT(noop->Commit(), IsOk()); + } + ASSERT_THAT(txn->Commit(), IsOk()); + EXPECT_FALSE(ReloadMetadata()->SnapshotById(kExpiredSnapshotId).has_value()); + EXPECT_EQ(deletes, 0); + for (const auto& path : {data_path, manifest_path, expired_list}) { + EXPECT_THAT(file_io_->ReadFile(path, std::nullopt), IsOk()); + } + EXPECT_THAT(txn->Commit(), IsError(ErrorKind::kValidationFailed)); + EXPECT_THAT(txn->Abort(), IsError(ErrorKind::kValidationFailed)); + EXPECT_EQ(deletes, 0); +} + +INSTANTIATE_TEST_SUITE_P( + CleanupPolicies, ExpirationLifecycleTest, + ::testing::Combine(::testing::Values(CleanupLevel::kAll, CleanupLevel::kMetadataOnly), + ::testing::Bool(), ::testing::Bool(), ::testing::Bool())); + +TEST_F(ExpireSnapshotsCleanupTest, RetryRecomputesExpirationAgainstRefreshedMetadata) { + const auto expired_list = table_location_ + "/metadata/expire-before-retry.avro"; + const auto current_list = table_location_ + "/metadata/current-before-retry.avro"; + WriteManifestList(expired_list, kExpiredSnapshotId, 0, kExpiredSequenceNumber, {}); + WriteManifestList(current_list, kCurrentSnapshotId, kExpiredSnapshotId, + kCurrentSequenceNumber, {}); + RewriteTableWithManifestLists(expired_list, current_list); + auto data_file = MakeDataFile(table_location_ + "/data/concurrent.parquet"); + data_file->partition_spec_id = DefaultSpec()->spec_id(); + FailCommits(1, [&](int attempt) { + if (attempt == 0) { + ICEBERG_UNWRAP_OR_FAIL(auto latest, catalog_->LoadTable(table_ident_)); + ICEBERG_UNWRAP_OR_FAIL(auto append, latest->NewFastAppend()); + append->AppendFile(data_file); + ASSERT_THAT(append->Commit(), IsOk()); + } + }); + ICEBERG_UNWRAP_OR_FAIL(auto expire, table_->NewExpireSnapshots()); + expire + ->ExpireOlderThan( + (CurrentTimePointMs() + std::chrono::hours(1)).time_since_epoch().count()) + .RetainLast(1); + std::vector deleted; + expire->DeleteWith([&](const std::string& path) { deleted.push_back(path); }); + ICEBERG_UNWRAP_OR_FAIL(auto preview, expire->Validate()); + EXPECT_THAT(preview.snapshot_ids_to_remove, ::testing::ElementsAre(kExpiredSnapshotId)); + ASSERT_THAT(expire->Commit(), IsOk()); + auto metadata = ReloadMetadata(); + EXPECT_FALSE(metadata->SnapshotById(kExpiredSnapshotId).has_value()); + EXPECT_FALSE(metadata->SnapshotById(kCurrentSnapshotId).has_value()); + EXPECT_EQ(metadata->snapshots.size(), 1U); + EXPECT_THAT(deleted, ::testing::Contains(expired_list)); + EXPECT_THAT(deleted, ::testing::Contains(current_list)); +} + } // namespace iceberg diff --git a/src/iceberg/test/fast_append_test.cc b/src/iceberg/test/fast_append_test.cc index 15cc78c40..f0b50069d 100644 --- a/src/iceberg/test/fast_append_test.cc +++ b/src/iceberg/test/fast_append_test.cc @@ -20,9 +20,11 @@ #include "iceberg/update/fast_append.h" #include +#include #include #include #include +#include #include #include #include @@ -34,6 +36,8 @@ #include "iceberg/avro/avro_register.h" #include "iceberg/constants.h" +#include "iceberg/exception.h" +#include "iceberg/expression/expressions.h" #include "iceberg/manifest/manifest_entry.h" #include "iceberg/manifest/manifest_reader.h" #include "iceberg/manifest/manifest_writer.h" @@ -45,12 +49,16 @@ #include "iceberg/snapshot.h" #include "iceberg/table_metadata.h" #include "iceberg/table_properties.h" +#include "iceberg/table_update.h" #include "iceberg/test/executor.h" #include "iceberg/test/matchers.h" #include "iceberg/test/mock_catalog.h" #include "iceberg/test/update_test_base.h" #include "iceberg/transaction.h" +#include "iceberg/update/delete_files.h" #include "iceberg/update/merge_append.h" +#include "iceberg/update/overwrite_files.h" +#include "iceberg/update/rewrite_files.h" #include "iceberg/update/update_properties.h" #include "iceberg/util/uuid.h" @@ -64,13 +72,44 @@ class TestSnapshotUpdate : public SnapshotUpdate { : SnapshotUpdate(std::move(ctx)) {} using SnapshotUpdate::ManifestPath; + using SnapshotUpdate::SnapshotId; + std::function apply_callback; + std::function finalize_callback; + std::function report_callback; + int applies = 0; + int finalizes = 0; + int reports = 0; + bool write_partial = false; + std::vector partial_paths; + + protected: Status CleanUncommitted(const std::unordered_set&) override { return {}; } - std::string operation() override { return "test"; } + std::string operation() override { return "append"; } Result> Apply(const TableMetadata&, const std::shared_ptr&) override { + ++applies; + if (write_partial) { + auto path = ManifestPath(); + partial_paths.push_back(path); + ICEBERG_RETURN_UNEXPECTED(ctx_->table->io()->WriteFile(path, "partial manifest")); + } + if (apply_callback) { + ICEBERG_RETURN_UNEXPECTED(apply_callback(applies)); + } return std::vector{}; } + Status Finalize(const TableMetadata& committed) override { + ++finalizes; + if (finalize_callback) { + ICEBERG_RETURN_UNEXPECTED(finalize_callback()); + } + return SnapshotUpdate::Finalize(committed); + } + Status ReportCommitted() override { + ++reports; + return report_callback ? report_callback() : Status{}; + } std::unordered_map Summary() override { return {}; } }; @@ -303,16 +342,20 @@ TEST_F(FastAppendTest, AppendNullFile) { EXPECT_THAT(table_->current_snapshot(), HasErrorMessage("No current snapshot")); } -TEST_F(FastAppendTest, FinalizeIgnoresCleanupDeleteFailure) { - std::shared_ptr fast_append; - ICEBERG_UNWRAP_OR_FAIL(fast_append, table_->NewFastAppend()); +TEST_F(FastAppendTest, AbortIgnoresCleanupDeleteFailure) { + ICEBERG_UNWRAP_OR_FAIL(auto txn, table_->NewTransaction()); + ICEBERG_UNWRAP_OR_FAIL(auto fast_append, txn->NewFastAppend()); fast_append->AppendFile(file_a_); - fast_append->DeleteWith([](const std::string&) { return IOError("delete failed"); }); - - EXPECT_THAT(static_cast(*fast_append).Apply(), IsOk()); - EXPECT_THAT(fast_append->Finalize(Result( - std::unexpected(CommitFailed("commit failed").error()))), - IsOk()); + int deletes = 0; + fast_append->DeleteWith([&](const std::string&) { + ++deletes; + return IOError("delete failed"); + }); + EXPECT_THAT(fast_append->Commit(), IsOk()); + EXPECT_THAT(txn->Abort(), IsOk()); + EXPECT_EQ(deletes, 2); + EXPECT_THAT(txn->Abort(), IsOk()); + EXPECT_EQ(deletes, 2); } TEST_F(FastAppendTest, TransactionApplyFailureCleansUpStagedFiles) { @@ -324,49 +367,49 @@ TEST_F(FastAppendTest, TransactionApplyFailureCleansUpStagedFiles) { return file_io_->DeleteFile(path); }); fast_append->AppendFile(file_a_); + EXPECT_THAT(fast_append->Commit(), IsOk()); - EXPECT_THAT(static_cast(*fast_append).Apply(), IsOk()); - fast_append->AppendFile(nullptr); - - EXPECT_THAT(fast_append->Commit(), IsError(ErrorKind::kValidationFailed)); + ICEBERG_UNWRAP_OR_FAIL(auto failed_append, txn->NewFastAppend()); + failed_append->AppendFile(nullptr); + EXPECT_THAT(failed_append->Commit(), IsError(ErrorKind::kValidationFailed)); + EXPECT_THAT(deleted_paths, ::testing::SizeIs(2U)); + EXPECT_EQ(txn->state(), TransactionState::kFailed); + EXPECT_THAT(txn->Commit(), IsError(ErrorKind::kValidationFailed)); + EXPECT_THAT(txn->Abort(), IsOk()); EXPECT_THAT(deleted_paths, ::testing::SizeIs(2U)); - EXPECT_THAT(txn->Commit(), - ::testing::AllOf(IsError(ErrorKind::kValidationFailed), - HasErrorMessage("Transaction already finalized"))); } TEST_F(FastAppendTest, RetryCopiesAppendManifestAgain) { table_->metadata()->format_version = 1; + ASSERT_THAT( + TableMetadataUtil::Write(*file_io_, std::string(table_->metadata_file_location()), + *table_->metadata()), + IsOk()); const auto path = table_location_ + "/metadata/input.avro"; ICEBERG_UNWRAP_OR_FAIL(auto manifest, WriteManifest(path, {file_a_})); - - std::shared_ptr fast_append; - ICEBERG_UNWRAP_OR_FAIL(fast_append, table_->NewFastAppend()); + FailCommits(2); + ICEBERG_UNWRAP_OR_FAIL(auto fast_append, table_->NewFastAppend()); std::vector deleted_paths; fast_append->DeleteWith([&](const std::string& deleted_path) { deleted_paths.push_back(deleted_path); + if (deleted_paths.size() == 1) { + throw std::runtime_error("cleanup callback threw"); + } + if (deleted_paths.size() == 2) { + return Status(IOError("cleanup returned an error")); + } return file_io_->DeleteFile(deleted_path); }); fast_append->AppendManifest(manifest); - - auto& update = static_cast(*fast_append); - // First Apply() copies the input manifest because v1 cannot inherit snapshot IDs. - ICEBERG_UNWRAP_OR_FAIL(auto first_apply, update.Apply()); - SnapshotCache first_cache(first_apply.snapshot.get()); - ICEBERG_UNWRAP_OR_FAIL(auto first_manifests, first_cache.Manifests(file_io_)); - ASSERT_EQ(first_manifests.size(), 1U); - const auto first_rewritten_path = first_manifests[0].manifest_path; - EXPECT_NE(first_rewritten_path, path); - - // Second Apply() simulates retry cleanup, then copies the original manifest again. - ICEBERG_UNWRAP_OR_FAIL(auto second_apply, update.Apply()); - EXPECT_THAT(deleted_paths, testing::Contains(first_rewritten_path)); - - SnapshotCache second_cache(second_apply.snapshot.get()); - ICEBERG_UNWRAP_OR_FAIL(auto second_manifests, second_cache.Manifests(file_io_)); - ASSERT_EQ(second_manifests.size(), 1U); - EXPECT_NE(second_manifests[0].manifest_path, path); - EXPECT_NE(second_manifests[0].manifest_path, first_rewritten_path); + EXPECT_THAT(fast_append->Commit(), IsOk()); + EXPECT_THAT(table_->Refresh(), IsOk()); + ICEBERG_UNWRAP_OR_FAIL(auto manifests, CurrentDataManifests()); + ASSERT_EQ(manifests.size(), 1U); + EXPECT_NE(manifests[0].manifest_path, path); + EXPECT_THAT(deleted_paths, ::testing::SizeIs(4U)); + EXPECT_THAT(deleted_paths, ::testing::Not(::testing::Contains(path))); + EXPECT_THAT(deleted_paths, + ::testing::Not(::testing::Contains(manifests[0].manifest_path))); } TEST_F(FastAppendTest, AppendDuplicateFile) { @@ -758,4 +801,464 @@ TEST_F(FastAppendMetricsTest, CommitStateUnknownDoesNotReport) { EXPECT_TRUE(reporter_->reports().empty()); } +TEST_F(FastAppendTest, ReplayApplyFailureStopsRetryAndCleansPartialFiles) { + for (auto replay_kind : + {ErrorKind::kRetryableValidationFailed, ErrorKind::kCommitStateUnknown}) { + SCOPED_TRACE(static_cast(replay_kind)); + auto mock = std::make_shared<::testing::NiceMock>(); + EXPECT_CALL(*mock, UpdateTable(::testing::_, ::testing::_, ::testing::_)) + .Times(1) + .WillOnce(::testing::Return(CommitFailed("first conflict"))); + EXPECT_CALL(*mock, LoadTable(::testing::_)).Times(1).WillOnce([&](const auto& name) { + return catalog_->LoadTable(name); + }); + ICEBERG_UNWRAP_OR_FAIL( + auto table, + Table::Make(table_->name(), table_->metadata(), + std::string(table_->metadata_file_location()), file_io_, mock)); + ICEBERG_UNWRAP_OR_FAIL(auto ctx, + TransactionContext::Make(table, TransactionKind::kUpdate)); + auto update = std::make_shared(ctx); + update->write_partial = true; + update->apply_callback = [replay_kind](int attempt) -> Status { + if (attempt == 2) { + return std::unexpected( + Error{.kind = replay_kind, .message = "replay validation failed"}); + } + return {}; + }; + std::vector deleted; + update->DeleteWith([&](const std::string& path) { + deleted.push_back(path); + EXPECT_THAT(update->Commit(), IsError(ErrorKind::kValidationFailed)); + EXPECT_THROW(update->Set("reenter", "value"), IcebergError); + return file_io_->DeleteFile(path); + }); + EXPECT_THAT(update->Commit(), + ::testing::AllOf(IsError(replay_kind), + HasErrorMessage("replay validation failed"))); + EXPECT_EQ(update->applies, 2); + EXPECT_EQ(update->finalizes, 0); + EXPECT_EQ(update->reports, 0); + EXPECT_THAT(deleted, ::testing::SizeIs(3U)); + for (const auto& path : update->partial_paths) { + EXPECT_THAT(deleted, ::testing::Contains(path)); + EXPECT_FALSE(file_io_->ReadFile(path, std::nullopt).has_value()); + } + EXPECT_THAT(update->Commit(), IsError(ErrorKind::kValidationFailed)); + EXPECT_FALSE(ctx->transaction.has_value()); + EXPECT_THAT(deleted, ::testing::SizeIs(3U)); + } +} + +TEST_F(FastAppendTest, StandaloneHooksAreTerminalAndIsolated) { + ICEBERG_UNWRAP_OR_FAIL(auto ctx, + TransactionContext::Make(table_, TransactionKind::kUpdate)); + auto update = std::make_shared(ctx); + update->finalize_callback = [&]() -> Status { + EXPECT_THAT(update->Commit(), IsError(ErrorKind::kValidationFailed)); + EXPECT_THROW(update->DeleteWith({}), IcebergError); + throw std::runtime_error("finalize failure"); + }; + update->report_callback = [&]() -> Status { + EXPECT_THAT(update->Commit(), IsError(ErrorKind::kValidationFailed)); + throw std::runtime_error("report failure"); + }; + ASSERT_THAT(update->Commit(), IsOk()); + EXPECT_EQ(update->finalizes, 1); + EXPECT_EQ(update->reports, 1); + EXPECT_THAT(update->Commit(), IsError(ErrorKind::kValidationFailed)); + EXPECT_EQ(update->finalizes, 1); + EXPECT_EQ(update->reports, 1); +} + +class CallbackReporter : public MetricsReporter { + public: + explicit CallbackReporter(std::function callback) + : callback_(std::move(callback)) {} + Status Report(const MetricsReport&) override { return callback_(); } + + private: + std::function callback_; +}; + +TEST_F(FastAppendTest, ExplicitSuccessPublishesAllTerminalMarkersBeforeReporting) { + ICEBERG_UNWRAP_OR_FAIL(auto txn, table_->NewTransaction()); + ICEBERG_UNWRAP_OR_FAIL(auto first, txn->NewFastAppend()); + std::shared_ptr second; + int reports = 0; + first->AppendFile(file_a_).ReportWith( + std::make_shared([&]() -> Status { + ++reports; + EXPECT_EQ(txn->state(), TransactionState::kCommitted); + EXPECT_THAT(first->Commit(), IsError(ErrorKind::kValidationFailed)); + EXPECT_THAT(second->Commit(), IsError(ErrorKind::kValidationFailed)); + EXPECT_THAT(txn->Commit(), IsError(ErrorKind::kValidationFailed)); + EXPECT_THAT(txn->NewFastAppend(), IsError(ErrorKind::kValidationFailed)); + EXPECT_THAT(txn->Abort(), IsError(ErrorKind::kValidationFailed)); + throw std::runtime_error("report failure"); + })); + ASSERT_THAT(first->Commit(), IsOk()); + ICEBERG_UNWRAP_OR_FAIL(second, txn->NewFastAppend()); + second->AppendFile(file_b_).ReportWith( + std::make_shared([&]() -> Status { + ++reports; + return IOError("another report failure"); + })); + ASSERT_THAT(second->Commit(), IsOk()); + ASSERT_THAT(txn->Commit(), IsOk()); + EXPECT_EQ(reports, 2); + EXPECT_THAT(txn->Commit(), IsError(ErrorKind::kValidationFailed)); + EXPECT_EQ(reports, 2); +} + +TEST_F(FastAppendTest, FrozenFileAliasesCannotChangeReplay) { + FailCommits(2); + ICEBERG_UNWRAP_OR_FAIL(auto txn, table_->NewTransaction()); + ICEBERG_UNWRAP_OR_FAIL(auto append, txn->NewFastAppend()); + append->AppendFile(file_a_); + const auto original_path = file_a_->file_path; + const auto original_records = file_a_->record_count; + ASSERT_THAT(append->Commit(), IsOk()); + EXPECT_THROW(append->AppendFile(file_b_), IcebergError); + EXPECT_THROW(append->DeleteWith({}), IcebergError); + file_a_->file_path = "/changed.parquet"; + file_a_->record_count = 123456; + file_a_->partition = PartitionValues({Literal::Long(42)}); + ASSERT_THAT(txn->Commit(), IsOk()); + EXPECT_THAT(table_->Refresh(), IsOk()); + ICEBERG_UNWRAP_OR_FAIL(auto manifests, CurrentDataManifests()); + ASSERT_EQ(manifests.size(), 1U); + ICEBERG_UNWRAP_OR_FAIL(auto entries, ReadEntries(manifests[0])); + ASSERT_EQ(entries.size(), 1U); + EXPECT_EQ(entries[0].data_file->file_path, original_path); + EXPECT_EQ(entries[0].data_file->record_count, original_records); + ICEBERG_UNWRAP_OR_FAIL(auto snapshot, table_->current_snapshot()); + EXPECT_EQ(snapshot->summary.at("added-records"), std::to_string(original_records)); +} + +TEST_F(FastAppendTest, ConflictThenUnknownPreservesLastAttempt) { + auto mock = std::make_shared<::testing::NiceMock>(); + EXPECT_CALL(*mock, UpdateTable(::testing::_, ::testing::_, ::testing::_)) + .WillOnce(::testing::Return(CommitFailed("conflict"))) + .WillOnce(::testing::Return(CommitStateUnknown("unknown"))); + EXPECT_CALL(*mock, LoadTable(::testing::_)).WillOnce([&](const auto& name) { + return catalog_->LoadTable(name); + }); + ICEBERG_UNWRAP_OR_FAIL( + auto table, + Table::Make(table_->name(), table_->metadata(), + std::string(table_->metadata_file_location()), file_io_, mock)); + ICEBERG_UNWRAP_OR_FAIL(auto txn, table->NewTransaction()); + ICEBERG_UNWRAP_OR_FAIL(auto append, txn->NewFastAppend()); + int deletes = 0; + append->AppendFile(file_a_).DeleteWith([&](const std::string& path) { + ++deletes; + EXPECT_THAT(txn->Abort(), IsError(ErrorKind::kValidationFailed)); + EXPECT_THAT(txn->NewFastAppend(), IsError(ErrorKind::kValidationFailed)); + return file_io_->DeleteFile(path); + }); + ASSERT_THAT(append->Commit(), IsOk()); + EXPECT_THAT(txn->Commit(), IsError(ErrorKind::kCommitStateUnknown)); + EXPECT_EQ(txn->state(), TransactionState::kCommitStateUnknown); + EXPECT_EQ(deletes, 2); + ICEBERG_UNWRAP_OR_FAIL(auto snapshot, txn->current().Snapshot()); + EXPECT_THAT(file_io_->ReadFile(snapshot->manifest_list, std::nullopt), IsOk()); + SnapshotCache cache(snapshot.get()); + ICEBERG_UNWRAP_OR_FAIL(auto manifests, cache.Manifests(file_io_)); + ASSERT_EQ(manifests.size(), 1U); + EXPECT_THAT(file_io_->ReadFile(manifests[0].manifest_path, std::nullopt), IsOk()); + EXPECT_THAT(txn->Abort(), IsError(ErrorKind::kValidationFailed)); + EXPECT_THAT(txn->Commit(), IsError(ErrorKind::kValidationFailed)); + EXPECT_THAT(append->Commit(), IsError(ErrorKind::kValidationFailed)); + EXPECT_EQ(deletes, 2); +} + +TEST_F(FastAppendTest, ApplyFailureAfterWritingCleansEveryUpdate) { + ICEBERG_UNWRAP_OR_FAIL(auto txn, table_->NewTransaction()); + int deletes = 0; + auto delete_file = [&](const std::string& path) -> Status { + ++deletes; + EXPECT_EQ(txn->state(), TransactionState::kFailed); + EXPECT_THAT(txn->Abort(), IsError(ErrorKind::kValidationFailed)); + if (deletes == 1) { + throw std::runtime_error("delete callback threw"); + } + return IOError("delete failure"); + }; + ICEBERG_UNWRAP_OR_FAIL(auto first, txn->NewFastAppend()); + first->AppendFile(file_a_).DeleteWith(delete_file); + ASSERT_THAT(first->Commit(), IsOk()); + ICEBERG_UNWRAP_OR_FAIL(auto failed, txn->NewRewriteFiles()); + failed->DeleteDataFile(file_a_).AddDataFile(file_b_).DeleteWith(delete_file); + EXPECT_THAT(failed->Commit(), HasErrorMessage("Invalid REPLACE operation")); + EXPECT_EQ(txn->state(), TransactionState::kFailed); + EXPECT_GE(deletes, 4); + const int cleanup_attempts = deletes; + EXPECT_THAT(txn->Commit(), IsError(ErrorKind::kValidationFailed)); + EXPECT_THAT(txn->Abort(), IsOk()); + EXPECT_THAT(txn->Abort(), IsOk()); + EXPECT_EQ(deletes, cleanup_attempts); +} + +TEST_F(FastAppendTest, ReplayBecomingNoopCleansStagingWithoutReporting) { + for (bool other_update : {false, true}) { + SCOPED_TRACE(other_update); + auto mock = std::make_shared<::testing::NiceMock>(); + std::shared_ptr
refreshed; + std::shared_ptr refreshed_metadata; + int attempts = 0; + EXPECT_CALL(*mock, UpdateTable(::testing::_, ::testing::_, ::testing::_)) + .Times(other_update ? 2 : 1) + .WillRepeatedly([&](const auto&, const auto&, + const auto& changes) -> Result> { + const bool first_attempt = ++attempts == 1; + auto builder = TableMetadataBuilder::BuildFrom( + first_attempt ? table_->metadata().get() : refreshed_metadata.get()); + for (const auto& change : changes) { + if (first_attempt && change->kind() == TableUpdate::Kind::kSetProperties) { + continue; + } + if (!first_attempt) { + EXPECT_EQ(change->kind(), TableUpdate::Kind::kSetProperties); + } + change->ApplyTo(*builder); + } + ICEBERG_ASSIGN_OR_RAISE(std::shared_ptr metadata, + builder->Build()); + if (first_attempt) { + // Use independent files for the refreshed snapshot; generated paths are + // owned by the update that created them. + metadata->snapshots.back() = + std::make_shared(*metadata->snapshots.back()); + auto snapshot = metadata->snapshots.back(); + snapshot->manifest_list = table_location_ + "/metadata/concurrent-list.avro"; + ICEBERG_ASSIGN_OR_RAISE( + auto writer, ManifestListWriter::MakeWriter( + metadata->format_version, snapshot->snapshot_id, + snapshot->parent_snapshot_id, snapshot->manifest_list, + file_io_, snapshot->sequence_number)); + ICEBERG_RETURN_UNEXPECTED(writer->Close()); + } + // Simulate the same logical snapshot already becoming current on refresh. + refreshed_metadata = metadata; + ICEBERG_ASSIGN_OR_RAISE( + refreshed, + Table::Make(table_->name(), std::move(metadata), + std::string(table_->metadata_file_location()) + ".refreshed", + file_io_, mock)); + if (first_attempt) { + return CommitFailed("conflict"); + } + return refreshed; + }); + EXPECT_CALL(*mock, LoadTable(::testing::_)) + .WillOnce( + [&](const auto&) -> Result> { return refreshed; }); + ICEBERG_UNWRAP_OR_FAIL( + auto table, + Table::Make(table_->name(), table_->metadata(), + std::string(table_->metadata_file_location()), file_io_, mock)); + std::shared_ptr txn; + if (other_update) { + ICEBERG_UNWRAP_OR_FAIL(txn, table->NewTransaction()); + } + ICEBERG_UNWRAP_OR_FAIL(auto append, + txn ? txn->NewFastAppend() : table->NewFastAppend()); + int reports = 0; + int deletes = 0; + append->AppendFile(file_a_) + .ReportWith(std::make_shared([&]() -> Status { + ++reports; + return {}; + })) + .DeleteWith([&](const std::string& path) { + ++deletes; + return file_io_->DeleteFile(path); + }); + ASSERT_THAT(append->Commit(), IsOk()); + if (txn) { + ICEBERG_UNWRAP_OR_FAIL(auto props, txn->NewUpdateProperties()); + props->Set("effective", "value"); + ASSERT_THAT(props->Commit(), IsOk()); + ICEBERG_UNWRAP_OR_FAIL(auto committed, txn->Commit()); + EXPECT_EQ(committed->properties().configs().at("effective"), "value"); + EXPECT_EQ(txn->state(), TransactionState::kCommitted); + } + EXPECT_EQ(deletes, 4); + EXPECT_EQ(reports, 0); + EXPECT_THAT(append->Commit(), IsError(ErrorKind::kValidationFailed)); + EXPECT_EQ(deletes, 4); + } +} + +TEST_F(FastAppendTest, FrozenFilterAliasesCannotChangeReplay) { + file_a_->partition = PartitionValues({Literal::Long(1)}); + file_b_->partition = PartitionValues({Literal::Long(2)}); + ICEBERG_UNWRAP_OR_FAIL(auto first, table_->NewFastAppend()); + first->AppendFile(file_a_).AppendFile(file_b_); + ASSERT_THAT(first->Commit(), IsOk()); + ASSERT_THAT(table_->Refresh(), IsOk()); + FailCommits(2); + ICEBERG_UNWRAP_OR_FAIL(auto txn, table_->NewTransaction()); + ICEBERG_UNWRAP_OR_FAIL(auto update, txn->NewDeleteFiles()); + auto reference = Expressions::Ref("x"); + auto predicate = Expressions::Equal(reference, Literal::Long(1)); + update->DeleteFromRowFilter(predicate); + ASSERT_THAT(update->Commit(), IsOk()); + *reference = *Expressions::Ref("missing"); + ASSERT_THAT(txn->Commit(), IsOk()); + ASSERT_THAT(table_->Refresh(), IsOk()); + ICEBERG_UNWRAP_OR_FAIL(auto snapshot, table_->current_snapshot()); + EXPECT_EQ(snapshot->summary.at("deleted-data-files"), "1"); +} + +TEST_F(FastAppendTest, StagedNoopIsCleanedAndCanBecomeEffectiveOnRebase) { + for (bool rebase : {false, true}) { + SCOPED_TRACE(rebase); + auto mock = std::make_shared<::testing::NiceMock>(); + ON_CALL(*mock, LoadTable(::testing::_)).WillByDefault([&](const auto& name) { + return catalog_->LoadTable(name); + }); + ON_CALL(*mock, UpdateTable(::testing::_, ::testing::_, ::testing::_)) + .WillByDefault( + [&](const auto& name, const auto& requirements, const auto& changes) { + return catalog_->UpdateTable(name, requirements, changes); + }); + EXPECT_CALL(*mock, LoadTable(::testing::_)).Times(rebase ? 1 : 0); + EXPECT_CALL(*mock, UpdateTable(::testing::_, ::testing::_, ::testing::_)) + .Times(rebase ? 1 : 0); + ICEBERG_UNWRAP_OR_FAIL(auto ctx, + TransactionContext::Make(table_, TransactionKind::kUpdate)); + auto update = std::make_shared(ctx); + update->write_partial = true; + auto builder = TableMetadataBuilder::BuildFrom(table_->metadata().get()); + auto existing = std::make_shared(Snapshot{ + .sequence_number = table_->metadata()->NextSequenceNumber(), + .snapshot_id = update->SnapshotId(), + .timestamp_ms = CurrentTimePointMs(), + .manifest_list = table_location_ + "/metadata/existing-noop-list.avro", + .summary = {{SnapshotSummaryFields::kOperation, DataOperation::kAppend}}, + .schema_id = table_->metadata()->current_schema_id, + }); + ICEBERG_UNWRAP_OR_FAIL( + auto writer, ManifestListWriter::MakeWriter(table_->metadata()->format_version, + existing->snapshot_id, std::nullopt, + existing->manifest_list, file_io_, + existing->sequence_number)); + ASSERT_THAT(writer->Close(), IsOk()); + builder->SetBranchSnapshot(existing, std::string(SnapshotRef::kMainBranch)); + ICEBERG_UNWRAP_OR_FAIL(std::shared_ptr metadata, builder->Build()); + // Prepare a base whose current snapshot has this update's logical ID. + ICEBERG_UNWRAP_OR_FAIL( + ctx->table, + Table::Make(table_->name(), metadata, + std::string(table_->metadata_file_location()) + ".synthetic", + file_io_, mock)); + ctx->metadata_builder = TableMetadataBuilder::BuildFrom(metadata.get()); + std::vector deleted; + update->DeleteWith([&](const std::string& path) { + deleted.push_back(path); + return file_io_->DeleteFile(path); + }); + if (rebase) { + // The table refreshes before the update commits its original builder. + // Initial Apply is a no-op; internal replay sees the real empty table. + ASSERT_THAT(ctx->table->Refresh(), IsOk()); + } + ASSERT_THAT(update->Commit(), IsOk()); + EXPECT_EQ(update->applies, rebase ? 2 : 1); + EXPECT_EQ(update->finalizes, rebase ? 1 : 0); + EXPECT_EQ(update->reports, rebase ? 1 : 0); + EXPECT_THAT(deleted, ::testing::SizeIs(rebase ? 3U : 2U)); + EXPECT_THAT(file_io_->ReadFile(existing->manifest_list, std::nullopt), IsOk()); + EXPECT_THAT(update->Commit(), IsError(ErrorKind::kValidationFailed)); + } +} + +TEST_F(FastAppendTest, ReplayFailurePartwayThroughCleansAllUncommittedGenerations) { + file_a_->partition = PartitionValues({Literal::Long(1)}); + file_b_->partition = PartitionValues({Literal::Long(3)}); + ICEBERG_UNWRAP_OR_FAIL(auto initial, table_->NewFastAppend()); + initial->AppendFile(file_a_); + ASSERT_THAT(initial->Commit(), IsOk()); + ASSERT_THAT(table_->Refresh(), IsOk()); + ICEBERG_UNWRAP_OR_FAIL(auto original_snapshot, table_->current_snapshot()); + auto concurrent_file = CreateDataFile("/data/concurrent.parquet", 1, 10, 1); + auto replacement = CreateDataFile("/data/replacement.parquet", 100, 50, 1); + auto mock = std::make_shared<::testing::NiceMock>(); + EXPECT_CALL(*mock, UpdateTable(::testing::_, ::testing::_, ::testing::_)) + .Times(1) + .WillOnce( + [&](const auto&, const auto&, const auto&) -> Result> { + ICEBERG_ASSIGN_OR_RAISE(auto latest, catalog_->LoadTable(table_ident_)); + ICEBERG_ASSIGN_OR_RAISE(auto concurrent, latest->NewFastAppend()); + concurrent->AppendFile(concurrent_file); + ICEBERG_RETURN_UNEXPECTED(concurrent->Commit()); + return CommitFailed("concurrent append"); + }); + EXPECT_CALL(*mock, LoadTable(::testing::_)).Times(1).WillOnce([&](const auto& name) { + return catalog_->LoadTable(name); + }); + ICEBERG_UNWRAP_OR_FAIL( + auto table, + Table::Make(table_->name(), table_->metadata(), + std::string(table_->metadata_file_location()), file_io_, mock)); + ICEBERG_UNWRAP_OR_FAIL(auto txn, table->NewTransaction()); + std::vector deleted; + std::shared_ptr append; + auto delete_file = [&](const std::string& path) { + deleted.push_back(path); + const auto state = txn->state(); + EXPECT_THAT(append->Commit(), IsError(ErrorKind::kValidationFailed)); + EXPECT_THAT(txn->Commit(), IsError(ErrorKind::kValidationFailed)); + EXPECT_THAT(txn->NewFastAppend(), IsError(ErrorKind::kValidationFailed)); + EXPECT_THAT(txn->Abort(), IsError(ErrorKind::kValidationFailed)); + EXPECT_EQ(txn->state(), state); + return file_io_->DeleteFile(path); + }; + ICEBERG_UNWRAP_OR_FAIL(append, txn->NewFastAppend()); + append->AppendFile(file_b_).DeleteWith(delete_file); + ASSERT_THAT(append->Commit(), IsOk()); + ICEBERG_UNWRAP_OR_FAIL(auto overwrite, txn->NewOverwrite()); + overwrite->DeleteFile(file_a_) + .AddFile(replacement) + .ValidateFromSnapshot(original_snapshot->snapshot_id) + .ConflictDetectionFilter(Expressions::Equal("x", Literal::Long(1))) + .ValidateNoConflictingData() + .DeleteWith(delete_file); + ASSERT_THAT(overwrite->Commit(), IsOk()); + EXPECT_THAT(txn->Commit(), HasErrorMessage("Found conflicting files")); + EXPECT_EQ(txn->state(), TransactionState::kFailed); + EXPECT_GE(deleted.size(), 6U); + std::unordered_set unique_deletes(deleted.begin(), deleted.end()); + EXPECT_EQ(unique_deletes.size(), deleted.size()); + EXPECT_THAT(txn->Abort(), IsOk()); + EXPECT_EQ(deleted.size(), unique_deletes.size()); + + std::unordered_set committed_paths; + auto metadata = ReloadMetadata(); + for (const auto& snapshot : metadata->snapshots) { + committed_paths.insert(snapshot->manifest_list); + SnapshotCache cache(snapshot.get()); + ICEBERG_UNWRAP_OR_FAIL(auto manifests, cache.Manifests(file_io_)); + for (const auto& manifest : manifests) { + committed_paths.insert(manifest.manifest_path); + } + } + auto& io = static_cast(*file_io_); + ::arrow::fs::FileSelector selector; + selector.base_dir = table_location_ + "/metadata"; + selector.recursive = true; + auto files = io.fs()->GetFileInfo(selector); + ASSERT_TRUE(files.ok()) << files.status(); + std::unordered_set remaining; + for (const auto& file : *files) { + if (file.path().ends_with(".avro")) { + remaining.insert(file.path()); + } + } + EXPECT_EQ(remaining, committed_paths); +} + } // namespace iceberg diff --git a/src/iceberg/test/merge_append_test.cc b/src/iceberg/test/merge_append_test.cc index bd4527904..3bf0473c3 100644 --- a/src/iceberg/test/merge_append_test.cc +++ b/src/iceberg/test/merge_append_test.cc @@ -1212,8 +1212,9 @@ TEST_P(MergeAppendTest, Recovery) { ICEBERG_UNWRAP_OR_FAIL(auto second_snapshot, CurrentSnapshot()); ICEBERG_UNWRAP_OR_FAIL(auto data_manifests, CurrentDataManifests()); ASSERT_EQ(data_manifests.size(), 1U); - EXPECT_EQ(data_manifests[0].manifest_path, pending_manifest.manifest_path); - EXPECT_TRUE(FileExists(pending_manifest.manifest_path)); + EXPECT_NE(data_manifests[0].manifest_path, pending_manifest.manifest_path); + EXPECT_FALSE(FileExists(pending_manifest.manifest_path)); + EXPECT_TRUE(FileExists(data_manifests[0].manifest_path)); ExpectManifestEntries( data_manifests[0], {file_b_, file_a_}, {ManifestStatus::kAdded, ManifestStatus::kExisting}, diff --git a/src/iceberg/test/merging_snapshot_update_test.cc b/src/iceberg/test/merging_snapshot_update_test.cc index f293b7923..1526d2339 100644 --- a/src/iceberg/test/merging_snapshot_update_test.cc +++ b/src/iceberg/test/merging_snapshot_update_test.cc @@ -55,6 +55,7 @@ #include "iceberg/test/retry.h" #include "iceberg/test/update_test_base.h" #include "iceberg/transaction.h" +#include "iceberg/update/delete_files.h" #include "iceberg/update/fast_append.h" #include "iceberg/update/merge_append.h" #include "iceberg/update/row_delta.h" @@ -92,9 +93,13 @@ class TestMergeAppend : public MergingSnapshotUpdate { std::string operation() override { return "append"; } // Expose protected API for test access - using MergingSnapshotUpdate::Apply; - using MergingSnapshotUpdate::CleanUncommitted; - using MergingSnapshotUpdate::Summary; + Result> CommitManifests() { + ICEBERG_RETURN_UNEXPECTED(Commit()); + ICEBERG_ASSIGN_OR_RAISE(auto snapshot, ctx_->table->current_snapshot()); + SnapshotCache cache(snapshot.get()); + ICEBERG_ASSIGN_OR_RAISE(auto manifests, cache.Manifests(ctx_->table->io())); + return std::vector(manifests.begin(), manifests.end()); + } Status AddFile(std::shared_ptr file) { return AddDataFile(std::move(file)); } Status AddDelete(std::shared_ptr file) { @@ -125,16 +130,6 @@ class TestMergeAppend : public MergingSnapshotUpdate { Result> DataSpec() const { return MergingSnapshotUpdate::DataSpec(); } - Result> WriteDeletesForTest( - std::span> files, - const std::shared_ptr& spec) { - auto entries = files | std::views::transform([](const auto& file) { - return ContentFileWithSequenceNumber{ - .file = file, .data_sequence_number = std::nullopt}; - }) | - std::ranges::to(); - return WriteDeleteManifests(entries, spec); - } int64_t GeneratedSnapshotId() { return SnapshotId(); } void SetDataSeqNumber(int64_t seq) { SetNewDataFilesDataSequenceNumber(seq); } void SetCaseSensitive(bool case_sensitive) { CaseSensitive(case_sensitive); } @@ -242,7 +237,13 @@ class TestOverwriteUpdate : public MergingSnapshotUpdate { std::string operation() override { return DataOperation::kOverwrite; } int64_t GeneratedSnapshotId() { return SnapshotId(); } - using MergingSnapshotUpdate::Apply; + Result> CommitManifests() { + ICEBERG_RETURN_UNEXPECTED(Commit()); + ICEBERG_ASSIGN_OR_RAISE(auto snapshot, ctx_->table->current_snapshot()); + SnapshotCache cache(snapshot.get()); + ICEBERG_ASSIGN_OR_RAISE(auto manifests, cache.Manifests(ctx_->table->io())); + return std::vector(manifests.begin(), manifests.end()); + } Status AddDelete(std::shared_ptr file) { return AddDeleteFile(std::move(file)); @@ -892,8 +893,7 @@ TEST_F(MergingSnapshotUpdateTest, CleanUncommittedAfterSuccessfulCommitDoesNotCr EXPECT_THAT(op->AddFile(file_a_), IsOk()); EXPECT_THAT(op->Commit(), IsOk()); - // Cleanup may run from an error handler even after commit success. - EXPECT_THAT(op->CleanUncommitted({}), IsOk()); + EXPECT_THAT(op->Commit(), IsError(ErrorKind::kValidationFailed)); } TEST_F(MergingSnapshotUpdateTest, @@ -905,18 +905,15 @@ TEST_F(MergingSnapshotUpdateTest, EXPECT_THAT(table_->Refresh(), IsOk()); std::vector deleted_paths; - ICEBERG_UNWRAP_OR_FAIL(auto op, NewMergeAppend()); + ICEBERG_UNWRAP_OR_FAIL(auto txn, table_->NewTransaction()); + ICEBERG_UNWRAP_OR_FAIL(auto op, txn->NewDeleteFiles()); op->DeleteWith([&deleted_paths](const std::string& path) { deleted_paths.push_back(path); return Status{}; }); - EXPECT_THAT(op->RemoveDataFile(file_a_), IsOk()); - - ICEBERG_UNWRAP_OR_FAIL( - auto manifests, op->Apply(*table_->metadata(), table_->current_snapshot().value())); - EXPECT_THAT(manifests, ::testing::SizeIs(1)); - - EXPECT_THAT(op->CleanUncommitted({}), IsOk()); + op->DeleteFile(file_a_->file_path); + EXPECT_THAT(op->Commit(), IsOk()); + EXPECT_THAT(txn->Abort(), IsOk()); EXPECT_THAT(deleted_paths, ::testing::Contains(::testing::HasSubstr("/metadata/"))); } @@ -957,7 +954,7 @@ TEST_F(MergingSnapshotUpdateTest, AddDeleteFileWithExplicitSequenceWritesSequenc ICEBERG_UNWRAP_OR_FAIL(auto op, NewMergeAppend()); EXPECT_THAT(op->AddDelete(del_file, 17), IsOk()); - ICEBERG_UNWRAP_OR_FAIL(auto manifests, op->Apply(*table_->metadata(), nullptr)); + ICEBERG_UNWRAP_OR_FAIL(auto manifests, op->CommitManifests()); auto delete_manifest_it = std::ranges::find_if(manifests, [](const ManifestFile& manifest) { return manifest.content == ManifestContent::kDeletes; @@ -985,7 +982,10 @@ TEST_F(MergingSnapshotUpdateTest, WriteDeleteGroups) { static_cast(index % 2)); }) | std::ranges::to(); - ICEBERG_UNWRAP_OR_FAIL(auto manifests, op->WriteDeletesForTest(files, spec_)); + for (const auto& file : files) { + EXPECT_THAT(op->AddDelete(file), IsOk()); + } + ICEBERG_UNWRAP_OR_FAIL(auto manifests, op->CommitManifests()); EXPECT_EQ(executor.submit_count(), 2); ASSERT_EQ(manifests.size(), 2U); @@ -994,24 +994,17 @@ TEST_F(MergingSnapshotUpdateTest, WriteDeleteGroups) { } } -TEST_F(MergingSnapshotUpdateTest, ApplyRebuildsDeleteSummaryAfterPreparingDeletes) { +TEST_F(MergingSnapshotUpdateTest, RetryRebuildsDeleteSummary) { + FailCommits(2); auto del_file = MakeDeleteFile("/delete/del_a.parquet", 1L); - ICEBERG_UNWRAP_OR_FAIL(auto op, NewMergeAppend()); EXPECT_THAT(op->AddDelete(del_file), IsOk()); EXPECT_THAT(op->AddDelete(del_file), IsOk()); - - ICEBERG_UNWRAP_OR_FAIL(auto first_manifests, op->Apply(*table_->metadata(), nullptr)); - EXPECT_THAT(first_manifests, ::testing::Contains(::testing::Field( - &ManifestFile::content, ManifestContent::kDeletes))); - - ICEBERG_UNWRAP_OR_FAIL(auto second_manifests, op->Apply(*table_->metadata(), nullptr)); - EXPECT_THAT(second_manifests, ::testing::Contains(::testing::Field( - &ManifestFile::content, ManifestContent::kDeletes))); - - auto summary = op->Summary(); - EXPECT_EQ(summary.at(SnapshotSummaryFields::kAddedDeleteFiles), "1"); - EXPECT_EQ(summary.at(SnapshotSummaryFields::kAddedPosDeleteFiles), "1"); + EXPECT_THAT(op->Commit(), IsOk()); + EXPECT_THAT(table_->Refresh(), IsOk()); + ICEBERG_UNWRAP_OR_FAIL(auto snapshot, table_->current_snapshot()); + EXPECT_EQ(snapshot->summary.at(SnapshotSummaryFields::kAddedDeleteFiles), "1"); + EXPECT_EQ(snapshot->summary.at(SnapshotSummaryFields::kAddedPosDeleteFiles), "1"); } // Covers the bug where deleted delete files were not tracked in the snapshot summary. @@ -1179,7 +1172,7 @@ TEST_F(MergingSnapshotUpdateTest, ApplyMergesDuplicateDeletionVectors) { EXPECT_THAT(op->AddDelete(dv_a2, 7), IsOk()); EXPECT_THAT(op->AddDelete(dv_b, 8), IsOk()); - ICEBERG_UNWRAP_OR_FAIL(auto manifests, op->Apply(*table_->metadata(), nullptr)); + ICEBERG_UNWRAP_OR_FAIL(auto manifests, op->CommitManifests()); auto delete_manifest_it = std::ranges::find_if(manifests, [](const ManifestFile& manifest) { return manifest.content == ManifestContent::kDeletes; @@ -1230,7 +1223,7 @@ TEST_F(MergingSnapshotUpdateTest, ApplyMergesDuplicateDeletionVectorsWithNullPar EXPECT_THAT(op->AddDelete(dv_a1, 7), IsOk()); EXPECT_THAT(op->AddDelete(dv_a2, 7), IsOk()); - EXPECT_THAT(op->Apply(*table_->metadata(), nullptr), IsOk()); + EXPECT_THAT(op->CommitManifests(), IsOk()); } TEST_F(MergingSnapshotUpdateTest, ValidateNewDeleteFileRejectsUnsupportedVersion) { @@ -1252,9 +1245,11 @@ TEST_F(MergingSnapshotUpdateTest, ApplyRejectsV2StagedPositionDeleteAfterV3Upgra ICEBERG_UNWRAP_OR_FAIL(auto op, NewMergeAppend()); EXPECT_THAT(op->AddDelete(del_file), IsOk()); - auto metadata = std::make_shared(*table_->metadata()); - metadata->format_version = 3; - EXPECT_THAT(op->Apply(*metadata, nullptr), IsError(ErrorKind::kInvalidArgument)); + ICEBERG_UNWRAP_OR_FAIL(auto properties, table_->NewUpdateProperties()); + properties->Set("format-version", "3"); + ASSERT_THAT(properties->Commit(), IsOk()); + ASSERT_THAT(table_->Refresh(), IsOk()); + EXPECT_THAT(op->Commit(), IsError(ErrorKind::kInvalidArgument)); } // ------------------------------------------------------------------------- @@ -1314,31 +1309,22 @@ TEST_F(MergingSnapshotUpdateTest, AddManifestRetryCopiesManifestAgain) { auto path = table_location_ + "/metadata/retry-input.avro"; ICEBERG_UNWRAP_OR_FAIL(auto manifest, WriteManifest(path, {file_a_})); manifest.added_snapshot_id = 12345; - + FailCommits(2); ICEBERG_UNWRAP_OR_FAIL(auto op, NewMergeAppend()); + std::vector deleted; + op->DeleteWith([&](const std::string& path) { + deleted.push_back(path); + return file_io_->DeleteFile(path); + }); EXPECT_THAT(op->AppendManifest(manifest), IsOk()); - - ICEBERG_UNWRAP_OR_FAIL(auto first_apply, static_cast(*op).Apply()); - SnapshotCache first_snapshot_cache(first_apply.snapshot.get()); - ICEBERG_UNWRAP_OR_FAIL(auto first_manifests, - first_snapshot_cache.DataManifests(file_io_)); - ASSERT_EQ(first_manifests.size(), 1U); - EXPECT_NE(first_manifests[0].manifest_path, path); - - ICEBERG_UNWRAP_OR_FAIL(auto second_apply, static_cast(*op).Apply()); - SnapshotCache second_snapshot_cache(second_apply.snapshot.get()); - ICEBERG_UNWRAP_OR_FAIL(auto second_manifests, - second_snapshot_cache.DataManifests(file_io_)); - ASSERT_EQ(second_manifests.size(), 1U); - EXPECT_NE(second_manifests[0].manifest_path, path); - EXPECT_NE(second_manifests[0].manifest_path, first_manifests[0].manifest_path); - - std::vector second_manifest_vector(second_manifests.begin(), - second_manifests.end()); - ICEBERG_UNWRAP_OR_FAIL(auto entries, - ReadAllEntries(second_manifest_vector, *table_->metadata())); + ICEBERG_UNWRAP_OR_FAIL(auto manifests, op->CommitManifests()); + ASSERT_EQ(manifests.size(), 1U); + EXPECT_NE(manifests[0].manifest_path, path); + EXPECT_THAT(deleted, ::testing::SizeIs(4U)); + EXPECT_THAT(deleted, ::testing::Not(::testing::Contains(path))); + EXPECT_THAT(deleted, ::testing::Not(::testing::Contains(manifests[0].manifest_path))); + ICEBERG_UNWRAP_OR_FAIL(auto entries, ReadAllEntries(manifests, *table_->metadata())); ASSERT_EQ(entries.size(), 1U); - ASSERT_NE(entries[0].data_file, nullptr); EXPECT_EQ(entries[0].data_file->file_path, file_a_->file_path); } @@ -1722,7 +1708,7 @@ TEST_F(MergingSnapshotUpdateTest, ValidateDataFilesExistUsesRowFilter) { ICEBERG_UNWRAP_OR_FAIL(auto op, NewOverwriteUpdate()); EXPECT_THAT(op->RemoveDataFile(file_a_), IsOk()); const int64_t second_snapshot_id = op->GeneratedSnapshotId(); - ICEBERG_UNWRAP_OR_FAIL(auto manifests, op->Apply(*table_->metadata(), first_snapshot)); + ICEBERG_UNWRAP_OR_FAIL(auto manifests, op->CommitManifests()); ICEBERG_UNWRAP_OR_FAIL( auto second_snapshot, MakeSyntheticSnapshot(DataOperation::kOverwrite, second_snapshot_id, @@ -1756,7 +1742,7 @@ TEST_F(MergingSnapshotUpdateTest, ICEBERG_UNWRAP_OR_FAIL(auto op, NewOverwriteUpdate()); EXPECT_THAT(op->AddDelete(del_file), IsOk()); const int64_t second_snapshot_id = op->GeneratedSnapshotId(); - ICEBERG_UNWRAP_OR_FAIL(auto manifests, op->Apply(*table_->metadata(), first_snapshot)); + ICEBERG_UNWRAP_OR_FAIL(auto manifests, op->CommitManifests()); ICEBERG_UNWRAP_OR_FAIL( auto second_snapshot, MakeSyntheticSnapshot(DataOperation::kOverwrite, second_snapshot_id, @@ -1784,7 +1770,7 @@ TEST_F(MergingSnapshotUpdateTest, ValidateNoNewDeletesForDataFilesDetectsConflic ICEBERG_UNWRAP_OR_FAIL(auto op, NewOverwriteUpdate()); EXPECT_THAT(op->AddDelete(del_file), IsOk()); const int64_t second_snapshot_id = op->GeneratedSnapshotId(); - ICEBERG_UNWRAP_OR_FAIL(auto manifests, op->Apply(*table_->metadata(), first_snapshot)); + ICEBERG_UNWRAP_OR_FAIL(auto manifests, op->CommitManifests()); ICEBERG_UNWRAP_OR_FAIL( auto second_snapshot, MakeSyntheticSnapshot(DataOperation::kOverwrite, second_snapshot_id, @@ -1813,7 +1799,7 @@ TEST_F(MergingSnapshotUpdateTest, ICEBERG_UNWRAP_OR_FAIL(auto op, NewOverwriteUpdate()); EXPECT_THAT(op->AddDelete(del_file), IsOk()); const int64_t second_snapshot_id = op->GeneratedSnapshotId(); - ICEBERG_UNWRAP_OR_FAIL(auto manifests, op->Apply(*table_->metadata(), first_snapshot)); + ICEBERG_UNWRAP_OR_FAIL(auto manifests, op->CommitManifests()); ICEBERG_UNWRAP_OR_FAIL( auto second_snapshot, MakeSyntheticSnapshot(DataOperation::kOverwrite, second_snapshot_id, @@ -1848,7 +1834,7 @@ TEST_F(MergingSnapshotUpdateTest, ICEBERG_UNWRAP_OR_FAIL(auto op, NewOverwriteUpdate()); EXPECT_THAT(op->AddDelete(del_file), IsOk()); const int64_t second_snapshot_id = op->GeneratedSnapshotId(); - ICEBERG_UNWRAP_OR_FAIL(auto manifests, op->Apply(*table_->metadata(), first_snapshot)); + ICEBERG_UNWRAP_OR_FAIL(auto manifests, op->CommitManifests()); ICEBERG_UNWRAP_OR_FAIL( auto second_snapshot, MakeSyntheticSnapshot(DataOperation::kOverwrite, second_snapshot_id, @@ -1879,8 +1865,7 @@ TEST_F(MergingSnapshotUpdateTest, ICEBERG_UNWRAP_OR_FAIL(auto overwrite, NewOverwriteUpdate()); EXPECT_THAT(overwrite->AddDelete(del_file), IsOk()); const int64_t second_snapshot_id = overwrite->GeneratedSnapshotId(); - ICEBERG_UNWRAP_OR_FAIL(auto manifests, - overwrite->Apply(*table_->metadata(), first_snapshot)); + ICEBERG_UNWRAP_OR_FAIL(auto manifests, overwrite->CommitManifests()); ICEBERG_UNWRAP_OR_FAIL( auto second_snapshot, MakeSyntheticSnapshot(DataOperation::kOverwrite, second_snapshot_id, @@ -1912,7 +1897,7 @@ TEST_F(MergingSnapshotUpdateTest, ICEBERG_UNWRAP_OR_FAIL(auto op, NewOverwriteUpdate()); EXPECT_THAT(op->AddDelete(del_file), IsOk()); const int64_t second_snapshot_id = op->GeneratedSnapshotId(); - ICEBERG_UNWRAP_OR_FAIL(auto manifests, op->Apply(*table_->metadata(), first_snapshot)); + ICEBERG_UNWRAP_OR_FAIL(auto manifests, op->CommitManifests()); ICEBERG_UNWRAP_OR_FAIL( auto second_snapshot, MakeSyntheticSnapshot(DataOperation::kOverwrite, second_snapshot_id, @@ -1940,7 +1925,7 @@ TEST_F(MergingSnapshotUpdateTest, ValidateNoNewDeleteFilesWithExpressionDetectsC ICEBERG_UNWRAP_OR_FAIL(auto op, NewOverwriteUpdate()); EXPECT_THAT(op->AddDelete(del_file), IsOk()); const int64_t second_snapshot_id = op->GeneratedSnapshotId(); - ICEBERG_UNWRAP_OR_FAIL(auto manifests, op->Apply(*table_->metadata(), first_snapshot)); + ICEBERG_UNWRAP_OR_FAIL(auto manifests, op->CommitManifests()); ICEBERG_UNWRAP_OR_FAIL( auto second_snapshot, MakeSyntheticSnapshot(DataOperation::kOverwrite, second_snapshot_id, @@ -1967,7 +1952,7 @@ TEST_F(MergingSnapshotUpdateTest, ICEBERG_UNWRAP_OR_FAIL(auto op, NewOverwriteUpdate()); EXPECT_THAT(op->AddDelete(del_file), IsOk()); const int64_t second_snapshot_id = op->GeneratedSnapshotId(); - ICEBERG_UNWRAP_OR_FAIL(auto manifests, op->Apply(*table_->metadata(), first_snapshot)); + ICEBERG_UNWRAP_OR_FAIL(auto manifests, op->CommitManifests()); ICEBERG_UNWRAP_OR_FAIL( auto second_snapshot, MakeSyntheticSnapshot(DataOperation::kOverwrite, second_snapshot_id, @@ -1994,7 +1979,7 @@ TEST_F(MergingSnapshotUpdateTest, ICEBERG_UNWRAP_OR_FAIL(auto op, NewOverwriteUpdate()); EXPECT_THAT(op->AddDelete(del_file), IsOk()); const int64_t second_snapshot_id = op->GeneratedSnapshotId(); - ICEBERG_UNWRAP_OR_FAIL(auto manifests, op->Apply(*table_->metadata(), first_snapshot)); + ICEBERG_UNWRAP_OR_FAIL(auto manifests, op->CommitManifests()); ICEBERG_UNWRAP_OR_FAIL( auto second_snapshot, MakeSyntheticSnapshot(DataOperation::kOverwrite, second_snapshot_id, @@ -2138,7 +2123,7 @@ TEST_F(MergingSnapshotUpdateTest, ValidateDeletedDataFilesWithExpressionDetectsC ICEBERG_UNWRAP_OR_FAIL(auto op, NewOverwriteUpdate()); EXPECT_THAT(op->RemoveDataFile(file_a_), IsOk()); const int64_t second_snapshot_id = op->GeneratedSnapshotId(); - ICEBERG_UNWRAP_OR_FAIL(auto manifests, op->Apply(*table_->metadata(), first_snapshot)); + ICEBERG_UNWRAP_OR_FAIL(auto manifests, op->CommitManifests()); ICEBERG_UNWRAP_OR_FAIL( auto second_snapshot, MakeSyntheticSnapshot(DataOperation::kOverwrite, second_snapshot_id, @@ -2164,7 +2149,7 @@ TEST_F(MergingSnapshotUpdateTest, ICEBERG_UNWRAP_OR_FAIL(auto op, NewOverwriteUpdate()); EXPECT_THAT(op->RemoveDataFile(file_a_), IsOk()); const int64_t second_snapshot_id = op->GeneratedSnapshotId(); - ICEBERG_UNWRAP_OR_FAIL(auto manifests, op->Apply(*table_->metadata(), first_snapshot)); + ICEBERG_UNWRAP_OR_FAIL(auto manifests, op->CommitManifests()); ICEBERG_UNWRAP_OR_FAIL( auto second_snapshot, MakeSyntheticSnapshot(DataOperation::kOverwrite, second_snapshot_id, diff --git a/src/iceberg/test/transaction_test.cc b/src/iceberg/test/transaction_test.cc index 0998beacd..52cfb11e4 100644 --- a/src/iceberg/test/transaction_test.cc +++ b/src/iceberg/test/transaction_test.cc @@ -25,8 +25,10 @@ #include #include +#include "iceberg/exception.h" #include "iceberg/expression/expressions.h" #include "iceberg/expression/term.h" +#include "iceberg/schema.h" #include "iceberg/sort_order.h" #include "iceberg/table_metadata.h" #include "iceberg/test/matchers.h" @@ -113,7 +115,7 @@ TEST_F(TransactionTest, ApplyFailureFinalizesTransaction) { EXPECT_THAT(update->Commit(), IsError(ErrorKind::kValidationFailed)); EXPECT_THAT(txn->Commit(), ::testing::AllOf(IsError(ErrorKind::kValidationFailed), - HasErrorMessage("Transaction already finalized"))); + HasErrorMessage("Transaction is not ready"))); } TEST_F(TransactionTest, CommitTransactionWithPropertyUpdate) { @@ -298,7 +300,7 @@ TEST_F(TransactionRetryTest, CommitNonRetryableErrorStopsImmediately) { EXPECT_EQ(update_call_count, 1); // Should not retry } -TEST_F(TransactionRetryTest, CommitExceptionRestoresLifecycleState) { +TEST_F(TransactionRetryTest, CommitExceptionMakesOutcomeUnknown) { int update_call_count = 0; ON_CALL(*mock_catalog_, UpdateTable(::testing::_, ::testing::_, ::testing::_)) .WillByDefault( @@ -315,14 +317,12 @@ TEST_F(TransactionRetryTest, CommitExceptionRestoresLifecycleState) { properties->Set("exception.test", "value"); EXPECT_THAT(properties->Commit(), IsOk()); - EXPECT_THROW(std::ignore = txn->Commit(), std::runtime_error); - - ICEBERG_UNWRAP_OR_FAIL(auto append, txn->NewFastAppend()); - append->AppendFile(nullptr); - EXPECT_THAT(append->Commit(), IsError(ErrorKind::kValidationFailed)); - EXPECT_THAT(txn->Commit(), - ::testing::AllOf(IsError(ErrorKind::kValidationFailed), - HasErrorMessage("Transaction already finalized"))); + EXPECT_THAT(txn->Commit(), IsError(ErrorKind::kCommitStateUnknown)); + EXPECT_EQ(txn->state(), TransactionState::kCommitStateUnknown); + EXPECT_THAT(txn->NewFastAppend(), IsError(ErrorKind::kValidationFailed)); + EXPECT_THAT(properties->Commit(), IsError(ErrorKind::kValidationFailed)); + EXPECT_THAT(txn->Commit(), IsError(ErrorKind::kValidationFailed)); + EXPECT_THAT(txn->Abort(), IsError(ErrorKind::kValidationFailed)); EXPECT_EQ(update_call_count, 1); } @@ -371,4 +371,124 @@ TEST_F(TransactionRetryTest, NonRetryableUpdatePreventsRetry) { EXPECT_EQ(update_call_count, 1); } +TEST_F(TransactionTest, AppliedUpdateCannotCompleteAnotherPendingOperation) { + ICEBERG_UNWRAP_OR_FAIL(auto txn, table_->NewTransaction()); + ICEBERG_UNWRAP_OR_FAIL(auto first, txn->NewUpdateProperties()); + first->Set("first", "1"); + ASSERT_THAT(first->Commit(), IsOk()); + EXPECT_EQ(txn->state(), TransactionState::kReady); + ICEBERG_UNWRAP_OR_FAIL(auto second, txn->NewUpdateProperties()); + EXPECT_THAT(first->Commit(), HasErrorMessage("not the current pending operation")); + EXPECT_EQ(txn->state(), TransactionState::kUpdatePending); + EXPECT_THAT(txn->Commit(), IsError(ErrorKind::kValidationFailed)); + EXPECT_THAT(txn->NewUpdateProperties(), IsError(ErrorKind::kValidationFailed)); + second->Set("second", "2"); + ASSERT_THAT(second->Commit(), IsOk()); + ASSERT_THAT(txn->Commit(), IsOk()); + EXPECT_EQ(txn->state(), TransactionState::kCommitted); + EXPECT_EQ(ReloadMetadata()->properties.configs().at("first"), "1"); + EXPECT_EQ(ReloadMetadata()->properties.configs().at("second"), "2"); + EXPECT_THAT(txn->Abort(), IsError(ErrorKind::kValidationFailed)); + EXPECT_THAT(first->Commit(), IsError(ErrorKind::kValidationFailed)); +} + +TEST_F(TransactionTest, StandaloneCommitIsTerminal) { + ICEBERG_UNWRAP_OR_FAIL(auto update, table_->NewUpdateProperties()); + update->Set("once", "value"); + ASSERT_THAT(update->Commit(), IsOk()); + EXPECT_THAT(update->Commit(), HasErrorMessage("Update is terminal")); + EXPECT_THROW(update->Set("twice", "value"), IcebergError); +} + +TEST_F(TransactionTest, AbortReadyAndPendingIsIdempotent) { + for (bool add_update : {false, true}) { + ICEBERG_UNWRAP_OR_FAIL(auto txn, table_->NewTransaction()); + std::shared_ptr update; + if (add_update) { + ICEBERG_UNWRAP_OR_FAIL(update, txn->NewUpdateProperties()); + update->Set("discard", "value"); + } + ASSERT_THAT(txn->Abort(), IsOk()); + EXPECT_EQ(txn->state(), TransactionState::kAborted); + EXPECT_THAT(txn->Abort(), IsOk()); + EXPECT_THAT(txn->Commit(), IsError(ErrorKind::kValidationFailed)); + EXPECT_THAT(txn->NewFastAppend(), IsError(ErrorKind::kValidationFailed)); + if (update) { + EXPECT_THAT(update->Commit(), IsError(ErrorKind::kValidationFailed)); + EXPECT_THROW(update->Set("reuse", "value"), IcebergError); + } + } +} + +TEST_F(TransactionTest, ApplyFailureCannotBeCorrected) { + ICEBERG_UNWRAP_OR_FAIL(auto txn, table_->NewTransaction()); + ICEBERG_UNWRAP_OR_FAIL(auto update, txn->NewUpdateProperties()); + update->Set("format-version", "100"); + EXPECT_THAT(update->Commit(), IsError(ErrorKind::kInvalidArgument)); + EXPECT_EQ(txn->state(), TransactionState::kFailed); + EXPECT_THROW(update->Set("format-version", "2"), IcebergError); + EXPECT_THAT(update->Commit(), IsError(ErrorKind::kValidationFailed)); + EXPECT_THAT(txn->Commit(), IsError(ErrorKind::kValidationFailed)); + EXPECT_THAT(txn->NewUpdateProperties(), IsError(ErrorKind::kValidationFailed)); + EXPECT_THAT(txn->Abort(), IsOk()); + EXPECT_THAT(txn->Abort(), IsOk()); +} + +TEST_F(TransactionRetryTest, NonRetryableStandaloneUpdateStopsAtFirstConflict) { + EXPECT_CALL(*mock_catalog_, UpdateTable(::testing::_, ::testing::_, ::testing::_)) + .Times(1) + .WillOnce(::testing::Return(CommitFailed("conflict"))); + EXPECT_CALL(*mock_catalog_, LoadTable(::testing::_)).Times(0); + ICEBERG_UNWRAP_OR_FAIL(auto update, mock_table_->NewUpdateSchema()); + update->AddColumn("new_column", int64()); + EXPECT_THAT(update->Commit(), IsError(ErrorKind::kCommitFailed)); + EXPECT_THAT(update->Commit(), IsError(ErrorKind::kValidationFailed)); +} + +TEST_F(TransactionRetryTest, FrozenMutationDoesNotPoisonReplay) { + EXPECT_CALL(*mock_catalog_, UpdateTable(::testing::_, ::testing::_, ::testing::_)) + .WillOnce(::testing::Return(CommitFailed("conflict"))) + .WillOnce([this](const auto& name, const auto& requirements, const auto& updates) { + return catalog_->UpdateTable(name, requirements, updates); + }); + ICEBERG_UNWRAP_OR_FAIL(auto txn, mock_table_->NewTransaction()); + ICEBERG_UNWRAP_OR_FAIL(auto update, txn->NewUpdateProperties()); + update->Set("frozen", "original"); + ASSERT_THAT(update->Commit(), IsOk()); + EXPECT_THROW(update->Set("frozen", "changed"), IcebergError); + ASSERT_THAT(txn->Commit(), IsOk()); + EXPECT_EQ(ReloadMetadata()->properties.configs().at("frozen"), "original"); +} + +TEST_F(TransactionTest, FreezeCopiesSortTransformAliases) { + FailCommits(2); + ICEBERG_UNWRAP_OR_FAIL(auto txn, table_->NewTransaction()); + ICEBERG_UNWRAP_OR_FAIL(auto update, txn->NewUpdateSortOrder()); + auto transform = Transform::Bucket(16); + ICEBERG_UNWRAP_OR_FAIL(auto term, + UnboundTransform::Make(Expressions::Ref("x"), transform)); + update->AddSortField(std::move(term), SortDirection::kAscending, NullOrder::kFirst); + ASSERT_THAT(update->Commit(), IsOk()); + *transform = *Transform::Bucket(32); + ICEBERG_UNWRAP_OR_FAIL(auto preview, update->Validate()); + *preview->fields()[0].transform() = *Transform::Bucket(64); + ASSERT_THAT(txn->Commit(), IsOk()); + ICEBERG_UNWRAP_OR_FAIL(auto order, ReloadMetadata()->SortOrder()); + EXPECT_EQ(*order->fields()[0].transform(), *Transform::Bucket(16)); +} + +TEST_F(TransactionTest, FreezeCopiesNestedSchemaTypes) { + ICEBERG_UNWRAP_OR_FAIL(auto txn, table_->NewTransaction()); + ICEBERG_UNWRAP_OR_FAIL(auto update, txn->NewUpdateSchema()); + auto type = decimal(9, 2); + update->AddColumn("amount", type); + ASSERT_THAT(update->Commit(), IsOk()); + *type = DecimalType(18, 4); + ASSERT_THAT(txn->Commit(), IsOk()); + ICEBERG_UNWRAP_OR_FAIL(auto schema, ReloadMetadata()->Schema()); + ICEBERG_UNWRAP_OR_FAIL(auto field, schema->FindFieldByName("amount")); + ASSERT_TRUE(field.has_value()); + EXPECT_EQ(*field->get().type(), *decimal(9, 2)); +} + } // namespace iceberg diff --git a/src/iceberg/test/update_location_test.cc b/src/iceberg/test/update_location_test.cc index a208209b3..403bb4af4 100644 --- a/src/iceberg/test/update_location_test.cc +++ b/src/iceberg/test/update_location_test.cc @@ -47,7 +47,7 @@ TEST_F(UpdateLocationTest, SetLocationSuccess) { ICEBERG_UNWRAP_OR_FAIL(auto update, table_->NewUpdateLocation()); update->SetLocation(new_location); - ICEBERG_UNWRAP_OR_FAIL(auto result, update->Apply()); + ICEBERG_UNWRAP_OR_FAIL(auto result, update->Validate()); EXPECT_EQ(result, new_location); // Commit and verify the location was persisted @@ -71,7 +71,7 @@ TEST_F(UpdateLocationTest, SetLocationMultipleTimes) { .SetLocation("/warehouse/second_location") .SetLocation(final_location); - ICEBERG_UNWRAP_OR_FAIL(auto result, update->Apply()); + ICEBERG_UNWRAP_OR_FAIL(auto result, update->Validate()); EXPECT_EQ(result, final_location); // Commit and verify the final location was persisted @@ -84,7 +84,7 @@ TEST_F(UpdateLocationTest, SetEmptyLocation) { ICEBERG_UNWRAP_OR_FAIL(auto update, table_->NewUpdateLocation()); update->SetLocation(""); - auto result = update->Apply(); + auto result = update->Validate(); EXPECT_THAT(result, IsError(ErrorKind::kValidationFailed)); EXPECT_THAT(result, HasErrorMessage("Location cannot be empty")); } @@ -92,7 +92,7 @@ TEST_F(UpdateLocationTest, SetEmptyLocation) { TEST_F(UpdateLocationTest, ApplyWithoutSettingLocation) { ICEBERG_UNWRAP_OR_FAIL(auto update, table_->NewUpdateLocation()); - auto result = update->Apply(); + auto result = update->Validate(); EXPECT_THAT(result, IsError(ErrorKind::kInvalidArgument)); EXPECT_THAT(result, HasErrorMessage("Location must be set before applying")); } @@ -109,7 +109,7 @@ TEST_F(UpdateLocationTest, MultipleUpdatesSequentially) { ICEBERG_UNWRAP_OR_FAIL(auto update, table_->NewUpdateLocation()); update->SetLocation(first_location); - ICEBERG_UNWRAP_OR_FAIL(auto result, update->Apply()); + ICEBERG_UNWRAP_OR_FAIL(auto result, update->Validate()); EXPECT_EQ(result, first_location); EXPECT_THAT(update->Commit(), IsOk()); @@ -123,7 +123,7 @@ TEST_F(UpdateLocationTest, MultipleUpdatesSequentially) { ICEBERG_UNWRAP_OR_FAIL(update, reloaded->NewUpdateLocation()); update->SetLocation(second_location); - ICEBERG_UNWRAP_OR_FAIL(result, update->Apply()); + ICEBERG_UNWRAP_OR_FAIL(result, update->Validate()); EXPECT_EQ(result, second_location); EXPECT_THAT(update->Commit(), IsOk()); diff --git a/src/iceberg/test/update_partition_spec_test.cc b/src/iceberg/test/update_partition_spec_test.cc index ebbb80243..ef5a49a3a 100644 --- a/src/iceberg/test/update_partition_spec_test.cc +++ b/src/iceberg/test/update_partition_spec_test.cc @@ -173,7 +173,7 @@ class UpdatePartitionSpecTest : public ::testing::TestWithParam { // Helper to apply update and get the resulting spec std::shared_ptr ApplyUpdateAndGetSpec( std::shared_ptr update) { - auto result = update->Apply(); + auto result = update->Validate(); if (!result.has_value()) { ADD_FAILURE() << "Failed to apply update: " << result.error().message; return nullptr; @@ -207,7 +207,7 @@ class UpdatePartitionSpecTest : public ::testing::TestWithParam { // Helper to expect an error with a specific message void ExpectError(std::shared_ptr update, ErrorKind expected_kind, const std::string& expected_message) { - auto result = update->Apply(); + auto result = update->Validate(); ASSERT_THAT(result, IsError(expected_kind)); ASSERT_THAT(result, HasErrorMessage(expected_message)); } @@ -529,7 +529,7 @@ TEST_P(UpdatePartitionSpecTest, TestMultipleChanges) { TEST_P(UpdatePartitionSpecTest, TestAddDeletedName) { ICEBERG_UNWRAP_OR_FAIL(auto update, partitioned_table_->NewUpdatePartitionSpec()); update->RemoveField(Expressions::Bucket("id", 16)); - ICEBERG_UNWRAP_OR_FAIL(auto result, update->Apply()); + ICEBERG_UNWRAP_OR_FAIL(auto result, update->Validate()); auto updated_spec = result.spec; if (format_version_ == 1) { @@ -608,14 +608,14 @@ TEST_P(UpdatePartitionSpecTest, TestNoEffectAddDeletedSameFieldWithSameName) { ICEBERG_UNWRAP_OR_FAIL(auto update1, partitioned_table_->NewUpdatePartitionSpec()); update1->RemoveField("shard"); update1->AddField(Expressions::Bucket("id", 16), "shard"); - ICEBERG_UNWRAP_OR_FAIL(auto result1, update1->Apply()); + ICEBERG_UNWRAP_OR_FAIL(auto result1, update1->Validate()); auto spec1 = result1.spec; AssertPartitionSpecEquals(*partitioned_spec_, *spec1); ICEBERG_UNWRAP_OR_FAIL(auto update2, partitioned_table_->NewUpdatePartitionSpec()); update2->RemoveField("shard"); update2->AddField(Expressions::Bucket("id", 16)); - ICEBERG_UNWRAP_OR_FAIL(auto result2, update2->Apply()); + ICEBERG_UNWRAP_OR_FAIL(auto result2, update2->Validate()); auto spec2 = result2.spec; AssertPartitionSpecEquals(*partitioned_spec_, *spec2); } @@ -624,7 +624,7 @@ TEST_P(UpdatePartitionSpecTest, TestGenerateNewSpecAddDeletedSameFieldWithDiffer ICEBERG_UNWRAP_OR_FAIL(auto update, partitioned_table_->NewUpdatePartitionSpec()); update->RemoveField("shard"); update->AddField(Expressions::Bucket("id", 16), "new_shard"); - ICEBERG_UNWRAP_OR_FAIL(auto result, update->Apply()); + ICEBERG_UNWRAP_OR_FAIL(auto result, update->Validate()); auto updated_spec = result.spec; ASSERT_EQ(updated_spec->fields().size(), 3); diff --git a/src/iceberg/test/update_partition_statistics_test.cc b/src/iceberg/test/update_partition_statistics_test.cc index 5ed84cc0a..23ec44bf7 100644 --- a/src/iceberg/test/update_partition_statistics_test.cc +++ b/src/iceberg/test/update_partition_statistics_test.cc @@ -57,7 +57,7 @@ class UpdatePartitionStatisticsTest : public UpdateTestBase { TEST_F(UpdatePartitionStatisticsTest, EmptyUpdate) { ICEBERG_UNWRAP_OR_FAIL(auto update, table_->NewUpdatePartitionStatistics()); - ICEBERG_UNWRAP_OR_FAIL(auto result, update->Apply()); + ICEBERG_UNWRAP_OR_FAIL(auto result, update->Validate()); EXPECT_TRUE(result.to_set.empty()); EXPECT_TRUE(result.to_remove.empty()); } @@ -68,7 +68,7 @@ TEST_F(UpdatePartitionStatisticsTest, SetPartitionStatistics) { 1, "/warehouse/test_table/metadata/partition-stats-1.parquet"); update->SetPartitionStatistics(partition_stats_file); - ICEBERG_UNWRAP_OR_FAIL(auto result, update->Apply()); + ICEBERG_UNWRAP_OR_FAIL(auto result, update->Validate()); EXPECT_EQ(result.to_set.size(), 1); EXPECT_TRUE(result.to_remove.empty()); @@ -90,7 +90,7 @@ TEST_F(UpdatePartitionStatisticsTest, SetMultiplePartitionStatistics) { update->SetPartitionStatistics(partition_stats_file1); update->SetPartitionStatistics(partition_stats_file2); - ICEBERG_UNWRAP_OR_FAIL(auto result, update->Apply()); + ICEBERG_UNWRAP_OR_FAIL(auto result, update->Validate()); EXPECT_EQ(result.to_set.size(), 2); EXPECT_TRUE(result.to_remove.empty()); @@ -115,7 +115,7 @@ TEST_F(UpdatePartitionStatisticsTest, ReplacePartitionStatistics) { update->SetPartitionStatistics(partition_stats_file1); update->SetPartitionStatistics(partition_stats_file2); - ICEBERG_UNWRAP_OR_FAIL(auto result, update->Apply()); + ICEBERG_UNWRAP_OR_FAIL(auto result, update->Validate()); EXPECT_EQ(result.to_set.size(), 1); EXPECT_TRUE(result.to_remove.empty()); @@ -130,7 +130,7 @@ TEST_F(UpdatePartitionStatisticsTest, RemovePartitionStatistics) { ICEBERG_UNWRAP_OR_FAIL(auto update, table_->NewUpdatePartitionStatistics()); update->RemovePartitionStatistics(1); - ICEBERG_UNWRAP_OR_FAIL(auto result, update->Apply()); + ICEBERG_UNWRAP_OR_FAIL(auto result, update->Validate()); EXPECT_TRUE(result.to_set.empty()); EXPECT_EQ(result.to_remove.size(), 1); EXPECT_EQ(result.to_remove[0], 1); @@ -144,7 +144,7 @@ TEST_F(UpdatePartitionStatisticsTest, SetThenRemovePartitionStatistics) { update->SetPartitionStatistics(partition_stats_file); update->RemovePartitionStatistics(1); - ICEBERG_UNWRAP_OR_FAIL(auto result, update->Apply()); + ICEBERG_UNWRAP_OR_FAIL(auto result, update->Validate()); EXPECT_TRUE(result.to_set.empty()); EXPECT_EQ(result.to_remove.size(), 1); EXPECT_EQ(result.to_remove[0], 1); @@ -155,7 +155,7 @@ TEST_F(UpdatePartitionStatisticsTest, SetNullPartitionStatistics) { update->SetPartitionStatistics(nullptr); - auto result = update->Apply(); + auto result = update->Validate(); EXPECT_THAT(result, IsError(ErrorKind::kValidationFailed)); EXPECT_THAT(result, HasErrorMessage("Statistics file cannot be null")); } @@ -172,7 +172,7 @@ TEST_F(UpdatePartitionStatisticsTest, SetAndRemoveMixed) { update->SetPartitionStatistics(partition_stats_file2); update->RemovePartitionStatistics(3); - ICEBERG_UNWRAP_OR_FAIL(auto result, update->Apply()); + ICEBERG_UNWRAP_OR_FAIL(auto result, update->Validate()); EXPECT_EQ(result.to_set.size(), 2); EXPECT_EQ(result.to_remove.size(), 1); EXPECT_EQ(result.to_remove[0], 3); diff --git a/src/iceberg/test/update_properties_test.cc b/src/iceberg/test/update_properties_test.cc index 2a22ff949..8920fb3c3 100644 --- a/src/iceberg/test/update_properties_test.cc +++ b/src/iceberg/test/update_properties_test.cc @@ -28,7 +28,7 @@ class UpdatePropertiesTest : public UpdateTestBase {}; TEST_F(UpdatePropertiesTest, EmptyUpdate) { ICEBERG_UNWRAP_OR_FAIL(auto update, table_->NewUpdateProperties()); - ICEBERG_UNWRAP_OR_FAIL(auto result, update->Apply()); + ICEBERG_UNWRAP_OR_FAIL(auto result, update->Validate()); EXPECT_THAT(result.updates.empty(), true); } @@ -36,7 +36,7 @@ TEST_F(UpdatePropertiesTest, SetProperty) { ICEBERG_UNWRAP_OR_FAIL(auto update, table_->NewUpdateProperties()); update->Set("key1", "value1").Set("key2", "value2"); - ICEBERG_UNWRAP_OR_FAIL(auto result, update->Apply()); + ICEBERG_UNWRAP_OR_FAIL(auto result, update->Validate()); EXPECT_EQ(result.updates.size(), 2); EXPECT_EQ(result.updates.at("key1"), "value1"); EXPECT_EQ(result.updates.at("key2"), "value2"); @@ -54,7 +54,7 @@ TEST_F(UpdatePropertiesTest, RemoveProperty) { ICEBERG_UNWRAP_OR_FAIL(auto update, reloaded->NewUpdateProperties()); update->Remove("key1").Remove("key2"); - ICEBERG_UNWRAP_OR_FAIL(auto result, update->Apply()); + ICEBERG_UNWRAP_OR_FAIL(auto result, update->Validate()); EXPECT_TRUE(result.updates.empty()); EXPECT_EQ(result.removals.size(), 2); EXPECT_TRUE(result.removals.contains("key1")); @@ -65,7 +65,7 @@ TEST_F(UpdatePropertiesTest, SetThenRemoveSameKey) { ICEBERG_UNWRAP_OR_FAIL(auto update, table_->NewUpdateProperties()); update->Set("key1", "value1").Remove("key1"); - auto result = update->Apply(); + auto result = update->Validate(); EXPECT_THAT(result, IsError(ErrorKind::kValidationFailed)); EXPECT_THAT(result, HasErrorMessage("already marked for update")); } @@ -74,7 +74,7 @@ TEST_F(UpdatePropertiesTest, RemoveThenSetSameKey) { ICEBERG_UNWRAP_OR_FAIL(auto update, table_->NewUpdateProperties()); update->Remove("key1").Set("key1", "value1"); - auto result = update->Apply(); + auto result = update->Validate(); EXPECT_THAT(result, IsError(ErrorKind::kValidationFailed)); EXPECT_THAT(result, HasErrorMessage("already marked for removal")); } @@ -83,7 +83,7 @@ TEST_F(UpdatePropertiesTest, SetAndRemoveDifferentKeys) { ICEBERG_UNWRAP_OR_FAIL(auto update, table_->NewUpdateProperties()); update->Set("key1", "value1").Remove("key2"); - ICEBERG_UNWRAP_OR_FAIL(auto result, update->Apply()); + ICEBERG_UNWRAP_OR_FAIL(auto result, update->Validate()); EXPECT_EQ(result.updates.size(), 1); EXPECT_EQ(result.updates.at("key1"), "value1"); EXPECT_EQ(result.removals.size(), 1); @@ -94,7 +94,7 @@ TEST_F(UpdatePropertiesTest, UpgradeFormatVersionValid) { ICEBERG_UNWRAP_OR_FAIL(auto update, table_->NewUpdateProperties()); update->Set("format-version", "3"); - ICEBERG_UNWRAP_OR_FAIL(auto result, update->Apply()); + ICEBERG_UNWRAP_OR_FAIL(auto result, update->Validate()); EXPECT_TRUE(result.updates.empty()); EXPECT_TRUE(result.removals.empty()); ASSERT_TRUE(result.format_version.has_value()); @@ -105,7 +105,7 @@ TEST_F(UpdatePropertiesTest, UpgradeFormatVersionInvalidString) { ICEBERG_UNWRAP_OR_FAIL(auto update, table_->NewUpdateProperties()); update->Set("format-version", "invalid"); - auto result = update->Apply(); + auto result = update->Validate(); EXPECT_THAT(result, IsError(ErrorKind::kInvalidArgument)); EXPECT_THAT(result, HasErrorMessage("invalid argument")); } @@ -114,7 +114,7 @@ TEST_F(UpdatePropertiesTest, UpgradeFormatVersionOutOfRange) { ICEBERG_UNWRAP_OR_FAIL(auto update, table_->NewUpdateProperties()); update->Set("format-version", "5000000000"); - auto result = update->Apply(); + auto result = update->Validate(); EXPECT_THAT(result, IsError(ErrorKind::kInvalidArgument)); EXPECT_THAT(result, HasErrorMessage("out of range")); } @@ -124,7 +124,7 @@ TEST_F(UpdatePropertiesTest, UpgradeFormatVersionUnsupported) { update->Set("format-version", std::to_string(TableMetadata::kSupportedTableFormatVersion + 1)); - auto result = update->Apply(); + auto result = update->Validate(); EXPECT_THAT(result, IsError(ErrorKind::kInvalidArgument)); EXPECT_THAT(result, HasErrorMessage("unsupported format version")); } @@ -133,7 +133,7 @@ TEST_F(UpdatePropertiesTest, SetReservedPropertyUuid) { ICEBERG_UNWRAP_OR_FAIL(auto update, table_->NewUpdateProperties()); update->Set("uuid", "some-uuid"); - auto result = update->Apply(); + auto result = update->Validate(); EXPECT_THAT(result, IsError(ErrorKind::kValidationFailed)); EXPECT_THAT(result, HasErrorMessage("Cannot set reserved property")); } @@ -142,7 +142,7 @@ TEST_F(UpdatePropertiesTest, SetReservedPropertyCurrentSchema) { ICEBERG_UNWRAP_OR_FAIL(auto update, table_->NewUpdateProperties()); update->Set("current-schema", R"({"type": "struct"})"); - auto result = update->Apply(); + auto result = update->Validate(); EXPECT_THAT(result, IsError(ErrorKind::kValidationFailed)); EXPECT_THAT(result, HasErrorMessage("Cannot set reserved property")); } @@ -151,7 +151,7 @@ TEST_F(UpdatePropertiesTest, SetReservedPropertyCurrentSnapshotId) { ICEBERG_UNWRAP_OR_FAIL(auto update, table_->NewUpdateProperties()); update->Set("current-snapshot-id", "12345"); - auto result = update->Apply(); + auto result = update->Validate(); EXPECT_THAT(result, IsError(ErrorKind::kValidationFailed)); EXPECT_THAT(result, HasErrorMessage("Cannot set reserved property")); } @@ -160,7 +160,7 @@ TEST_F(UpdatePropertiesTest, SetFormatVersionStillAllowed) { ICEBERG_UNWRAP_OR_FAIL(auto update, table_->NewUpdateProperties()); update->Set("format-version", "3"); - ICEBERG_UNWRAP_OR_FAIL(auto result, update->Apply()); + ICEBERG_UNWRAP_OR_FAIL(auto result, update->Validate()); EXPECT_TRUE(result.updates.empty()); ASSERT_TRUE(result.format_version.has_value()); EXPECT_EQ(result.format_version.value(), 3); @@ -171,7 +171,7 @@ TEST_F(UpdatePropertiesTest, SetValidAndReservedProperties) { update->Set("valid.key", "valid.value"); update->Set("snapshot-count", "10"); - auto result = update->Apply(); + auto result = update->Validate(); EXPECT_THAT(result, IsError(ErrorKind::kValidationFailed)); EXPECT_THAT(result, HasErrorMessage("Cannot set reserved property")); } diff --git a/src/iceberg/test/update_schema_test.cc b/src/iceberg/test/update_schema_test.cc index 4057c52c8..1ee828ce7 100644 --- a/src/iceberg/test/update_schema_test.cc +++ b/src/iceberg/test/update_schema_test.cc @@ -23,6 +23,7 @@ #include #include #include +#include #include #include #include @@ -40,6 +41,7 @@ #include "iceberg/test/matchers.h" #include "iceberg/test/mock_io.h" #include "iceberg/test/test_resource.h" +#include "iceberg/transaction.h" #include "iceberg/type.h" #include "iceberg/util/checked_cast.h" #include "iceberg/util/uuid.h" @@ -107,7 +109,7 @@ TEST_F(UpdateSchemaTest, AddOptionalColumn) { ICEBERG_UNWRAP_OR_FAIL(auto update, table_->NewUpdateSchema()); update->AddColumn("new_col", int32(), "A new integer column"); - ICEBERG_UNWRAP_OR_FAIL(auto result, update->Apply()); + ICEBERG_UNWRAP_OR_FAIL(auto result, update->Validate()); ASSERT_TRUE(result.schema != nullptr); ICEBERG_UNWRAP_OR_FAIL(auto new_field_opt, result.schema->FindFieldByName("new_col")); @@ -115,7 +117,7 @@ TEST_F(UpdateSchemaTest, AddOptionalColumn) { const auto& new_field = new_field_opt->get(); EXPECT_EQ(new_field.name(), "new_col"); - EXPECT_EQ(new_field.type(), int32()); + EXPECT_EQ(*new_field.type(), *int32()); EXPECT_TRUE(new_field.optional()); EXPECT_EQ(new_field.doc(), "A new integer column"); } @@ -124,7 +126,7 @@ TEST_F(UpdateSchemaTest, AddRequiredColumnFails) { ICEBERG_UNWRAP_OR_FAIL(auto update, table_->NewUpdateSchema()); update->AddRequiredColumn("required_col", string(), "A required string column"); - auto result = update->Apply(); + auto result = update->Validate(); EXPECT_THAT(result, IsError(ErrorKind::kValidationFailed)); EXPECT_THAT(result, HasErrorMessage("Incompatible change")); } @@ -134,7 +136,7 @@ TEST_F(UpdateSchemaTest, AddRequiredColumnWithAllowIncompatible) { update->AllowIncompatibleChanges().AddRequiredColumn("required_col", string(), "A required string column"); - ICEBERG_UNWRAP_OR_FAIL(auto result, update->Apply()); + ICEBERG_UNWRAP_OR_FAIL(auto result, update->Validate()); ASSERT_TRUE(result.schema != nullptr); ICEBERG_UNWRAP_OR_FAIL(auto new_field_opt, @@ -143,7 +145,7 @@ TEST_F(UpdateSchemaTest, AddRequiredColumnWithAllowIncompatible) { const auto& new_field = new_field_opt->get(); EXPECT_EQ(new_field.name(), "required_col"); - EXPECT_EQ(new_field.type(), string()); + EXPECT_EQ(*new_field.type(), *string()); EXPECT_FALSE(new_field.optional()); EXPECT_EQ(new_field.doc(), "A required string column"); } @@ -169,7 +171,7 @@ TEST_F(UpdateSchemaTest, AddColumnWithDefaultValueRequiresV3) { ICEBERG_UNWRAP_OR_FAIL(auto update, table_->NewUpdateSchema()); update->AddColumn("new_col", int32(), "An integer column", Literal::Int(42)); - auto result = update->Apply(); + auto result = update->Validate(); EXPECT_THAT(result, IsError(ErrorKind::kInvalidSchema)); EXPECT_THAT(result, HasErrorMessage("is not supported until v3")); } @@ -178,7 +180,7 @@ TEST_F(UpdateSchemaDefaultValueTest, AddColumnWithDefaultValue) { ICEBERG_UNWRAP_OR_FAIL(auto update, table_->NewUpdateSchema()); update->AddColumn("new_col", int32(), "An integer column", Literal::Int(42)); - ICEBERG_UNWRAP_OR_FAIL(auto result, update->Apply()); + ICEBERG_UNWRAP_OR_FAIL(auto result, update->Validate()); ICEBERG_UNWRAP_OR_FAIL(auto new_field_opt, result.schema->FindFieldByName("new_col")); ASSERT_TRUE(new_field_opt.has_value()); @@ -194,7 +196,7 @@ TEST_F(UpdateSchemaDefaultValueTest, AddRequiredColumnWithDefaultValue) { update->AddRequiredColumn("required_col", string(), "A required string column", Literal::String("n/a")); - ICEBERG_UNWRAP_OR_FAIL(auto result, update->Apply()); + ICEBERG_UNWRAP_OR_FAIL(auto result, update->Validate()); ICEBERG_UNWRAP_OR_FAIL(auto new_field_opt, result.schema->FindFieldByName("required_col")); ASSERT_TRUE(new_field_opt.has_value()); @@ -211,7 +213,7 @@ TEST_F(UpdateSchemaDefaultValueTest, AddColumnWithMismatchedDefaultValueFails) { ICEBERG_UNWRAP_OR_FAIL(auto update, table_->NewUpdateSchema()); update->AddColumn("new_col", int32(), "An integer column", Literal::String("oops")); - auto result = update->Apply(); + auto result = update->Validate(); EXPECT_THAT(result, IsError(ErrorKind::kValidationFailed)); EXPECT_THAT(result, HasErrorMessage("Cannot cast default value")); } @@ -221,7 +223,7 @@ TEST_F(UpdateSchemaDefaultValueTest, AddColumnWithNarrowingDefaultValueFails) { update->AddColumn("new_col", int32(), "An integer column", Literal::Long(std::numeric_limits::max())); - auto result = update->Apply(); + auto result = update->Validate(); EXPECT_THAT(result, IsError(ErrorKind::kValidationFailed)); EXPECT_THAT(result, HasErrorMessage("Cannot cast default value")); } @@ -231,7 +233,7 @@ TEST_F(UpdateSchemaDefaultValueTest, UpdateColumnDefault) { update->AddColumn("new_col", int32(), "An integer column", Literal::Int(42)) .UpdateColumnDefault("new_col", Literal::Int(7)); - ICEBERG_UNWRAP_OR_FAIL(auto result, update->Apply()); + ICEBERG_UNWRAP_OR_FAIL(auto result, update->Validate()); ICEBERG_UNWRAP_OR_FAIL(auto new_field_opt, result.schema->FindFieldByName("new_col")); ASSERT_TRUE(new_field_opt.has_value()); @@ -246,7 +248,7 @@ TEST_F(UpdateSchemaDefaultValueTest, UpdateColumnDefaultOnExistingColumn) { ICEBERG_UNWRAP_OR_FAIL(auto update, table_->NewUpdateSchema()); update->UpdateColumnDefault("x", Literal::Long(0)); - ICEBERG_UNWRAP_OR_FAIL(auto result, update->Apply()); + ICEBERG_UNWRAP_OR_FAIL(auto result, update->Validate()); ICEBERG_UNWRAP_OR_FAIL(auto field_opt, result.schema->FindFieldByName("x")); ASSERT_TRUE(field_opt.has_value()); @@ -261,7 +263,7 @@ TEST_F(UpdateSchemaDefaultValueTest, UpdateColumnDefaultClearsWithNullopt) { update->AddColumn("new_col", int32(), "An integer column", Literal::Int(42)) .UpdateColumnDefault("new_col", std::nullopt); - ICEBERG_UNWRAP_OR_FAIL(auto result, update->Apply()); + ICEBERG_UNWRAP_OR_FAIL(auto result, update->Validate()); ICEBERG_UNWRAP_OR_FAIL(auto field_opt, result.schema->FindFieldByName("new_col")); ASSERT_TRUE(field_opt.has_value()); @@ -280,7 +282,7 @@ TEST_F(UpdateSchemaDefaultValueTest, AddNestedColumnPreservesNestedDefaults) { ICEBERG_UNWRAP_OR_FAIL(auto update, table_->NewUpdateSchema()); update->AddColumn("outer", nested_type, "A nested column"); - ICEBERG_UNWRAP_OR_FAIL(auto result, update->Apply()); + ICEBERG_UNWRAP_OR_FAIL(auto result, update->Validate()); ICEBERG_UNWRAP_OR_FAIL(auto outer_opt, result.schema->FindFieldByName("outer")); ASSERT_TRUE(outer_opt.has_value()); @@ -294,11 +296,91 @@ TEST_F(UpdateSchemaDefaultValueTest, AddNestedColumnPreservesNestedDefaults) { EXPECT_EQ(*inner.write_default(), Literal::Int(9)); } +TEST_F(UpdateSchemaDefaultValueTest, FrozenNestedInputsAndPreviewsAreIsolated) { + auto amount_type = decimal(9, 2); + auto key_type = fixed(4); + auto initial = std::make_shared(Literal::Decimal(1234, 9, 2)); + auto write = std::make_shared(Literal::Decimal(5678, 9, 2)); + auto element_type = std::make_shared(std::vector{ + SchemaField(100, "amount", amount_type, false, "amount doc", initial, write)}); + auto list_type = + std::make_shared(SchemaField(101, "element", element_type, true)); + auto map_type = std::make_shared(SchemaField(102, "key", key_type, false), + SchemaField(103, "value", list_type, true)); + + ICEBERG_UNWRAP_OR_FAIL(auto txn, table_->NewTransaction()); + ICEBERG_UNWRAP_OR_FAIL(auto update, txn->NewUpdateSchema()); + update->AddColumn("nested", map_type, "nested doc") + .AddRequiredColumn("copy_id", int64(), "identifier doc", Literal::Long(42)); + std::string_view identifier_names[] = {"copy_id"}; + std::span identifiers(identifier_names); + update->SetIdentifierFields(identifiers); + ICEBERG_UNWRAP_OR_FAIL(auto expected, update->Validate()); + const auto identifier_ids = expected.schema->IdentifierFieldIds(); + ASSERT_THAT(update->Commit(), IsOk()); + + auto verify = [&](const Schema& schema) { + EXPECT_THAT(schema.fields(), ::testing::ElementsAreArray(expected.schema->fields())); + EXPECT_EQ(schema.IdentifierFieldIds(), identifier_ids); + ICEBERG_UNWRAP_OR_FAIL(auto nested, schema.FindFieldByName("nested")); + ASSERT_TRUE(nested.has_value()); + EXPECT_EQ(nested->get().doc(), "nested doc"); + const auto& map = checked_cast(*nested->get().type()); + EXPECT_EQ(*map.key().type(), *fixed(4)); + const auto& list = checked_cast(*map.value().type()); + const auto& element = checked_cast(*list.element().type()); + ASSERT_EQ(element.fields().size(), 1U); + const auto& amount = element.fields()[0]; + EXPECT_EQ(*amount.type(), *decimal(9, 2)); + EXPECT_EQ(amount.doc(), "amount doc"); + ASSERT_NE(amount.initial_default(), nullptr); + EXPECT_EQ(*amount.initial_default(), Literal::Decimal(1234, 9, 2)); + ASSERT_NE(amount.write_default(), nullptr); + EXPECT_EQ(*amount.write_default(), Literal::Decimal(5678, 9, 2)); + }; + + // Mutating retained input aliases must not change the frozen operation. + *amount_type = DecimalType(18, 4); + *key_type = FixedType(8); + *initial = Literal::Decimal(9999, 18, 4); + *write = Literal::Decimal(8888, 18, 4); + ICEBERG_UNWRAP_OR_FAIL(auto preview, update->Validate()); + EXPECT_EQ(preview.schema->schema_id(), expected.schema->schema_id()); + verify(*preview.schema); + + // A public preview must also be independent of the frozen inputs and the + // metadata already staged in the transaction. + ICEBERG_UNWRAP_OR_FAIL(auto nested, preview.schema->FindFieldByName("nested")); + ASSERT_TRUE(nested.has_value()); + auto& map = checked_cast(*nested->get().type()); + checked_cast(*map.key().type()) = FixedType(16); + auto& list = checked_cast(*map.value().type()); + auto& element = checked_cast(*list.element().type()); + const auto& amount = element.fields()[0]; + checked_cast(*amount.type()) = DecimalType(20, 5); + checked_cast(*amount.initial_default()->type()) = DecimalType(21, 6); + checked_cast(*amount.write_default()->type()) = DecimalType(22, 7); + list = ListType(SchemaField(101, "element", int64(), true)); + map = MapType(SchemaField(102, "key", string(), false), + SchemaField(103, "value", int64(), false)); + + ICEBERG_UNWRAP_OR_FAIL(auto fresh_preview, update->Validate()); + EXPECT_EQ(fresh_preview.schema->schema_id(), expected.schema->schema_id()); + verify(*fresh_preview.schema); + ICEBERG_UNWRAP_OR_FAIL(auto staged, txn->current().Schema()); + verify(*staged); + ASSERT_THAT(txn->Commit(), IsOk()); + ICEBERG_UNWRAP_OR_FAIL(auto reloaded, catalog_->LoadTable(table_ident_)); + ICEBERG_UNWRAP_OR_FAIL(auto committed, reloaded->schema()); + EXPECT_EQ(committed->schema_id(), staged->schema_id()); + verify(*committed); +} + TEST_F(UpdateSchemaDefaultValueTest, UpdateColumnDefaultCastsToColumnType) { ICEBERG_UNWRAP_OR_FAIL(auto update, table_->NewUpdateSchema()); update->UpdateColumnDefault("x", Literal::Int(5)); - ICEBERG_UNWRAP_OR_FAIL(auto result, update->Apply()); + ICEBERG_UNWRAP_OR_FAIL(auto result, update->Validate()); ICEBERG_UNWRAP_OR_FAIL(auto field_opt, result.schema->FindFieldByName("x")); ASSERT_TRUE(field_opt.has_value()); @@ -312,7 +394,7 @@ TEST_F(UpdateSchemaDefaultValueTest, RequireColumnAddedWithDefault) { update->AddColumn("new_col", int32(), "An integer column", Literal::Int(42)) .RequireColumn("new_col"); - ICEBERG_UNWRAP_OR_FAIL(auto result, update->Apply()); + ICEBERG_UNWRAP_OR_FAIL(auto result, update->Validate()); ICEBERG_UNWRAP_OR_FAIL(auto new_field_opt, result.schema->FindFieldByName("new_col")); ASSERT_TRUE(new_field_opt.has_value()); EXPECT_FALSE(new_field_opt->get().optional()); @@ -345,7 +427,7 @@ TEST_F(UpdateSchemaDefaultValueTest, RequireNestedMapListColumnAddedWithDefault) .AddColumn("points", "z", int64(), "z coordinate", Literal::Long(0)) .RequireColumn("points.z"); - ICEBERG_UNWRAP_OR_FAIL(auto result, update->Apply()); + ICEBERG_UNWRAP_OR_FAIL(auto result, update->Validate()); ICEBERG_UNWRAP_OR_FAIL(auto locations_opt, result.schema->FindFieldByName("locations")); ASSERT_TRUE(locations_opt.has_value()); @@ -371,7 +453,7 @@ TEST_F(UpdateSchemaDefaultValueTest, UpdateColumnDocPreservesDefaultValues) { update->AddColumn("new_col", int32(), "An integer column", Literal::Int(42)) .UpdateColumnDoc("new_col", "updated doc"); - ICEBERG_UNWRAP_OR_FAIL(auto result, update->Apply()); + ICEBERG_UNWRAP_OR_FAIL(auto result, update->Validate()); ICEBERG_UNWRAP_OR_FAIL(auto field_opt, result.schema->FindFieldByName("new_col")); ASSERT_TRUE(field_opt.has_value()); @@ -388,12 +470,12 @@ TEST_F(UpdateSchemaDefaultValueTest, UpdateColumnTypePromotesDefaultValues) { update->AddColumn("new_col", int32(), "An integer column", Literal::Int(42)) .UpdateColumn("new_col", int64()); - ICEBERG_UNWRAP_OR_FAIL(auto result, update->Apply()); + ICEBERG_UNWRAP_OR_FAIL(auto result, update->Validate()); ICEBERG_UNWRAP_OR_FAIL(auto field_opt, result.schema->FindFieldByName("new_col")); ASSERT_TRUE(field_opt.has_value()); const auto& field = field_opt->get(); - EXPECT_EQ(field.type(), int64()); + EXPECT_EQ(*field.type(), *int64()); ASSERT_NE(field.initial_default(), nullptr); EXPECT_EQ(*field.initial_default(), Literal::Long(42)); ASSERT_NE(field.write_default(), nullptr); @@ -407,7 +489,7 @@ TEST_F(UpdateSchemaDefaultValueTest, UpdateColumnTypePromotesDecimalDefault) { Literal::Decimal(1234, 9, 2)) .UpdateColumn("new_col", decimal(18, 2)); - ICEBERG_UNWRAP_OR_FAIL(auto result, update->Apply()); + ICEBERG_UNWRAP_OR_FAIL(auto result, update->Validate()); ICEBERG_UNWRAP_OR_FAIL(auto field_opt, result.schema->FindFieldByName("new_col")); ASSERT_TRUE(field_opt.has_value()); @@ -424,7 +506,7 @@ TEST_F(UpdateSchemaDefaultValueTest, AddColumnWithWiderPrecisionDecimalDefault) update->AddColumn("new_col", decimal(18, 2), "A decimal column", Literal::Decimal(1234, 9, 2)); - ICEBERG_UNWRAP_OR_FAIL(auto result, update->Apply()); + ICEBERG_UNWRAP_OR_FAIL(auto result, update->Validate()); ICEBERG_UNWRAP_OR_FAIL(auto field_opt, result.schema->FindFieldByName("new_col")); ASSERT_TRUE(field_opt.has_value()); @@ -440,7 +522,7 @@ TEST_F(UpdateSchemaDefaultValueTest, UpdateColumnDefaultWiderPrecisionDecimal) { update->AddColumn("new_col", decimal(18, 2), "A decimal column") .UpdateColumnDefault("new_col", Literal::Decimal(1234, 9, 2)); - ICEBERG_UNWRAP_OR_FAIL(auto result, update->Apply()); + ICEBERG_UNWRAP_OR_FAIL(auto result, update->Validate()); ICEBERG_UNWRAP_OR_FAIL(auto field_opt, result.schema->FindFieldByName("new_col")); ASSERT_TRUE(field_opt.has_value()); @@ -454,7 +536,7 @@ TEST_F(UpdateSchemaDefaultValueTest, AddColumnWithDifferentScaleDecimalDefaultFa update->AddColumn("new_col", decimal(18, 2), "A decimal column", Literal::Decimal(1234, 9, 3)); - auto result = update->Apply(); + auto result = update->Validate(); EXPECT_THAT(result, IsError(ErrorKind::kValidationFailed)); EXPECT_THAT(result, HasErrorMessage("Cannot cast default value")); } @@ -464,7 +546,7 @@ TEST_F(UpdateSchemaDefaultValueTest, UpdateColumnDefaultDifferentScaleDecimalFai update->AddColumn("new_col", decimal(18, 2), "A decimal column") .UpdateColumnDefault("new_col", Literal::Decimal(1234, 9, 3)); - auto result = update->Apply(); + auto result = update->Validate(); EXPECT_THAT(result, IsError(ErrorKind::kValidationFailed)); EXPECT_THAT(result, HasErrorMessage("Cannot cast default value")); } @@ -474,7 +556,7 @@ TEST_F(UpdateSchemaDefaultValueTest, AddColumnWithOutOfPrecisionDecimalDefaultFa update->AddColumn("new_col", decimal(4, 2), "A decimal column", Literal::Decimal(1234567, 9, 2)); - auto result = update->Apply(); + auto result = update->Validate(); EXPECT_THAT(result, IsError(ErrorKind::kValidationFailed)); EXPECT_THAT(result, HasErrorMessage("Cannot cast default value")); } @@ -484,7 +566,7 @@ TEST_F(UpdateSchemaDefaultValueTest, AddColumnWithTypedNullDecimalDefaultFails) update->AddColumn("new_col", decimal(18, 2), "A decimal column", Literal::Null(decimal(18, 2))); - auto result = update->Apply(); + auto result = update->Validate(); EXPECT_THAT(result, IsError(ErrorKind::kValidationFailed)); EXPECT_THAT(result, HasErrorMessage("Cannot cast default value")); } @@ -495,7 +577,7 @@ TEST_F(UpdateSchemaTest, AddMultipleColumns) { .AddColumn("col2", string(), "Second column") .AddColumn("col3", boolean(), "Third column"); - ICEBERG_UNWRAP_OR_FAIL(auto result, update->Apply()); + ICEBERG_UNWRAP_OR_FAIL(auto result, update->Validate()); ASSERT_TRUE(result.schema != nullptr); ICEBERG_UNWRAP_OR_FAIL(auto col1_opt, result.schema->FindFieldByName("col1")); @@ -506,16 +588,16 @@ TEST_F(UpdateSchemaTest, AddMultipleColumns) { ASSERT_TRUE(col2_opt.has_value()); ASSERT_TRUE(col3_opt.has_value()); - EXPECT_EQ(col1_opt->get().type(), int32()); - EXPECT_EQ(col2_opt->get().type(), string()); - EXPECT_EQ(col3_opt->get().type(), boolean()); + EXPECT_EQ(*col1_opt->get().type(), *int32()); + EXPECT_EQ(*col2_opt->get().type(), *string()); + EXPECT_EQ(*col3_opt->get().type(), *boolean()); } TEST_F(UpdateSchemaTest, AddColumnWithDotInNameFails) { ICEBERG_UNWRAP_OR_FAIL(auto update, table_->NewUpdateSchema()); update->AddColumn("col.with.dots", int32(), "Column with dots"); - auto result = update->Apply(); + auto result = update->Validate(); EXPECT_THAT(result, IsError(ErrorKind::kValidationFailed)); EXPECT_THAT(result, HasErrorMessage("Cannot add column with ambiguous name")); } @@ -532,7 +614,7 @@ TEST_F(UpdateSchemaTest, AddColumnToNestedStruct) { ICEBERG_UNWRAP_OR_FAIL(auto update, reloaded->NewUpdateSchema()); update->AddColumn("struct_col", "new_nested_field", string(), "New nested field"); - ICEBERG_UNWRAP_OR_FAIL(auto result, update->Apply()); + ICEBERG_UNWRAP_OR_FAIL(auto result, update->Validate()); ASSERT_TRUE(result.schema != nullptr); ICEBERG_UNWRAP_OR_FAIL(auto struct_field_opt, @@ -549,14 +631,14 @@ TEST_F(UpdateSchemaTest, AddColumnToNestedStruct) { const auto& nested_field = nested_field_opt->get(); EXPECT_EQ(nested_field.name(), "new_nested_field"); - EXPECT_EQ(nested_field.type(), string()); + EXPECT_EQ(*nested_field.type(), *string()); } TEST_F(UpdateSchemaTest, AddColumnToNonExistentParentFails) { ICEBERG_UNWRAP_OR_FAIL(auto update, table_->NewUpdateSchema()); update->AddColumn("non_existent_parent", "new_field", int32(), "New field"); - auto result = update->Apply(); + auto result = update->Validate(); EXPECT_THAT(result, IsError(ErrorKind::kValidationFailed)); EXPECT_THAT(result, HasErrorMessage("Cannot find parent struct")); } @@ -570,7 +652,7 @@ TEST_F(UpdateSchemaTest, AddColumnToNonStructParentFails) { ICEBERG_UNWRAP_OR_FAIL(auto update, reloaded->NewUpdateSchema()); update->AddColumn("primitive_col", "nested_field", string(), "Should fail"); - auto result = update->Apply(); + auto result = update->Validate(); EXPECT_THAT(result, IsError(ErrorKind::kValidationFailed)); EXPECT_THAT(result, HasErrorMessage("Cannot add to non-struct column")); } @@ -580,7 +662,7 @@ TEST_F(UpdateSchemaTest, AddDuplicateColumnNameFails) { update->AddColumn("duplicate_col", int32(), "First column") .AddColumn("duplicate_col", string(), "Duplicate column"); - auto result = update->Apply(); + auto result = update->Validate(); EXPECT_THAT(result, IsError(ErrorKind::kInvalidSchema)); EXPECT_THAT(result, HasErrorMessage("Duplicate path found")); } @@ -593,7 +675,7 @@ TEST_F(UpdateSchemaTest, ColumnIdAssignment) { update->AddColumn("new_col1", int32(), "First new column") .AddColumn("new_col2", string(), "Second new column"); - ICEBERG_UNWRAP_OR_FAIL(auto result, update->Apply()); + ICEBERG_UNWRAP_OR_FAIL(auto result, update->Validate()); EXPECT_EQ(result.new_last_column_id, original_last_id + 2); @@ -615,7 +697,7 @@ TEST_F(UpdateSchemaTest, AddNestedStructColumn) { ICEBERG_UNWRAP_OR_FAIL(auto update, table_->NewUpdateSchema()); update->AddColumn("complex_struct", nested_struct, "A complex struct column"); - ICEBERG_UNWRAP_OR_FAIL(auto result, update->Apply()); + ICEBERG_UNWRAP_OR_FAIL(auto result, update->Validate()); ASSERT_TRUE(result.schema != nullptr); ICEBERG_UNWRAP_OR_FAIL(auto struct_field_opt, @@ -636,8 +718,8 @@ TEST_F(UpdateSchemaTest, AddNestedStructColumn) { ASSERT_TRUE(field1_opt.has_value()); ASSERT_TRUE(field2_opt.has_value()); - EXPECT_EQ(field1_opt->get().type(), int32()); - EXPECT_EQ(field2_opt->get().type(), string()); + EXPECT_EQ(*field1_opt->get().type(), *int32()); + EXPECT_EQ(*field2_opt->get().type(), *string()); EXPECT_TRUE(field1_opt->get().optional()); EXPECT_FALSE(field2_opt->get().optional()); } @@ -648,7 +730,7 @@ TEST_F(UpdateSchemaTest, CaseSensitiveColumnNames) { .AddColumn("Column", int32(), "Uppercase column") .AddColumn("column", string(), "Lowercase column"); - ICEBERG_UNWRAP_OR_FAIL(auto result, update->Apply()); + ICEBERG_UNWRAP_OR_FAIL(auto result, update->Validate()); ASSERT_TRUE(result.schema != nullptr); ICEBERG_UNWRAP_OR_FAIL(auto upper_opt, result.schema->FindFieldByName("Column", true)); @@ -657,8 +739,8 @@ TEST_F(UpdateSchemaTest, CaseSensitiveColumnNames) { ASSERT_TRUE(upper_opt.has_value()); ASSERT_TRUE(lower_opt.has_value()); - EXPECT_EQ(upper_opt->get().type(), int32()); - EXPECT_EQ(lower_opt->get().type(), string()); + EXPECT_EQ(*upper_opt->get().type(), *int32()); + EXPECT_EQ(*lower_opt->get().type(), *string()); } TEST_F(UpdateSchemaTest, CaseInsensitiveDuplicateDetection) { @@ -667,7 +749,7 @@ TEST_F(UpdateSchemaTest, CaseInsensitiveDuplicateDetection) { .AddColumn("Column", int32(), "First column") .AddColumn("COLUMN", string(), "Duplicate column"); - auto result = update->Apply(); + auto result = update->Validate(); EXPECT_THAT(result, IsError(ErrorKind::kInvalidSchema)); EXPECT_THAT(result, HasErrorMessage("Duplicate path found")); } @@ -676,7 +758,7 @@ TEST_F(UpdateSchemaTest, EmptyUpdate) { ICEBERG_UNWRAP_OR_FAIL(auto original_schema, table_->schema()); ICEBERG_UNWRAP_OR_FAIL(auto update, table_->NewUpdateSchema()); - ICEBERG_UNWRAP_OR_FAIL(auto result, update->Apply()); + ICEBERG_UNWRAP_OR_FAIL(auto result, update->Validate()); EXPECT_EQ(*result.schema, *original_schema); EXPECT_EQ(result.new_last_column_id, table_->metadata()->last_column_id); @@ -730,7 +812,7 @@ TEST_F(UpdateSchemaTest, AddFieldsToMapAndList) { update->AddColumn("locations", "alt", float32(), "altitude") .AddColumn("points", "z", int64(), "z coordinate"); - ICEBERG_UNWRAP_OR_FAIL(auto result, update->Apply()); + ICEBERG_UNWRAP_OR_FAIL(auto result, update->Validate()); ICEBERG_UNWRAP_OR_FAIL(auto locations_opt, result.schema->FindFieldByName("locations")); ASSERT_TRUE(locations_opt.has_value()); @@ -741,7 +823,7 @@ TEST_F(UpdateSchemaTest, AddFieldsToMapAndList) { const auto& value_struct = checked_cast(*map.value().type()); ICEBERG_UNWRAP_OR_FAIL(auto alt_opt, value_struct.GetFieldByName("alt")); ASSERT_TRUE(alt_opt.has_value()); - EXPECT_EQ(alt_opt->get().type(), float32()); + EXPECT_EQ(*alt_opt->get().type(), *float32()); ICEBERG_UNWRAP_OR_FAIL(auto points_opt, result.schema->FindFieldByName("points")); ASSERT_TRUE(points_opt.has_value()); @@ -752,7 +834,7 @@ TEST_F(UpdateSchemaTest, AddFieldsToMapAndList) { const auto& element_struct = checked_cast(*list.element().type()); ICEBERG_UNWRAP_OR_FAIL(auto z_opt, element_struct.GetFieldByName("z")); ASSERT_TRUE(z_opt.has_value()); - EXPECT_EQ(z_opt->get().type(), int64()); + EXPECT_EQ(*z_opt->get().type(), *int64()); } TEST_F(UpdateSchemaTest, AddNestedStructWithIdReassignment) { @@ -762,7 +844,7 @@ TEST_F(UpdateSchemaTest, AddNestedStructWithIdReassignment) { ICEBERG_UNWRAP_OR_FAIL(auto update, table_->NewUpdateSchema()); update->AddColumn("location", nested_struct); - ICEBERG_UNWRAP_OR_FAIL(auto result, update->Apply()); + ICEBERG_UNWRAP_OR_FAIL(auto result, update->Validate()); ICEBERG_UNWRAP_OR_FAIL(auto location_opt, result.schema->FindFieldByName("location")); ASSERT_TRUE(location_opt.has_value()); @@ -798,7 +880,7 @@ TEST_F(UpdateSchemaTest, AddNestedMapOfStructs) { ICEBERG_UNWRAP_OR_FAIL(auto update, table_->NewUpdateSchema()); update->AddColumn("locations", map_type); - ICEBERG_UNWRAP_OR_FAIL(auto result, update->Apply()); + ICEBERG_UNWRAP_OR_FAIL(auto result, update->Validate()); ICEBERG_UNWRAP_OR_FAIL(auto locations_opt, result.schema->FindFieldByName("locations")); ASSERT_TRUE(locations_opt.has_value()); @@ -831,7 +913,7 @@ TEST_F(UpdateSchemaTest, AddNestedListOfStructs) { ICEBERG_UNWRAP_OR_FAIL(auto update, table_->NewUpdateSchema()); update->AddColumn("locations", list_type); - ICEBERG_UNWRAP_OR_FAIL(auto result, update->Apply()); + ICEBERG_UNWRAP_OR_FAIL(auto result, update->Validate()); ICEBERG_UNWRAP_OR_FAIL(auto locations_opt, result.schema->FindFieldByName("locations")); ASSERT_TRUE(locations_opt.has_value()); @@ -864,7 +946,7 @@ TEST_F(UpdateSchemaTest, AddFieldWithDotsInName) { ICEBERG_UNWRAP_OR_FAIL(auto update, reloaded->NewUpdateSchema()); update->AddColumn("struct_col", "field.with.dots", int64(), "Field with dots in name"); - ICEBERG_UNWRAP_OR_FAIL(auto result, update->Apply()); + ICEBERG_UNWRAP_OR_FAIL(auto result, update->Validate()); ICEBERG_UNWRAP_OR_FAIL(auto struct_field_opt, result.schema->FindFieldByName("struct_col")); @@ -877,7 +959,7 @@ TEST_F(UpdateSchemaTest, AddFieldWithDotsInName) { nested_struct.GetFieldByName("field.with.dots")); ASSERT_TRUE(dotted_field_opt.has_value()); EXPECT_EQ(dotted_field_opt->get().name(), "field.with.dots"); - EXPECT_EQ(dotted_field_opt->get().type(), int64()); + EXPECT_EQ(*dotted_field_opt->get().type(), *int64()); } TEST_F(UpdateSchemaTest, AddFieldToMapKeyFails) { @@ -899,7 +981,7 @@ TEST_F(UpdateSchemaTest, AddFieldToMapKeyFails) { ICEBERG_UNWRAP_OR_FAIL(auto update, reloaded->NewUpdateSchema()); update->AddColumn("locations.key", "city", string(), "Should fail"); - auto result = update->Apply(); + auto result = update->Validate(); EXPECT_THAT(result, IsError(ErrorKind::kValidationFailed)); EXPECT_THAT(result, HasErrorMessage("Cannot add fields to map keys")); } @@ -913,7 +995,7 @@ TEST_F(UpdateSchemaTest, DeleteColumn) { ICEBERG_UNWRAP_OR_FAIL(auto update, reloaded->NewUpdateSchema()); update->DeleteColumn("to_delete"); - ICEBERG_UNWRAP_OR_FAIL(auto result, update->Apply()); + ICEBERG_UNWRAP_OR_FAIL(auto result, update->Validate()); ICEBERG_UNWRAP_OR_FAIL(auto field_opt, result.schema->FindFieldByName("to_delete")); EXPECT_FALSE(field_opt.has_value()); @@ -932,7 +1014,7 @@ TEST_F(UpdateSchemaTest, DeleteNestedColumn) { ICEBERG_UNWRAP_OR_FAIL(auto update, reloaded->NewUpdateSchema()); update->DeleteColumn("struct_col.field1"); - ICEBERG_UNWRAP_OR_FAIL(auto result, update->Apply()); + ICEBERG_UNWRAP_OR_FAIL(auto result, update->Validate()); ICEBERG_UNWRAP_OR_FAIL(auto struct_field_opt, result.schema->FindFieldByName("struct_col")); @@ -952,7 +1034,7 @@ TEST_F(UpdateSchemaTest, DeleteMissingColumnFails) { ICEBERG_UNWRAP_OR_FAIL(auto update, table_->NewUpdateSchema()); update->DeleteColumn("non_existent"); - auto result = update->Apply(); + auto result = update->Validate(); EXPECT_THAT(result, IsError(ErrorKind::kValidationFailed)); EXPECT_THAT(result, HasErrorMessage("Cannot delete missing column")); } @@ -967,13 +1049,13 @@ TEST_F(UpdateSchemaTest, DeleteThenAdd) { ICEBERG_UNWRAP_OR_FAIL(auto update, reloaded->NewUpdateSchema()); update->DeleteColumn("col").AddColumn("col", string(), "Now optional string"); - ICEBERG_UNWRAP_OR_FAIL(auto result, update->Apply()); + ICEBERG_UNWRAP_OR_FAIL(auto result, update->Validate()); ICEBERG_UNWRAP_OR_FAIL(auto field_opt, result.schema->FindFieldByName("col")); ASSERT_TRUE(field_opt.has_value()); const auto& field = field_opt->get(); - EXPECT_EQ(field.type(), string()); + EXPECT_EQ(*field.type(), *string()); EXPECT_TRUE(field.optional()); } @@ -990,7 +1072,7 @@ TEST_F(UpdateSchemaTest, DeleteThenAddNested) { update->DeleteColumn("struct_col.field1") .AddColumn("struct_col", "field1", int32(), "Re-added field"); - ICEBERG_UNWRAP_OR_FAIL(auto result, update->Apply()); + ICEBERG_UNWRAP_OR_FAIL(auto result, update->Validate()); ICEBERG_UNWRAP_OR_FAIL(auto struct_field_opt, result.schema->FindFieldByName("struct_col")); @@ -1001,14 +1083,14 @@ TEST_F(UpdateSchemaTest, DeleteThenAddNested) { ICEBERG_UNWRAP_OR_FAIL(auto field1_opt, nested_struct.GetFieldByName("field1")); ASSERT_TRUE(field1_opt.has_value()); - EXPECT_EQ(field1_opt->get().type(), int32()); + EXPECT_EQ(*field1_opt->get().type(), *int32()); } TEST_F(UpdateSchemaTest, AddDeleteConflict) { ICEBERG_UNWRAP_OR_FAIL(auto update, table_->NewUpdateSchema()); update->AddColumn("new_col", int32()).DeleteColumn("new_col"); - auto result = update->Apply(); + auto result = update->Validate(); EXPECT_THAT(result, IsError(ErrorKind::kValidationFailed)); EXPECT_THAT(result, HasErrorMessage("Cannot delete missing column")); } @@ -1025,7 +1107,7 @@ TEST_F(UpdateSchemaTest, DeleteColumnWithAdditionsFails) { ICEBERG_UNWRAP_OR_FAIL(auto update, reloaded->NewUpdateSchema()); update->AddColumn("struct_col", "field2", string()).DeleteColumn("struct_col"); - auto result = update->Apply(); + auto result = update->Validate(); EXPECT_THAT(result, IsError(ErrorKind::kValidationFailed)); EXPECT_THAT(result, HasErrorMessage("Cannot delete a column that has additions")); } @@ -1042,7 +1124,7 @@ TEST_F(UpdateSchemaTest, DeleteMapKeyFails) { ICEBERG_UNWRAP_OR_FAIL(auto update, reloaded->NewUpdateSchema()); update->DeleteColumn("map_col.key"); - auto result = update->Apply(); + auto result = update->Validate(); EXPECT_THAT(result, IsError(ErrorKind::kValidationFailed)); EXPECT_THAT(result, HasErrorMessage("Cannot delete map keys")); } @@ -1056,7 +1138,7 @@ TEST_F(UpdateSchemaTest, DeleteColumnCaseInsensitive) { ICEBERG_UNWRAP_OR_FAIL(auto update, reloaded->NewUpdateSchema()); update->CaseSensitive(false).DeleteColumn("mycolumn"); - ICEBERG_UNWRAP_OR_FAIL(auto result, update->Apply()); + ICEBERG_UNWRAP_OR_FAIL(auto result, update->Validate()); ICEBERG_UNWRAP_OR_FAIL(auto field_opt, result.schema->FindFieldByName("MyColumn", false)); @@ -1072,7 +1154,7 @@ TEST_F(UpdateSchemaTest, RenameColumn) { ICEBERG_UNWRAP_OR_FAIL(auto update, reloaded->NewUpdateSchema()); update->RenameColumn("old_name", "new_name"); - ICEBERG_UNWRAP_OR_FAIL(auto result, update->Apply()); + ICEBERG_UNWRAP_OR_FAIL(auto result, update->Validate()); ICEBERG_UNWRAP_OR_FAIL(auto old_field_opt, result.schema->FindFieldByName("old_name")); ICEBERG_UNWRAP_OR_FAIL(auto new_field_opt, result.schema->FindFieldByName("new_name")); @@ -1095,7 +1177,7 @@ TEST_F(UpdateSchemaTest, RenameNestedColumn) { ICEBERG_UNWRAP_OR_FAIL(auto update, reloaded->NewUpdateSchema()); update->RenameColumn("struct_col.field1", "renamed_field"); - ICEBERG_UNWRAP_OR_FAIL(auto result, update->Apply()); + ICEBERG_UNWRAP_OR_FAIL(auto result, update->Validate()); ICEBERG_UNWRAP_OR_FAIL(auto struct_field_opt, result.schema->FindFieldByName("struct_col")); @@ -1121,7 +1203,7 @@ TEST_F(UpdateSchemaTest, RenameColumnWithDotsInName) { ICEBERG_UNWRAP_OR_FAIL(auto update, reloaded->NewUpdateSchema()); update->RenameColumn("simple_name", "name.with.dots"); - ICEBERG_UNWRAP_OR_FAIL(auto result, update->Apply()); + ICEBERG_UNWRAP_OR_FAIL(auto result, update->Validate()); ICEBERG_UNWRAP_OR_FAIL(auto new_field_opt, result.schema->FindFieldByName("name.with.dots")); @@ -1133,7 +1215,7 @@ TEST_F(UpdateSchemaTest, RenameMissingColumnFails) { ICEBERG_UNWRAP_OR_FAIL(auto update, table_->NewUpdateSchema()); update->RenameColumn("non_existent", "new_name"); - auto result = update->Apply(); + auto result = update->Validate(); EXPECT_THAT(result, IsError(ErrorKind::kValidationFailed)); EXPECT_THAT(result, HasErrorMessage("Cannot rename missing column")); } @@ -1147,7 +1229,7 @@ TEST_F(UpdateSchemaTest, RenameDeletedColumnFails) { ICEBERG_UNWRAP_OR_FAIL(auto update, reloaded->NewUpdateSchema()); update->DeleteColumn("col").RenameColumn("col", "new_name"); - auto result = update->Apply(); + auto result = update->Validate(); EXPECT_THAT(result, IsError(ErrorKind::kValidationFailed)); EXPECT_THAT(result, HasErrorMessage("Cannot rename a column that will be deleted")); } @@ -1161,7 +1243,7 @@ TEST_F(UpdateSchemaTest, RenameColumnCaseInsensitive) { ICEBERG_UNWRAP_OR_FAIL(auto update, reloaded->NewUpdateSchema()); update->CaseSensitive(false).RenameColumn("mycolumn", "NewName"); - ICEBERG_UNWRAP_OR_FAIL(auto result, update->Apply()); + ICEBERG_UNWRAP_OR_FAIL(auto result, update->Validate()); ICEBERG_UNWRAP_OR_FAIL(auto old_field_opt, result.schema->FindFieldByName("MyColumn", false)); @@ -1181,7 +1263,7 @@ TEST_F(UpdateSchemaTest, RenameThenDeleteOldNameFails) { ICEBERG_UNWRAP_OR_FAIL(auto update, reloaded->NewUpdateSchema()); update->RenameColumn("old_name", "new_name").DeleteColumn("old_name"); - auto result = update->Apply(); + auto result = update->Validate(); EXPECT_THAT(result, IsError(ErrorKind::kValidationFailed)); EXPECT_THAT(result, HasErrorMessage("Cannot delete a column that has updates")); } @@ -1195,7 +1277,7 @@ TEST_F(UpdateSchemaTest, RenameThenDeleteNewNameFails) { ICEBERG_UNWRAP_OR_FAIL(auto update, reloaded->NewUpdateSchema()); update->RenameColumn("old_name", "new_name").DeleteColumn("new_name"); - auto result = update->Apply(); + auto result = update->Validate(); EXPECT_THAT(result, IsError(ErrorKind::kValidationFailed)); EXPECT_THAT(result, HasErrorMessage("Cannot delete missing column")); } @@ -1209,7 +1291,7 @@ TEST_F(UpdateSchemaTest, RenameThenAddWithOldName) { ICEBERG_UNWRAP_OR_FAIL(auto update, reloaded->NewUpdateSchema()); update->RenameColumn("old_name", "new_name").AddColumn("old_name", string()); - auto result = update->Apply(); + auto result = update->Validate(); EXPECT_THAT(result, IsError(ErrorKind::kValidationFailed)); EXPECT_THAT(result, HasErrorMessage("Cannot add column, name already exists")); } @@ -1218,7 +1300,7 @@ TEST_F(UpdateSchemaTest, AddThenRename) { ICEBERG_UNWRAP_OR_FAIL(auto update, table_->NewUpdateSchema()); update->AddColumn("temp_name", string()).RenameColumn("temp_name", "final_name"); - auto result = update->Apply(); + auto result = update->Validate(); EXPECT_THAT(result, IsError(ErrorKind::kValidationFailed)); EXPECT_THAT(result, HasErrorMessage("Cannot rename missing column")); } @@ -1234,7 +1316,7 @@ TEST_F(UpdateSchemaTest, DeleteThenAddThenRename) { .AddColumn("col", string(), "New column with same name") .RenameColumn("col", "renamed_col"); - auto result = update->Apply(); + auto result = update->Validate(); EXPECT_THAT(result, IsError(ErrorKind::kValidationFailed)); EXPECT_THAT(result, HasErrorMessage("Cannot rename a column that will be deleted")); } @@ -1248,7 +1330,7 @@ TEST_F(UpdateSchemaTest, MakeColumnOptional) { ICEBERG_UNWRAP_OR_FAIL(auto update, reloaded->NewUpdateSchema()); update->MakeColumnOptional("id"); - ICEBERG_UNWRAP_OR_FAIL(auto result, update->Apply()); + ICEBERG_UNWRAP_OR_FAIL(auto result, update->Validate()); ICEBERG_UNWRAP_OR_FAIL(auto field_opt, result.schema->FindFieldByName("id")); ASSERT_TRUE(field_opt.has_value()); @@ -1264,7 +1346,7 @@ TEST_F(UpdateSchemaTest, RequireColumn) { ICEBERG_UNWRAP_OR_FAIL(auto update, reloaded->NewUpdateSchema()); update->RequireColumn("id"); - auto result = update->Apply(); + auto result = update->Validate(); EXPECT_THAT(result, IsError(ErrorKind::kValidationFailed)); EXPECT_THAT(result, HasErrorMessage("Cannot change column nullability")); EXPECT_THAT(result, HasErrorMessage("optional -> required")); @@ -1273,7 +1355,7 @@ TEST_F(UpdateSchemaTest, RequireColumn) { ICEBERG_UNWRAP_OR_FAIL(auto update2, reloaded2->NewUpdateSchema()); update2->AllowIncompatibleChanges().RequireColumn("id"); - ICEBERG_UNWRAP_OR_FAIL(auto result2, update2->Apply()); + ICEBERG_UNWRAP_OR_FAIL(auto result2, update2->Validate()); ICEBERG_UNWRAP_OR_FAIL(auto field_opt, result2.schema->FindFieldByName("id")); ASSERT_TRUE(field_opt.has_value()); @@ -1289,7 +1371,7 @@ TEST_F(UpdateSchemaTest, RequireColumnNoop) { ICEBERG_UNWRAP_OR_FAIL(auto update, reloaded->NewUpdateSchema()); update->RequireColumn("id"); - ICEBERG_UNWRAP_OR_FAIL(auto result, update->Apply()); + ICEBERG_UNWRAP_OR_FAIL(auto result, update->Validate()); ICEBERG_UNWRAP_OR_FAIL(auto field_opt, result.schema->FindFieldByName("id")); ASSERT_TRUE(field_opt.has_value()); @@ -1305,7 +1387,7 @@ TEST_F(UpdateSchemaTest, MakeColumnOptionalNoop) { ICEBERG_UNWRAP_OR_FAIL(auto update, reloaded->NewUpdateSchema()); update->MakeColumnOptional("id"); - ICEBERG_UNWRAP_OR_FAIL(auto result, update->Apply()); + ICEBERG_UNWRAP_OR_FAIL(auto result, update->Validate()); ICEBERG_UNWRAP_OR_FAIL(auto field_opt, result.schema->FindFieldByName("id")); ASSERT_TRUE(field_opt.has_value()); @@ -1321,7 +1403,7 @@ TEST_F(UpdateSchemaTest, RequireColumnCaseInsensitive) { ICEBERG_UNWRAP_OR_FAIL(auto update, reloaded->NewUpdateSchema()); update->CaseSensitive(false).AllowIncompatibleChanges().RequireColumn("id"); - ICEBERG_UNWRAP_OR_FAIL(auto result, update->Apply()); + ICEBERG_UNWRAP_OR_FAIL(auto result, update->Validate()); ICEBERG_UNWRAP_OR_FAIL(auto field_opt, result.schema->FindFieldByName("ID", false)); ASSERT_TRUE(field_opt.has_value()); @@ -1332,7 +1414,7 @@ TEST_F(UpdateSchemaTest, MakeColumnOptionalMissingFails) { ICEBERG_UNWRAP_OR_FAIL(auto update, table_->NewUpdateSchema()); update->MakeColumnOptional("non_existent"); - auto result = update->Apply(); + auto result = update->Validate(); EXPECT_THAT(result, IsError(ErrorKind::kValidationFailed)); EXPECT_THAT(result, HasErrorMessage("Cannot update missing column")); } @@ -1341,7 +1423,7 @@ TEST_F(UpdateSchemaTest, RequireColumnMissingFails) { ICEBERG_UNWRAP_OR_FAIL(auto update, table_->NewUpdateSchema()); update->AllowIncompatibleChanges().RequireColumn("non_existent"); - auto result = update->Apply(); + auto result = update->Validate(); EXPECT_THAT(result, IsError(ErrorKind::kValidationFailed)); EXPECT_THAT(result, HasErrorMessage("Cannot update missing column")); } @@ -1355,7 +1437,7 @@ TEST_F(UpdateSchemaTest, MakeColumnOptionalDeletedFails) { ICEBERG_UNWRAP_OR_FAIL(auto update, reloaded->NewUpdateSchema()); update->DeleteColumn("col").MakeColumnOptional("col"); - auto result = update->Apply(); + auto result = update->Validate(); EXPECT_THAT(result, IsError(ErrorKind::kValidationFailed)); EXPECT_THAT(result, HasErrorMessage("Cannot update a column that will be deleted")); } @@ -1369,7 +1451,7 @@ TEST_F(UpdateSchemaTest, RequireColumnDeletedFails) { ICEBERG_UNWRAP_OR_FAIL(auto update, reloaded->NewUpdateSchema()); update->DeleteColumn("col").AllowIncompatibleChanges().RequireColumn("col"); - auto result = update->Apply(); + auto result = update->Validate(); EXPECT_THAT(result, IsError(ErrorKind::kValidationFailed)); EXPECT_THAT(result, HasErrorMessage("Cannot update a column that will be deleted")); } @@ -1379,7 +1461,7 @@ TEST_F(UpdateSchemaTest, UpdateColumnDoc) { update->AddColumn("col", int32(), "original doc"); update->UpdateColumnDoc("col", "updated doc"); - ICEBERG_UNWRAP_OR_FAIL(auto result, update->Apply()); + ICEBERG_UNWRAP_OR_FAIL(auto result, update->Validate()); ICEBERG_UNWRAP_OR_FAIL(auto field_opt, result.schema->FindFieldByName("col")); ASSERT_TRUE(field_opt.has_value()); @@ -1390,7 +1472,7 @@ TEST_F(UpdateSchemaTest, UpdateColumnDocMissingFails) { ICEBERG_UNWRAP_OR_FAIL(auto update, table_->NewUpdateSchema()); update->UpdateColumnDoc("non_existent", "some doc"); - auto result = update->Apply(); + auto result = update->Validate(); EXPECT_THAT(result, IsError(ErrorKind::kValidationFailed)); EXPECT_THAT(result, HasErrorMessage("Cannot update missing column")); } @@ -1404,7 +1486,7 @@ TEST_F(UpdateSchemaTest, UpdateColumnDocDeletedFails) { ICEBERG_UNWRAP_OR_FAIL(auto update, reloaded->NewUpdateSchema()); update->DeleteColumn("col").UpdateColumnDoc("col", "new doc"); - auto result = update->Apply(); + auto result = update->Validate(); EXPECT_THAT(result, IsError(ErrorKind::kValidationFailed)); EXPECT_THAT(result, HasErrorMessage("Cannot update a column that will be deleted")); } @@ -1418,7 +1500,7 @@ TEST_F(UpdateSchemaTest, UpdateColumnDocNoop) { ICEBERG_UNWRAP_OR_FAIL(auto update, reloaded->NewUpdateSchema()); update->UpdateColumnDoc("col", "same doc"); - ICEBERG_UNWRAP_OR_FAIL(auto result, update->Apply()); + ICEBERG_UNWRAP_OR_FAIL(auto result, update->Validate()); ICEBERG_UNWRAP_OR_FAIL(auto field_opt, result.schema->FindFieldByName("col")); ASSERT_TRUE(field_opt.has_value()); @@ -1430,7 +1512,7 @@ TEST_F(UpdateSchemaTest, UpdateColumnDocEmptyString) { update->AddColumn("col", int32(), "original doc"); update->UpdateColumnDoc("col", ""); - ICEBERG_UNWRAP_OR_FAIL(auto result, update->Apply()); + ICEBERG_UNWRAP_OR_FAIL(auto result, update->Validate()); ICEBERG_UNWRAP_OR_FAIL(auto field_opt, result.schema->FindFieldByName("col")); ASSERT_TRUE(field_opt.has_value()); @@ -1442,7 +1524,7 @@ TEST_F(UpdateSchemaTest, UpdateColumnIntToLong) { update->AddColumn("id", int32(), "An integer ID"); update->UpdateColumn("id", int64()); - ICEBERG_UNWRAP_OR_FAIL(auto result, update->Apply()); + ICEBERG_UNWRAP_OR_FAIL(auto result, update->Validate()); ICEBERG_UNWRAP_OR_FAIL(auto field_opt, result.schema->FindFieldByName("id")); ASSERT_TRUE(field_opt.has_value()); @@ -1455,7 +1537,7 @@ TEST_F(UpdateSchemaTest, UpdateColumnFloatToDouble) { update->AddColumn("value", float32(), "A float value"); update->UpdateColumn("value", float64()); - ICEBERG_UNWRAP_OR_FAIL(auto result, update->Apply()); + ICEBERG_UNWRAP_OR_FAIL(auto result, update->Validate()); ICEBERG_UNWRAP_OR_FAIL(auto field_opt, result.schema->FindFieldByName("value")); ASSERT_TRUE(field_opt.has_value()); @@ -1467,7 +1549,7 @@ TEST_F(UpdateSchemaTest, UpdateColumnUnknownToPrimitive) { update->AddColumn("mystery", unknown(), "A null-only placeholder"); update->UpdateColumn("mystery", string()); - ICEBERG_UNWRAP_OR_FAIL(auto result, update->Apply()); + ICEBERG_UNWRAP_OR_FAIL(auto result, update->Validate()); ICEBERG_UNWRAP_OR_FAIL(auto field_opt, result.schema->FindFieldByName("mystery")); ASSERT_TRUE(field_opt.has_value()); @@ -1480,7 +1562,7 @@ TEST_F(UpdateSchemaTest, AddRequiredUnknownColumnFails) { ICEBERG_UNWRAP_OR_FAIL(auto update, table_->NewUpdateSchema()); update->AllowIncompatibleChanges().AddRequiredColumn("mystery", unknown()); - auto result = update->Apply(); + auto result = update->Validate(); EXPECT_THAT(result, IsError(ErrorKind::kInvalidArgument)); EXPECT_THAT(result, HasErrorMessage("Unknown type field 'mystery' must be optional")); } @@ -1491,7 +1573,7 @@ TEST_F(UpdateSchemaTest, AddColumnWithRequiredNestedUnknownFails) { SchemaField::MakeRequired(3, "mystery", unknown()), })); - auto result = update->Apply(); + auto result = update->Validate(); EXPECT_THAT(result, IsError(ErrorKind::kInvalidArgument)); EXPECT_THAT(result, HasErrorMessage("Unknown type field 'mystery' must be optional")); } @@ -1501,7 +1583,7 @@ TEST_F(UpdateSchemaTest, UpdateColumnSameType) { update->AddColumn("id", int32()); update->UpdateColumn("id", int32()); - ICEBERG_UNWRAP_OR_FAIL(auto result, update->Apply()); + ICEBERG_UNWRAP_OR_FAIL(auto result, update->Validate()); ICEBERG_UNWRAP_OR_FAIL(auto field_opt, result.schema->FindFieldByName("id")); ASSERT_TRUE(field_opt.has_value()); @@ -1512,7 +1594,7 @@ TEST_F(UpdateSchemaTest, UpdateColumnMissingFails) { ICEBERG_UNWRAP_OR_FAIL(auto update, table_->NewUpdateSchema()); update->UpdateColumn("non_existent", int64()); - auto result = update->Apply(); + auto result = update->Validate(); EXPECT_THAT(result, IsError(ErrorKind::kValidationFailed)); EXPECT_THAT(result, HasErrorMessage("Cannot update missing column")); } @@ -1526,7 +1608,7 @@ TEST_F(UpdateSchemaTest, UpdateColumnDeletedFails) { ICEBERG_UNWRAP_OR_FAIL(auto update, reloaded->NewUpdateSchema()); update->DeleteColumn("col").UpdateColumn("col", int64()); - auto result = update->Apply(); + auto result = update->Validate(); EXPECT_THAT(result, IsError(ErrorKind::kValidationFailed)); EXPECT_THAT(result, HasErrorMessage("Cannot update a column that will be deleted")); } @@ -1536,7 +1618,7 @@ TEST_F(UpdateSchemaTest, UpdateColumnInvalidPromotionFails) { update->AddColumn("id", int64()); update->UpdateColumn("id", int32()); - auto result = update->Apply(); + auto result = update->Validate(); EXPECT_THAT(result, IsError(ErrorKind::kValidationFailed)); EXPECT_THAT(result, HasErrorMessage("Cannot change column type")); } @@ -1546,7 +1628,7 @@ TEST_F(UpdateSchemaTest, UpdateColumnInvalidPromotionDoubleToFloatFails) { update->AddColumn("value", float64()); update->UpdateColumn("value", float32()); - auto result = update->Apply(); + auto result = update->Validate(); EXPECT_THAT(result, IsError(ErrorKind::kValidationFailed)); EXPECT_THAT(result, HasErrorMessage("Cannot change column type")); } @@ -1556,7 +1638,7 @@ TEST_F(UpdateSchemaTest, UpdateColumnIncompatibleTypesFails) { update->AddColumn("id", int32()); update->UpdateColumn("id", string()); - auto result = update->Apply(); + auto result = update->Validate(); EXPECT_THAT(result, IsError(ErrorKind::kValidationFailed)); EXPECT_THAT(result, HasErrorMessage("Cannot change column type")); } @@ -1567,7 +1649,7 @@ TEST_F(UpdateSchemaTest, RenameAndUpdateColumnInSameTransaction) { update->UpdateColumn("old_name", int64()); update->RenameColumn("old_name", "new_name"); - auto result = update->Apply(); + auto result = update->Validate(); EXPECT_THAT(result, IsError(ErrorKind::kValidationFailed)); EXPECT_THAT(result, HasErrorMessage("Cannot rename missing column")); } @@ -1579,7 +1661,7 @@ TEST_F(UpdateSchemaTest, UpdateColumnDecimalPrecisionWidening) { update->AddColumn("price", decimal_10_2); update->UpdateColumn("price", decimal_20_2); - ICEBERG_UNWRAP_OR_FAIL(auto result, update->Apply()); + ICEBERG_UNWRAP_OR_FAIL(auto result, update->Validate()); ICEBERG_UNWRAP_OR_FAIL(auto field_opt, result.schema->FindFieldByName("price")); ASSERT_TRUE(field_opt.has_value()); @@ -1593,7 +1675,7 @@ TEST_F(UpdateSchemaTest, UpdateColumnDecimalDifferentScaleFails) { update->AddColumn("price", decimal_10_2); update->UpdateColumn("price", decimal_10_3); - auto result = update->Apply(); + auto result = update->Validate(); EXPECT_THAT(result, IsError(ErrorKind::kValidationFailed)); EXPECT_THAT(result, HasErrorMessage("Cannot change column type")); } @@ -1605,7 +1687,7 @@ TEST_F(UpdateSchemaTest, UpdateColumnDecimalPrecisionNarrowingFails) { update->AddColumn("price", decimal_20_2); update->UpdateColumn("price", decimal_10_2); - auto result = update->Apply(); + auto result = update->Validate(); EXPECT_THAT(result, IsError(ErrorKind::kValidationFailed)); EXPECT_THAT(result, HasErrorMessage("Cannot change column type")); } @@ -1616,7 +1698,7 @@ TEST_F(UpdateSchemaTest, UpdateTypePreservesOtherMetadata) { update->UpdateColumn("value", int64()); - ICEBERG_UNWRAP_OR_FAIL(auto result, update->Apply()); + ICEBERG_UNWRAP_OR_FAIL(auto result, update->Validate()); ICEBERG_UNWRAP_OR_FAIL(auto field_opt, result.schema->FindFieldByName("value")); ASSERT_TRUE(field_opt.has_value()); @@ -1633,7 +1715,7 @@ TEST_F(UpdateSchemaTest, UpdateDocPreservesOtherMetadata) { update->UpdateColumnDoc("id", "new doc"); - ICEBERG_UNWRAP_OR_FAIL(auto result, update->Apply()); + ICEBERG_UNWRAP_OR_FAIL(auto result, update->Validate()); ICEBERG_UNWRAP_OR_FAIL(auto field_opt, result.schema->FindFieldByName("id")); ASSERT_TRUE(field_opt.has_value()); @@ -1653,7 +1735,7 @@ TEST_F(UpdateSchemaTest, RenameDeleteConflict) { ICEBERG_UNWRAP_OR_FAIL(auto update, reloaded->NewUpdateSchema()); update->RenameColumn("col", "new_name").DeleteColumn("col"); - auto result = update->Apply(); + auto result = update->Validate(); EXPECT_THAT(result, IsError(ErrorKind::kValidationFailed)); EXPECT_THAT(result, HasErrorMessage("Cannot delete a column that has updates")); } @@ -1667,7 +1749,7 @@ TEST_F(UpdateSchemaTest, DeleteRenameConflict) { ICEBERG_UNWRAP_OR_FAIL(auto update, reloaded->NewUpdateSchema()); update->DeleteColumn("col").RenameColumn("col", "new_name"); - auto result = update->Apply(); + auto result = update->Validate(); EXPECT_THAT(result, IsError(ErrorKind::kValidationFailed)); EXPECT_THAT(result, HasErrorMessage("Cannot rename a column that will be deleted")); } @@ -1678,7 +1760,7 @@ TEST_F(UpdateSchemaTest, CaseInsensitiveAddThenUpdate) { .AddColumn("Foo", int32(), "A column with uppercase name") .UpdateColumn("foo", int64()); - ICEBERG_UNWRAP_OR_FAIL(auto result, update->Apply()); + ICEBERG_UNWRAP_OR_FAIL(auto result, update->Validate()); ICEBERG_UNWRAP_OR_FAIL(auto field_opt, result.schema->FindFieldByName("Foo", false)); ASSERT_TRUE(field_opt.has_value()); @@ -1691,7 +1773,7 @@ TEST_F(UpdateSchemaTest, CaseInsensitiveAddThenUpdateDoc) { .AddColumn("Foo", int32(), "original doc") .UpdateColumnDoc("foo", "updated doc"); - ICEBERG_UNWRAP_OR_FAIL(auto result, update->Apply()); + ICEBERG_UNWRAP_OR_FAIL(auto result, update->Validate()); ICEBERG_UNWRAP_OR_FAIL(auto field_opt, result.schema->FindFieldByName("Foo", false)); ASSERT_TRUE(field_opt.has_value()); @@ -1705,7 +1787,7 @@ TEST_F(UpdateSchemaTest, CaseInsensitiveAddThenMakeOptional) { .AddRequiredColumn("Foo", int32(), "required column") .MakeColumnOptional("foo"); - ICEBERG_UNWRAP_OR_FAIL(auto result, update->Apply()); + ICEBERG_UNWRAP_OR_FAIL(auto result, update->Validate()); ICEBERG_UNWRAP_OR_FAIL(auto field_opt, result.schema->FindFieldByName("Foo", false)); ASSERT_TRUE(field_opt.has_value()); @@ -1719,7 +1801,7 @@ TEST_F(UpdateSchemaTest, CaseInsensitiveAddThenRequire) { .AddColumn("Foo", int32(), "optional column") .RequireColumn("foo"); // Require using lowercase - ICEBERG_UNWRAP_OR_FAIL(auto result, update->Apply()); + ICEBERG_UNWRAP_OR_FAIL(auto result, update->Validate()); ICEBERG_UNWRAP_OR_FAIL(auto field_opt, result.schema->FindFieldByName("Foo", false)); ASSERT_TRUE(field_opt.has_value()); @@ -1791,7 +1873,7 @@ TEST_F(UpdateSchemaTest, MixedChanges) { .RequireColumn("data") .AddRequiredColumn("locations", "description", string(), "Location description"); - ICEBERG_UNWRAP_OR_FAIL(auto result, update->Apply()); + ICEBERG_UNWRAP_OR_FAIL(auto result, update->Validate()); ICEBERG_UNWRAP_OR_FAIL(auto id_opt, result.schema->FindFieldByName("id")); ASSERT_TRUE(id_opt.has_value()); @@ -1913,7 +1995,7 @@ TEST_F(UpdateSchemaTest, TestMultipleMoves) { update->MoveFirst("w").MoveFirst("z").MoveAfter("y", "w").MoveBefore("w", "x"); - ICEBERG_UNWRAP_OR_FAIL(auto result, update->Apply()); + ICEBERG_UNWRAP_OR_FAIL(auto result, update->Validate()); ASSERT_TRUE(result.schema != nullptr); const auto& fields = result.schema->fields(); @@ -1935,7 +2017,7 @@ TEST_F(UpdateSchemaTest, TestMoveTopLevelColumnFirst) { ICEBERG_UNWRAP_OR_FAIL(auto update, table_->NewUpdateSchema()); update->MoveFirst("y"); - ICEBERG_UNWRAP_OR_FAIL(auto result, update->Apply()); + ICEBERG_UNWRAP_OR_FAIL(auto result, update->Validate()); ASSERT_TRUE(result.schema != nullptr); const auto& fields = result.schema->fields(); @@ -1946,7 +2028,7 @@ TEST_F(UpdateSchemaTest, TestMoveTopLevelColumnBeforeFirst) { ICEBERG_UNWRAP_OR_FAIL(auto update, table_->NewUpdateSchema()); update->MoveBefore("y", "x"); - ICEBERG_UNWRAP_OR_FAIL(auto result, update->Apply()); + ICEBERG_UNWRAP_OR_FAIL(auto result, update->Validate()); ASSERT_TRUE(result.schema != nullptr); const auto& fields = result.schema->fields(); @@ -1965,7 +2047,7 @@ TEST_F(UpdateSchemaTest, TestMoveTopLevelColumnAfterLast) { ICEBERG_UNWRAP_OR_FAIL(auto update, table_->NewUpdateSchema()); update->MoveAfter("x", "z"); - ICEBERG_UNWRAP_OR_FAIL(auto result, update->Apply()); + ICEBERG_UNWRAP_OR_FAIL(auto result, update->Validate()); ASSERT_TRUE(result.schema != nullptr); const auto& fields = result.schema->fields(); @@ -1981,7 +2063,7 @@ TEST_F(UpdateSchemaTest, TestMoveTopLevelColumnAfter) { ICEBERG_UNWRAP_OR_FAIL(auto update, reloaded->NewUpdateSchema()); update->MoveAfter("w", "x"); - ICEBERG_UNWRAP_OR_FAIL(auto result, update->Apply()); + ICEBERG_UNWRAP_OR_FAIL(auto result, update->Validate()); ASSERT_TRUE(result.schema != nullptr); const auto& fields = result.schema->fields(); @@ -2005,7 +2087,7 @@ TEST_F(UpdateSchemaTest, TestMoveTopLevelColumnBefore) { ICEBERG_UNWRAP_OR_FAIL(auto update, reloaded->NewUpdateSchema()); update->MoveBefore("w", "z"); - ICEBERG_UNWRAP_OR_FAIL(auto result, update->Apply()); + ICEBERG_UNWRAP_OR_FAIL(auto result, update->Validate()); ASSERT_TRUE(result.schema != nullptr); const auto& fields = result.schema->fields(); @@ -2031,7 +2113,7 @@ TEST_F(UpdateSchemaTest, TestMoveNestedFieldFirst) { ICEBERG_UNWRAP_OR_FAIL(auto update, reloaded->NewUpdateSchema()); update->MoveFirst("s.b"); - ICEBERG_UNWRAP_OR_FAIL(auto result, update->Apply()); + ICEBERG_UNWRAP_OR_FAIL(auto result, update->Validate()); ASSERT_TRUE(result.schema != nullptr); ICEBERG_UNWRAP_OR_FAIL(auto s_opt, result.schema->FindFieldByName("s")); @@ -2053,7 +2135,7 @@ TEST_F(UpdateSchemaTest, TestMoveNestedFieldBeforeFirst) { ICEBERG_UNWRAP_OR_FAIL(auto update, reloaded->NewUpdateSchema()); update->MoveBefore("s.b", "s.a"); - ICEBERG_UNWRAP_OR_FAIL(auto result, update->Apply()); + ICEBERG_UNWRAP_OR_FAIL(auto result, update->Validate()); ASSERT_TRUE(result.schema != nullptr); ICEBERG_UNWRAP_OR_FAIL(auto s_opt, result.schema->FindFieldByName("s")); @@ -2083,7 +2165,7 @@ TEST_F(UpdateSchemaTest, TestMoveNestedFieldAfterLast) { ICEBERG_UNWRAP_OR_FAIL(auto update, reloaded->NewUpdateSchema()); update->MoveAfter("s.a", "s.b"); - ICEBERG_UNWRAP_OR_FAIL(auto result, update->Apply()); + ICEBERG_UNWRAP_OR_FAIL(auto result, update->Validate()); ASSERT_TRUE(result.schema != nullptr); ICEBERG_UNWRAP_OR_FAIL(auto s_opt, result.schema->FindFieldByName("s")); @@ -2106,7 +2188,7 @@ TEST_F(UpdateSchemaTest, TestMoveNestedFieldAfter) { ICEBERG_UNWRAP_OR_FAIL(auto update, reloaded->NewUpdateSchema()); update->MoveAfter("s.c", "s.a"); - ICEBERG_UNWRAP_OR_FAIL(auto result, update->Apply()); + ICEBERG_UNWRAP_OR_FAIL(auto result, update->Validate()); ASSERT_TRUE(result.schema != nullptr); ICEBERG_UNWRAP_OR_FAIL(auto s_opt, result.schema->FindFieldByName("s")); @@ -2137,7 +2219,7 @@ TEST_F(UpdateSchemaTest, TestMoveNestedFieldBefore) { ICEBERG_UNWRAP_OR_FAIL(auto update, reloaded->NewUpdateSchema()); update->MoveBefore("s.c", "s.b"); - ICEBERG_UNWRAP_OR_FAIL(auto result, update->Apply()); + ICEBERG_UNWRAP_OR_FAIL(auto result, update->Validate()); ASSERT_TRUE(result.schema != nullptr); ICEBERG_UNWRAP_OR_FAIL(auto s_opt, result.schema->FindFieldByName("s")); @@ -2169,7 +2251,7 @@ TEST_F(UpdateSchemaTest, TestMoveListElementField) { ICEBERG_UNWRAP_OR_FAIL(auto update, reloaded->NewUpdateSchema()); update->MoveAfter("list.a", "list.b"); - ICEBERG_UNWRAP_OR_FAIL(auto result, update->Apply()); + ICEBERG_UNWRAP_OR_FAIL(auto result, update->Validate()); ASSERT_TRUE(result.schema != nullptr); ICEBERG_UNWRAP_OR_FAIL(auto list_opt, result.schema->FindFieldByName("list")); @@ -2205,7 +2287,7 @@ TEST_F(UpdateSchemaTest, TestMoveMapValueStructField) { ICEBERG_UNWRAP_OR_FAIL(auto update, reloaded->NewUpdateSchema()); update->MoveAfter("locations.lat", "locations.long"); - ICEBERG_UNWRAP_OR_FAIL(auto result, update->Apply()); + ICEBERG_UNWRAP_OR_FAIL(auto result, update->Validate()); ASSERT_TRUE(result.schema != nullptr); ICEBERG_UNWRAP_OR_FAIL(auto locs_opt, result.schema->FindFieldByName("locations")); @@ -2230,7 +2312,7 @@ TEST_F(UpdateSchemaTest, TestMoveAddedTopLevelColumn) { ICEBERG_UNWRAP_OR_FAIL(auto update, table_->NewUpdateSchema()); update->AddColumn("ts", timestamp_tz()).MoveAfter("ts", "x"); - ICEBERG_UNWRAP_OR_FAIL(auto result, update->Apply()); + ICEBERG_UNWRAP_OR_FAIL(auto result, update->Validate()); ASSERT_TRUE(result.schema != nullptr); const auto& fields = result.schema->fields(); @@ -2252,7 +2334,7 @@ TEST_F(UpdateSchemaTest, TestMoveAddedTopLevelColumnAfterAddedColumn) { .MoveAfter("ts", "x") .MoveAfter("count", "ts"); - ICEBERG_UNWRAP_OR_FAIL(auto result, update->Apply()); + ICEBERG_UNWRAP_OR_FAIL(auto result, update->Validate()); ASSERT_TRUE(result.schema != nullptr); const auto& fields = result.schema->fields(); @@ -2284,7 +2366,7 @@ TEST_F(UpdateSchemaTest, TestMoveAddedNestedStructField) { update->AddColumn("preferences", "ts", timestamp_tz()) .MoveBefore("preferences.ts", "preferences.feature1"); - ICEBERG_UNWRAP_OR_FAIL(auto result, update->Apply()); + ICEBERG_UNWRAP_OR_FAIL(auto result, update->Validate()); ASSERT_TRUE(result.schema != nullptr); ICEBERG_UNWRAP_OR_FAIL(auto prefs_opt, result.schema->FindFieldByName("preferences")); @@ -2310,7 +2392,7 @@ TEST_F(UpdateSchemaTest, TestMoveAddedNestedStructFieldBeforeAddedColumn) { .MoveBefore("preferences.ts", "preferences.feature1") .MoveBefore("preferences.size", "preferences.ts"); - ICEBERG_UNWRAP_OR_FAIL(auto result, update->Apply()); + ICEBERG_UNWRAP_OR_FAIL(auto result, update->Validate()); ASSERT_TRUE(result.schema != nullptr); ICEBERG_UNWRAP_OR_FAIL(auto prefs_opt, result.schema->FindFieldByName("preferences")); @@ -2325,13 +2407,13 @@ TEST_F(UpdateSchemaTest, TestMoveAddedNestedStructFieldBeforeAddedColumn) { TEST_F(UpdateSchemaTest, TestMoveSelfReferenceFails) { ICEBERG_UNWRAP_OR_FAIL(auto update1, table_->NewUpdateSchema()); update1->MoveBefore("x", "x"); - auto result1 = update1->Apply(); + auto result1 = update1->Validate(); EXPECT_THAT(result1, IsError(ErrorKind::kValidationFailed)); EXPECT_THAT(result1, HasErrorMessage("Cannot move x before itself")); ICEBERG_UNWRAP_OR_FAIL(auto update2, table_->NewUpdateSchema()); update2->MoveAfter("x", "x"); - auto result2 = update2->Apply(); + auto result2 = update2->Validate(); EXPECT_THAT(result2, IsError(ErrorKind::kValidationFailed)); EXPECT_THAT(result2, HasErrorMessage("Cannot move x after itself")); } @@ -2339,19 +2421,19 @@ TEST_F(UpdateSchemaTest, TestMoveSelfReferenceFails) { TEST_F(UpdateSchemaTest, TestMoveMissingColumnFails) { ICEBERG_UNWRAP_OR_FAIL(auto update1, table_->NewUpdateSchema()); update1->MoveFirst("items"); - auto result1 = update1->Apply(); + auto result1 = update1->Validate(); EXPECT_THAT(result1, IsError(ErrorKind::kValidationFailed)); EXPECT_THAT(result1, HasErrorMessage("Cannot move missing column: items")); ICEBERG_UNWRAP_OR_FAIL(auto update2, table_->NewUpdateSchema()); update2->MoveBefore("items", "x"); - auto result2 = update2->Apply(); + auto result2 = update2->Validate(); EXPECT_THAT(result2, IsError(ErrorKind::kValidationFailed)); EXPECT_THAT(result2, HasErrorMessage("Cannot move missing column: items")); ICEBERG_UNWRAP_OR_FAIL(auto update3, table_->NewUpdateSchema()); update3->MoveAfter("items", "y"); - auto result3 = update3->Apply(); + auto result3 = update3->Validate(); EXPECT_THAT(result3, IsError(ErrorKind::kValidationFailed)); EXPECT_THAT(result3, HasErrorMessage("Cannot move missing column: items")); } @@ -2359,7 +2441,7 @@ TEST_F(UpdateSchemaTest, TestMoveMissingColumnFails) { TEST_F(UpdateSchemaTest, TestMoveBeforeAddFails) { ICEBERG_UNWRAP_OR_FAIL(auto update, table_->NewUpdateSchema()); update->MoveBefore("ts", "x").AddColumn("ts", timestamp_tz()); - auto result = update->Apply(); + auto result = update->Validate(); EXPECT_THAT(result, IsError(ErrorKind::kValidationFailed)); EXPECT_THAT(result, HasErrorMessage("Cannot move missing column: ts")); } @@ -2367,13 +2449,13 @@ TEST_F(UpdateSchemaTest, TestMoveBeforeAddFails) { TEST_F(UpdateSchemaTest, TestMoveMissingReferenceColumnFails) { ICEBERG_UNWRAP_OR_FAIL(auto update1, table_->NewUpdateSchema()); update1->MoveBefore("x", "items"); - auto result1 = update1->Apply(); + auto result1 = update1->Validate(); EXPECT_THAT(result1, IsError(ErrorKind::kValidationFailed)); EXPECT_THAT(result1, HasErrorMessage("Cannot move x before missing column: items")); ICEBERG_UNWRAP_OR_FAIL(auto update2, table_->NewUpdateSchema()); update2->MoveAfter("y", "items"); - auto result2 = update2->Apply(); + auto result2 = update2->Validate(); EXPECT_THAT(result2, IsError(ErrorKind::kValidationFailed)); EXPECT_THAT(result2, HasErrorMessage("Cannot move y after missing column: items")); } @@ -2389,7 +2471,7 @@ TEST_F(UpdateSchemaTest, TestMovePrimitiveMapKeyFails) { ICEBERG_UNWRAP_OR_FAIL(auto reloaded, catalog_->LoadTable(table_ident_)); ICEBERG_UNWRAP_OR_FAIL(auto update, reloaded->NewUpdateSchema()); update->MoveBefore("properties.key", "properties.value"); - auto result = update->Apply(); + auto result = update->Validate(); EXPECT_THAT(result, IsError(ErrorKind::kValidationFailed)); EXPECT_THAT(result, HasErrorMessage("Cannot move fields in non-struct type")); } @@ -2405,7 +2487,7 @@ TEST_F(UpdateSchemaTest, TestMovePrimitiveMapValueFails) { ICEBERG_UNWRAP_OR_FAIL(auto reloaded, catalog_->LoadTable(table_ident_)); ICEBERG_UNWRAP_OR_FAIL(auto update, reloaded->NewUpdateSchema()); update->MoveBefore("properties.value", "properties.key"); - auto result = update->Apply(); + auto result = update->Validate(); EXPECT_THAT(result, IsError(ErrorKind::kValidationFailed)); EXPECT_THAT(result, HasErrorMessage("Cannot move fields in non-struct type")); } @@ -2421,7 +2503,7 @@ TEST_F(UpdateSchemaTest, TestMovePrimitiveListElementFails) { ICEBERG_UNWRAP_OR_FAIL(auto reloaded, catalog_->LoadTable(table_ident_)); ICEBERG_UNWRAP_OR_FAIL(auto update, reloaded->NewUpdateSchema()); update->MoveBefore("doubles.element", "doubles"); - auto result = update->Apply(); + auto result = update->Validate(); EXPECT_THAT(result, IsError(ErrorKind::kValidationFailed)); EXPECT_THAT(result, HasErrorMessage("Cannot move fields in non-struct type")); } @@ -2436,7 +2518,7 @@ TEST_F(UpdateSchemaTest, TestMoveTopLevelBetweenStructsFails) { ICEBERG_UNWRAP_OR_FAIL(auto reloaded, catalog_->LoadTable(table_ident_)); ICEBERG_UNWRAP_OR_FAIL(auto update, reloaded->NewUpdateSchema()); update->MoveBefore("x", "preferences.feature1"); - auto result = update->Apply(); + auto result = update->Validate(); EXPECT_THAT(result, IsError(ErrorKind::kValidationFailed)); EXPECT_THAT(result, HasErrorMessage("Cannot move field x to a different struct")); } @@ -2454,7 +2536,7 @@ TEST_F(UpdateSchemaTest, TestMoveBetweenStructsFails) { ICEBERG_UNWRAP_OR_FAIL(auto reloaded, catalog_->LoadTable(table_ident_)); ICEBERG_UNWRAP_OR_FAIL(auto update, reloaded->NewUpdateSchema()); update->MoveBefore("points.a", "preferences.feature1"); - auto result = update->Apply(); + auto result = update->Validate(); EXPECT_THAT(result, IsError(ErrorKind::kValidationFailed)); EXPECT_THAT(result, HasErrorMessage("Cannot move field points.a to a different struct")); @@ -2467,7 +2549,7 @@ TEST_F(UpdateSchemaTest, TestMoveTopDeletedColumnAfterAnotherColumn) { .AddRequiredColumn("z", int32()) .MoveAfter("z", "y"); - ICEBERG_UNWRAP_OR_FAIL(auto result, update->Apply()); + ICEBERG_UNWRAP_OR_FAIL(auto result, update->Validate()); ASSERT_TRUE(result.schema != nullptr); const auto& fields = result.schema->fields(); @@ -2489,7 +2571,7 @@ TEST_F(UpdateSchemaTest, TestMoveTopDeletedColumnBeforeAnotherColumn) { .AddRequiredColumn("z", int32()) .MoveBefore("z", "x"); - ICEBERG_UNWRAP_OR_FAIL(auto result, update->Apply()); + ICEBERG_UNWRAP_OR_FAIL(auto result, update->Validate()); ASSERT_TRUE(result.schema != nullptr); const auto& fields = result.schema->fields(); @@ -2511,7 +2593,7 @@ TEST_F(UpdateSchemaTest, TestMoveTopDeletedColumnToFirst) { .AddRequiredColumn("z", int32()) .MoveFirst("z"); - ICEBERG_UNWRAP_OR_FAIL(auto result, update->Apply()); + ICEBERG_UNWRAP_OR_FAIL(auto result, update->Validate()); ASSERT_TRUE(result.schema != nullptr); const auto& fields = result.schema->fields(); @@ -2533,7 +2615,7 @@ TEST_F(UpdateSchemaTest, TestMoveDeletedNestedStructFieldAfterAnotherColumn) { .AddRequiredColumn("preferences", "feature1", boolean()) .MoveAfter("preferences.feature1", "preferences.feature2"); - ICEBERG_UNWRAP_OR_FAIL(auto result, update->Apply()); + ICEBERG_UNWRAP_OR_FAIL(auto result, update->Validate()); ASSERT_TRUE(result.schema != nullptr); ICEBERG_UNWRAP_OR_FAIL(auto prefs_opt, result.schema->FindFieldByName("preferences")); @@ -2559,7 +2641,7 @@ TEST_F(UpdateSchemaTest, TestMoveDeletedNestedStructFieldBeforeAnotherColumn) { .AddColumn("preferences", "feature2", boolean()) .MoveBefore("preferences.feature2", "preferences.feature1"); - ICEBERG_UNWRAP_OR_FAIL(auto result, update->Apply()); + ICEBERG_UNWRAP_OR_FAIL(auto result, update->Validate()); ASSERT_TRUE(result.schema != nullptr); ICEBERG_UNWRAP_OR_FAIL(auto prefs_opt, result.schema->FindFieldByName("preferences")); @@ -2585,7 +2667,7 @@ TEST_F(UpdateSchemaTest, TestMoveDeletedNestedStructFieldToFirst) { .AddColumn("preferences", "feature2", boolean()) .MoveFirst("preferences.feature2"); - ICEBERG_UNWRAP_OR_FAIL(auto result, update->Apply()); + ICEBERG_UNWRAP_OR_FAIL(auto result, update->Validate()); ASSERT_TRUE(result.schema != nullptr); ICEBERG_UNWRAP_OR_FAIL(auto prefs_opt, result.schema->FindFieldByName("preferences")); @@ -2600,7 +2682,7 @@ TEST_F(UpdateSchemaTest, TestCaseInsensitiveAddTopLevelAndMove) { ICEBERG_UNWRAP_OR_FAIL(auto update, table_->NewUpdateSchema()); update->CaseSensitive(false).AddColumn("TS", timestamp_tz()).MoveAfter("ts", "X"); - ICEBERG_UNWRAP_OR_FAIL(auto result, update->Apply()); + ICEBERG_UNWRAP_OR_FAIL(auto result, update->Validate()); ASSERT_TRUE(result.schema != nullptr); const auto& fields = result.schema->fields(); @@ -2628,7 +2710,7 @@ TEST_F(UpdateSchemaTest, TestCaseInsensitiveAddNestedAndMove) { .AddColumn("Preferences", "TS", timestamp_tz()) .MoveBefore("preferences.ts", "PREFERENCES.Feature1"); - ICEBERG_UNWRAP_OR_FAIL(auto result, update->Apply()); + ICEBERG_UNWRAP_OR_FAIL(auto result, update->Validate()); ASSERT_TRUE(result.schema != nullptr); ICEBERG_UNWRAP_OR_FAIL(auto prefs_opt, result.schema->FindFieldByName("Preferences")); @@ -2647,7 +2729,7 @@ TEST_F(UpdateSchemaTest, TestCaseInsensitiveMoveAfterNewlyAddedField) { .MoveAfter("ts", "X") .MoveAfter("count", "TS"); - ICEBERG_UNWRAP_OR_FAIL(auto result, update->Apply()); + ICEBERG_UNWRAP_OR_FAIL(auto result, update->Validate()); ASSERT_TRUE(result.schema != nullptr); const auto& fields = result.schema->fields(); diff --git a/src/iceberg/test/update_sort_order_test.cc b/src/iceberg/test/update_sort_order_test.cc index 4d83d381d..30f5d3c9a 100644 --- a/src/iceberg/test/update_sort_order_test.cc +++ b/src/iceberg/test/update_sort_order_test.cc @@ -41,7 +41,7 @@ class UpdateSortOrderTest : public UpdateTestBase { // Helper function to apply update and verify the resulting sort order void ApplyAndExpectSortOrder(UpdateSortOrder* update, std::vector expected_fields) { - ICEBERG_UNWRAP_OR_FAIL(auto result, update->Apply()); + ICEBERG_UNWRAP_OR_FAIL(auto result, update->Validate()); ICEBERG_UNWRAP_OR_FAIL( auto expected_sort_order, SortOrder::Make(result->order_id(), std::move(expected_fields))); @@ -51,7 +51,7 @@ class UpdateSortOrderTest : public UpdateTestBase { TEST_F(UpdateSortOrderTest, EmptySortOrder) { ICEBERG_UNWRAP_OR_FAIL(auto update, table_->NewUpdateSortOrder()); - ICEBERG_UNWRAP_OR_FAIL(auto result, update->Apply()); + ICEBERG_UNWRAP_OR_FAIL(auto result, update->Validate()); // Should succeed with an unsorted order EXPECT_TRUE(result->fields().empty()); } @@ -144,7 +144,7 @@ TEST_F(UpdateSortOrderTest, AddSortFieldNullTerm) { update->AddSortField(nullptr, SortDirection::kAscending, NullOrder::kFirst); - auto result = update->Apply(); + auto result = update->Validate(); EXPECT_THAT(result, IsError(ErrorKind::kValidationFailed)); EXPECT_THAT(result, HasErrorMessage("Term cannot be null")); } @@ -158,7 +158,7 @@ TEST_F(UpdateSortOrderTest, AddSortFieldInvalidTransform) { update->AddSortField(std::move(term), SortDirection::kAscending, NullOrder::kFirst); - auto result = update->Apply(); + auto result = update->Validate(); EXPECT_THAT(result, IsError(ErrorKind::kValidationFailed)); EXPECT_THAT(result, HasErrorMessage("not a valid input type")); } @@ -171,7 +171,7 @@ TEST_F(UpdateSortOrderTest, AddSortFieldNonExistentField) { update->AddSortField(std::move(term), SortDirection::kAscending, NullOrder::kFirst); - auto result = update->Apply(); + auto result = update->Validate(); EXPECT_THAT(result, IsError(ErrorKind::kValidationFailed)); EXPECT_THAT(result, HasErrorMessage("Cannot find")); } @@ -185,7 +185,7 @@ TEST_F(UpdateSortOrderTest, CaseSensitiveTrue) { update->CaseSensitive(true).AddSortField(std::move(term), SortDirection::kAscending, NullOrder::kFirst); - auto result = update->Apply(); + auto result = update->Validate(); // Should fail because schema has "x" (lowercase) EXPECT_THAT(result, IsError(ErrorKind::kValidationFailed)); } diff --git a/src/iceberg/test/update_statistics_test.cc b/src/iceberg/test/update_statistics_test.cc index d6721e5bd..3010fb9fe 100644 --- a/src/iceberg/test/update_statistics_test.cc +++ b/src/iceberg/test/update_statistics_test.cc @@ -31,6 +31,7 @@ #include "iceberg/test/matchers.h" #include "iceberg/test/mock_catalog.h" #include "iceberg/test/update_test_base.h" +#include "iceberg/transaction.h" namespace iceberg { @@ -99,7 +100,7 @@ class UpdateStatisticsRetryTest : public UpdateStatisticsTest { TEST_F(UpdateStatisticsTest, EmptyUpdate) { ICEBERG_UNWRAP_OR_FAIL(auto update, table_->NewUpdateStatistics()); - ICEBERG_UNWRAP_OR_FAIL(auto result, update->Apply()); + ICEBERG_UNWRAP_OR_FAIL(auto result, update->Validate()); EXPECT_TRUE(result.to_set.empty()); EXPECT_TRUE(result.to_remove.empty()); } @@ -110,10 +111,11 @@ TEST_F(UpdateStatisticsTest, SetStatistics) { MakeStatisticsFile(1, "/warehouse/test_table/metadata/stats-1.puffin"); update->SetStatistics(stats_file); - ICEBERG_UNWRAP_OR_FAIL(auto result, update->Apply()); + ICEBERG_UNWRAP_OR_FAIL(auto result, update->Validate()); EXPECT_EQ(result.to_set.size(), 1); EXPECT_TRUE(result.to_remove.empty()); - EXPECT_EQ(FindStatistics(result.to_set, 1), stats_file); + EXPECT_THAT(FindStatistics(result.to_set, 1), + ::testing::Pointee(::testing::Eq(*stats_file))); } TEST_F(UpdateStatisticsTest, SetMultipleStatistics) { @@ -125,18 +127,20 @@ TEST_F(UpdateStatisticsTest, SetMultipleStatistics) { update->SetStatistics(stats_file_1).SetStatistics(stats_file_2); - ICEBERG_UNWRAP_OR_FAIL(auto result, update->Apply()); + ICEBERG_UNWRAP_OR_FAIL(auto result, update->Validate()); EXPECT_EQ(result.to_set.size(), 2); EXPECT_TRUE(result.to_remove.empty()); - EXPECT_EQ(FindStatistics(result.to_set, 1), stats_file_1); - EXPECT_EQ(FindStatistics(result.to_set, 2), stats_file_2); + EXPECT_THAT(FindStatistics(result.to_set, 1), + ::testing::Pointee(::testing::Eq(*stats_file_1))); + EXPECT_THAT(FindStatistics(result.to_set, 2), + ::testing::Pointee(::testing::Eq(*stats_file_2))); } TEST_F(UpdateStatisticsTest, RemoveStatistics) { ICEBERG_UNWRAP_OR_FAIL(auto update, table_->NewUpdateStatistics()); update->RemoveStatistics(1); - ICEBERG_UNWRAP_OR_FAIL(auto result, update->Apply()); + ICEBERG_UNWRAP_OR_FAIL(auto result, update->Validate()); EXPECT_TRUE(result.to_set.empty()); EXPECT_EQ(result.to_remove.size(), 1); EXPECT_THAT(result.to_remove, ::testing::Contains(1)); @@ -146,7 +150,7 @@ TEST_F(UpdateStatisticsTest, RemoveMultipleStatistics) { ICEBERG_UNWRAP_OR_FAIL(auto update, table_->NewUpdateStatistics()); update->RemoveStatistics(1).RemoveStatistics(2).RemoveStatistics(3); - ICEBERG_UNWRAP_OR_FAIL(auto result, update->Apply()); + ICEBERG_UNWRAP_OR_FAIL(auto result, update->Validate()); EXPECT_TRUE(result.to_set.empty()); EXPECT_EQ(result.to_remove.size(), 3); EXPECT_THAT(result.to_remove, ::testing::UnorderedElementsAre(1, 2, 3)); @@ -159,9 +163,10 @@ TEST_F(UpdateStatisticsTest, SetAndRemoveDifferentSnapshots) { update->SetStatistics(stats_file).RemoveStatistics(2); - ICEBERG_UNWRAP_OR_FAIL(auto result, update->Apply()); + ICEBERG_UNWRAP_OR_FAIL(auto result, update->Validate()); EXPECT_EQ(result.to_set.size(), 1); - EXPECT_EQ(FindStatistics(result.to_set, 1), stats_file); + EXPECT_THAT(FindStatistics(result.to_set, 1), + ::testing::Pointee(::testing::Eq(*stats_file))); EXPECT_EQ(result.to_remove.size(), 1); EXPECT_THAT(result.to_remove, ::testing::Contains(2)); } @@ -176,11 +181,12 @@ TEST_F(UpdateStatisticsTest, ReplaceStatistics) { // Set statistics for snapshot 1, then replace it update->SetStatistics(stats_file_1).SetStatistics(stats_file_2); - ICEBERG_UNWRAP_OR_FAIL(auto result, update->Apply()); + ICEBERG_UNWRAP_OR_FAIL(auto result, update->Validate()); EXPECT_EQ(result.to_set.size(), 1); EXPECT_TRUE(result.to_remove.empty()); // Should have the second one (replacement) - EXPECT_EQ(FindStatistics(result.to_set, 1), stats_file_2); + EXPECT_THAT(FindStatistics(result.to_set, 1), + ::testing::Pointee(::testing::Eq(*stats_file_2))); EXPECT_NE(FindStatistics(result.to_set, 1), stats_file_1); } @@ -192,7 +198,7 @@ TEST_F(UpdateStatisticsTest, SetThenRemoveSameSnapshot) { // Set statistics for snapshot 1, then remove it update->SetStatistics(stats_file).RemoveStatistics(1); - ICEBERG_UNWRAP_OR_FAIL(auto result, update->Apply()); + ICEBERG_UNWRAP_OR_FAIL(auto result, update->Validate()); EXPECT_TRUE(result.to_set.empty()); EXPECT_EQ(result.to_remove.size(), 1); EXPECT_THAT(result.to_remove, ::testing::Contains(1)); @@ -206,10 +212,11 @@ TEST_F(UpdateStatisticsTest, RemoveThenSetSameSnapshot) { // Remove statistics for snapshot 1, then set new ones update->RemoveStatistics(1).SetStatistics(stats_file); - ICEBERG_UNWRAP_OR_FAIL(auto result, update->Apply()); + ICEBERG_UNWRAP_OR_FAIL(auto result, update->Validate()); EXPECT_EQ(result.to_set.size(), 1); EXPECT_TRUE(result.to_remove.empty()); - EXPECT_EQ(FindStatistics(result.to_set, 1), stats_file); + EXPECT_THAT(FindStatistics(result.to_set, 1), + ::testing::Pointee(::testing::Eq(*stats_file))); } TEST_F(UpdateStatisticsTest, SetNullStatistics) { @@ -217,7 +224,7 @@ TEST_F(UpdateStatisticsTest, SetNullStatistics) { update->SetStatistics(nullptr); - auto result = update->Apply(); + auto result = update->Validate(); EXPECT_THAT(result, IsError(ErrorKind::kValidationFailed)); EXPECT_THAT(result, HasErrorMessage("Statistics file cannot be null")); } @@ -281,4 +288,29 @@ TEST_F(UpdateStatisticsRetryTest, StandaloneCommitRetriesAfterConflict) { EXPECT_EQ(load_table_count_, 1); } +TEST_F(UpdateStatisticsTest, FreezeCopiesInputsAndPreviewResults) { + FailCommits(2); + ICEBERG_UNWRAP_OR_FAIL(auto snapshot, table_->current_snapshot()); + const auto snapshot_id = snapshot->snapshot_id; + ICEBERG_UNWRAP_OR_FAIL(auto txn, table_->NewTransaction()); + ICEBERG_UNWRAP_OR_FAIL(auto update, txn->NewUpdateStatistics()); + auto statistics = + MakeStatisticsFile(snapshot_id, table_location_ + "/metadata/frozen.puffin"); + const auto expected = *statistics; + update->SetStatistics(statistics); + ASSERT_THAT(update->Commit(), IsOk()); + statistics->path = "/changed.puffin"; + statistics->blob_metadata[0].properties["ndv"] = "999"; + ICEBERG_UNWRAP_OR_FAIL(auto preview, update->Validate()); + ASSERT_EQ(preview.to_set.size(), 1U); + preview.to_set[0].second->path = "/changed-preview.puffin"; + ASSERT_THAT(txn->Commit(), IsOk()); + auto metadata = ReloadMetadata(); + auto it = std::ranges::find_if(metadata->statistics, [&](const auto& file) { + return file->snapshot_id == snapshot_id; + }); + ASSERT_NE(it, metadata->statistics.end()); + EXPECT_EQ(**it, expected); +} + } // namespace iceberg diff --git a/src/iceberg/test/update_test_base.h b/src/iceberg/test/update_test_base.h index 232060d2e..e0ea53c3b 100644 --- a/src/iceberg/test/update_test_base.h +++ b/src/iceberg/test/update_test_base.h @@ -20,6 +20,7 @@ #pragma once #include +#include #include #include @@ -34,6 +35,7 @@ #include "iceberg/table_identifier.h" #include "iceberg/table_metadata.h" #include "iceberg/test/matchers.h" +#include "iceberg/test/mock_catalog.h" #include "iceberg/test/test_resource.h" #include "iceberg/util/uuid.h" @@ -85,6 +87,41 @@ class UpdateTestBase : public ::testing::Test { catalog_->RegisterTable(table_ident_, metadata_location)); } + // Exercise the public commit retry path against real catalog metadata. + void FailCommits(int conflicts, std::function before_attempt = {}) { + auto mock = std::make_shared<::testing::NiceMock>(); + EXPECT_CALL(*mock, LoadTable(::testing::_)) + .Times(::testing::AtLeast(conflicts)) + .WillRepeatedly([catalog = catalog_](const TableIdentifier& name) { + return catalog->LoadTable(name); + }); + EXPECT_CALL(*mock, UpdateTable(::testing::_, ::testing::_, ::testing::_)) + .Times(conflicts + 1) + .WillRepeatedly( + [catalog = catalog_, conflicts, before_attempt, + attempt = std::make_shared(0)]( + const auto& name, const auto& requirements, + const auto& updates) mutable -> Result> { + if (before_attempt) { + before_attempt(*attempt); + } + if ((*attempt)++ < conflicts) { + return CommitFailed("injected conflict"); + } + auto result = catalog->UpdateTable(name, requirements, updates); + if (!result) { + ADD_FAILURE() << result.error().message; + return ValidationFailed("Test catalog failed: {}", + result.error().message); + } + return result; + }); + ICEBERG_UNWRAP_OR_FAIL( + table_, Table::Make(table_->name(), table_->metadata(), + std::string(table_->metadata_file_location()), file_io_, mock, + table_->full_name(), table_->reporter())); + } + /// \brief Reload the table from catalog and return its metadata. std::shared_ptr ReloadMetadata() { auto result = catalog_->LoadTable(table_ident_); diff --git a/src/iceberg/transaction.cc b/src/iceberg/transaction.cc index b26232376..a067449ff 100644 --- a/src/iceberg/transaction.cc +++ b/src/iceberg/transaction.cc @@ -18,11 +18,14 @@ */ #include "iceberg/transaction.h" +#include #include #include +#include #include "iceberg/catalog.h" #include "iceberg/location_provider.h" +#include "iceberg/logging/log_macros.h" #include "iceberg/schema.h" #include "iceberg/snapshot.h" #include "iceberg/statistics_file.h" @@ -87,6 +90,7 @@ Result> TransactionContext::Make( auto ctx = std::make_shared(); ctx->kind = kind; ctx->table = std::move(table); + ctx->base_metadata_ = ctx->table->metadata(); if (kind == TransactionKind::kCreate) { ctx->metadata_builder = TableMetadataBuilder::BuildFromEmpty(); std::ignore = ctx->metadata_builder->ApplyChangesForCreate(*ctx->table->metadata()); @@ -151,21 +155,77 @@ std::string Transaction::MetadataFileLocation(std::string_view filename) const { return ctx_->MetadataFileLocation(filename); } -Status Transaction::AddUpdate(const std::shared_ptr& update) { - ICEBERG_CHECK(!committed_, "Cannot add update to a committed transaction"); - ICEBERG_CHECK(!finalized_, "Cannot add update to a finalized transaction"); - ICEBERG_CHECK(last_update_committed_, - "Cannot add update when previous update is not committed"); +Status Transaction::CheckActive() const { + ICEBERG_CHECK(!ctx_->in_progress_, "Cannot reenter a transaction operation"); + ICEBERG_CHECK( + state_ == TransactionState::kReady || state_ == TransactionState::kUpdatePending, + "Transaction is terminal"); + return {}; +} +Status Transaction::CheckReady() const { + ICEBERG_CHECK(!ctx_->in_progress_, "Cannot reenter a transaction operation"); + ICEBERG_CHECK(state_ == TransactionState::kReady, "Transaction is not ready (state {})", + static_cast(state_)); + return {}; +} + +Status Transaction::AddUpdate(const std::shared_ptr& update) { + ICEBERG_RETURN_UNEXPECTED(CheckReady()); + ICEBERG_PRECHECK(update && update->ctx_.get() == ctx_.get(), + "Update must belong to this transaction context"); + ICEBERG_CHECK(update->phase_ == PendingUpdate::Phase::kMutable, + "Update has already been used"); pending_updates_.push_back(update); - last_update_committed_ = false; + state_ = TransactionState::kUpdatePending; return {}; } Status Transaction::Apply(PendingUpdate& update) { - ICEBERG_CHECK(!committed_, "Cannot apply update to a committed transaction"); - ICEBERG_CHECK(!finalized_, "Cannot apply update to a finalized transaction"); + ICEBERG_CHECK(!ctx_->in_progress_, "Cannot reenter a transaction operation"); + ICEBERG_CHECK(state_ == TransactionState::kUpdatePending, + "Transaction has no pending operation (state {})", + static_cast(state_)); + ICEBERG_CHECK(!pending_updates_.empty() && pending_updates_.back().get() == &update, + "Update is not the current pending operation"); + ScopedTrue running(ctx_->in_progress_); + Status status; + try { + update.phase_ = PendingUpdate::Phase::kFrozen; + update.staged_ = true; + status = update.Freeze(); + if (status) { + status = ApplyRegistered(update); + } + } catch (const std::exception& e) { + status = ValidationFailed("Update Apply threw: {}", e.what()); + } catch (...) { + status = ValidationFailed("Update Apply threw an unknown exception"); + } + if (!status) { + SetTerminalState(TransactionState::kFailed); + CleanupUpdates(); + return status; + } + state_ = TransactionState::kReady; + return {}; +} + +Status Transaction::ReplayApply(PendingUpdate& update) { + ICEBERG_CHECK(state_ == TransactionState::kReady && ctx_->in_progress_, + "Replay requires an active transaction commit"); + ICEBERG_CHECK(std::ranges::any_of(pending_updates_, + [&update](const auto& registered) { + return registered.get() == &update; + }), + "Cannot replay an unregistered update"); + ICEBERG_CHECK(update.phase_ == PendingUpdate::Phase::kFrozen, + "Cannot replay this update"); + update.staged_ = true; + return ApplyRegistered(update); +} +Status Transaction::ApplyRegistered(PendingUpdate& update) { switch (update.kind()) { case PendingUpdate::Kind::kExpireSnapshots: ICEBERG_RETURN_UNEXPECTED( @@ -216,9 +276,7 @@ Status Transaction::Apply(PendingUpdate& update) { static_cast(update.kind())); } - last_update_committed_ = true; - - return {}; + return ctx_->metadata_builder->CheckErrors(); } Status Transaction::ApplyExpireSnapshots(ExpireSnapshots& update) { @@ -243,7 +301,7 @@ Status Transaction::ApplyExpireSnapshots(ExpireSnapshots& update) { } Status Transaction::ApplySetSnapshot(SetSnapshot& update) { - ICEBERG_ASSIGN_OR_RAISE(auto snapshot_id, update.Apply()); + ICEBERG_ASSIGN_OR_RAISE(auto snapshot_id, update.Validate()); ctx_->metadata_builder->SetBranchSnapshot(snapshot_id, std::string(SnapshotRef::kMainBranch)); ICEBERG_RETURN_UNEXPECTED(ctx_->metadata_builder->CheckErrors()); @@ -251,13 +309,13 @@ Status Transaction::ApplySetSnapshot(SetSnapshot& update) { } Status Transaction::ApplyUpdateLocation(UpdateLocation& update) { - ICEBERG_ASSIGN_OR_RAISE(auto location, update.Apply()); + ICEBERG_ASSIGN_OR_RAISE(auto location, update.Validate()); ctx_->metadata_builder->SetLocation(location); return {}; } Status Transaction::ApplyUpdatePartitionSpec(UpdatePartitionSpec& update) { - ICEBERG_ASSIGN_OR_RAISE(auto result, update.Apply()); + ICEBERG_ASSIGN_OR_RAISE(auto result, update.Validate()); if (result.set_as_default) { ctx_->metadata_builder->SetDefaultPartitionSpec(std::move(result.spec)); } else { @@ -268,7 +326,7 @@ Status Transaction::ApplyUpdatePartitionSpec(UpdatePartitionSpec& update) { } Status Transaction::ApplyUpdateProperties(UpdateProperties& update) { - ICEBERG_ASSIGN_OR_RAISE(auto result, update.Apply()); + ICEBERG_ASSIGN_OR_RAISE(auto result, update.Validate()); if (!result.updates.empty()) { ctx_->metadata_builder->SetProperties(std::move(result.updates)); } @@ -283,7 +341,7 @@ Status Transaction::ApplyUpdateProperties(UpdateProperties& update) { } Status Transaction::ApplyUpdateSchema(UpdateSchema& update) { - ICEBERG_ASSIGN_OR_RAISE(auto result, update.Apply()); + ICEBERG_ASSIGN_OR_RAISE(auto result, update.Validate()); ctx_->metadata_builder->SetCurrentSchema(std::move(result.schema), result.new_last_column_id); if (!result.updated_props.empty()) { @@ -312,9 +370,9 @@ Status Transaction::ApplyUpdateSnapshot(SnapshotUpdate& update) { ICEBERG_RETURN_UNEXPECTED(temp_update->CheckErrors()); if (temp_update->changes().empty()) { - // Do not commit if the metadata has not changed. for example, this may happen - // when setting the current snapshot to an ID that is already current. note that - // this check uses identity. + // Apply may already have written files. Consume this generation immediately, + // while retaining its frozen intent for a possible replay against new metadata. + update.Cleanup(); return {}; } @@ -333,7 +391,7 @@ Status Transaction::ApplyUpdateSnapshot(SnapshotUpdate& update) { } Status Transaction::ApplyUpdateSnapshotReference(UpdateSnapshotReference& update) { - ICEBERG_ASSIGN_OR_RAISE(auto result, update.Apply()); + ICEBERG_ASSIGN_OR_RAISE(auto result, update.Validate()); for (const auto& name : result.to_remove) { ctx_->metadata_builder->RemoveRef(name); } @@ -345,14 +403,14 @@ Status Transaction::ApplyUpdateSnapshotReference(UpdateSnapshotReference& update } Status Transaction::ApplyUpdateSortOrder(UpdateSortOrder& update) { - ICEBERG_ASSIGN_OR_RAISE(auto sort_order, update.Apply()); + ICEBERG_ASSIGN_OR_RAISE(auto sort_order, update.Validate()); ctx_->metadata_builder->SetDefaultSortOrder(std::move(sort_order)); ICEBERG_RETURN_UNEXPECTED(ctx_->metadata_builder->CheckErrors()); return {}; } Status Transaction::ApplyUpdateStatistics(UpdateStatistics& update) { - ICEBERG_ASSIGN_OR_RAISE(auto result, update.Apply()); + ICEBERG_ASSIGN_OR_RAISE(auto result, update.Validate()); for (auto&& [_, stat_file] : result.to_set) { ctx_->metadata_builder->SetStatistics(std::move(stat_file)); } @@ -364,7 +422,7 @@ Status Transaction::ApplyUpdateStatistics(UpdateStatistics& update) { } Status Transaction::ApplyUpdatePartitionStatistics(UpdatePartitionStatistics& update) { - ICEBERG_ASSIGN_OR_RAISE(auto result, update.Apply()); + ICEBERG_ASSIGN_OR_RAISE(auto result, update.Validate()); for (auto&& [_, partition_stat_file] : result.to_set) { ctx_->metadata_builder->SetPartitionStatistics(std::move(partition_stat_file)); } @@ -376,85 +434,170 @@ Status Transaction::ApplyUpdatePartitionStatistics(UpdatePartitionStatistics& up } Result> Transaction::Commit() { - ICEBERG_CHECK(!committed_, "Transaction already committed"); - ICEBERG_CHECK(!finalized_, "Transaction already finalized"); - ICEBERG_CHECK(last_update_committed_, - "Cannot commit transaction when previous update is not committed"); - - const auto& updates = ctx_->metadata_builder->changes(); + ICEBERG_RETURN_UNEXPECTED(CheckReady()); + ScopedTrue running(ctx_->in_progress_); Result> commit_result = ctx_->table; - if (!updates.empty()) { - const auto& props = ctx_->table->properties(); - int32_t num_retries = - CanRetry() ? static_cast(props.Get(TableProperties::kCommitNumRetries)) - : 0; - int32_t min_wait_ms = props.Get(TableProperties::kCommitMinRetryWaitMs); - int32_t max_wait_ms = props.Get(TableProperties::kCommitMaxRetryWaitMs); - int32_t total_timeout_ms = props.Get(TableProperties::kCommitTotalRetryTimeMs); - - bool is_first_attempt = true; - ScopedTrue committing(committing_); + bool catalog_state_unknown = false; + try { + ConfigureExpirationCleanup(); + auto builder_status = ctx_->metadata_builder->CheckErrors(); + if (!builder_status) { + commit_result = std::unexpected(builder_status.error()); + } else { + const auto& props = ctx_->table->properties(); + const int32_t num_retries = + CanRetry() ? static_cast(props.Get(TableProperties::kCommitNumRetries)) + : 0; + bool is_first_attempt = true; + std::optional replay_error; + commit_result = + MakeCommitRetryRunner(num_retries, + props.Get(TableProperties::kCommitMinRetryWaitMs), + props.Get(TableProperties::kCommitMaxRetryWaitMs), + props.Get(TableProperties::kCommitTotalRetryTimeMs)) + .Run([this, &is_first_attempt, &replay_error, + &catalog_state_unknown]() -> Result> { + auto result = + CommitOnce(is_first_attempt, replay_error, catalog_state_unknown); + is_first_attempt = false; + // A replay failure is not another catalog conflict. Stop the + // runner, then restore the original Apply error for the + // caller below. + if (replay_error) { + return ValidationFailed("Transaction replay failed"); + } + return result; + }); + if (replay_error) { + commit_result = std::unexpected(std::move(*replay_error)); + } + } + } catch (const std::exception& e) { + // CommitOnce catches exceptions at the catalog boundary separately. + commit_result = ValidationFailed("Transaction preparation threw: {}", e.what()); + } catch (...) { commit_result = - MakeCommitRetryRunner(num_retries, min_wait_ms, max_wait_ms, total_timeout_ms) - .Run([this, &is_first_attempt]() -> Result> { - auto result = CommitOnce(is_first_attempt); - is_first_attempt = false; - return result; - }); + ValidationFailed("Transaction preparation threw an unknown exception"); } - Result finalize_result = - commit_result.has_value() - ? Result(commit_result.value()->metadata().get()) - : std::unexpected(commit_result.error()); - FinalizeUpdates(finalize_result); - - ICEBERG_RETURN_UNEXPECTED(commit_result); + if (!commit_result) { + if (catalog_state_unknown) { + SetTerminalState(TransactionState::kCommitStateUnknown); + } else { + SetTerminalState(TransactionState::kFailed); + CleanupUpdates(); + } + return commit_result; + } - // Mark as committed and update table reference - committed_ = true; ctx_->table = std::move(commit_result.value()); - + SetTerminalState(TransactionState::kCommitted); + FinalizeUpdates(*ctx_->table->metadata()); return ctx_->table; } -void Transaction::FinalizeUpdates(const Result& commit_result) { - if (finalized_) { - return; +Status Transaction::Abort() { + ICEBERG_CHECK(!ctx_->in_progress_, "Cannot reenter a transaction operation"); + if (state_ == TransactionState::kAborted) { + return {}; } - finalized_ = true; + ICEBERG_CHECK(state_ == TransactionState::kReady || + state_ == TransactionState::kUpdatePending || + state_ == TransactionState::kFailed, + "Cannot abort a committed or unknown transaction"); + ScopedTrue running(ctx_->in_progress_); + SetTerminalState(TransactionState::kAborted); + CleanupUpdates(); + return {}; +} + +void Transaction::SetTerminalState(TransactionState state) { + state_ = state; + // Publish every marker before invoking even the first user callback. for (const auto& update : pending_updates_) { - std::ignore = update->Finalize(commit_result); + update->phase_ = PendingUpdate::Phase::kTerminal; } } -Result> Transaction::CommitOnce(bool is_first_attempt) { - std::vector> requirements; +void Transaction::CleanupUpdates() noexcept { + for (const auto& update : pending_updates_) { + update->Cleanup(); + } +} - switch (ctx_->kind) { - case TransactionKind::kCreate: { - ICEBERG_ASSIGN_OR_RAISE(requirements, TableRequirements::ForCreateTable( - ctx_->metadata_builder->changes())); - } break; - case TransactionKind::kUpdate: { - if (!is_first_attempt) { - ICEBERG_RETURN_UNEXPECTED(ctx_->table->Refresh()); - } - if (ctx_->metadata_builder->base() != ctx_->table->metadata().get()) { - ctx_->metadata_builder = - TableMetadataBuilder::BuildFrom(ctx_->table->metadata().get()); - for (const auto& update : pending_updates_) { - ICEBERG_RETURN_UNEXPECTED(update->Commit()); +void Transaction::FinalizeUpdates(const TableMetadata& committed) noexcept { + for (const auto& update : pending_updates_) { + update->FinalizeOnce(committed); + } +} + +void Transaction::ConfigureExpirationCleanup() { + bool may_add_references = false; + for (const auto& update : pending_updates_ | std::views::reverse) { + if (update->kind() == PendingUpdate::Kind::kExpireSnapshots) { + internal::checked_cast(*update).skip_physical_cleanup_ = + may_add_references; + } + may_add_references |= update->MayAddFileReferences(); + } +} + +Result> Transaction::CommitOnce(bool is_first_attempt, + std::optional& replay_error, + bool& catalog_state_unknown) { + std::vector> requirements; + if (ctx_->kind == TransactionKind::kUpdate) { + if (!is_first_attempt) { + ICEBERG_RETURN_UNEXPECTED(ctx_->table->Refresh()); + } + if (!is_first_attempt || + ctx_->metadata_builder->base() != ctx_->table->metadata().get()) { + ICEBERG_CHECK(CanRetry(), + "Cannot rebase a transaction containing a non-retryable update"); + CleanupUpdates(); + ctx_->metadata_builder = + TableMetadataBuilder::BuildFrom(ctx_->table->metadata().get()); + ctx_->base_metadata_ = ctx_->table->metadata(); + for (const auto& update : pending_updates_) { + Status applied; + try { + applied = ReplayApply(*update); + } catch (const std::exception& e) { + applied = ValidationFailed("Replay Apply threw: {}", e.what()); + } catch (...) { + applied = ValidationFailed("Replay Apply threw an unknown exception"); + } + if (!applied) { + replay_error = applied.error(); + return std::unexpected(applied.error()); } } - ICEBERG_ASSIGN_OR_RAISE(requirements, TableRequirements::ForUpdateTable( - *ctx_->metadata_builder->base(), - ctx_->metadata_builder->changes())); - } break; + } + if (ctx_->metadata_builder->changes().empty()) { + return ctx_->table; + } + ICEBERG_ASSIGN_OR_RAISE(requirements, TableRequirements::ForUpdateTable( + *ctx_->metadata_builder->base(), + ctx_->metadata_builder->changes())); + } else { + ICEBERG_ASSIGN_OR_RAISE(requirements, TableRequirements::ForCreateTable( + ctx_->metadata_builder->changes())); } - return ctx_->table->catalog()->UpdateTable(ctx_->table->name(), requirements, - ctx_->metadata_builder->changes()); + // Only this boundary can turn an uncaught exception into an unknown commit. + try { + auto result = ctx_->table->catalog()->UpdateTable(ctx_->table->name(), requirements, + ctx_->metadata_builder->changes()); + catalog_state_unknown = + !result && result.error().kind == ErrorKind::kCommitStateUnknown; + return result; + } catch (const std::exception& e) { + catalog_state_unknown = true; + return CommitStateUnknown("Catalog commit threw: {}", e.what()); + } catch (...) { + catalog_state_unknown = true; + return CommitStateUnknown("Catalog commit threw an unknown exception"); + } } bool Transaction::CanRetry() const { @@ -470,6 +613,7 @@ bool Transaction::CanRetry() const { } Result> Transaction::NewUpdatePartitionSpec() { + ICEBERG_RETURN_UNEXPECTED(CheckReady()); ICEBERG_ASSIGN_OR_RAISE(std::shared_ptr update_spec, UpdatePartitionSpec::Make(ctx_)); ICEBERG_RETURN_UNEXPECTED(AddUpdate(update_spec)); @@ -477,6 +621,7 @@ Result> Transaction::NewUpdatePartitionSpec } Result> Transaction::NewUpdateProperties() { + ICEBERG_RETURN_UNEXPECTED(CheckReady()); ICEBERG_ASSIGN_OR_RAISE(std::shared_ptr update_properties, UpdateProperties::Make(ctx_)); ICEBERG_RETURN_UNEXPECTED(AddUpdate(update_properties)); @@ -484,6 +629,7 @@ Result> Transaction::NewUpdateProperties() { } Result> Transaction::NewUpdateSortOrder() { + ICEBERG_RETURN_UNEXPECTED(CheckReady()); ICEBERG_ASSIGN_OR_RAISE(std::shared_ptr update_sort_order, UpdateSortOrder::Make(ctx_)); ICEBERG_RETURN_UNEXPECTED(AddUpdate(update_sort_order)); @@ -491,6 +637,7 @@ Result> Transaction::NewUpdateSortOrder() { } Result> Transaction::NewUpdateSchema() { + ICEBERG_RETURN_UNEXPECTED(CheckReady()); ICEBERG_ASSIGN_OR_RAISE(std::shared_ptr update_schema, UpdateSchema::Make(ctx_)); ICEBERG_RETURN_UNEXPECTED(AddUpdate(update_schema)); @@ -498,6 +645,7 @@ Result> Transaction::NewUpdateSchema() { } Result> Transaction::NewExpireSnapshots() { + ICEBERG_RETURN_UNEXPECTED(CheckReady()); ICEBERG_ASSIGN_OR_RAISE(std::shared_ptr expire_snapshots, ExpireSnapshots::Make(ctx_)); ICEBERG_RETURN_UNEXPECTED(AddUpdate(expire_snapshots)); @@ -505,6 +653,7 @@ Result> Transaction::NewExpireSnapshots() { } Result> Transaction::NewUpdateLocation() { + ICEBERG_RETURN_UNEXPECTED(CheckReady()); ICEBERG_ASSIGN_OR_RAISE(std::shared_ptr update_location, UpdateLocation::Make(ctx_)); ICEBERG_RETURN_UNEXPECTED(AddUpdate(update_location)); @@ -512,6 +661,7 @@ Result> Transaction::NewUpdateLocation() { } Result> Transaction::NewSetSnapshot() { + ICEBERG_RETURN_UNEXPECTED(CheckReady()); ICEBERG_ASSIGN_OR_RAISE(std::shared_ptr set_snapshot, SetSnapshot::Make(ctx_)); ICEBERG_RETURN_UNEXPECTED(AddUpdate(set_snapshot)); @@ -519,6 +669,7 @@ Result> Transaction::NewSetSnapshot() { } Result> Transaction::NewFastAppend() { + ICEBERG_RETURN_UNEXPECTED(CheckReady()); ICEBERG_ASSIGN_OR_RAISE(std::shared_ptr fast_append, FastAppend::Make(ctx_->table->name().name, ctx_)); ICEBERG_RETURN_UNEXPECTED(AddUpdate(fast_append)); @@ -526,6 +677,7 @@ Result> Transaction::NewFastAppend() { } Result> Transaction::NewMergeAppend() { + ICEBERG_RETURN_UNEXPECTED(CheckReady()); ICEBERG_ASSIGN_OR_RAISE(std::shared_ptr merge_append, MergeAppend::Make(ctx_->table->name().name, ctx_)); ICEBERG_RETURN_UNEXPECTED(AddUpdate(merge_append)); @@ -533,6 +685,7 @@ Result> Transaction::NewMergeAppend() { } Result> Transaction::NewDeleteFiles() { + ICEBERG_RETURN_UNEXPECTED(CheckReady()); ICEBERG_ASSIGN_OR_RAISE(std::shared_ptr delete_files, DeleteFiles::Make(ctx_->table->name().name, ctx_)); ICEBERG_RETURN_UNEXPECTED(AddUpdate(delete_files)); @@ -540,6 +693,7 @@ Result> Transaction::NewDeleteFiles() { } Result> Transaction::NewRowDelta() { + ICEBERG_RETURN_UNEXPECTED(CheckReady()); ICEBERG_ASSIGN_OR_RAISE(std::shared_ptr row_delta, RowDelta::Make(ctx_->table->name().name, ctx_)); ICEBERG_RETURN_UNEXPECTED(AddUpdate(row_delta)); @@ -547,6 +701,7 @@ Result> Transaction::NewRowDelta() { } Result> Transaction::NewOverwrite() { + ICEBERG_RETURN_UNEXPECTED(CheckReady()); ICEBERG_ASSIGN_OR_RAISE(std::shared_ptr overwrite, OverwriteFiles::Make(ctx_->table->name().name, ctx_)); ICEBERG_RETURN_UNEXPECTED(AddUpdate(overwrite)); @@ -554,6 +709,7 @@ Result> Transaction::NewOverwrite() { } Result> Transaction::NewRewriteFiles() { + ICEBERG_RETURN_UNEXPECTED(CheckReady()); ICEBERG_ASSIGN_OR_RAISE(std::shared_ptr rewrite_files, RewriteFiles::Make(ctx_->table->name().name, ctx_)); ICEBERG_RETURN_UNEXPECTED(AddUpdate(rewrite_files)); @@ -561,6 +717,7 @@ Result> Transaction::NewRewriteFiles() { } Result> Transaction::NewReplacePartitions() { + ICEBERG_RETURN_UNEXPECTED(CheckReady()); ICEBERG_ASSIGN_OR_RAISE(std::shared_ptr replace_partitions, ReplacePartitions::Make(ctx_->table->name().name, ctx_)); ICEBERG_RETURN_UNEXPECTED(AddUpdate(replace_partitions)); @@ -568,6 +725,7 @@ Result> Transaction::NewReplacePartitions() { } Result> Transaction::NewUpdateStatistics() { + ICEBERG_RETURN_UNEXPECTED(CheckReady()); ICEBERG_ASSIGN_OR_RAISE(std::shared_ptr update_statistics, UpdateStatistics::Make(ctx_)); ICEBERG_RETURN_UNEXPECTED(AddUpdate(update_statistics)); @@ -576,6 +734,7 @@ Result> Transaction::NewUpdateStatistics() { Result> Transaction::NewUpdatePartitionStatistics() { + ICEBERG_RETURN_UNEXPECTED(CheckReady()); ICEBERG_ASSIGN_OR_RAISE( std::shared_ptr update_partition_statistics, UpdatePartitionStatistics::Make(ctx_)); @@ -585,6 +744,7 @@ Transaction::NewUpdatePartitionStatistics() { Result> Transaction::NewUpdateSnapshotReference() { + ICEBERG_RETURN_UNEXPECTED(CheckReady()); ICEBERG_ASSIGN_OR_RAISE(std::shared_ptr update_ref, UpdateSnapshotReference::Make(ctx_)); ICEBERG_RETURN_UNEXPECTED(AddUpdate(update_ref)); @@ -592,6 +752,7 @@ Transaction::NewUpdateSnapshotReference() { } Result> Transaction::NewSnapshotManager() { + ICEBERG_RETURN_UNEXPECTED(CheckReady()); // SnapshotManager has its own commit logic, so it is not added to the pending updates. return SnapshotManager::Make(shared_from_this()); } diff --git a/src/iceberg/transaction.h b/src/iceberg/transaction.h index 4dd2b03bb..cfe61e957 100644 --- a/src/iceberg/transaction.h +++ b/src/iceberg/transaction.h @@ -39,6 +39,16 @@ namespace iceberg { /// \brief Whether a transaction creates a new table or updates an existing one. enum class TransactionKind : uint8_t { kCreate, kUpdate }; +/// \brief Lifecycle outcomes of a transaction. Failed transactions may only be aborted. +enum class TransactionState : uint8_t { + kReady, + kUpdatePending, + kCommitted, + kFailed, + kAborted, + kCommitStateUnknown, +}; + /// \brief A transaction for performing multiple updates to a table class ICEBERG_EXPORT Transaction : public std::enable_shared_from_this { public: @@ -77,6 +87,13 @@ class ICEBERG_EXPORT Transaction : public std::enable_shared_from_this> Commit(); + /// \brief Discard staged changes and clean owned files best effort. + /// Repeated Abort succeeds without repeating cleanup. Committed and unknown + /// transactions cannot be aborted. Destructors never perform cleanup. + Status Abort(); + + TransactionState state() const { return state_; } + /// \brief Create a new UpdatePartitionSpec to update the partition spec of this table /// and commit the changes. Result> NewUpdatePartitionSpec(); @@ -146,10 +163,14 @@ class ICEBERG_EXPORT Transaction : public std::enable_shared_from_this ctx); + Status CheckReady() const; + Status CheckActive() const; Status AddUpdate(const std::shared_ptr& update); /// \brief Apply the pending changes to current table. - Status Apply(PendingUpdate& updates); + Status Apply(PendingUpdate& update); + Status ApplyRegistered(PendingUpdate& update); + Status ReplayApply(PendingUpdate& update); // Helper methods for applying different types of updates Status ApplyExpireSnapshots(ExpireSnapshots& update); @@ -165,32 +186,28 @@ class ICEBERG_EXPORT Transaction : public std::enable_shared_from_this> CommitOnce(bool is_first_attempt); + Result> CommitOnce(bool is_first_attempt, + std::optional& replay_error, + bool& catalog_state_unknown); /// \brief Whether this transaction can retry after a commit conflict. bool CanRetry() const; - /// \brief Finalize all registered updates exactly once. - void FinalizeUpdates(const Result& commit_result); + void SetTerminalState(TransactionState state); + void CleanupUpdates() noexcept; + void FinalizeUpdates(const TableMetadata& committed) noexcept; + void ConfigureExpirationCleanup(); private: friend class PendingUpdate; + friend class SnapshotManager; // Shared context owning the table, metadata builder, and kind. std::shared_ptr ctx_; // Keep track of all created pending updates. std::vector> pending_updates_; - // To make the state simple, we require updates are added and committed in order. - bool last_update_committed_ = true; - // Tracks if transaction has been committed to prevent double-commit - bool committed_ = false; - // Tracks whether registered updates have reached a terminal state. - bool finalized_ = false; - // True while Commit() is running its retry loop. Re-applying an update may fail - // with a retryable error; PendingUpdate::Commit() must not finalize the - // transaction in that window or the retry would see a finalized transaction. - bool committing_ = false; + TransactionState state_ = TransactionState::kReady; }; /// \brief Shared context between Transaction and PendingUpdate instances. @@ -213,6 +230,14 @@ class ICEBERG_EXPORT TransactionContext { // If PendingUpdate is created directly from Table, this is nullopt; // otherwise, it holds a weak pointer to the Transaction that created it. std::optional> transaction; + + private: + friend class Transaction; + friend class PendingUpdate; + + // Keep the builder's base alive when Table::Refresh replaces table metadata. + std::shared_ptr base_metadata_; + bool in_progress_ = false; }; } // namespace iceberg diff --git a/src/iceberg/update/delete_files.cc b/src/iceberg/update/delete_files.cc index e738108de..d2bbac71e 100644 --- a/src/iceberg/update/delete_files.cc +++ b/src/iceberg/update/delete_files.cc @@ -42,27 +42,32 @@ DeleteFiles::DeleteFiles(std::string table_name, std::shared_ptr& file) { + EnsureMutable(); ICEBERG_BUILDER_RETURN_IF_ERROR(DeleteDataFile(file)); return *this; } DeleteFiles& DeleteFiles::DeleteFromRowFilter(std::shared_ptr expr) { + EnsureMutable(); ICEBERG_BUILDER_RETURN_IF_ERROR(DeleteByRowFilter(std::move(expr))); return *this; } DeleteFiles& DeleteFiles::CaseSensitive(bool case_sensitive) { + EnsureMutable(); MergingSnapshotUpdate::CaseSensitive(case_sensitive); return *this; } DeleteFiles& DeleteFiles::ValidateFilesExist() { + EnsureMutable(); validate_files_to_delete_exist_ = true; return *this; } diff --git a/src/iceberg/update/expire_snapshots.cc b/src/iceberg/update/expire_snapshots.cc index 5573efa77..fbd21b16c 100644 --- a/src/iceberg/update/expire_snapshots.cc +++ b/src/iceberg/update/expire_snapshots.cc @@ -31,6 +31,7 @@ #include #include "iceberg/file_io.h" +#include "iceberg/logging/log_macros.h" #include "iceberg/manifest/manifest_entry.h" #include "iceberg/manifest/manifest_reader.h" #include "iceberg/result.h" @@ -108,24 +109,18 @@ class FileCleanupStrategy { return; } - if (!delete_func_) { - std::vector path_list(paths.begin(), paths.end()); - - TaskGroup> group(kDeleteRetryConfig); - group.Submit([this, paths = std::move(path_list)]() -> Status { - return file_io_->DeleteFiles(paths); - }); - std::ignore = std::move(group).Run(); - return; - } - TaskGroup> group(kDeleteRetryConfig); - group.SetExecutor(delete_executor_); + if (delete_func_) { + group.SetExecutor(delete_executor_); + } for (const auto& path : paths) { group.Submit([this, path]() -> Status { try { - delete_func_(path); - return {}; + if (delete_func_) { + delete_func_(path); + return {}; + } + return file_io_->DeleteFile(path); } catch (const std::exception& e) { return IOError("Delete callback failed for {}: {}", path, e.what()); } catch (...) { @@ -133,7 +128,9 @@ class FileCleanupStrategy { } }); } - std::ignore = std::move(group).Run(); + if (auto status = std::move(group).Run(); !status) { + ICEBERG_LOG_WARN("Expiration deletion failed: {}", status.error().message); + } } bool HasAnyStatisticsFiles(const TableMetadata& metadata) const { @@ -754,17 +751,20 @@ ExpireSnapshots::ExpireSnapshots(std::shared_ptr ctx) ExpireSnapshots::~ExpireSnapshots() = default; ExpireSnapshots& ExpireSnapshots::ExpireSnapshotId(int64_t snapshot_id) { + EnsureMutable(); snapshot_ids_to_expire_.push_back(snapshot_id); specified_snapshot_id_ = true; return *this; } ExpireSnapshots& ExpireSnapshots::ExpireOlderThan(int64_t timestamp_millis) { + EnsureMutable(); default_expire_older_than_ = TimePointMsFromUnixMs(timestamp_millis); return *this; } ExpireSnapshots& ExpireSnapshots::RetainLast(int num_snapshots) { + EnsureMutable(); ICEBERG_BUILDER_CHECK(num_snapshots > 0, "Number of snapshots to retain must be positive: {}", num_snapshots); @@ -774,26 +774,31 @@ ExpireSnapshots& ExpireSnapshots::RetainLast(int num_snapshots) { ExpireSnapshots& ExpireSnapshots::DeleteWith( std::function delete_func) { + EnsureMutable(); delete_func_ = std::move(delete_func); return *this; } ExpireSnapshots& ExpireSnapshots::PlanWith(Executor& executor) { + EnsureMutable(); plan_executor_ = std::ref(executor); return *this; } ExpireSnapshots& ExpireSnapshots::CleanupLevel(enum CleanupLevel level) { + EnsureMutable(); cleanup_level_ = level; return *this; } ExpireSnapshots& ExpireSnapshots::CleanExpiredMetadata(bool clean) { + EnsureMutable(); clean_expired_metadata_ = clean; return *this; } ExpireSnapshots& ExpireSnapshots::ExecuteDeleteWith(Executor& executor) { + EnsureMutable(); delete_executor_ = std::ref(executor); return *this; } @@ -908,7 +913,7 @@ Result ExpireSnapshots::ComputeRetainedRefs( return retained_refs; } -Result ExpireSnapshots::Apply() { +Result ExpireSnapshots::Validate() const { ICEBERG_RETURN_UNEXPECTED(CheckErrors()); const TableMetadata& base = this->base(); @@ -999,33 +1004,35 @@ Result ExpireSnapshots::Apply() { std::ranges::to>(); } - // Cache the result for use during Finalize() - apply_result_ = result; - return result; } -Status ExpireSnapshots::Finalize(Result commit_result) { - if (!commit_result.has_value()) { - return {}; - } +Result ExpireSnapshots::Apply() { + ICEBERG_ASSIGN_OR_RAISE(auto result, Validate()); + apply_result_ = result; + return result; +} - if (cleanup_level_ == CleanupLevel::kNone) { +Status ExpireSnapshots::Finalize(const TableMetadata& metadata_after_expiration) { + // The cached Apply result belongs to this generation only; consume it regardless + // of whether any physical cleanup happens. + auto apply_result = std::exchange(apply_result_, std::nullopt); + if (cleanup_level_ == CleanupLevel::kNone || !apply_result.has_value() || + apply_result->snapshot_ids_to_remove.empty()) { return {}; } - if (!apply_result_.has_value() || apply_result_->snapshot_ids_to_remove.empty()) { + if (skip_physical_cleanup_) { + ICEBERG_LOG_WARN( + "Skipping expiration file deletion because a later update may add file " + "references"); return {}; } - ICEBERG_PRECHECK(apply_result_->metadata_before_expiration != nullptr, + ICEBERG_PRECHECK(apply_result->metadata_before_expiration != nullptr, "Missing pre-expiration table metadata for cleanup"); - ICEBERG_PRECHECK(commit_result.value() != nullptr, - "Missing committed table metadata for cleanup"); - auto metadata_before_expiration_ptr = apply_result_->metadata_before_expiration; - const TableMetadata& metadata_before_expiration = *metadata_before_expiration_ptr; - const TableMetadata& metadata_after_expiration = *commit_result.value(); - apply_result_.reset(); + const TableMetadata& metadata_before_expiration = + *apply_result->metadata_before_expiration; // Pick incremental cleanup when the expiration is a simple linear-ancestry walk: // no explicit snapshot IDs, no removed snapshots outside main ancestry, and no diff --git a/src/iceberg/update/expire_snapshots.h b/src/iceberg/update/expire_snapshots.h index 215ad85fd..cba08672c 100644 --- a/src/iceberg/update/expire_snapshots.h +++ b/src/iceberg/update/expire_snapshots.h @@ -64,7 +64,7 @@ enum class CleanupLevel : uint8_t { /// that were deleted by snapshots that are expired will be deleted. DeleteWith() can be /// used to pass an alternative deletion method. /// -/// Apply() returns a list of the snapshots that will be removed. +/// Validate() previews the snapshots that will be removed without staging cleanup. class ICEBERG_EXPORT ExpireSnapshots : public PendingUpdate { public: static Result> Make( @@ -158,22 +158,21 @@ class ICEBERG_EXPORT ExpireSnapshots : public PendingUpdate { Kind kind() const final { return Kind::kExpireSnapshots; } bool IsRetryable() const override { return true; } - /// \brief Apply the pending changes and return the results - /// \return The results of changes - Result Apply(); - - /// \brief Finalize the expire snapshots update, cleaning up expired files. - /// - /// After a successful commit, this method deletes manifest files, manifest lists, - /// data files, and statistics files that are no longer referenced by any valid - /// snapshot. The cleanup behavior is controlled by the CleanupLevel setting. - /// - /// \param commit_result The committed table metadata when the commit succeeds, or the - /// commit error when it fails. - /// \return Status indicating success or failure - Status Finalize(Result commit_result) override; + /// \brief Validate and preview changes without staging or modifying this update. + Result Validate() const; private: + friend class Transaction; + + bool MayAddFileReferences() const override { return false; } + Status Finalize(const TableMetadata& committed) override; + Status CleanStaged() override { + apply_result_.reset(); + return {}; + } + bool skip_physical_cleanup_ = false; + Result Apply(); + explicit ExpireSnapshots(std::shared_ptr ctx); using SnapshotToRef = std::unordered_map>; diff --git a/src/iceberg/update/fast_append.cc b/src/iceberg/update/fast_append.cc index 3e21c67c1..688edd034 100644 --- a/src/iceberg/update/fast_append.cc +++ b/src/iceberg/update/fast_append.cc @@ -30,6 +30,7 @@ #include "iceberg/table_metadata.h" #include "iceberg/table_properties.h" #include "iceberg/transaction.h" +#include "iceberg/update/update_util_internal.h" #include "iceberg/util/error_collector.h" #include "iceberg/util/macros.h" @@ -47,6 +48,7 @@ FastAppend::FastAppend(std::string table_name, std::shared_ptr& file) { + EnsureMutable(); ICEBERG_BUILDER_CHECK(file != nullptr, "Invalid data file: null"); ICEBERG_BUILDER_CHECK(file->partition_spec_id.has_value(), "Data file must have partition spec ID"); @@ -65,6 +67,7 @@ FastAppend& FastAppend::AppendFile(const std::shared_ptr& file) { } FastAppend& FastAppend::AppendManifest(const ManifestFile& manifest) { + EnsureMutable(); ICEBERG_BUILDER_CHECK(!manifest.has_existing_files(), "Cannot append manifest with existing files"); ICEBERG_BUILDER_CHECK(!manifest.has_deleted_files(), @@ -79,20 +82,38 @@ FastAppend& FastAppend::AppendManifest(const ManifestFile& manifest) { append_manifests_.push_back(manifest); } else { // The manifest must be rewritten with this update's snapshot ID - ICEBERG_BUILDER_ASSIGN_OR_RETURN(auto copied_manifest, - CopyManifest(manifest, /*update_summary=*/true)); append_manifests_to_copy_.push_back(manifest); - rewritten_append_manifests_.push_back(std::move(copied_manifest)); } return *this; } +Status FastAppend::Freeze() { + std::unordered_map frozen; + added_data_files_summary_.Clear(); + for (const auto& [_, files] : new_data_files_by_spec_) { + for (const auto& file : files) { + ICEBERG_ASSIGN_OR_RAISE(auto copy, internal::CopyUpdateDataFile(*file)); + ICEBERG_PRECHECK(copy->partition_spec_id.has_value(), + "Data file must have partition spec ID"); + ICEBERG_ASSIGN_OR_RAISE(auto spec, Spec(*copy->partition_spec_id)); + frozen[*copy->partition_spec_id].insert(copy); + ICEBERG_RETURN_UNEXPECTED(added_data_files_summary_.AddedFile(*spec, *copy)); + } + } + new_data_files_by_spec_ = std::move(frozen); + return {}; +} + std::string FastAppend::operation() { return DataOperation::kAppend; } Result> FastAppend::Apply( const TableMetadata& metadata_to_update, const std::shared_ptr& snapshot) { std::vector manifests; + appended_manifests_summary_.Clear(); + for (const auto& manifest : append_manifests_) { + appended_manifests_summary_.AddedManifest(manifest); + } ICEBERG_ASSIGN_OR_RAISE(auto new_written_manifests, WriteNewManifests()); // A retry cleanup deletes copied append manifests and clears the rewritten @@ -100,7 +121,7 @@ Result> FastAppend::Apply( if (rewritten_append_manifests_.empty() && !append_manifests_to_copy_.empty()) { for (const auto& manifest : append_manifests_to_copy_) { ICEBERG_ASSIGN_OR_RAISE(auto copied_manifest, - CopyManifest(manifest, /*update_summary=*/false)); + CopyManifest(manifest, /*update_summary=*/true)); rewritten_append_manifests_.push_back(std::move(copied_manifest)); } } @@ -184,15 +205,6 @@ Status FastAppend::CleanUncommitted(const std::unordered_set& commi return {}; } -bool FastAppend::CleanupAfterCommit() const { - // Cleanup after committing is disabled for FastAppend unless append manifests - // were copied or need to be copied on retry because: - // 1.) Directly appended manifests are never rewritten - // 2.) Manifests which are written out as part of AppendFile are already cleaned - // up between commit attempts in WriteNewManifests - return !rewritten_append_manifests_.empty() || !append_manifests_to_copy_.empty(); -} - Result> FastAppend::Spec(int32_t spec_id) { return base().PartitionSpecById(spec_id); } diff --git a/src/iceberg/update/fast_append.h b/src/iceberg/update/fast_append.h index c28c61cdd..eeb2e374e 100644 --- a/src/iceberg/update/fast_append.h +++ b/src/iceberg/update/fast_append.h @@ -73,6 +73,8 @@ class ICEBERG_EXPORT FastAppend : public SnapshotUpdate { /// \return This FastAppend for method chaining. FastAppend& AppendManifest(const ManifestFile& manifest); + protected: + Status Freeze() override; std::string operation() override; Result> Apply( @@ -81,7 +83,6 @@ class ICEBERG_EXPORT FastAppend : public SnapshotUpdate { std::unordered_map Summary() override; void SetSummaryProperty(const std::string& property, const std::string& value) override; Status CleanUncommitted(const std::unordered_set& committed) override; - bool CleanupAfterCommit() const override; private: explicit FastAppend(std::string table_name, std::shared_ptr ctx); diff --git a/src/iceberg/update/merge_append.cc b/src/iceberg/update/merge_append.cc index fd6cc35ae..85ec48590 100644 --- a/src/iceberg/update/merge_append.cc +++ b/src/iceberg/update/merge_append.cc @@ -41,11 +41,13 @@ MergeAppend::MergeAppend(std::string table_name, std::shared_ptr& file) { + EnsureMutable(); ICEBERG_BUILDER_RETURN_IF_ERROR(AddDataFile(file)); return *this; } MergeAppend& MergeAppend::AppendManifest(const ManifestFile& manifest) { + EnsureMutable(); ICEBERG_BUILDER_CHECK(!manifest.has_existing_files(), "Cannot append manifest with existing files"); ICEBERG_BUILDER_CHECK(!manifest.has_deleted_files(), diff --git a/src/iceberg/update/merging_snapshot_update.cc b/src/iceberg/update/merging_snapshot_update.cc index 7ce577076..54994c28f 100644 --- a/src/iceberg/update/merging_snapshot_update.cc +++ b/src/iceberg/update/merging_snapshot_update.cc @@ -51,6 +51,7 @@ #include "iceberg/table_metadata.h" #include "iceberg/table_properties.h" #include "iceberg/transaction.h" +#include "iceberg/update/update_util_internal.h" #include "iceberg/util/content_file_util.h" #include "iceberg/util/macros.h" #include "iceberg/util/snapshot_util_internal.h" @@ -476,7 +477,7 @@ Status MergingSnapshotUpdate::AddDataFile(std::shared_ptr file) { // Suppress first_row_id in the staged copy. The commit assigns row IDs for newly // added files and must not mutate the caller-owned file object. - auto staged_file = std::make_shared(*file); + ICEBERG_ASSIGN_OR_RAISE(auto staged_file, internal::CopyUpdateDataFile(*file)); staged_file->first_row_id = std::nullopt; auto& data_files = new_data_files_by_spec_[spec_id]; @@ -552,7 +553,7 @@ Status MergingSnapshotUpdate::AddDeleteFile(std::shared_ptr file, } ICEBERG_RETURN_UNEXPECTED(base().PartitionSpecById(file->partition_spec_id.value())); - auto staged_file = std::make_shared(*file); + ICEBERG_ASSIGN_OR_RAISE(auto staged_file, internal::CopyUpdateDataFile(*file)); has_new_delete_files_ = true; PendingDeleteFile pending_file{.file = std::move(staged_file), .data_sequence_number = std::move(data_sequence_number)}; @@ -572,7 +573,7 @@ Status MergingSnapshotUpdate::DeleteDataFile(std::shared_ptr file) { if (!file) { return InvalidArgument("Cannot delete a null data file"); } - auto staged_file = std::make_shared(*file); + ICEBERG_ASSIGN_OR_RAISE(auto staged_file, internal::CopyUpdateDataFile(*file)); return data_filter_manager_->DeleteFile(std::move(staged_file)); } @@ -580,7 +581,7 @@ Status MergingSnapshotUpdate::DeleteDeleteFile(std::shared_ptr file) { if (!file) { return InvalidArgument("Cannot delete a null delete file"); } - auto staged_file = std::make_shared(*file); + ICEBERG_ASSIGN_OR_RAISE(auto staged_file, internal::CopyUpdateDataFile(*file)); return delete_filter_manager_->DeleteFile(std::move(staged_file)); } @@ -589,6 +590,7 @@ Status MergingSnapshotUpdate::DeleteByPath(std::string_view path) { } Status MergingSnapshotUpdate::DeleteByRowFilter(std::shared_ptr expr) { + ICEBERG_ASSIGN_OR_RAISE(expr, internal::CopyUpdateExpression(expr)); // If a delete file matches the row filter, it can also be removed because the rows // it references will also be deleted. Both filter managers receive the expression. delete_expression_ = expr; @@ -665,9 +667,7 @@ Status MergingSnapshotUpdate::AddManifest(ManifestFile manifest) { appended_manifests_summary_.AddedManifest(manifest); append_manifests_.push_back(std::move(manifest)); } else { - ICEBERG_ASSIGN_OR_RAISE(auto copied, CopyManifest(manifest, /*update_summary=*/true)); append_manifests_to_copy_.push_back(std::move(manifest)); - rewritten_append_manifests_.push_back(std::move(copied)); } return {}; } @@ -821,9 +821,10 @@ MergingSnapshotUpdate::MergeDVs() { } ICEBERG_ASSIGN_OR_RAISE(auto location_provider, ctx_->NewLocationProvider()); - auto output_path = location_provider->NewDataLocation( - std::format("merged-dvs-{}-{}.puffin", SnapshotId(), ++dv_merge_attempt_)); + auto output_path = location_provider->NewDataLocation(std::format( + "merged-dvs-{}-{}-{}.puffin", SnapshotId(), commit_uuid(), ++dv_merge_attempt_)); + RegisterStagedFile(output_path, /*data_file=*/true); auto merged_files = DVUtil::MergeAndWriteDVs(groups, output_path, ctx_->table->io()); if (!merged_files) { std::ignore = DeleteFile(output_path); @@ -924,6 +925,11 @@ Result> MergingSnapshotUpdate::WriteNewDeleteManifests Result> MergingSnapshotUpdate::Apply( const TableMetadata& metadata_to_update, const std::shared_ptr& snapshot) { + appended_manifests_summary_.Clear(); + for (const auto& manifest : append_manifests_) { + appended_manifests_summary_.AddedManifest(manifest); + } + ICEBERG_RETURN_UNEXPECTED(ManagersReady()); // Re-validate buffered delete files against the current format version. A format @@ -989,7 +995,7 @@ Result> MergingSnapshotUpdate::Apply( if (rewritten_append_manifests_.empty() && !append_manifests_to_copy_.empty()) { for (const auto& manifest : append_manifests_to_copy_) { ICEBERG_ASSIGN_OR_RAISE(auto copied, CopyManifest(manifest, - /*update_summary=*/false)); + /*update_summary=*/true)); rewritten_append_manifests_.push_back(std::move(copied)); } } diff --git a/src/iceberg/update/overwrite_files.cc b/src/iceberg/update/overwrite_files.cc index cba6e028c..e35a725bf 100644 --- a/src/iceberg/update/overwrite_files.cc +++ b/src/iceberg/update/overwrite_files.cc @@ -34,6 +34,7 @@ #include "iceberg/table_metadata.h" #include "iceberg/transaction.h" #include "iceberg/type.h" +#include "iceberg/update/update_util_internal.h" #include "iceberg/util/error_collector.h" #include "iceberg/util/macros.h" @@ -54,6 +55,7 @@ OverwriteFiles::OverwriteFiles(std::string table_name, OverwriteFiles::~OverwriteFiles() = default; OverwriteFiles& OverwriteFiles::AddFile(const std::shared_ptr& file) { + EnsureMutable(); ICEBERG_BUILDER_CHECK(file != nullptr, "Invalid data file: null"); ICEBERG_BUILDER_CHECK(file->content == DataFile::Content::kData, "Invalid data file to add: {} has delete-file content", @@ -63,17 +65,20 @@ OverwriteFiles& OverwriteFiles::AddFile(const std::shared_ptr& file) { } OverwriteFiles& OverwriteFiles::DeleteFile(const std::shared_ptr& file) { + EnsureMutable(); ICEBERG_BUILDER_CHECK(file != nullptr, "Invalid data file: null"); ICEBERG_BUILDER_CHECK(file->content == DataFile::Content::kData, "Invalid data file to delete: {} has delete-file content", file->file_path); - deleted_data_files_.insert(file); + ICEBERG_BUILDER_ASSIGN_OR_RETURN(auto frozen, internal::CopyUpdateDataFile(*file)); + deleted_data_files_.insert(std::move(frozen)); ICEBERG_BUILDER_RETURN_IF_ERROR(DeleteDataFile(file)); return *this; } OverwriteFiles& OverwriteFiles::DeleteFiles(const DataFileSet& data_files_to_delete, const DeleteFileSet& delete_files_to_delete) { + EnsureMutable(); // Both sets use DataFile pointers, so validate content before forwarding to the // data-file and delete-file removal paths. for (const auto& file : data_files_to_delete) { @@ -81,7 +86,8 @@ OverwriteFiles& OverwriteFiles::DeleteFiles(const DataFileSet& data_files_to_del ICEBERG_BUILDER_CHECK(file->content == DataFile::Content::kData, "Invalid data file to delete: {} has delete-file content", file->file_path); - deleted_data_files_.insert(file); + ICEBERG_BUILDER_ASSIGN_OR_RETURN(auto frozen, internal::CopyUpdateDataFile(*file)); + deleted_data_files_.insert(std::move(frozen)); ICEBERG_BUILDER_RETURN_IF_ERROR(DeleteDataFile(file)); } for (const auto& file : delete_files_to_delete) { @@ -95,12 +101,14 @@ OverwriteFiles& OverwriteFiles::DeleteFiles(const DataFileSet& data_files_to_del } OverwriteFiles& OverwriteFiles::OverwriteByRowFilter(std::shared_ptr expr) { + EnsureMutable(); ICEBERG_BUILDER_CHECK(expr != nullptr, "Invalid row filter expression: null"); ICEBERG_BUILDER_RETURN_IF_ERROR(DeleteByRowFilter(std::move(expr))); return *this; } OverwriteFiles& OverwriteFiles::ValidateFromSnapshot(int64_t snapshot_id) { + EnsureMutable(); ICEBERG_BUILDER_CHECK(snapshot_id >= 0, "Invalid snapshot id: {}", snapshot_id); starting_snapshot_id_ = snapshot_id; return *this; @@ -108,29 +116,35 @@ OverwriteFiles& OverwriteFiles::ValidateFromSnapshot(int64_t snapshot_id) { OverwriteFiles& OverwriteFiles::ConflictDetectionFilter( std::shared_ptr expr) { + EnsureMutable(); ICEBERG_BUILDER_CHECK(expr != nullptr, "Invalid conflict detection filter: null"); - conflict_detection_filter_ = std::move(expr); + ICEBERG_BUILDER_ASSIGN_OR_RETURN(conflict_detection_filter_, + internal::CopyUpdateExpression(expr)); return *this; } OverwriteFiles& OverwriteFiles::CaseSensitive(bool case_sensitive) { + EnsureMutable(); MergingSnapshotUpdate::CaseSensitive(case_sensitive); return *this; } OverwriteFiles& OverwriteFiles::ValidateNoConflictingData() { + EnsureMutable(); validate_new_data_files_ = true; FailMissingDeletePaths(); return *this; } OverwriteFiles& OverwriteFiles::ValidateNoConflictingDeletes() { + EnsureMutable(); validate_new_deletes_ = true; FailMissingDeletePaths(); return *this; } OverwriteFiles& OverwriteFiles::ValidateAddedFilesMatchOverwriteFilter() { + EnsureMutable(); validate_added_files_match_overwrite_filter_ = true; return *this; } diff --git a/src/iceberg/update/pending_update.cc b/src/iceberg/update/pending_update.cc index 71e148b62..b0eb60f0a 100644 --- a/src/iceberg/update/pending_update.cc +++ b/src/iceberg/update/pending_update.cc @@ -19,6 +19,8 @@ #include "iceberg/update/pending_update.h" +#include "iceberg/exception.h" +#include "iceberg/logging/log_macros.h" #include "iceberg/result.h" #include "iceberg/table.h" #include "iceberg/transaction.h" @@ -27,19 +29,18 @@ namespace iceberg { namespace { -class ScopedTransactionBinding { - public: - ScopedTransactionBinding(TransactionContext& ctx, - const std::shared_ptr& txn) - : ctx_(ctx) { - ctx_.transaction = txn; +template +void BestEffort(std::string_view name, Hook&& hook) noexcept { + try { + if (auto result = hook(); !result) { + ICEBERG_LOG_WARN("Update {} failed: {}", name, result.error().message); + } + } catch (const std::exception& e) { + ICEBERG_LOG_WARN("Update {} threw: {}", name, e.what()); + } catch (...) { + ICEBERG_LOG_WARN("Update {} threw an unknown exception", name); } - - ~ScopedTransactionBinding() { ctx_.transaction.reset(); } - - private: - TransactionContext& ctx_; -}; +} } // namespace @@ -48,51 +49,67 @@ PendingUpdate::PendingUpdate(std::shared_ptr ctx) PendingUpdate::~PendingUpdate() = default; -Status PendingUpdate::Commit() { - if (!ctx_->transaction) { - // Standalone update path: no transaction is attached to the context, so create a - // temporary one for this Commit() call. - ICEBERG_ASSIGN_OR_RAISE(auto txn, Transaction::Make(ctx_)); - auto self = weak_from_this().lock(); - ICEBERG_PRECHECK(self != nullptr, "PendingUpdate must be owned by std::shared_ptr"); - ICEBERG_RETURN_UNEXPECTED(txn->AddUpdate(self)); - // Keep Transaction::Make(ctx_) detached, but expose this live transaction while - // Commit() runs so an internal retry can reapply through update->Commit(). - ScopedTransactionBinding binding(*ctx_, txn); - - auto apply_status = txn->Apply(*this); - if (!apply_status.has_value()) { - txn->FinalizeUpdates(std::unexpected(apply_status.error())); - return apply_status; - } - - auto commit_result = txn->Commit(); - ICEBERG_RETURN_UNEXPECTED(commit_result); - return {}; +void PendingUpdate::EnsureMutable() const { + ICEBERG_CHECK_OR_DIE(phase_ == Phase::kMutable, + "Update configuration is frozen or terminal"); + ICEBERG_CHECK_OR_DIE(!ctx_->in_progress_, + "Cannot mutate an update during an operation"); + if (ctx_->transaction) { + auto txn = ctx_->transaction->lock(); + ICEBERG_CHECK_OR_DIE(txn != nullptr, "Transaction has been destroyed"); + ICEBERG_CHECK_OR_DIE(txn->state() == TransactionState::kReady || + txn->state() == TransactionState::kUpdatePending, + "Transaction is terminal"); } +} - auto txn = ctx_->transaction->lock(); - if (!txn) { - return CommitFailed("Transaction has been destroyed"); - } +Status PendingUpdate::CheckCommitAllowed() const { + ICEBERG_CHECK(phase_ != Phase::kTerminal, "Update is terminal"); + ICEBERG_CHECK(!ctx_->in_progress_, "Cannot reenter an update or transaction operation"); + return {}; +} - auto apply_status = txn->Apply(*this); - if (!apply_status.has_value() && !txn->committing_) { - // Finalize eagerly so a failed update cleans up its staged files even if the - // caller never commits the transaction. When the transaction is mid-commit, - // leave finalization to Transaction::Commit(): the failure may be retryable - // (e.g. RetryableValidationFailed from a stale sequence number), and - // finalizing here would destroy staged state before the retry runs. - txn->FinalizeUpdates(std::unexpected(apply_status.error())); +Status PendingUpdate::Commit() { + ICEBERG_RETURN_UNEXPECTED(CheckCommitAllowed()); + if (ctx_->transaction) { + auto txn = ctx_->transaction->lock(); + ICEBERG_CHECK(txn != nullptr, "Transaction has been destroyed"); + return txn->Apply(*this); } - return apply_status; + + auto self = weak_from_this().lock(); + ICEBERG_PRECHECK(self != nullptr, "PendingUpdate must be owned by std::shared_ptr"); + ICEBERG_ASSIGN_OR_RAISE(auto txn, Transaction::Make(ctx_)); + ICEBERG_RETURN_UNEXPECTED(txn->AddUpdate(self)); + ICEBERG_RETURN_UNEXPECTED(txn->Apply(*this)); + ICEBERG_RETURN_UNEXPECTED(txn->Commit()); + return {}; } -Status PendingUpdate::Finalize( - [[maybe_unused]] Result commit_result) { +Status PendingUpdate::Finalize([[maybe_unused]] const TableMetadata& committed) { return {}; } +void PendingUpdate::Cleanup() noexcept { + if (!staged_) { + return; + } + // Consume the generation before any callback can throw or reenter. + staged_ = false; + BestEffort("staging cleanup", [this] { return CleanStaged(); }); +} + +void PendingUpdate::FinalizeOnce(const TableMetadata& committed) noexcept { + // A generation already consumed by Cleanup (a staged snapshot that produced no + // metadata change) has nothing to finalize or report. + if (!staged_) { + return; + } + staged_ = false; + BestEffort("finalization", [this, &committed] { return Finalize(committed); }); + BestEffort("reporting", [this] { return ReportCommitted(); }); +} + const TableMetadata& PendingUpdate::base() const { return ctx_->current(); } } // namespace iceberg diff --git a/src/iceberg/update/pending_update.h b/src/iceberg/update/pending_update.h index dc9e705aa..becc33625 100644 --- a/src/iceberg/update/pending_update.h +++ b/src/iceberg/update/pending_update.h @@ -31,16 +31,34 @@ namespace iceberg { +enum class TransactionState : uint8_t; + /// \brief Base class for all kinds of table metadata updates. /// /// Any created `PendingUpdate` instance is tracked by the `Transaction` instance /// and commit is also delegated to the `Transaction` instance. /// +/// Lifecycle: an update is configured through its fluent mutators, then committed +/// exactly once. `Commit()` freezes the configuration before the first Apply so a +/// commit retry replays the same intent against refreshed metadata. After that point +/// the update is either frozen (explicit transaction, awaiting `Transaction::Commit()`) +/// or terminal (standalone commit finished, or the owning transaction reached a +/// terminal state). +/// /// \note Implementations are expected to use builder pattern and errors -/// should be handled by the ErrorCollector base class. -class ICEBERG_EXPORT PendingUpdate : public ErrorCollector, +/// should be handled by the ErrorCollector base class. Configuration errors are +/// collected and surfaced by `Commit()`. Lifecycle misuse is different: calling a +/// mutator on a frozen or terminal update, or while an operation is in progress, +/// throws `IcebergError` rather than adding a collected error, so the frozen replay +/// intent can never be poisoned. +class ICEBERG_EXPORT PendingUpdate : protected ErrorCollector, public std::enable_shared_from_this { public: + using ErrorCollector::CheckErrors; + using ErrorCollector::error_count; + using ErrorCollector::errors; + using ErrorCollector::has_errors; + enum class Kind : uint8_t { kExpireSnapshots, kSetSnapshot, @@ -68,23 +86,14 @@ class ICEBERG_EXPORT PendingUpdate : public ErrorCollector, /// - CommitFailed: if it cannot be committed due to conflicts. /// - CommitStateUnknown: unknown status, no cleanup should be done. /// \note The update must be owned by a `std::shared_ptr` before calling Commit(). + /// Commit freezes its configuration. Later mutators throw IcebergError; an Apply + /// failure is terminal and cannot be corrected within the same transaction. virtual Status Commit(); - /// \brief Finalize the pending update. - /// - /// This method is called after the update is committed. - /// Implementations should override this method to clean up any resources. - /// - /// \param commit_result The committed table metadata when the commit succeeds, or the - /// commit error when it fails. - /// \return Status indicating success or failure - virtual Status Finalize(Result commit_result); - - // Non-copyable, movable PendingUpdate(const PendingUpdate&) = delete; PendingUpdate& operator=(const PendingUpdate&) = delete; - PendingUpdate(PendingUpdate&&) noexcept = default; - PendingUpdate& operator=(PendingUpdate&&) noexcept = default; + PendingUpdate(PendingUpdate&&) = delete; + PendingUpdate& operator=(PendingUpdate&&) = delete; ~PendingUpdate() override; @@ -93,7 +102,50 @@ class ICEBERG_EXPORT PendingUpdate : public ErrorCollector, const TableMetadata& base() const; + /// \brief Reject lifecycle misuse without poisoning the builder's collected errors. + /// Fluent mutators throw IcebergError when frozen, terminal, or reentered. + void EnsureMutable() const; + Status CheckCommitAllowed() const; + + /// \brief Capture mutable input values before the first Apply. + virtual Status Freeze() { return {}; } + /// \brief Discard this generation's staging, retaining frozen operation intent. + virtual Status CleanStaged() { return {}; } + /// \brief Complete a known successful commit. Called only by Transaction with the + /// final committed table metadata. + virtual Status Finalize(const TableMetadata& committed); + virtual Status ReportCommitted() { return {}; } + virtual bool MayAddFileReferences() const { return true; } + + using ErrorCollector::AddError; + using ErrorCollector::ClearErrors; + std::shared_ptr ctx_; + + private: + friend class Transaction; + // ErrorCollector's explicit-object helpers access the derived builder's errors. + friend class ErrorCollector; + + void Cleanup() noexcept; + void FinalizeOnce(const TableMetadata& committed) noexcept; + + /// \brief Lifecycle of this update, advanced only by Transaction. + /// + /// kMutable: configurable, not yet applied. + /// kFrozen: public configuration is closed. Entered before Freeze and the first + /// Apply; reaching this phase does not imply either step has succeeded. + /// kTerminal: the owning transaction reached a terminal state (an update that was + /// never applied can also become terminal when its transaction is aborted). + enum class Phase : uint8_t { kMutable, kFrozen, kTerminal }; + Phase phase_ = Phase::kMutable; + + // True while the current Apply generation has not been consumed by Cleanup or + // FinalizeOnce. Set before Freeze/Apply, even before any output exists, so a + // failure can clean partially staged resources. Cleared before cleanup or + // finalization hooks run to make them idempotent per generation and to skip + // finalization/reporting for a snapshot generation already cleaned as a no-op. + bool staged_ = false; }; } // namespace iceberg diff --git a/src/iceberg/update/replace_partitions.cc b/src/iceberg/update/replace_partitions.cc index 4487751bf..22dc2909e 100644 --- a/src/iceberg/update/replace_partitions.cc +++ b/src/iceberg/update/replace_partitions.cc @@ -47,6 +47,7 @@ ReplacePartitions::ReplacePartitions(std::string table_name, } ReplacePartitions& ReplacePartitions::AddFile(const std::shared_ptr& file) { + EnsureMutable(); ICEBERG_BUILDER_CHECK(file != nullptr, "Invalid data file: null"); ICEBERG_BUILDER_CHECK(file->partition_spec_id.has_value(), "Data file must have partition spec ID"); @@ -59,21 +60,25 @@ ReplacePartitions& ReplacePartitions::AddFile(const std::shared_ptr& f } ReplacePartitions& ReplacePartitions::ValidateAppendOnly() { + EnsureMutable(); FailAnyDelete(); return *this; } ReplacePartitions& ReplacePartitions::ValidateFromSnapshot(int64_t snapshot_id) { + EnsureMutable(); starting_snapshot_id_ = snapshot_id; return *this; } ReplacePartitions& ReplacePartitions::ValidateNoConflictingData() { + EnsureMutable(); validate_conflicting_data_ = true; return *this; } ReplacePartitions& ReplacePartitions::ValidateNoConflictingDeletes() { + EnsureMutable(); validate_conflicting_deletes_ = true; return *this; } diff --git a/src/iceberg/update/rewrite_files.cc b/src/iceberg/update/rewrite_files.cc index 3c743c01d..a3c9a2395 100644 --- a/src/iceberg/update/rewrite_files.cc +++ b/src/iceberg/update/rewrite_files.cc @@ -28,6 +28,7 @@ #include "iceberg/snapshot.h" #include "iceberg/table.h" #include "iceberg/transaction.h" +#include "iceberg/update/update_util_internal.h" #include "iceberg/util/macros.h" namespace iceberg { @@ -48,17 +49,20 @@ Result> RewriteFiles::Make( } RewriteFiles& RewriteFiles::DeleteDataFile(const std::shared_ptr& data_file) { + EnsureMutable(); ICEBERG_BUILDER_CHECK(data_file != nullptr, "Invalid data file: null"); ICEBERG_BUILDER_CHECK(data_file->content == DataFile::Content::kData, "Invalid data file to delete: {} has delete-file content", data_file->file_path); ICEBERG_BUILDER_RETURN_IF_ERROR(MergingSnapshotUpdate::DeleteDataFile(data_file)); - replaced_data_files_.insert(std::make_shared(*data_file)); + ICEBERG_BUILDER_ASSIGN_OR_RETURN(auto frozen, internal::CopyUpdateDataFile(*data_file)); + replaced_data_files_.insert(std::move(frozen)); return *this; } RewriteFiles& RewriteFiles::DeleteDeleteFile( const std::shared_ptr& delete_file) { + EnsureMutable(); ICEBERG_BUILDER_CHECK(delete_file != nullptr, "Invalid delete file: null"); ICEBERG_BUILDER_CHECK(delete_file->content != DataFile::Content::kData, "Invalid delete file to delete: {} has data-file content", @@ -68,6 +72,7 @@ RewriteFiles& RewriteFiles::DeleteDeleteFile( } RewriteFiles& RewriteFiles::AddDataFile(const std::shared_ptr& file) { + EnsureMutable(); ICEBERG_BUILDER_CHECK(file != nullptr, "Invalid data file: null"); ICEBERG_BUILDER_CHECK(file->content == DataFile::Content::kData, "Invalid data file to add: {} has delete-file content", @@ -77,6 +82,7 @@ RewriteFiles& RewriteFiles::AddDataFile(const std::shared_ptr& file) { } RewriteFiles& RewriteFiles::AddDeleteFile(const std::shared_ptr& delete_file) { + EnsureMutable(); ICEBERG_BUILDER_CHECK(delete_file != nullptr, "Invalid delete file: null"); ICEBERG_BUILDER_CHECK(delete_file->content != DataFile::Content::kData, "Invalid delete file to add: {} has data-file content", @@ -87,6 +93,7 @@ RewriteFiles& RewriteFiles::AddDeleteFile(const std::shared_ptr& delet RewriteFiles& RewriteFiles::AddDeleteFile(const std::shared_ptr& delete_file, int64_t data_sequence_number) { + EnsureMutable(); ICEBERG_BUILDER_CHECK(delete_file != nullptr, "Invalid delete file: null"); ICEBERG_BUILDER_CHECK(delete_file->content != DataFile::Content::kData, "Invalid delete file to add: {} has data-file content", @@ -97,6 +104,7 @@ RewriteFiles& RewriteFiles::AddDeleteFile(const std::shared_ptr& delet } RewriteFiles& RewriteFiles::SetDataSequenceNumber(int64_t sequence_number) { + EnsureMutable(); SetNewDataFilesDataSequenceNumber(sequence_number); return *this; } @@ -104,6 +112,7 @@ RewriteFiles& RewriteFiles::SetDataSequenceNumber(int64_t sequence_number) { RewriteFiles& RewriteFiles::RewriteDataFiles( const std::vector>& files_to_delete, const std::vector>& files_to_add, int64_t sequence_number) { + EnsureMutable(); SetNewDataFilesDataSequenceNumber(sequence_number); Rewrite(files_to_delete, {}, files_to_add, {}); return *this; @@ -114,6 +123,7 @@ RewriteFiles& RewriteFiles::Rewrite( const std::vector>& delete_files_to_replace, const std::vector>& data_files_to_add, const std::vector>& delete_files_to_add) { + EnsureMutable(); for (const auto& data_file : data_files_to_replace) { DeleteDataFile(data_file); } @@ -134,6 +144,7 @@ RewriteFiles& RewriteFiles::Rewrite( } RewriteFiles& RewriteFiles::ValidateFromSnapshot(int64_t snapshot_id) { + EnsureMutable(); starting_snapshot_id_ = snapshot_id; return *this; } diff --git a/src/iceberg/update/row_delta.cc b/src/iceberg/update/row_delta.cc index f239ea495..64fe583e9 100644 --- a/src/iceberg/update/row_delta.cc +++ b/src/iceberg/update/row_delta.cc @@ -31,6 +31,7 @@ #include "iceberg/table.h" #include "iceberg/table_metadata.h" #include "iceberg/transaction.h" +#include "iceberg/update/update_util_internal.h" #include "iceberg/util/error_collector.h" #include "iceberg/util/formatter_internal.h" #include "iceberg/util/macros.h" @@ -50,38 +51,46 @@ RowDelta::RowDelta(std::string table_name, std::shared_ptr c conflict_detection_filter_(Expressions::AlwaysTrue()) {} RowDelta& RowDelta::AddRows(const std::shared_ptr& inserts) { + EnsureMutable(); ICEBERG_BUILDER_RETURN_IF_ERROR(AddDataFile(inserts)); return *this; } RowDelta& RowDelta::AddDeletes(const std::shared_ptr& deletes) { + EnsureMutable(); ICEBERG_BUILDER_RETURN_IF_ERROR(AddDeleteFile(deletes)); return *this; } RowDelta& RowDelta::RemoveRows(const std::shared_ptr& file) { + EnsureMutable(); ICEBERG_BUILDER_RETURN_IF_ERROR(DeleteDataFile(file)); - removed_data_files_.insert(file); + ICEBERG_BUILDER_ASSIGN_OR_RETURN(auto frozen, internal::CopyUpdateDataFile(*file)); + removed_data_files_.insert(std::move(frozen)); return *this; } RowDelta& RowDelta::RemoveDeletes(const std::shared_ptr& deletes) { + EnsureMutable(); ICEBERG_BUILDER_RETURN_IF_ERROR(DeleteDeleteFile(deletes)); return *this; } RowDelta& RowDelta::ValidateFromSnapshot(int64_t snapshot_id) { + EnsureMutable(); starting_snapshot_id_ = snapshot_id; return *this; } RowDelta& RowDelta::CaseSensitive(bool case_sensitive) { + EnsureMutable(); MergingSnapshotUpdate::CaseSensitive(case_sensitive); return *this; } RowDelta& RowDelta::ValidateDataFilesExist( std::span referenced_files) { + EnsureMutable(); for (const auto& file : referenced_files) { referenced_data_files_.insert(file); } @@ -89,22 +98,27 @@ RowDelta& RowDelta::ValidateDataFilesExist( } RowDelta& RowDelta::ValidateDeletedFiles() { + EnsureMutable(); validate_deletes_ = true; return *this; } RowDelta& RowDelta::ConflictDetectionFilter(std::shared_ptr filter) { + EnsureMutable(); ICEBERG_BUILDER_CHECK(filter != nullptr, "Conflict detection filter cannot be null"); - conflict_detection_filter_ = std::move(filter); + ICEBERG_BUILDER_ASSIGN_OR_RETURN(conflict_detection_filter_, + internal::CopyUpdateExpression(filter)); return *this; } RowDelta& RowDelta::ValidateNoConflictingDataFiles() { + EnsureMutable(); validate_new_data_files_ = true; return *this; } RowDelta& RowDelta::ValidateNoConflictingDeleteFiles() { + EnsureMutable(); validate_new_delete_files_ = true; return *this; } diff --git a/src/iceberg/update/set_snapshot.cc b/src/iceberg/update/set_snapshot.cc index 79662890b..966842850 100644 --- a/src/iceberg/update/set_snapshot.cc +++ b/src/iceberg/update/set_snapshot.cc @@ -45,6 +45,7 @@ SetSnapshot::SetSnapshot(std::shared_ptr ctx) SetSnapshot::~SetSnapshot() = default; SetSnapshot& SetSnapshot::SetCurrentSnapshot(int64_t snapshot_id) { + EnsureMutable(); // Validate that the snapshot exists ICEBERG_BUILDER_ASSIGN_OR_RETURN(auto snapshot, base().SnapshotById(snapshot_id)); ICEBERG_BUILDER_CHECK(snapshot != nullptr, @@ -54,6 +55,7 @@ SetSnapshot& SetSnapshot::SetCurrentSnapshot(int64_t snapshot_id) { } SetSnapshot& SetSnapshot::RollbackToTime(int64_t timestamp_ms) { + EnsureMutable(); // Find the latest snapshot by timestamp older than timestamp_ms ICEBERG_BUILDER_ASSIGN_OR_RETURN(auto snapshot_id_opt, FindLatestAncestorOlderThan(timestamp_ms)); @@ -69,6 +71,7 @@ SetSnapshot& SetSnapshot::RollbackToTime(int64_t timestamp_ms) { } SetSnapshot& SetSnapshot::RollbackTo(int64_t snapshot_id) { + EnsureMutable(); // Validate that the snapshot exists auto snapshot_result = base().SnapshotById(snapshot_id); ICEBERG_BUILDER_CHECK(snapshot_result.has_value(), @@ -85,7 +88,7 @@ SetSnapshot& SetSnapshot::RollbackTo(int64_t snapshot_id) { return SetCurrentSnapshot(snapshot_id); } -Result SetSnapshot::Apply() { +Result SetSnapshot::Validate() const { ICEBERG_RETURN_UNEXPECTED(CheckErrors()); const TableMetadata& base_metadata = ctx_->current(); diff --git a/src/iceberg/update/set_snapshot.h b/src/iceberg/update/set_snapshot.h index 431e636b2..9dcf33433 100644 --- a/src/iceberg/update/set_snapshot.h +++ b/src/iceberg/update/set_snapshot.h @@ -53,8 +53,8 @@ class ICEBERG_EXPORT SetSnapshot : public PendingUpdate { Kind kind() const final { return Kind::kSetSnapshot; } bool IsRetryable() const override { return true; } - /// \brief Apply the pending changes and return the target snapshot ID. - Result Apply(); + /// \brief Validate and preview changes without staging or modifying this update. + Result Validate() const; private: explicit SetSnapshot(std::shared_ptr ctx); diff --git a/src/iceberg/update/snapshot_manager.cc b/src/iceberg/update/snapshot_manager.cc index 5473f3033..81bee7161 100644 --- a/src/iceberg/update/snapshot_manager.cc +++ b/src/iceberg/update/snapshot_manager.cc @@ -51,9 +51,16 @@ SnapshotManager::SnapshotManager(std::shared_ptr transaction, : transaction_(std::move(transaction)), is_external_transaction_(is_external_transaction) {} +void SnapshotManager::EnsureMutable() const { + const auto status = transaction_->CheckActive(); + ICEBERG_CHECK_OR_DIE(status.has_value(), "Cannot mutate snapshot manager: {}", + status.error().message); +} + SnapshotManager::~SnapshotManager() = default; SnapshotManager& SnapshotManager::Cherrypick(int64_t snapshot_id) { + EnsureMutable(); ICEBERG_BUILDER_RETURN_IF_ERROR(CommitIfRefUpdatesExist()); // TODO(anyone): Implement cherrypick operation ICEBERG_BUILDER_CHECK(false, "Cherrypick operation not yet implemented"); @@ -61,6 +68,7 @@ SnapshotManager& SnapshotManager::Cherrypick(int64_t snapshot_id) { } SnapshotManager& SnapshotManager::SetCurrentSnapshot(int64_t snapshot_id) { + EnsureMutable(); ICEBERG_BUILDER_RETURN_IF_ERROR(CommitIfRefUpdatesExist()); ICEBERG_BUILDER_ASSIGN_OR_RETURN(auto set_snapshot, transaction_->NewSetSnapshot()); set_snapshot->SetCurrentSnapshot(snapshot_id); @@ -69,6 +77,7 @@ SnapshotManager& SnapshotManager::SetCurrentSnapshot(int64_t snapshot_id) { } SnapshotManager& SnapshotManager::RollbackToTime(int64_t timestamp_ms) { + EnsureMutable(); ICEBERG_BUILDER_RETURN_IF_ERROR(CommitIfRefUpdatesExist()); ICEBERG_BUILDER_ASSIGN_OR_RETURN(auto set_snapshot, transaction_->NewSetSnapshot()); set_snapshot->RollbackToTime(timestamp_ms); @@ -77,6 +86,7 @@ SnapshotManager& SnapshotManager::RollbackToTime(int64_t timestamp_ms) { } SnapshotManager& SnapshotManager::RollbackTo(int64_t snapshot_id) { + EnsureMutable(); ICEBERG_BUILDER_RETURN_IF_ERROR(CommitIfRefUpdatesExist()); ICEBERG_BUILDER_ASSIGN_OR_RETURN(auto set_snapshot, transaction_->NewSetSnapshot()); set_snapshot->RollbackTo(snapshot_id); @@ -85,6 +95,7 @@ SnapshotManager& SnapshotManager::RollbackTo(int64_t snapshot_id) { } SnapshotManager& SnapshotManager::CreateBranch(const std::string& name) { + EnsureMutable(); const auto& base = transaction_->current(); if (base.current_snapshot_id != kInvalidSnapshotId) { ICEBERG_BUILDER_ASSIGN_OR_RETURN(auto current_snapshot, base.Snapshot()); @@ -99,6 +110,7 @@ SnapshotManager& SnapshotManager::CreateBranch(const std::string& name) { SnapshotManager& SnapshotManager::CreateBranch(const std::string& name, int64_t snapshot_id) { + EnsureMutable(); ICEBERG_BUILDER_ASSIGN_OR_RETURN(auto update_ref, UpdateSnapshotReferencesOperation()); update_ref->CreateBranch(name, snapshot_id); return *this; @@ -106,18 +118,21 @@ SnapshotManager& SnapshotManager::CreateBranch(const std::string& name, SnapshotManager& SnapshotManager::CreateTag(const std::string& name, int64_t snapshot_id) { + EnsureMutable(); ICEBERG_BUILDER_ASSIGN_OR_RETURN(auto update_ref, UpdateSnapshotReferencesOperation()); update_ref->CreateTag(name, snapshot_id); return *this; } SnapshotManager& SnapshotManager::RemoveBranch(const std::string& name) { + EnsureMutable(); ICEBERG_BUILDER_ASSIGN_OR_RETURN(auto update_ref, UpdateSnapshotReferencesOperation()); update_ref->RemoveBranch(name); return *this; } SnapshotManager& SnapshotManager::RemoveTag(const std::string& name) { + EnsureMutable(); ICEBERG_BUILDER_ASSIGN_OR_RETURN(auto update_ref, UpdateSnapshotReferencesOperation()); update_ref->RemoveTag(name); return *this; @@ -125,6 +140,7 @@ SnapshotManager& SnapshotManager::RemoveTag(const std::string& name) { SnapshotManager& SnapshotManager::ReplaceTag(const std::string& name, int64_t snapshot_id) { + EnsureMutable(); ICEBERG_BUILDER_ASSIGN_OR_RETURN(auto update_ref, UpdateSnapshotReferencesOperation()); update_ref->ReplaceTag(name, snapshot_id); return *this; @@ -132,6 +148,7 @@ SnapshotManager& SnapshotManager::ReplaceTag(const std::string& name, SnapshotManager& SnapshotManager::ReplaceBranch(const std::string& name, int64_t snapshot_id) { + EnsureMutable(); ICEBERG_BUILDER_ASSIGN_OR_RETURN(auto update_ref, UpdateSnapshotReferencesOperation()); update_ref->ReplaceBranch(name, snapshot_id); return *this; @@ -139,6 +156,7 @@ SnapshotManager& SnapshotManager::ReplaceBranch(const std::string& name, SnapshotManager& SnapshotManager::ReplaceBranch(const std::string& from, const std::string& to) { + EnsureMutable(); ICEBERG_BUILDER_ASSIGN_OR_RETURN(auto update_ref, UpdateSnapshotReferencesOperation()); update_ref->ReplaceBranch(from, to); return *this; @@ -146,6 +164,7 @@ SnapshotManager& SnapshotManager::ReplaceBranch(const std::string& from, SnapshotManager& SnapshotManager::FastForwardBranch(const std::string& from, const std::string& to) { + EnsureMutable(); ICEBERG_BUILDER_ASSIGN_OR_RETURN(auto update_ref, UpdateSnapshotReferencesOperation()); update_ref->FastForward(from, to); return *this; @@ -153,6 +172,7 @@ SnapshotManager& SnapshotManager::FastForwardBranch(const std::string& from, SnapshotManager& SnapshotManager::RenameBranch(const std::string& name, const std::string& new_name) { + EnsureMutable(); ICEBERG_BUILDER_ASSIGN_OR_RETURN(auto update_ref, UpdateSnapshotReferencesOperation()); update_ref->RenameBranch(name, new_name); return *this; @@ -160,6 +180,7 @@ SnapshotManager& SnapshotManager::RenameBranch(const std::string& name, SnapshotManager& SnapshotManager::SetMinSnapshotsToKeep(const std::string& branch_name, int32_t min_snapshots_to_keep) { + EnsureMutable(); ICEBERG_BUILDER_ASSIGN_OR_RETURN(auto update_ref, UpdateSnapshotReferencesOperation()); update_ref->SetMinSnapshotsToKeep(branch_name, min_snapshots_to_keep); return *this; @@ -167,6 +188,7 @@ SnapshotManager& SnapshotManager::SetMinSnapshotsToKeep(const std::string& branc SnapshotManager& SnapshotManager::SetMaxSnapshotAgeMs(const std::string& branch_name, int64_t max_snapshot_age_ms) { + EnsureMutable(); ICEBERG_BUILDER_ASSIGN_OR_RETURN(auto update_ref, UpdateSnapshotReferencesOperation()); update_ref->SetMaxSnapshotAgeMs(branch_name, max_snapshot_age_ms); return *this; @@ -174,6 +196,7 @@ SnapshotManager& SnapshotManager::SetMaxSnapshotAgeMs(const std::string& branch_ SnapshotManager& SnapshotManager::SetMaxRefAgeMs(const std::string& name, int64_t max_ref_age_ms) { + EnsureMutable(); ICEBERG_BUILDER_ASSIGN_OR_RETURN(auto update_ref, UpdateSnapshotReferencesOperation()); update_ref->SetMaxRefAgeMs(name, max_ref_age_ms); return *this; @@ -181,6 +204,7 @@ SnapshotManager& SnapshotManager::SetMaxRefAgeMs(const std::string& name, Status SnapshotManager::Commit() { ICEBERG_RETURN_UNEXPECTED(CheckErrors()); + ICEBERG_RETURN_UNEXPECTED(transaction_->CheckActive()); ICEBERG_RETURN_UNEXPECTED(CommitIfRefUpdatesExist()); if (!is_external_transaction_) { ICEBERG_RETURN_UNEXPECTED(transaction_->Commit()); diff --git a/src/iceberg/update/snapshot_manager.h b/src/iceberg/update/snapshot_manager.h index fd81f8339..9e7509afa 100644 --- a/src/iceberg/update/snapshot_manager.h +++ b/src/iceberg/update/snapshot_manager.h @@ -187,6 +187,7 @@ class ICEBERG_EXPORT SnapshotManager : public ErrorCollector { Status Commit(); private: + void EnsureMutable() const; SnapshotManager(std::shared_ptr transaction, bool is_external_transaction); /// \brief Get or create the UpdateSnapshotReference operation. diff --git a/src/iceberg/update/snapshot_update.cc b/src/iceberg/update/snapshot_update.cc index 6e8d28bd5..0095d843c 100644 --- a/src/iceberg/update/snapshot_update.cc +++ b/src/iceberg/update/snapshot_update.cc @@ -26,6 +26,7 @@ #include "iceberg/constants.h" #include "iceberg/file_io.h" +#include "iceberg/logging/log_macros.h" #include "iceberg/manifest/manifest_entry.h" #include "iceberg/manifest/manifest_list.h" #include "iceberg/manifest/manifest_reader.h" @@ -202,17 +203,17 @@ SnapshotUpdate::SnapshotUpdate(std::shared_ptr ctx) reporter_(ctx_->table->reporter()) {} Status SnapshotUpdate::Commit() { - commit_metrics_->attempts->Increment(); + ICEBERG_RETURN_UNEXPECTED(CheckCommitAllowed()); [[maybe_unused]] auto commit_timer = commit_metrics_->total_duration->Start(); return PendingUpdate::Commit(); } -void SnapshotUpdate::ReportCommit() const { +Status SnapshotUpdate::ReportCommitted() { ICEBERG_DCHECK(staged_snapshot_ != nullptr, "Staged snapshot is null after a successful commit"); if (!reporter_) { - return; + return {}; } const auto operation = staged_snapshot_->Operation(); @@ -225,7 +226,7 @@ void SnapshotUpdate::ReportCommit() const { CommitMetricsResult::From(*commit_metrics_, staged_snapshot_->summary), .metadata = {}, }; - std::ignore = reporter_->Report(report); + return reporter_->Report(report); } void SnapshotUpdate::SetSummaryProperty(const std::string& property, @@ -302,17 +303,11 @@ int64_t SnapshotUpdate::SnapshotId() { } Result SnapshotUpdate::Apply() { + commit_metrics_->attempts->Increment(); ICEBERG_RETURN_UNEXPECTED(CheckErrors()); - - if (staged_snapshot_ != nullptr) { - for (const auto& manifest_list : manifest_lists_) { - std::ignore = DeleteFile(manifest_list); - } - manifest_lists_.clear(); - ICEBERG_RETURN_UNEXPECTED(CleanUncommitted(std::unordered_set{})); - - staged_snapshot_ = nullptr; - summary_.Clear(); + { + std::lock_guard lock(staging_mutex_); + attempted_deletes_.clear(); } ICEBERG_ASSIGN_OR_RAISE(auto parent_snapshot, @@ -338,7 +333,6 @@ Result SnapshotUpdate::Apply() { ICEBERG_RETURN_UNEXPECTED(std::move(metadata_tasks).Run()); std::string manifest_list_path = ManifestListPath(); - manifest_lists_.push_back(manifest_list_path); ICEBERG_ASSIGN_OR_RAISE( auto writer, ManifestListWriter::MakeWriter(base().format_version, SnapshotId(), parent_snapshot_id, manifest_list_path, @@ -388,38 +382,42 @@ Result SnapshotUpdate::Apply() { .stage_only = stage_only_}; } -Status SnapshotUpdate::Finalize(Result commit_result) { - if (!commit_result.has_value()) { - if (commit_result.error().kind == ErrorKind::kCommitStateUnknown) { - return {}; +Status SnapshotUpdate::Finalize([[maybe_unused]] const TableMetadata& metadata) { + ICEBERG_CHECK(staged_snapshot_ != nullptr, "Missing staged snapshot after commit"); + auto cached_snapshot = SnapshotCache(staged_snapshot_.get()); + ICEBERG_ASSIGN_OR_RAISE(auto manifests, cached_snapshot.Manifests(ctx_->table->io())); + auto committed = + manifests | + std::views::transform([](const auto& manifest) { return manifest.manifest_path; }) | + std::ranges::to>(); + // Let derived updates release their caches and clean operation-specific files. + try { + if (auto status = CleanUncommitted(committed); !status) { + ICEBERG_LOG_WARN("Snapshot cleanup failed: {}", status.error().message); } - std::ignore = CleanAll(); - return {}; + } catch (const std::exception& e) { + ICEBERG_LOG_WARN("Snapshot cleanup threw: {}", e.what()); + } catch (...) { + ICEBERG_LOG_WARN("Snapshot cleanup threw an unknown exception"); } - - if (CleanupAfterCommit()) { - ICEBERG_CHECK(staged_snapshot_ != nullptr, - "Staged snapshot is null during finalize after commit"); - auto cached_snapshot = SnapshotCache(staged_snapshot_.get()); - if (auto manifests = cached_snapshot.Manifests(ctx_->table->io()); - manifests.has_value()) { - std::ignore = CleanUncommitted(manifests.value() | - std::views::transform([](const auto& manifest) { - return manifest.manifest_path; - }) | - std::ranges::to>()); + committed.insert(staged_snapshot_->manifest_list); + std::vector unused; + { + std::lock_guard lock(staging_mutex_); + for (const auto& path : staged_files_) { + if (!committed.contains(path) && !staged_data_files_.contains(path)) { + unused.push_back(path); + } } } - - // Also clean up unused manifest lists created by multiple attempts - for (const auto& manifest_list : manifest_lists_) { - if (manifest_list != staged_snapshot_->manifest_list) { - std::ignore = DeleteFile(manifest_list); - } + for (const auto& path : unused) { + std::ignore = DeleteFile(path); + } + { + std::lock_guard lock(staging_mutex_); + staged_files_.clear(); + staged_data_files_.clear(); } - - ReportCommit(); - return {}; } @@ -473,20 +471,59 @@ Result> SnapshotUpdate::ComputeSumm return summary; } -Status SnapshotUpdate::CleanAll() { - for (const auto& manifest_list : manifest_lists_) { - std::ignore = DeleteFile(manifest_list); +Status SnapshotUpdate::CleanStaged() { + try { + if (auto status = CleanUncommitted({}); !status) { + ICEBERG_LOG_WARN("Snapshot staging cleanup failed: {}", status.error().message); + } + } catch (const std::exception& e) { + ICEBERG_LOG_WARN("Snapshot staging cleanup threw: {}", e.what()); + } catch (...) { + ICEBERG_LOG_WARN("Snapshot staging cleanup threw an unknown exception"); + } + // Include paths whose writes failed before a derived cache recorded a manifest. + std::unordered_set paths; + { + std::lock_guard lock(staging_mutex_); + paths.swap(staged_files_); + staged_data_files_.clear(); } - manifest_lists_.clear(); - std::ignore = CleanUncommitted(std::unordered_set{}); + for (const auto& path : paths) { + std::ignore = DeleteFile(path); + } + staged_snapshot_.reset(); + summary_.Clear(); return {}; } -Status SnapshotUpdate::DeleteFile(const std::string& path) { - if (delete_func_) { - return delete_func_(path); +void SnapshotUpdate::RegisterStagedFile(const std::string& path, bool data_file) { + std::lock_guard lock(staging_mutex_); + staged_files_.insert(path); + if (data_file) { + staged_data_files_.insert(path); } - return ctx_->table->io()->DeleteFile(path); +} + +Status SnapshotUpdate::DeleteFile(const std::string& path) noexcept { + try { + { + std::lock_guard lock(staging_mutex_); + if (!attempted_deletes_.insert(path).second) { + return {}; + } + staged_files_.erase(path); + staged_data_files_.erase(path); + } + auto result = delete_func_ ? delete_func_(path) : ctx_->table->io()->DeleteFile(path); + if (!result) { + ICEBERG_LOG_WARN("Cannot clean staged file {}: {}", path, result.error().message); + } + } catch (const std::exception& e) { + ICEBERG_LOG_WARN("Cannot clean staged file {}: {}", path, e.what()); + } catch (...) { + ICEBERG_LOG_WARN("Cannot clean staged file {}: unknown exception", path); + } + return {}; } std::string SnapshotUpdate::ManifestListPath() { @@ -496,7 +533,9 @@ std::string SnapshotUpdate::ManifestListPath() { auto attempt = attempt_.fetch_add(1, std::memory_order_relaxed) + 1; std::string filename = std::format("snap-{}-{}-{}.avro", snapshot_id, attempt, commit_uuid_); - return ctx_->MetadataFileLocation(filename); + auto path = ctx_->MetadataFileLocation(filename); + RegisterStagedFile(path); + return path; } SnapshotSummaryBuilder SnapshotUpdate::BuildManifestCountSummary( @@ -526,7 +565,9 @@ std::string SnapshotUpdate::ManifestPath() { // Format: {metadata_location}/{uuid}-m{manifest_count}.avro auto manifest_count = manifest_count_.fetch_add(1, std::memory_order_relaxed); std::string filename = std::format("{}-m{}.avro", commit_uuid_, manifest_count); - return ctx_->MetadataFileLocation(filename); + auto path = ctx_->MetadataFileLocation(filename); + RegisterStagedFile(path); + return path; } } // namespace iceberg diff --git a/src/iceberg/update/snapshot_update.h b/src/iceberg/update/snapshot_update.h index 1812b7072..52b5bf6a9 100644 --- a/src/iceberg/update/snapshot_update.h +++ b/src/iceberg/update/snapshot_update.h @@ -25,6 +25,7 @@ #include #include #include +#include #include #include #include @@ -66,6 +67,7 @@ class ICEBERG_EXPORT SnapshotUpdate : public PendingUpdate { /// \param reporter The metrics reporter to use. /// \return Reference to this for method chaining. auto& ReportWith(this auto& self, std::shared_ptr reporter) { + self.EnsureMutable(); static_cast(self).reporter_ = std::move(reporter); return self; } @@ -77,6 +79,7 @@ class ICEBERG_EXPORT SnapshotUpdate : public PendingUpdate { /// \note Cannot be called more than once. auto& DeleteWith(this auto& self, std::function delete_func) { + self.EnsureMutable(); if (self.delete_func_) { return self.AddError(ErrorKind::kInvalidArgument, "Cannot set delete callback more than once"); @@ -92,6 +95,7 @@ class ICEBERG_EXPORT SnapshotUpdate : public PendingUpdate { /// /// \return This update for method chaining. auto& StageOnly(this auto& self) { + self.EnsureMutable(); self.stage_only_ = true; return self; } @@ -101,6 +105,7 @@ class ICEBERG_EXPORT SnapshotUpdate : public PendingUpdate { /// \param executor Executor to use while planning manifests. /// \return Reference to this for method chaining. auto& ScanManifestsWith(this auto& self, Executor& executor) { + self.EnsureMutable(); self.plan_executor_ = std::ref(executor); return self; } @@ -110,6 +115,7 @@ class ICEBERG_EXPORT SnapshotUpdate : public PendingUpdate { /// \param branch The name of a SnapshotRef of type branch. /// \return This update for method chaining. auto& ToBranch(this auto& self, const std::string& branch) { + self.EnsureMutable(); if (branch.empty()) [[unlikely]] { return self.AddError(ErrorKind::kInvalidArgument, "Branch name cannot be empty"); } @@ -133,6 +139,7 @@ class ICEBERG_EXPORT SnapshotUpdate : public PendingUpdate { /// \param value A String property value. /// \return This update for method chaining. auto& Set(this auto& self, const std::string& property, const std::string& value) { + self.EnsureMutable(); static_cast(self).SetSummaryProperty(property, value); return self; } @@ -145,6 +152,7 @@ class ICEBERG_EXPORT SnapshotUpdate : public PendingUpdate { /// \note Custom FileIO implementations and registered writer factories used for /// manifest writes must support concurrent calls when an executor is configured. auto& WriteManifestsWith(this auto& self, Executor& executor, int32_t parallelism) { + self.EnsureMutable(); if (parallelism <= 0) [[unlikely]] { return self.AddError( ErrorKind::kInvalidArgument, @@ -156,20 +164,14 @@ class ICEBERG_EXPORT SnapshotUpdate : public PendingUpdate { return self; } - /// \brief Apply the update's changes to create a new snapshot. - /// - /// This method validates the changes, applies them to the current base - /// metadata, and creates a new snapshot without committing it. Commit retries - /// call Apply() again with refreshed metadata so the same changes can be - /// applied to the new latest snapshot. - /// - /// \return A result containing the new snapshot, or an error. - Result Apply(); + protected: + friend class Transaction; - /// \brief Finalize the snapshot update, cleaning up any uncommitted files. - Status Finalize(Result commit_result) override; + Result Apply(); + Status Finalize(const TableMetadata& committed) override; + Status CleanStaged() override; + Status ReportCommitted() override; - protected: struct ContentFileWithSequenceNumber { std::shared_ptr file; std::optional data_sequence_number; @@ -249,11 +251,6 @@ class ICEBERG_EXPORT SnapshotUpdate : public PendingUpdate { /// retry-safe summary rebuilds. virtual void SetSummaryProperty(const std::string& property, const std::string& value); - /// \brief Check if cleanup should happen after commit - /// - /// \return True if cleanup should happen after commit - virtual bool CleanupAfterCommit() const { return true; } - /// \brief Get or generate the snapshot ID for the new snapshot. int64_t SnapshotId(); @@ -261,7 +258,8 @@ class ICEBERG_EXPORT SnapshotUpdate : public PendingUpdate { /// /// \param path The path of the file to delete /// \return A status indicating the result of the deletion - Status DeleteFile(const std::string& path); + Status DeleteFile(const std::string& path) noexcept; + void RegisterStagedFile(const std::string& path, bool data_file = false); std::string ManifestPath(); std::string ManifestListPath(); @@ -274,12 +272,6 @@ class ICEBERG_EXPORT SnapshotUpdate : public PendingUpdate { Result> ComputeSummary( const TableMetadata& previous); - /// \brief Clean up all uncommitted files - Status CleanAll(); - - /// \brief Report metrics for the most recently staged snapshot. - void ReportCommit() const; - protected: SnapshotSummaryBuilder summary_; @@ -290,7 +282,11 @@ class ICEBERG_EXPORT SnapshotUpdate : public PendingUpdate { int32_t write_manifest_parallelism_{1}; std::atomic manifest_count_{0}; std::atomic attempt_{0}; - std::vector manifest_lists_; + // All paths are registered before writes, including partial/failed writes. + std::mutex staging_mutex_; + std::unordered_set staged_files_; + std::unordered_set staged_data_files_; + std::unordered_set attempted_deletes_; const int64_t target_manifest_size_bytes_; std::optional snapshot_id_; OptionalExecutor plan_executor_; diff --git a/src/iceberg/update/update_location.cc b/src/iceberg/update/update_location.cc index 064bebc8c..7008dd89a 100644 --- a/src/iceberg/update/update_location.cc +++ b/src/iceberg/update/update_location.cc @@ -41,12 +41,13 @@ UpdateLocation::UpdateLocation(std::shared_ptr ctx) UpdateLocation::~UpdateLocation() = default; UpdateLocation& UpdateLocation::SetLocation(std::string_view location) { + EnsureMutable(); ICEBERG_BUILDER_CHECK(!location.empty(), "Location cannot be empty"); location_ = std::string(location); return *this; } -Result UpdateLocation::Apply() { +Result UpdateLocation::Validate() const { ICEBERG_RETURN_UNEXPECTED(CheckErrors()); if (location_.empty()) { return InvalidArgument("Location must be set before applying"); diff --git a/src/iceberg/update/update_location.h b/src/iceberg/update/update_location.h index 48fb84c21..2fb0df846 100644 --- a/src/iceberg/update/update_location.h +++ b/src/iceberg/update/update_location.h @@ -47,10 +47,12 @@ class ICEBERG_EXPORT UpdateLocation : public PendingUpdate { Kind kind() const final { return Kind::kUpdateLocation; } bool IsRetryable() const override { return true; } - /// \brief Apply the pending changes and return the new location. - Result Apply(); + /// \brief Validate and preview changes without staging or modifying this update. + Result Validate() const; private: + bool MayAddFileReferences() const override { return false; } + explicit UpdateLocation(std::shared_ptr ctx); std::string location_; diff --git a/src/iceberg/update/update_partition_spec.cc b/src/iceberg/update/update_partition_spec.cc index 2be6a59a5..a615a6388 100644 --- a/src/iceberg/update/update_partition_spec.cc +++ b/src/iceberg/update/update_partition_spec.cc @@ -86,16 +86,19 @@ UpdatePartitionSpec::UpdatePartitionSpec(std::shared_ptr ctx UpdatePartitionSpec::~UpdatePartitionSpec() = default; UpdatePartitionSpec& UpdatePartitionSpec::CaseSensitive(bool is_case_sensitive) { + EnsureMutable(); case_sensitive_ = is_case_sensitive; return *this; } UpdatePartitionSpec& UpdatePartitionSpec::AddNonDefaultSpec() { + EnsureMutable(); set_as_default_ = false; return *this; } UpdatePartitionSpec& UpdatePartitionSpec::AddField(std::string_view source_name) { + EnsureMutable(); // Find the source field in the schema ICEBERG_BUILDER_ASSIGN_OR_RETURN( auto field_opt, schema_->FindFieldByName(source_name, case_sensitive_)); @@ -108,6 +111,7 @@ UpdatePartitionSpec& UpdatePartitionSpec::AddField(std::string_view source_name) UpdatePartitionSpec& UpdatePartitionSpec::AddField(const std::shared_ptr& term, std::string_view part_name) { + EnsureMutable(); ICEBERG_BUILDER_CHECK(term->is_unbound(), "Cannot add bound term to partition spec"); // Bind the term to get the source id if (term->kind() == Term::Kind::kReference) { @@ -130,6 +134,7 @@ UpdatePartitionSpec& UpdatePartitionSpec::AddField(const std::shared_ptr& UpdatePartitionSpec& UpdatePartitionSpec::AddFieldInternal( std::string_view name, int32_t source_id, const std::shared_ptr& transform) { + EnsureMutable(); // Check for duplicate name in added fields ICEBERG_BUILDER_CHECK(name.empty() || !added_field_names_.contains(name), "Cannot add duplicate partition field: {}", name); @@ -212,6 +217,7 @@ UpdatePartitionSpec& UpdatePartitionSpec::AddFieldInternal( UpdatePartitionSpec& UpdatePartitionSpec::RewriteDeleteAndAddField( const PartitionField& existing, std::string_view name) { + EnsureMutable(); deletes_.erase(existing.field_id()); if (name.empty() || std::string(existing.name()) == name) { return *this; @@ -220,6 +226,7 @@ UpdatePartitionSpec& UpdatePartitionSpec::RewriteDeleteAndAddField( } UpdatePartitionSpec& UpdatePartitionSpec::RemoveField(std::string_view name) { + EnsureMutable(); // Cannot delete newly added fields ICEBERG_BUILDER_CHECK(!added_field_names_.contains(name), "Cannot delete newly added field: {}", name); @@ -237,6 +244,7 @@ UpdatePartitionSpec& UpdatePartitionSpec::RemoveField(std::string_view name) { } UpdatePartitionSpec& UpdatePartitionSpec::RemoveField(const std::shared_ptr& term) { + EnsureMutable(); ICEBERG_BUILDER_CHECK(term->is_unbound(), "Cannot remove bound term from partition spec"); // Bind the term to get the source id @@ -263,6 +271,7 @@ UpdatePartitionSpec& UpdatePartitionSpec::RemoveField(const std::shared_ptrsecond)) { @@ -306,7 +316,7 @@ UpdatePartitionSpec& UpdatePartitionSpec::RenameField(std::string_view name, return *this; } -Result UpdatePartitionSpec::Apply() { +Result UpdatePartitionSpec::Validate() const { ICEBERG_RETURN_UNEXPECTED(CheckErrors()); std::vector new_fields; @@ -341,6 +351,12 @@ Result UpdatePartitionSpec::Apply() { // Add new fields new_fields.insert(new_fields.end(), adds_.begin(), adds_.end()); + // Keep preview results independent of this operation's frozen transforms. + for (auto& field : new_fields) { + field = PartitionField(field.source_id(), field.field_id(), std::string(field.name()), + std::make_shared(*field.transform())); + } + // Use -1 as a placeholder for the spec id, the actual spec id will be assigned by // TableMetadataBuilder when the AddPartitionSpec update is applied. ICEBERG_ASSIGN_OR_RAISE(auto new_spec, @@ -446,4 +462,12 @@ void UpdatePartitionSpec::BuildHistoricalFieldsIndex() { } } +Status UpdatePartitionSpec::Freeze() { + for (auto& field : adds_) { + field = PartitionField(field.source_id(), field.field_id(), std::string(field.name()), + std::make_shared(*field.transform())); + } + return {}; +} + } // namespace iceberg diff --git a/src/iceberg/update/update_partition_spec.h b/src/iceberg/update/update_partition_spec.h index 67dcb1413..ef9e1f965 100644 --- a/src/iceberg/update/update_partition_spec.h +++ b/src/iceberg/update/update_partition_spec.h @@ -111,9 +111,14 @@ class ICEBERG_EXPORT UpdatePartitionSpec : public PendingUpdate { std::shared_ptr spec; bool set_as_default; }; - Result Apply(); + /// \brief Validate and preview changes without staging or modifying this update. + Result Validate() const; private: + Status Freeze() override; + + bool MayAddFileReferences() const override { return false; } + explicit UpdatePartitionSpec(std::shared_ptr ctx); /// \brief Pair of source ID and transform string for indexing. diff --git a/src/iceberg/update/update_partition_statistics.cc b/src/iceberg/update/update_partition_statistics.cc index 3a5ab4f8a..0fd989a79 100644 --- a/src/iceberg/update/update_partition_statistics.cc +++ b/src/iceberg/update/update_partition_statistics.cc @@ -47,6 +47,7 @@ UpdatePartitionStatistics::~UpdatePartitionStatistics() = default; UpdatePartitionStatistics& UpdatePartitionStatistics::SetPartitionStatistics( std::shared_ptr partition_statistics_file) { + EnsureMutable(); ICEBERG_BUILDER_CHECK(partition_statistics_file != nullptr, "Statistics file cannot be null"); @@ -57,17 +58,20 @@ UpdatePartitionStatistics& UpdatePartitionStatistics::SetPartitionStatistics( UpdatePartitionStatistics& UpdatePartitionStatistics::RemovePartitionStatistics( int64_t snapshot_id) { + EnsureMutable(); partition_statistics_to_set_[snapshot_id] = nullptr; return *this; } -Result UpdatePartitionStatistics::Apply() { +Result UpdatePartitionStatistics::Validate() + const { ICEBERG_RETURN_UNEXPECTED(CheckErrors()); ApplyResult result; for (const auto& [snapshot_id, partition_stats] : partition_statistics_to_set_) { if (partition_stats) { - result.to_set.emplace_back(snapshot_id, partition_stats); + result.to_set.emplace_back( + snapshot_id, std::make_shared(*partition_stats)); } else { result.to_remove.push_back(snapshot_id); } @@ -75,4 +79,13 @@ Result UpdatePartitionStatistics::Apply( return result; } +Status UpdatePartitionStatistics::Freeze() { + for (auto& [_, value] : partition_statistics_to_set_) { + if (value) { + value = std::make_shared(*value); + } + } + return {}; +} + } // namespace iceberg diff --git a/src/iceberg/update/update_partition_statistics.h b/src/iceberg/update/update_partition_statistics.h index 982b1bd39..5633c4da0 100644 --- a/src/iceberg/update/update_partition_statistics.h +++ b/src/iceberg/update/update_partition_statistics.h @@ -75,9 +75,12 @@ class ICEBERG_EXPORT UpdatePartitionStatistics : public PendingUpdate { std::vector to_remove; }; - Result Apply(); + /// \brief Validate and preview changes without staging or modifying this update. + Result Validate() const; private: + Status Freeze() override; + explicit UpdatePartitionStatistics(std::shared_ptr ctx); std::unordered_map> diff --git a/src/iceberg/update/update_properties.cc b/src/iceberg/update/update_properties.cc index d2e75c301..091bc047f 100644 --- a/src/iceberg/update/update_properties.cc +++ b/src/iceberg/update/update_properties.cc @@ -46,6 +46,7 @@ UpdateProperties::~UpdateProperties() = default; UpdateProperties& UpdateProperties::Set(const std::string& key, const std::string& value) { + EnsureMutable(); ICEBERG_BUILDER_CHECK(!removals_.contains(key), "Cannot set property '{}' that is already marked for removal", key); @@ -59,6 +60,7 @@ UpdateProperties& UpdateProperties::Set(const std::string& key, } UpdateProperties& UpdateProperties::Remove(const std::string& key) { + EnsureMutable(); ICEBERG_BUILDER_CHECK(!updates_.contains(key), "Cannot remove property '{}' that is already marked for update", key); @@ -66,8 +68,10 @@ UpdateProperties& UpdateProperties::Remove(const std::string& key) { return *this; } -Result UpdateProperties::Apply() { +Result UpdateProperties::Validate() const { ICEBERG_RETURN_UNEXPECTED(CheckErrors()); + auto updates = updates_; + std::optional format_version; const auto& current_props = base().properties.configs(); std::unordered_map new_properties; std::vector removals; @@ -91,17 +95,18 @@ Result UpdateProperties::Apply() { "Cannot upgrade table to unsupported format version: v{} (supported: v{})", parsed_version, TableMetadata::kSupportedTableFormatVersion); } - format_version_ = static_cast(parsed_version); + format_version = static_cast(parsed_version); - updates_.erase(TableProperties::kFormatVersion.key()); + updates.erase(TableProperties::kFormatVersion.key()); } if (auto schema = base().Schema(); schema.has_value()) { ICEBERG_RETURN_UNEXPECTED( MetricsConfig::VerifyReferencedColumns(new_properties, *schema.value())); } - return ApplyResult{ - .updates = updates_, .removals = removals_, .format_version = format_version_}; + return ApplyResult{.updates = std::move(updates), + .removals = removals_, + .format_version = format_version}; } } // namespace iceberg diff --git a/src/iceberg/update/update_properties.h b/src/iceberg/update/update_properties.h index 18eba427b..8ab41e318 100644 --- a/src/iceberg/update/update_properties.h +++ b/src/iceberg/update/update_properties.h @@ -53,7 +53,7 @@ class ICEBERG_EXPORT UpdateProperties : public PendingUpdate { /// /// The key must not have been previously marked for removal and must not be a /// reserved property key (except `format-version`). Setting a reserved property - /// will result in a validation error at Apply() time. + /// will result in an error during validation. /// /// \param key The property key to set /// \param value The property value to set @@ -69,15 +69,16 @@ class ICEBERG_EXPORT UpdateProperties : public PendingUpdate { Kind kind() const final { return Kind::kUpdateProperties; } bool IsRetryable() const override { return true; } - /// \brief Apply the pending changes and return the updates and removals. - Result Apply(); + /// \brief Validate and preview changes without staging or modifying this update. + Result Validate() const; private: + bool MayAddFileReferences() const override { return false; } + explicit UpdateProperties(std::shared_ptr ctx); std::unordered_map updates_; std::unordered_set removals_; - std::optional format_version_; }; } // namespace iceberg diff --git a/src/iceberg/update/update_schema.cc b/src/iceberg/update/update_schema.cc index 56e167e10..77d4b4ddd 100644 --- a/src/iceberg/update/update_schema.cc +++ b/src/iceberg/update/update_schema.cc @@ -38,6 +38,7 @@ #include "iceberg/table_properties.h" #include "iceberg/transaction.h" #include "iceberg/type.h" +#include "iceberg/update/update_util_internal.h" #include "iceberg/util/checked_cast.h" #include "iceberg/util/error_collector.h" #include "iceberg/util/formatter.h" // IWYU pragma: keep @@ -339,11 +340,13 @@ UpdateSchema::Move UpdateSchema::Move::After(int32_t field_id, } UpdateSchema& UpdateSchema::AllowIncompatibleChanges() { + EnsureMutable(); allow_incompatible_changes_ = true; return *this; } UpdateSchema& UpdateSchema::CaseSensitive(bool case_sensitive) { + EnsureMutable(); case_sensitive_ = case_sensitive; return *this; } @@ -351,6 +354,7 @@ UpdateSchema& UpdateSchema::CaseSensitive(bool case_sensitive) { UpdateSchema& UpdateSchema::AddColumn(std::string_view name, std::shared_ptr type, std::string_view doc, std::optional default_value) { + EnsureMutable(); ICEBERG_BUILDER_CHECK(!name.contains('.'), "Cannot add column with ambiguous name: {}, use " "AddColumn(parent, name, type, doc)", @@ -363,6 +367,7 @@ UpdateSchema& UpdateSchema::AddColumn(std::optional parent, std::string_view name, std::shared_ptr type, std::string_view doc, std::optional default_value) { + EnsureMutable(); return AddColumnInternal(std::move(parent), name, /*is_optional=*/true, std::move(type), doc, std::move(default_value)); } @@ -371,6 +376,7 @@ UpdateSchema& UpdateSchema::AddRequiredColumn(std::string_view name, std::shared_ptr type, std::string_view doc, std::optional default_value) { + EnsureMutable(); ICEBERG_BUILDER_CHECK(!name.contains('.'), "Cannot add column with ambiguous name: {}, use " "AddRequiredColumn(parent, name, type, doc)", @@ -384,12 +390,14 @@ UpdateSchema& UpdateSchema::AddRequiredColumn(std::optional pa std::shared_ptr type, std::string_view doc, std::optional default_value) { + EnsureMutable(); return AddColumnInternal(std::move(parent), name, /*is_optional=*/false, std::move(type), doc, std::move(default_value)); } UpdateSchema& UpdateSchema::UpdateColumn(std::string_view name, std::shared_ptr new_type) { + EnsureMutable(); ICEBERG_BUILDER_ASSIGN_OR_RETURN(auto field_opt, FindFieldForUpdate(name)); ICEBERG_BUILDER_CHECK(field_opt.has_value(), "Cannot update missing column: {}", name); @@ -432,6 +440,7 @@ UpdateSchema& UpdateSchema::UpdateColumn(std::string_view name, UpdateSchema& UpdateSchema::UpdateColumnDoc(std::string_view name, std::string_view new_doc) { + EnsureMutable(); ICEBERG_BUILDER_ASSIGN_OR_RETURN(auto field_opt, FindFieldForUpdate(name)); ICEBERG_BUILDER_CHECK(field_opt.has_value(), "Cannot update missing column: {}", name); @@ -452,6 +461,7 @@ UpdateSchema& UpdateSchema::UpdateColumnDoc(std::string_view name, UpdateSchema& UpdateSchema::UpdateColumnDefault(std::string_view name, std::optional new_default) { + EnsureMutable(); ICEBERG_BUILDER_ASSIGN_OR_RETURN(auto field_opt, FindFieldForUpdate(name)); ICEBERG_BUILDER_CHECK(field_opt.has_value(), "Cannot update missing column: {}", name); @@ -489,6 +499,7 @@ UpdateSchema& UpdateSchema::UpdateColumnDefault(std::string_view name, UpdateSchema& UpdateSchema::RenameColumn(std::string_view name, std::string_view new_name) { + EnsureMutable(); ICEBERG_BUILDER_ASSIGN_OR_RETURN(auto field_opt, FindField(name)); ICEBERG_BUILDER_CHECK(field_opt.has_value(), "Cannot rename missing column: {}", name); ICEBERG_BUILDER_CHECK(!new_name.empty(), "Cannot rename a column to null"); @@ -514,15 +525,18 @@ UpdateSchema& UpdateSchema::RenameColumn(std::string_view name, } UpdateSchema& UpdateSchema::MakeColumnOptional(std::string_view name) { + EnsureMutable(); return UpdateColumnRequirementInternal(name, /*is_optional=*/true); } UpdateSchema& UpdateSchema::RequireColumn(std::string_view name) { + EnsureMutable(); return UpdateColumnRequirementInternal(name, /*is_optional=*/false); } UpdateSchema& UpdateSchema::UpdateColumnRequirementInternal(std::string_view name, bool is_optional) { + EnsureMutable(); ICEBERG_BUILDER_ASSIGN_OR_RETURN(auto field_opt, FindFieldForUpdate(name)); ICEBERG_BUILDER_CHECK(field_opt.has_value(), "Cannot update missing column: {}", name); @@ -552,6 +566,7 @@ UpdateSchema& UpdateSchema::UpdateColumnRequirementInternal(std::string_view nam } UpdateSchema& UpdateSchema::DeleteColumn(std::string_view name) { + EnsureMutable(); ICEBERG_BUILDER_ASSIGN_OR_RETURN(auto field_opt, FindField(name)); ICEBERG_BUILDER_CHECK(field_opt.has_value(), "Cannot delete missing column: {}", name); @@ -569,6 +584,7 @@ UpdateSchema& UpdateSchema::DeleteColumn(std::string_view name) { } UpdateSchema& UpdateSchema::MoveFirst(std::string_view name) { + EnsureMutable(); ICEBERG_BUILDER_ASSIGN_OR_RETURN(auto field_id, FindFieldIdForMove(name)); return MoveInternal(name, Move::First(field_id)); @@ -576,6 +592,7 @@ UpdateSchema& UpdateSchema::MoveFirst(std::string_view name) { UpdateSchema& UpdateSchema::MoveBefore(std::string_view name, std::string_view before_name) { + EnsureMutable(); ICEBERG_BUILDER_ASSIGN_OR_RETURN(auto field_id, FindFieldIdForMove(name)); ICEBERG_BUILDER_ASSIGN_OR_RETURN_WITH_ERROR( auto before_id, FindFieldIdForMove(before_name), @@ -588,6 +605,7 @@ UpdateSchema& UpdateSchema::MoveBefore(std::string_view name, UpdateSchema& UpdateSchema::MoveAfter(std::string_view name, std::string_view after_name) { + EnsureMutable(); ICEBERG_BUILDER_ASSIGN_OR_RETURN(auto field_id, FindFieldIdForMove(name)); ICEBERG_BUILDER_ASSIGN_OR_RETURN_WITH_ERROR( auto after_id, FindFieldIdForMove(after_name), @@ -599,6 +617,7 @@ UpdateSchema& UpdateSchema::MoveAfter(std::string_view name, } UpdateSchema& UpdateSchema::UnionByNameWith(std::shared_ptr new_schema) { + EnsureMutable(); // TODO(Guotao Yu): Implement UnionByNameWith AddError(NotImplemented("UpdateSchema::UnionByNameWith not implemented")); return *this; @@ -606,11 +625,12 @@ UpdateSchema& UpdateSchema::UnionByNameWith(std::shared_ptr new_schema) UpdateSchema& UpdateSchema::SetIdentifierFields( const std::span& names) { + EnsureMutable(); identifier_field_names_ = names | std::ranges::to>(); return *this; } -Result UpdateSchema::Apply() { +Result UpdateSchema::Validate() const { ICEBERG_RETURN_UNEXPECTED(CheckErrors()); for (const auto& name : identifier_field_names_) { @@ -653,10 +673,17 @@ Result UpdateSchema::Apply() { fresh_identifier_ids.push_back(field_opt->get().field_id()); } - auto new_fields = temp_schema->fields() | std::ranges::to>(); - ICEBERG_ASSIGN_OR_RAISE( - auto new_schema, - Schema::Make(std::move(new_fields), schema_->schema_id(), fresh_identifier_ids)); + // The result must not alias the frozen replay parameters (or the base schema), so + // deep-copy the fields before building the schema that is returned. + std::vector new_fields; + new_fields.reserve(temp_schema->fields().size()); + for (const auto& field : temp_schema->fields()) { + ICEBERG_ASSIGN_OR_RAISE(auto copy, internal::CopyUpdateField(field)); + new_fields.push_back(std::move(copy)); + } + ICEBERG_ASSIGN_OR_RAISE(auto new_schema, + Schema::Make(std::move(new_fields), schema_->schema_id(), + std::move(fresh_identifier_ids))); ICEBERG_RETURN_UNEXPECTED(new_schema->Validate(base().format_version)); std::unordered_map updated_props; @@ -688,6 +715,7 @@ UpdateSchema& UpdateSchema::AddColumnInternal(std::optional pa std::shared_ptr type, std::string_view doc, std::optional default_value) { + EnsureMutable(); int32_t parent_id = kTableRootId; std::string full_name; // For map/list adds, this omits synthetic value/element path segments. @@ -847,6 +875,7 @@ Result UpdateSchema::FindFieldIdForMove(std::string_view name) const { } UpdateSchema& UpdateSchema::MoveInternal(std::string_view name, const Move& move) { + EnsureMutable(); auto parent_it = id_to_parent_.find(move.field_id); if (parent_it != id_to_parent_.end()) { @@ -881,4 +910,13 @@ UpdateSchema& UpdateSchema::MoveInternal(std::string_view name, const Move& move return *this; } +Status UpdateSchema::Freeze() { + // Field values may contain nested types and mutable aliases to defaults. + for (auto& [_, field] : updates_) { + ICEBERG_ASSIGN_OR_RAISE(auto copy, internal::CopyUpdateField(*field)); + field = std::make_shared(std::move(copy)); + } + return {}; +} + } // namespace iceberg diff --git a/src/iceberg/update/update_schema.h b/src/iceberg/update/update_schema.h index d0dd944ef..9e4ae29f7 100644 --- a/src/iceberg/update/update_schema.h +++ b/src/iceberg/update/update_schema.h @@ -364,14 +364,14 @@ class ICEBERG_EXPORT UpdateSchema : public PendingUpdate { std::unordered_map updated_props; }; - /// \brief Apply the pending changes to the original schema and return the result. - /// - /// This does not result in a permanent update. - /// - /// \return The result Schema and last column id when all pending updates are applied. - Result Apply(); + /// \brief Validate and preview changes without staging or modifying this update. + Result Validate() const; private: + Status Freeze() override; + + bool MayAddFileReferences() const override { return false; } + explicit UpdateSchema(std::shared_ptr ctx); /// \brief Internal implementation for adding a column with full control. diff --git a/src/iceberg/update/update_snapshot_reference.cc b/src/iceberg/update/update_snapshot_reference.cc index 908962ecd..e11e3acb3 100644 --- a/src/iceberg/update/update_snapshot_reference.cc +++ b/src/iceberg/update/update_snapshot_reference.cc @@ -47,6 +47,7 @@ UpdateSnapshotReference::~UpdateSnapshotReference() = default; UpdateSnapshotReference& UpdateSnapshotReference::CreateBranch(const std::string& name, int64_t snapshot_id) { + EnsureMutable(); ICEBERG_BUILDER_CHECK(!name.empty(), "Branch name cannot be empty"); ICEBERG_BUILDER_ASSIGN_OR_RETURN(auto branch, SnapshotRef::MakeBranch(snapshot_id)); auto [_, inserted] = updated_refs_.emplace(name, std::move(branch)); @@ -56,6 +57,7 @@ UpdateSnapshotReference& UpdateSnapshotReference::CreateBranch(const std::string UpdateSnapshotReference& UpdateSnapshotReference::CreateTag(const std::string& name, int64_t snapshot_id) { + EnsureMutable(); ICEBERG_BUILDER_CHECK(!name.empty(), "Tag name cannot be empty"); ICEBERG_BUILDER_ASSIGN_OR_RETURN(auto tag, SnapshotRef::MakeTag(snapshot_id)); auto [_, inserted] = updated_refs_.emplace(name, std::move(tag)); @@ -64,6 +66,7 @@ UpdateSnapshotReference& UpdateSnapshotReference::CreateTag(const std::string& n } UpdateSnapshotReference& UpdateSnapshotReference::RemoveBranch(const std::string& name) { + EnsureMutable(); ICEBERG_BUILDER_CHECK(!name.empty(), "Branch name cannot be empty"); ICEBERG_BUILDER_CHECK(name != SnapshotRef::kMainBranch, "Cannot remove main branch"); auto it = updated_refs_.find(name); @@ -75,6 +78,7 @@ UpdateSnapshotReference& UpdateSnapshotReference::RemoveBranch(const std::string } UpdateSnapshotReference& UpdateSnapshotReference::RemoveTag(const std::string& name) { + EnsureMutable(); ICEBERG_BUILDER_CHECK(!name.empty(), "Tag name cannot be empty"); auto it = updated_refs_.find(name); ICEBERG_BUILDER_CHECK(it != updated_refs_.end(), "Tag does not exist: {}", name); @@ -86,6 +90,7 @@ UpdateSnapshotReference& UpdateSnapshotReference::RemoveTag(const std::string& n UpdateSnapshotReference& UpdateSnapshotReference::RenameBranch( const std::string& name, const std::string& new_name) { + EnsureMutable(); ICEBERG_BUILDER_CHECK(!name.empty(), "Branch to rename cannot be empty"); ICEBERG_BUILDER_CHECK(!new_name.empty(), "New branch name cannot be empty"); ICEBERG_BUILDER_CHECK(name != SnapshotRef::kMainBranch, "Cannot rename main branch"); @@ -101,6 +106,7 @@ UpdateSnapshotReference& UpdateSnapshotReference::RenameBranch( UpdateSnapshotReference& UpdateSnapshotReference::ReplaceBranch(const std::string& name, int64_t snapshot_id) { + EnsureMutable(); ICEBERG_BUILDER_CHECK(!name.empty(), "Branch name cannot be empty"); auto it = updated_refs_.find(name); ICEBERG_BUILDER_CHECK(it != updated_refs_.end(), "Branch does not exist: {}", name); @@ -112,16 +118,19 @@ UpdateSnapshotReference& UpdateSnapshotReference::ReplaceBranch(const std::strin UpdateSnapshotReference& UpdateSnapshotReference::ReplaceBranch(const std::string& from, const std::string& to) { + EnsureMutable(); return ReplaceBranchInternal(from, to, /*fast_forward=*/false); } UpdateSnapshotReference& UpdateSnapshotReference::FastForward(const std::string& from, const std::string& to) { + EnsureMutable(); return ReplaceBranchInternal(from, to, /*fast_forward=*/true); } UpdateSnapshotReference& UpdateSnapshotReference::ReplaceBranchInternal( const std::string& from, const std::string& to, bool fast_forward) { + EnsureMutable(); ICEBERG_BUILDER_CHECK(!from.empty(), "Branch to update cannot be empty"); ICEBERG_BUILDER_CHECK(!to.empty(), "Destination ref cannot be empty"); auto to_it = updated_refs_.find(to); @@ -160,6 +169,7 @@ UpdateSnapshotReference& UpdateSnapshotReference::ReplaceBranchInternal( UpdateSnapshotReference& UpdateSnapshotReference::ReplaceTag(const std::string& name, int64_t snapshot_id) { + EnsureMutable(); ICEBERG_BUILDER_CHECK(!name.empty(), "Tag name cannot be empty"); auto it = updated_refs_.find(name); ICEBERG_BUILDER_CHECK(it != updated_refs_.end(), "Tag does not exist: {}", name); @@ -171,6 +181,7 @@ UpdateSnapshotReference& UpdateSnapshotReference::ReplaceTag(const std::string& UpdateSnapshotReference& UpdateSnapshotReference::SetMinSnapshotsToKeep( const std::string& name, int32_t min_snapshots_to_keep) { + EnsureMutable(); ICEBERG_BUILDER_CHECK(!name.empty(), "Branch name cannot be empty"); auto it = updated_refs_.find(name); ICEBERG_BUILDER_CHECK(it != updated_refs_.end(), "Branch does not exist: {}", name); @@ -187,6 +198,7 @@ UpdateSnapshotReference& UpdateSnapshotReference::SetMinSnapshotsToKeep( UpdateSnapshotReference& UpdateSnapshotReference::SetMaxSnapshotAgeMs( const std::string& name, int64_t max_snapshot_age_ms) { + EnsureMutable(); ICEBERG_BUILDER_CHECK(!name.empty(), "Branch name cannot be empty"); auto it = updated_refs_.find(name); ICEBERG_BUILDER_CHECK(it != updated_refs_.end(), "Branch does not exist: {}", name); @@ -203,6 +215,7 @@ UpdateSnapshotReference& UpdateSnapshotReference::SetMaxSnapshotAgeMs( UpdateSnapshotReference& UpdateSnapshotReference::SetMaxRefAgeMs(const std::string& name, int64_t max_ref_age_ms) { + EnsureMutable(); ICEBERG_BUILDER_CHECK(!name.empty(), "Reference name cannot be empty"); auto it = updated_refs_.find(name); ICEBERG_BUILDER_CHECK(it != updated_refs_.end(), "Ref does not exist: {}", name); @@ -217,7 +230,7 @@ UpdateSnapshotReference& UpdateSnapshotReference::SetMaxRefAgeMs(const std::stri return *this; } -Result UpdateSnapshotReference::Apply() { +Result UpdateSnapshotReference::Validate() const { ICEBERG_RETURN_UNEXPECTED(CheckErrors()); ApplyResult result; @@ -234,11 +247,20 @@ Result UpdateSnapshotReference::Apply() { for (const auto& [name, ref] : updated_refs_) { if (auto iter = current_refs.find(name); iter == current_refs.end() || *iter->second != *ref) { - result.to_set.emplace_back(name, ref); + result.to_set.emplace_back(name, std::make_shared(*ref)); } } return result; } +Status UpdateSnapshotReference::Freeze() { + for (auto& [_, value] : updated_refs_) { + if (value) { + value = std::make_shared(*value); + } + } + return {}; +} + } // namespace iceberg diff --git a/src/iceberg/update/update_snapshot_reference.h b/src/iceberg/update/update_snapshot_reference.h index 9ff0a5083..bd40938d4 100644 --- a/src/iceberg/update/update_snapshot_reference.h +++ b/src/iceberg/update/update_snapshot_reference.h @@ -148,10 +148,12 @@ class ICEBERG_EXPORT UpdateSnapshotReference : public PendingUpdate { std::vector to_remove; }; - /// \brief Apply the pending changes and return the updated and removed references. - Result Apply(); + /// \brief Validate and preview changes without staging or modifying this update. + Result Validate() const; private: + Status Freeze() override; + explicit UpdateSnapshotReference(std::shared_ptr ctx); UpdateSnapshotReference& ReplaceBranchInternal(const std::string& from, diff --git a/src/iceberg/update/update_sort_order.cc b/src/iceberg/update/update_sort_order.cc index 8086b903f..6c79ef5ae 100644 --- a/src/iceberg/update/update_sort_order.cc +++ b/src/iceberg/update/update_sort_order.cc @@ -47,6 +47,7 @@ UpdateSortOrder::~UpdateSortOrder() = default; UpdateSortOrder& UpdateSortOrder::AddSortField(const std::shared_ptr& term, SortDirection direction, NullOrder null_order) { + EnsureMutable(); ICEBERG_BUILDER_CHECK(term != nullptr, "Term cannot be null"); ICEBERG_BUILDER_CHECK(term->is_unbound(), "Term must be unbound"); @@ -75,17 +76,19 @@ UpdateSortOrder& UpdateSortOrder::AddSortField(const std::shared_ptr& term UpdateSortOrder& UpdateSortOrder::AddSortFieldByName(std::string_view name, SortDirection direction, NullOrder null_order) { + EnsureMutable(); ICEBERG_BUILDER_ASSIGN_OR_RETURN(auto named_ref, NamedReference::Make(std::string(name))); return AddSortField(std::move(named_ref), direction, null_order); } UpdateSortOrder& UpdateSortOrder::CaseSensitive(bool case_sensitive) { + EnsureMutable(); case_sensitive_ = case_sensitive; return *this; } -Result> UpdateSortOrder::Apply() { +Result> UpdateSortOrder::Validate() const { ICEBERG_RETURN_UNEXPECTED(CheckErrors()); // If no sort fields are specified, return an unsorted order (ID = 0). @@ -96,11 +99,25 @@ Result> UpdateSortOrder::Apply() { // Use -1 as a placeholder for non-empty sort orders. // The actual sort order ID will be assigned by TableMetadataBuilder when // the AddSortOrder update is applied. - ICEBERG_ASSIGN_OR_RAISE(order, SortOrder::Make(/*sort_id=*/-1, sort_fields_)); + std::vector fields; + for (const auto& field : sort_fields_) { + fields.emplace_back(field.source_id(), + std::make_shared(*field.transform()), + field.direction(), field.null_order()); + } + ICEBERG_ASSIGN_OR_RAISE(order, SortOrder::Make(/*sort_id=*/-1, std::move(fields))); ICEBERG_ASSIGN_OR_RAISE(auto schema, base().Schema()); ICEBERG_RETURN_UNEXPECTED(order->Validate(*schema)); } return order; } +Status UpdateSortOrder::Freeze() { + for (auto& field : sort_fields_) { + field = SortField(field.source_id(), std::make_shared(*field.transform()), + field.direction(), field.null_order()); + } + return {}; +} + } // namespace iceberg diff --git a/src/iceberg/update/update_sort_order.h b/src/iceberg/update/update_sort_order.h index 4696ae72d..11cf88387 100644 --- a/src/iceberg/update/update_sort_order.h +++ b/src/iceberg/update/update_sort_order.h @@ -68,10 +68,14 @@ class ICEBERG_EXPORT UpdateSortOrder : public PendingUpdate { Kind kind() const final { return Kind::kUpdateSortOrder; } bool IsRetryable() const override { return true; } - /// \brief Apply the pending changes and return the new SortOrder. - Result> Apply(); + /// \brief Validate and preview changes without staging or modifying this update. + Result> Validate() const; private: + Status Freeze() override; + + bool MayAddFileReferences() const override { return false; } + explicit UpdateSortOrder(std::shared_ptr ctx); std::vector sort_fields_; diff --git a/src/iceberg/update/update_statistics.cc b/src/iceberg/update/update_statistics.cc index afa6ea0c6..1e7208b0f 100644 --- a/src/iceberg/update/update_statistics.cc +++ b/src/iceberg/update/update_statistics.cc @@ -44,23 +44,25 @@ UpdateStatistics::~UpdateStatistics() = default; UpdateStatistics& UpdateStatistics::SetStatistics( std::shared_ptr statistics_file) { + EnsureMutable(); ICEBERG_BUILDER_CHECK(statistics_file != nullptr, "Statistics file cannot be null"); statistics_to_set_[statistics_file->snapshot_id] = std::move(statistics_file); return *this; } UpdateStatistics& UpdateStatistics::RemoveStatistics(int64_t snapshot_id) { + EnsureMutable(); statistics_to_set_[snapshot_id] = nullptr; return *this; } -Result UpdateStatistics::Apply() { +Result UpdateStatistics::Validate() const { ICEBERG_RETURN_UNEXPECTED(CheckErrors()); ApplyResult result; for (const auto& [snapshot_id, stats] : statistics_to_set_) { if (stats) { - result.to_set.emplace_back(snapshot_id, stats); + result.to_set.emplace_back(snapshot_id, std::make_shared(*stats)); } else { result.to_remove.push_back(snapshot_id); } @@ -68,4 +70,13 @@ Result UpdateStatistics::Apply() { return result; } +Status UpdateStatistics::Freeze() { + for (auto& [_, value] : statistics_to_set_) { + if (value) { + value = std::make_shared(*value); + } + } + return {}; +} + } // namespace iceberg diff --git a/src/iceberg/update/update_statistics.h b/src/iceberg/update/update_statistics.h index 4a6c12fc4..28ec30bb4 100644 --- a/src/iceberg/update/update_statistics.h +++ b/src/iceberg/update/update_statistics.h @@ -73,9 +73,12 @@ class ICEBERG_EXPORT UpdateStatistics : public PendingUpdate { std::vector to_remove; }; - Result Apply(); + /// \brief Validate and preview changes without staging or modifying this update. + Result Validate() const; private: + Status Freeze() override; + explicit UpdateStatistics(std::shared_ptr ctx); std::unordered_map> statistics_to_set_; diff --git a/src/iceberg/update/update_util_internal.h b/src/iceberg/update/update_util_internal.h new file mode 100644 index 000000000..4ef49fca3 --- /dev/null +++ b/src/iceberg/update/update_util_internal.h @@ -0,0 +1,191 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#pragma once + +#include "iceberg/expression/expression.h" +#include "iceberg/expression/literal.h" +#include "iceberg/expression/predicate.h" +#include "iceberg/expression/term.h" +#include "iceberg/manifest/manifest_entry.h" +#include "iceberg/schema_field.h" +#include "iceberg/transform.h" +#include "iceberg/type.h" +#include "iceberg/util/checked_cast.h" +#include "iceberg/util/macros.h" + +namespace iceberg::internal { + +// Most primitive types have no mutable state. Parameterized types must be copied +// even when they are reached through a const Literal or shared_ptr alias. +// Returns nullptr for stateless primitives, which are safe to share. +inline std::shared_ptr CopyParameterizedPrimitive(const Type& type) { + switch (type.type_id()) { + case TypeId::kDecimal: + return std::make_shared(checked_cast(type)); + case TypeId::kFixed: + return std::make_shared(checked_cast(type)); + case TypeId::kGeometry: + return std::make_shared(checked_cast(type)); + case TypeId::kGeography: + return std::make_shared(checked_cast(type)); + default: + return nullptr; + } +} + +inline Result CopyUpdateLiteral(const Literal& literal) { + auto type = CopyParameterizedPrimitive(*literal.type()); + if (!type) { + return literal; + } + if (literal.IsNull()) { + return Literal::Null(std::move(type)); + } + ICEBERG_ASSIGN_OR_RAISE(auto bytes, literal.Serialize()); + return Literal::Deserialize(bytes, std::move(type)); +} + +inline Result> CopyUpdateType(const std::shared_ptr& type); + +// Deep-copies a field, including nested types and default literals, so a caller +// keeping an alias to any of those objects cannot change frozen replay intent. +inline Result CopyUpdateField(const SchemaField& field) { + ICEBERG_ASSIGN_OR_RAISE(auto type, CopyUpdateType(field.type())); + std::shared_ptr initial_default; + if (field.initial_default()) { + ICEBERG_ASSIGN_OR_RAISE(auto copy, CopyUpdateLiteral(*field.initial_default())); + initial_default = std::make_shared(std::move(copy)); + } + std::shared_ptr write_default; + if (field.write_default()) { + ICEBERG_ASSIGN_OR_RAISE(auto copy, CopyUpdateLiteral(*field.write_default())); + write_default = std::make_shared(std::move(copy)); + } + return SchemaField(field.field_id(), field.name(), std::move(type), field.optional(), + field.doc(), std::move(initial_default), std::move(write_default)); +} + +inline Result> CopyUpdateType(const std::shared_ptr& type) { + if (!type) { + return nullptr; + } + switch (type->type_id()) { + case TypeId::kStruct: { + const auto source_fields = checked_cast(*type).fields(); + std::vector fields; + fields.reserve(source_fields.size()); + for (const auto& field : source_fields) { + ICEBERG_ASSIGN_OR_RAISE(auto copy, CopyUpdateField(field)); + fields.push_back(std::move(copy)); + } + return std::make_shared(std::move(fields)); + } + case TypeId::kList: { + ICEBERG_ASSIGN_OR_RAISE( + auto element, CopyUpdateField(checked_cast(*type).element())); + return std::make_shared(std::move(element)); + } + case TypeId::kMap: { + const auto& map = checked_cast(*type); + ICEBERG_ASSIGN_OR_RAISE(auto key, CopyUpdateField(map.key())); + ICEBERG_ASSIGN_OR_RAISE(auto value, CopyUpdateField(map.value())); + return std::make_shared(std::move(key), std::move(value)); + } + default: + if (auto copy = CopyParameterizedPrimitive(*type)) { + return copy; + } + return type; + } +} + +inline Result> CopyUpdateDataFile(const DataFile& file) { + auto copy = std::make_shared(file); + std::vector values; + values.reserve(file.partition.num_fields()); + for (const auto& value : file.partition.values()) { + ICEBERG_ASSIGN_OR_RAISE(auto frozen, CopyUpdateLiteral(value)); + values.push_back(std::move(frozen)); + } + copy->partition.Reset(std::move(values)); + return copy; +} + +inline Result> CopyUpdateExpression( + const std::shared_ptr& expression) { + if (!expression) { + return nullptr; + } + using Op = Expression::Operation; + switch (expression->op()) { + case Op::kTrue: + return True::Instance(); + case Op::kFalse: + return False::Instance(); + case Op::kAnd: { + const auto& node = checked_cast(*expression); + ICEBERG_ASSIGN_OR_RAISE(auto left, CopyUpdateExpression(node.left())); + ICEBERG_ASSIGN_OR_RAISE(auto right, CopyUpdateExpression(node.right())); + return And::Make(std::move(left), std::move(right)); + } + case Op::kOr: { + const auto& node = checked_cast(*expression); + ICEBERG_ASSIGN_OR_RAISE(auto left, CopyUpdateExpression(node.left())); + ICEBERG_ASSIGN_OR_RAISE(auto right, CopyUpdateExpression(node.right())); + return Or::Make(std::move(left), std::move(right)); + } + case Op::kNot: { + const auto& node = checked_cast(*expression); + ICEBERG_ASSIGN_OR_RAISE(auto child, CopyUpdateExpression(node.child())); + return Not::Make(std::move(child)); + } + default: + break; + } + if (!expression->is_unbound_predicate()) { + // Preserve the normal validation path for already-bound/invalid filters. + return expression; + } + const auto& predicate = checked_cast(*expression); + std::vector values; + for (const auto& value : predicate.literals()) { + ICEBERG_ASSIGN_OR_RAISE(auto copy, CopyUpdateLiteral(value)); + values.push_back(std::move(copy)); + } + const auto& term = predicate.unbound_term(); + if (term.kind() == Term::Kind::kReference) { + const auto& reference = checked_cast(term); + ICEBERG_ASSIGN_OR_RAISE(auto copy, + NamedReference::Make(std::string(reference.name()))); + return UnboundPredicateImpl::Make(expression->op(), std::move(copy), + std::move(values)); + } + const auto& transform = checked_cast(term); + ICEBERG_ASSIGN_OR_RAISE( + auto reference, NamedReference::Make(std::string(transform.reference()->name()))); + ICEBERG_ASSIGN_OR_RAISE( + auto copy, + UnboundTransform::Make(std::move(reference), + std::make_shared(*transform.transform()))); + return UnboundPredicateImpl::Make(expression->op(), std::move(copy), + std::move(values)); +} + +} // namespace iceberg::internal From a990fb6c65f62a8afc869070427612b144a6af6a Mon Sep 17 00:00:00 2001 From: Zhao Junwang Date: Thu, 10 Sep 2026 21:47:35 +0800 Subject: [PATCH 4/5] fix: resolve transaction update CI build failures Use dynamic_cast for virtual-base predicates in all build modes, and initialize Snapshot fields in declaration order. --- src/iceberg/test/fast_append_test.cc | 2 +- src/iceberg/update/update_util_internal.h | 4 +++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/src/iceberg/test/fast_append_test.cc b/src/iceberg/test/fast_append_test.cc index f0b50069d..72d1305af 100644 --- a/src/iceberg/test/fast_append_test.cc +++ b/src/iceberg/test/fast_append_test.cc @@ -1134,8 +1134,8 @@ TEST_F(FastAppendTest, StagedNoopIsCleanedAndCanBecomeEffectiveOnRebase) { update->write_partial = true; auto builder = TableMetadataBuilder::BuildFrom(table_->metadata().get()); auto existing = std::make_shared(Snapshot{ - .sequence_number = table_->metadata()->NextSequenceNumber(), .snapshot_id = update->SnapshotId(), + .sequence_number = table_->metadata()->NextSequenceNumber(), .timestamp_ms = CurrentTimePointMs(), .manifest_list = table_location_ + "/metadata/existing-noop-list.avro", .summary = {{SnapshotSummaryFields::kOperation, DataOperation::kAppend}}, diff --git a/src/iceberg/update/update_util_internal.h b/src/iceberg/update/update_util_internal.h index 4ef49fca3..cd7d1fe51 100644 --- a/src/iceberg/update/update_util_internal.h +++ b/src/iceberg/update/update_util_internal.h @@ -163,7 +163,9 @@ inline Result> CopyUpdateExpression( // Preserve the normal validation path for already-bound/invalid filters. return expression; } - const auto& predicate = checked_cast(*expression); + // Expression is a virtual base of UnboundPredicate, so this cast must stay + // dynamic in release builds too. + const auto& predicate = dynamic_cast(*expression); std::vector values; for (const auto& value : predicate.literals()) { ICEBERG_ASSIGN_OR_RAISE(auto copy, CopyUpdateLiteral(value)); From 309f4a5182caac1fbdf24e0978ba19a0ab6f79ef Mon Sep 17 00:00:00 2001 From: Zhao Junwang Date: Thu, 10 Sep 2026 23:18:50 +0800 Subject: [PATCH 5/5] test: use std::array for schema identifier names --- src/iceberg/test/update_schema_test.cc | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/iceberg/test/update_schema_test.cc b/src/iceberg/test/update_schema_test.cc index 1ee828ce7..91b9f5f9a 100644 --- a/src/iceberg/test/update_schema_test.cc +++ b/src/iceberg/test/update_schema_test.cc @@ -19,6 +19,7 @@ #include "iceberg/update/update_schema.h" +#include #include #include #include @@ -312,7 +313,7 @@ TEST_F(UpdateSchemaDefaultValueTest, FrozenNestedInputsAndPreviewsAreIsolated) { ICEBERG_UNWRAP_OR_FAIL(auto update, txn->NewUpdateSchema()); update->AddColumn("nested", map_type, "nested doc") .AddRequiredColumn("copy_id", int64(), "identifier doc", Literal::Long(42)); - std::string_view identifier_names[] = {"copy_id"}; + std::array identifier_names = {"copy_id"}; std::span identifiers(identifier_names); update->SetIdentifierFields(identifiers); ICEBERG_UNWRAP_OR_FAIL(auto expected, update->Validate());