diff --git a/src/iceberg/test/expire_snapshots_test.cc b/src/iceberg/test/expire_snapshots_test.cc index b96754ec0..bf9b1e7e5 100644 --- a/src/iceberg/test/expire_snapshots_test.cc +++ b/src/iceberg/test/expire_snapshots_test.cc @@ -40,6 +40,9 @@ #include "iceberg/test/executor.h" #include "iceberg/test/matchers.h" #include "iceberg/test/update_test_base.h" +#include "iceberg/transaction.h" +#include "iceberg/update/fast_append.h" +#include "iceberg/update/set_snapshot.h" namespace iceberg { @@ -221,7 +224,7 @@ class ExpireSnapshotsCleanupTest : public UpdateTestBase { TEST_F(ExpireSnapshotsTest, DefaultExpireByAge) { ICEBERG_UNWRAP_OR_FAIL(auto update, table_->NewExpireSnapshots()); - ICEBERG_UNWRAP_OR_FAIL(auto result, update->Apply()); + ICEBERG_UNWRAP_OR_FAIL(auto result, update->Validate()); EXPECT_EQ(result.snapshot_ids_to_remove.size(), 1); EXPECT_EQ(result.snapshot_ids_to_remove.at(0), 3051729675574597004); } @@ -229,7 +232,7 @@ TEST_F(ExpireSnapshotsTest, DefaultExpireByAge) { TEST_F(ExpireSnapshotsTest, KeepAll) { ICEBERG_UNWRAP_OR_FAIL(auto update, table_->NewExpireSnapshots()); update->RetainLast(2); - ICEBERG_UNWRAP_OR_FAIL(auto result, update->Apply()); + ICEBERG_UNWRAP_OR_FAIL(auto result, update->Validate()); EXPECT_TRUE(result.snapshot_ids_to_remove.empty()); EXPECT_TRUE(result.refs_to_remove.empty()); } @@ -237,7 +240,7 @@ TEST_F(ExpireSnapshotsTest, KeepAll) { TEST_F(ExpireSnapshotsTest, ExpireById) { ICEBERG_UNWRAP_OR_FAIL(auto update, table_->NewExpireSnapshots()); update->ExpireSnapshotId(3051729675574597004); - ICEBERG_UNWRAP_OR_FAIL(auto result, update->Apply()); + ICEBERG_UNWRAP_OR_FAIL(auto result, update->Validate()); EXPECT_EQ(result.snapshot_ids_to_remove.size(), 1); EXPECT_EQ(result.snapshot_ids_to_remove.at(0), 3051729675574597004); } @@ -247,7 +250,7 @@ TEST_F(ExpireSnapshotsTest, ExpireByIdOverridesRetainLast) { update->RetainLast(2); update->ExpireSnapshotId(3051729675574597004); - ICEBERG_UNWRAP_OR_FAIL(auto result, update->Apply()); + ICEBERG_UNWRAP_OR_FAIL(auto result, update->Validate()); EXPECT_THAT(result.snapshot_ids_to_remove, testing::ElementsAre(3051729675574597004)); } @@ -262,7 +265,7 @@ TEST_F(ExpireSnapshotsTest, ExpireOlderThan) { for (const auto& test_case : test_cases) { ICEBERG_UNWRAP_OR_FAIL(auto update, table_->NewExpireSnapshots()); update->ExpireOlderThan(test_case.expire_older_than); - ICEBERG_UNWRAP_OR_FAIL(auto result, update->Apply()); + ICEBERG_UNWRAP_OR_FAIL(auto result, update->Validate()); EXPECT_EQ(result.snapshot_ids_to_remove.size(), test_case.expected_num_expired); } } @@ -286,26 +289,20 @@ TEST_F(ExpireSnapshotsCleanupTest, RetainsUnreferencedSnapshotAtExpireThreshold) ICEBERG_UNWRAP_OR_FAIL(auto update, table_->NewExpireSnapshots()); update->ExpireOlderThan(expire_at_ms); - ICEBERG_UNWRAP_OR_FAIL(auto result, update->Apply()); + ICEBERG_UNWRAP_OR_FAIL(auto result, update->Validate()); EXPECT_THAT(result.snapshot_ids_to_remove, testing::Not(testing::Contains(unreferenced_snapshot_id))); } -TEST_F(ExpireSnapshotsTest, FinalizeRequiresCommittedMetadata) { +TEST_F(ExpireSnapshotsTest, ValidateDoesNotScheduleCleanup) { std::vector deleted_files; - ICEBERG_UNWRAP_OR_FAIL(auto update, table_->NewExpireSnapshots()); + ICEBERG_UNWRAP_OR_FAIL(auto txn, table_->NewTransaction()); + ICEBERG_UNWRAP_OR_FAIL(auto update, txn->NewExpireSnapshots()); update->DeleteWith( [&deleted_files](const std::string& path) { deleted_files.push_back(path); }); - - // Apply first so apply_result_ is cached - ICEBERG_UNWRAP_OR_FAIL(auto result, update->Apply()); + ICEBERG_UNWRAP_OR_FAIL(auto result, update->Validate()); EXPECT_EQ(result.snapshot_ids_to_remove.size(), 1); - - // A successful finalize now requires the committed metadata from the catalog. - auto finalize_status = update->Finalize(static_cast(nullptr)); - EXPECT_THAT(finalize_status, IsError(ErrorKind::kInvalidArgument)); - EXPECT_THAT(finalize_status, - HasErrorMessage("Missing committed table metadata for cleanup")); + EXPECT_THAT(txn->Abort(), IsOk()); EXPECT_TRUE(deleted_files.empty()); } @@ -316,28 +313,24 @@ TEST_F(ExpireSnapshotsTest, CleanupNoneSkipsDeletion) { update->DeleteWith( [&deleted_files](const std::string& path) { deleted_files.push_back(path); }); - ICEBERG_UNWRAP_OR_FAIL(auto result, update->Apply()); + ICEBERG_UNWRAP_OR_FAIL(auto result, update->Validate()); EXPECT_EQ(result.snapshot_ids_to_remove.size(), 1); // With kNone cleanup level, Finalize should skip all file deletion - auto finalize_status = update->Finalize(static_cast(nullptr)); + auto finalize_status = update->Commit(); EXPECT_THAT(finalize_status, IsOk()); EXPECT_TRUE(deleted_files.empty()); } -TEST_F(ExpireSnapshotsTest, FinalizeSkippedOnCommitError) { +TEST_F(ExpireSnapshotsTest, AbortSkipsExpirationDeletion) { std::vector deleted_files; - ICEBERG_UNWRAP_OR_FAIL(auto update, table_->NewExpireSnapshots()); + ICEBERG_UNWRAP_OR_FAIL(auto txn, table_->NewTransaction()); + ICEBERG_UNWRAP_OR_FAIL(auto update, txn->NewExpireSnapshots()); update->DeleteWith( [&deleted_files](const std::string& path) { deleted_files.push_back(path); }); - - ICEBERG_UNWRAP_OR_FAIL(auto result, update->Apply()); - EXPECT_EQ(result.snapshot_ids_to_remove.size(), 1); - - // Simulate a commit failure - Finalize should not delete any files - auto finalize_status = update->Finalize(Result(std::unexpected( - Error{.kind = ErrorKind::kCommitFailed, .message = "simulated failure"}))); - EXPECT_THAT(finalize_status, IsOk()); + EXPECT_THAT(update->Commit(), IsOk()); + EXPECT_THAT(txn->Abort(), IsOk()); + EXPECT_THAT(txn->Abort(), IsOk()); EXPECT_TRUE(deleted_files.empty()); } @@ -348,11 +341,11 @@ TEST_F(ExpireSnapshotsTest, FinalizeSkipsWhenNothingExpired) { update->DeleteWith( [&deleted_files](const std::string& path) { deleted_files.push_back(path); }); - ICEBERG_UNWRAP_OR_FAIL(auto result, update->Apply()); + ICEBERG_UNWRAP_OR_FAIL(auto result, update->Validate()); EXPECT_TRUE(result.snapshot_ids_to_remove.empty()); // No snapshots expired, so Finalize should not delete any files - auto finalize_status = update->Finalize(static_cast(nullptr)); + auto finalize_status = update->Commit(); EXPECT_THAT(finalize_status, IsOk()); EXPECT_TRUE(deleted_files.empty()); } @@ -1016,4 +1009,97 @@ TEST_F(ExpireSnapshotsCleanupTest, CommitIgnoresMalformedSourceSnapshotIdCleanup EXPECT_EQ(committed_metadata->snapshots.at(0)->snapshot_id, kCurrentSnapshotId); } +class ExpirationLifecycleTest + : public ExpireSnapshotsCleanupTest, + public ::testing::WithParamInterface> {}; + +TEST_P(ExpirationLifecycleTest, LaterReferencesSuppressPhysicalDeletion) { + auto [cleanup_level, custom_delete, reattach, retry] = GetParam(); + const auto data_path = table_location_ + "/data/reattached.parquet"; + const auto manifest_path = table_location_ + "/metadata/expired.avro"; + const auto expired_list = table_location_ + "/metadata/expired-list.avro"; + const auto current_list = table_location_ + "/metadata/current-list.avro"; + auto data_file = MakeDataFile(data_path); + data_file->partition_spec_id = DefaultSpec()->spec_id(); + ASSERT_THAT(file_io_->WriteFile(data_path, "data"), IsOk()); + auto manifest = WriteDataManifest(manifest_path, kExpiredSnapshotId, + {MakeEntry(ManifestStatus::kAdded, kExpiredSnapshotId, + kExpiredSequenceNumber, data_file)}); + WriteManifestList(expired_list, kExpiredSnapshotId, 0, kExpiredSequenceNumber, + {manifest}); + WriteManifestList(current_list, kCurrentSnapshotId, kExpiredSnapshotId, + kCurrentSequenceNumber, {}); + RewriteTableWithManifestLists(expired_list, current_list); + if (retry) { + FailCommits(1); + } + ICEBERG_UNWRAP_OR_FAIL(auto txn, table_->NewTransaction()); + ICEBERG_UNWRAP_OR_FAIL(auto expire, txn->NewExpireSnapshots()); + expire->ExpireSnapshotId(kExpiredSnapshotId).CleanupLevel(cleanup_level); + int deletes = 0; + if (custom_delete) { + expire->DeleteWith([&](const std::string&) { ++deletes; }); + } + ASSERT_THAT(expire->Commit(), IsOk()); + if (reattach) { + // FastAppend inherits the conservative default capability from PendingUpdate. + ICEBERG_UNWRAP_OR_FAIL(auto append, txn->NewFastAppend()); + append->AppendFile(data_file); + ASSERT_THAT(append->Commit(), IsOk()); + } else { + ICEBERG_UNWRAP_OR_FAIL(auto noop, txn->NewSetSnapshot()); + noop->SetCurrentSnapshot(kCurrentSnapshotId); + ASSERT_THAT(noop->Commit(), IsOk()); + } + ASSERT_THAT(txn->Commit(), IsOk()); + EXPECT_FALSE(ReloadMetadata()->SnapshotById(kExpiredSnapshotId).has_value()); + EXPECT_EQ(deletes, 0); + for (const auto& path : {data_path, manifest_path, expired_list}) { + EXPECT_THAT(file_io_->ReadFile(path, std::nullopt), IsOk()); + } + EXPECT_THAT(txn->Commit(), IsError(ErrorKind::kValidationFailed)); + EXPECT_THAT(txn->Abort(), IsError(ErrorKind::kValidationFailed)); + EXPECT_EQ(deletes, 0); +} + +INSTANTIATE_TEST_SUITE_P( + CleanupPolicies, ExpirationLifecycleTest, + ::testing::Combine(::testing::Values(CleanupLevel::kAll, CleanupLevel::kMetadataOnly), + ::testing::Bool(), ::testing::Bool(), ::testing::Bool())); + +TEST_F(ExpireSnapshotsCleanupTest, RetryRecomputesExpirationAgainstRefreshedMetadata) { + const auto expired_list = table_location_ + "/metadata/expire-before-retry.avro"; + const auto current_list = table_location_ + "/metadata/current-before-retry.avro"; + WriteManifestList(expired_list, kExpiredSnapshotId, 0, kExpiredSequenceNumber, {}); + WriteManifestList(current_list, kCurrentSnapshotId, kExpiredSnapshotId, + kCurrentSequenceNumber, {}); + RewriteTableWithManifestLists(expired_list, current_list); + auto data_file = MakeDataFile(table_location_ + "/data/concurrent.parquet"); + data_file->partition_spec_id = DefaultSpec()->spec_id(); + FailCommits(1, [&](int attempt) { + if (attempt == 0) { + ICEBERG_UNWRAP_OR_FAIL(auto latest, catalog_->LoadTable(table_ident_)); + ICEBERG_UNWRAP_OR_FAIL(auto append, latest->NewFastAppend()); + append->AppendFile(data_file); + ASSERT_THAT(append->Commit(), IsOk()); + } + }); + ICEBERG_UNWRAP_OR_FAIL(auto expire, table_->NewExpireSnapshots()); + expire + ->ExpireOlderThan( + (CurrentTimePointMs() + std::chrono::hours(1)).time_since_epoch().count()) + .RetainLast(1); + std::vector deleted; + expire->DeleteWith([&](const std::string& path) { deleted.push_back(path); }); + ICEBERG_UNWRAP_OR_FAIL(auto preview, expire->Validate()); + EXPECT_THAT(preview.snapshot_ids_to_remove, ::testing::ElementsAre(kExpiredSnapshotId)); + ASSERT_THAT(expire->Commit(), IsOk()); + auto metadata = ReloadMetadata(); + EXPECT_FALSE(metadata->SnapshotById(kExpiredSnapshotId).has_value()); + EXPECT_FALSE(metadata->SnapshotById(kCurrentSnapshotId).has_value()); + EXPECT_EQ(metadata->snapshots.size(), 1U); + EXPECT_THAT(deleted, ::testing::Contains(expired_list)); + EXPECT_THAT(deleted, ::testing::Contains(current_list)); +} + } // namespace iceberg diff --git a/src/iceberg/test/fast_append_test.cc b/src/iceberg/test/fast_append_test.cc index f88d2e011..72d1305af 100644 --- a/src/iceberg/test/fast_append_test.cc +++ b/src/iceberg/test/fast_append_test.cc @@ -20,9 +20,11 @@ #include "iceberg/update/fast_append.h" #include +#include #include #include #include +#include #include #include #include @@ -34,6 +36,8 @@ #include "iceberg/avro/avro_register.h" #include "iceberg/constants.h" +#include "iceberg/exception.h" +#include "iceberg/expression/expressions.h" #include "iceberg/manifest/manifest_entry.h" #include "iceberg/manifest/manifest_reader.h" #include "iceberg/manifest/manifest_writer.h" @@ -45,12 +49,16 @@ #include "iceberg/snapshot.h" #include "iceberg/table_metadata.h" #include "iceberg/table_properties.h" +#include "iceberg/table_update.h" #include "iceberg/test/executor.h" #include "iceberg/test/matchers.h" #include "iceberg/test/mock_catalog.h" #include "iceberg/test/update_test_base.h" #include "iceberg/transaction.h" +#include "iceberg/update/delete_files.h" #include "iceberg/update/merge_append.h" +#include "iceberg/update/overwrite_files.h" +#include "iceberg/update/rewrite_files.h" #include "iceberg/update/update_properties.h" #include "iceberg/util/uuid.h" @@ -64,13 +72,44 @@ class TestSnapshotUpdate : public SnapshotUpdate { : SnapshotUpdate(std::move(ctx)) {} using SnapshotUpdate::ManifestPath; + using SnapshotUpdate::SnapshotId; + std::function apply_callback; + std::function finalize_callback; + std::function report_callback; + int applies = 0; + int finalizes = 0; + int reports = 0; + bool write_partial = false; + std::vector partial_paths; + + protected: Status CleanUncommitted(const std::unordered_set&) override { return {}; } - std::string operation() override { return "test"; } + std::string operation() override { return "append"; } Result> Apply(const TableMetadata&, const std::shared_ptr&) override { + ++applies; + if (write_partial) { + auto path = ManifestPath(); + partial_paths.push_back(path); + ICEBERG_RETURN_UNEXPECTED(ctx_->table->io()->WriteFile(path, "partial manifest")); + } + if (apply_callback) { + ICEBERG_RETURN_UNEXPECTED(apply_callback(applies)); + } return std::vector{}; } + Status Finalize(const TableMetadata& committed) override { + ++finalizes; + if (finalize_callback) { + ICEBERG_RETURN_UNEXPECTED(finalize_callback()); + } + return SnapshotUpdate::Finalize(committed); + } + Status ReportCommitted() override { + ++reports; + return report_callback ? report_callback() : Status{}; + } std::unordered_map Summary() override { return {}; } }; @@ -303,50 +342,74 @@ TEST_F(FastAppendTest, AppendNullFile) { EXPECT_THAT(table_->current_snapshot(), HasErrorMessage("No current snapshot")); } -TEST_F(FastAppendTest, FinalizeIgnoresCleanupDeleteFailure) { - std::shared_ptr fast_append; - ICEBERG_UNWRAP_OR_FAIL(fast_append, table_->NewFastAppend()); +TEST_F(FastAppendTest, AbortIgnoresCleanupDeleteFailure) { + ICEBERG_UNWRAP_OR_FAIL(auto txn, table_->NewTransaction()); + ICEBERG_UNWRAP_OR_FAIL(auto fast_append, txn->NewFastAppend()); + fast_append->AppendFile(file_a_); + int deletes = 0; + fast_append->DeleteWith([&](const std::string&) { + ++deletes; + return IOError("delete failed"); + }); + EXPECT_THAT(fast_append->Commit(), IsOk()); + EXPECT_THAT(txn->Abort(), IsOk()); + EXPECT_EQ(deletes, 2); + EXPECT_THAT(txn->Abort(), IsOk()); + EXPECT_EQ(deletes, 2); +} + +TEST_F(FastAppendTest, TransactionApplyFailureCleansUpStagedFiles) { + ICEBERG_UNWRAP_OR_FAIL(auto txn, table_->NewTransaction()); + ICEBERG_UNWRAP_OR_FAIL(auto fast_append, txn->NewFastAppend()); + std::vector deleted_paths; + fast_append->DeleteWith([&](const std::string& path) { + deleted_paths.push_back(path); + return file_io_->DeleteFile(path); + }); fast_append->AppendFile(file_a_); - fast_append->DeleteWith([](const std::string&) { return IOError("delete failed"); }); + EXPECT_THAT(fast_append->Commit(), IsOk()); - EXPECT_THAT(static_cast(*fast_append).Apply(), IsOk()); - EXPECT_THAT(fast_append->Finalize(Result( - std::unexpected(CommitFailed("commit failed").error()))), - IsOk()); + ICEBERG_UNWRAP_OR_FAIL(auto failed_append, txn->NewFastAppend()); + failed_append->AppendFile(nullptr); + EXPECT_THAT(failed_append->Commit(), IsError(ErrorKind::kValidationFailed)); + EXPECT_THAT(deleted_paths, ::testing::SizeIs(2U)); + EXPECT_EQ(txn->state(), TransactionState::kFailed); + EXPECT_THAT(txn->Commit(), IsError(ErrorKind::kValidationFailed)); + EXPECT_THAT(txn->Abort(), IsOk()); + EXPECT_THAT(deleted_paths, ::testing::SizeIs(2U)); } TEST_F(FastAppendTest, RetryCopiesAppendManifestAgain) { table_->metadata()->format_version = 1; + ASSERT_THAT( + TableMetadataUtil::Write(*file_io_, std::string(table_->metadata_file_location()), + *table_->metadata()), + IsOk()); const auto path = table_location_ + "/metadata/input.avro"; ICEBERG_UNWRAP_OR_FAIL(auto manifest, WriteManifest(path, {file_a_})); - - std::shared_ptr fast_append; - ICEBERG_UNWRAP_OR_FAIL(fast_append, table_->NewFastAppend()); + FailCommits(2); + ICEBERG_UNWRAP_OR_FAIL(auto fast_append, table_->NewFastAppend()); std::vector deleted_paths; fast_append->DeleteWith([&](const std::string& deleted_path) { deleted_paths.push_back(deleted_path); + if (deleted_paths.size() == 1) { + throw std::runtime_error("cleanup callback threw"); + } + if (deleted_paths.size() == 2) { + return Status(IOError("cleanup returned an error")); + } return file_io_->DeleteFile(deleted_path); }); fast_append->AppendManifest(manifest); - - auto& update = static_cast(*fast_append); - // First Apply() copies the input manifest because v1 cannot inherit snapshot IDs. - ICEBERG_UNWRAP_OR_FAIL(auto first_apply, update.Apply()); - SnapshotCache first_cache(first_apply.snapshot.get()); - ICEBERG_UNWRAP_OR_FAIL(auto first_manifests, first_cache.Manifests(file_io_)); - ASSERT_EQ(first_manifests.size(), 1U); - const auto first_rewritten_path = first_manifests[0].manifest_path; - EXPECT_NE(first_rewritten_path, path); - - // Second Apply() simulates retry cleanup, then copies the original manifest again. - ICEBERG_UNWRAP_OR_FAIL(auto second_apply, update.Apply()); - EXPECT_THAT(deleted_paths, testing::Contains(first_rewritten_path)); - - SnapshotCache second_cache(second_apply.snapshot.get()); - ICEBERG_UNWRAP_OR_FAIL(auto second_manifests, second_cache.Manifests(file_io_)); - ASSERT_EQ(second_manifests.size(), 1U); - EXPECT_NE(second_manifests[0].manifest_path, path); - EXPECT_NE(second_manifests[0].manifest_path, first_rewritten_path); + EXPECT_THAT(fast_append->Commit(), IsOk()); + EXPECT_THAT(table_->Refresh(), IsOk()); + ICEBERG_UNWRAP_OR_FAIL(auto manifests, CurrentDataManifests()); + ASSERT_EQ(manifests.size(), 1U); + EXPECT_NE(manifests[0].manifest_path, path); + EXPECT_THAT(deleted_paths, ::testing::SizeIs(4U)); + EXPECT_THAT(deleted_paths, ::testing::Not(::testing::Contains(path))); + EXPECT_THAT(deleted_paths, + ::testing::Not(::testing::Contains(manifests[0].manifest_path))); } TEST_F(FastAppendTest, AppendDuplicateFile) { @@ -738,4 +801,464 @@ TEST_F(FastAppendMetricsTest, CommitStateUnknownDoesNotReport) { EXPECT_TRUE(reporter_->reports().empty()); } +TEST_F(FastAppendTest, ReplayApplyFailureStopsRetryAndCleansPartialFiles) { + for (auto replay_kind : + {ErrorKind::kRetryableValidationFailed, ErrorKind::kCommitStateUnknown}) { + SCOPED_TRACE(static_cast(replay_kind)); + auto mock = std::make_shared<::testing::NiceMock>(); + EXPECT_CALL(*mock, UpdateTable(::testing::_, ::testing::_, ::testing::_)) + .Times(1) + .WillOnce(::testing::Return(CommitFailed("first conflict"))); + EXPECT_CALL(*mock, LoadTable(::testing::_)).Times(1).WillOnce([&](const auto& name) { + return catalog_->LoadTable(name); + }); + ICEBERG_UNWRAP_OR_FAIL( + auto table, + Table::Make(table_->name(), table_->metadata(), + std::string(table_->metadata_file_location()), file_io_, mock)); + ICEBERG_UNWRAP_OR_FAIL(auto ctx, + TransactionContext::Make(table, TransactionKind::kUpdate)); + auto update = std::make_shared(ctx); + update->write_partial = true; + update->apply_callback = [replay_kind](int attempt) -> Status { + if (attempt == 2) { + return std::unexpected( + Error{.kind = replay_kind, .message = "replay validation failed"}); + } + return {}; + }; + std::vector deleted; + update->DeleteWith([&](const std::string& path) { + deleted.push_back(path); + EXPECT_THAT(update->Commit(), IsError(ErrorKind::kValidationFailed)); + EXPECT_THROW(update->Set("reenter", "value"), IcebergError); + return file_io_->DeleteFile(path); + }); + EXPECT_THAT(update->Commit(), + ::testing::AllOf(IsError(replay_kind), + HasErrorMessage("replay validation failed"))); + EXPECT_EQ(update->applies, 2); + EXPECT_EQ(update->finalizes, 0); + EXPECT_EQ(update->reports, 0); + EXPECT_THAT(deleted, ::testing::SizeIs(3U)); + for (const auto& path : update->partial_paths) { + EXPECT_THAT(deleted, ::testing::Contains(path)); + EXPECT_FALSE(file_io_->ReadFile(path, std::nullopt).has_value()); + } + EXPECT_THAT(update->Commit(), IsError(ErrorKind::kValidationFailed)); + EXPECT_FALSE(ctx->transaction.has_value()); + EXPECT_THAT(deleted, ::testing::SizeIs(3U)); + } +} + +TEST_F(FastAppendTest, StandaloneHooksAreTerminalAndIsolated) { + ICEBERG_UNWRAP_OR_FAIL(auto ctx, + TransactionContext::Make(table_, TransactionKind::kUpdate)); + auto update = std::make_shared(ctx); + update->finalize_callback = [&]() -> Status { + EXPECT_THAT(update->Commit(), IsError(ErrorKind::kValidationFailed)); + EXPECT_THROW(update->DeleteWith({}), IcebergError); + throw std::runtime_error("finalize failure"); + }; + update->report_callback = [&]() -> Status { + EXPECT_THAT(update->Commit(), IsError(ErrorKind::kValidationFailed)); + throw std::runtime_error("report failure"); + }; + ASSERT_THAT(update->Commit(), IsOk()); + EXPECT_EQ(update->finalizes, 1); + EXPECT_EQ(update->reports, 1); + EXPECT_THAT(update->Commit(), IsError(ErrorKind::kValidationFailed)); + EXPECT_EQ(update->finalizes, 1); + EXPECT_EQ(update->reports, 1); +} + +class CallbackReporter : public MetricsReporter { + public: + explicit CallbackReporter(std::function callback) + : callback_(std::move(callback)) {} + Status Report(const MetricsReport&) override { return callback_(); } + + private: + std::function callback_; +}; + +TEST_F(FastAppendTest, ExplicitSuccessPublishesAllTerminalMarkersBeforeReporting) { + ICEBERG_UNWRAP_OR_FAIL(auto txn, table_->NewTransaction()); + ICEBERG_UNWRAP_OR_FAIL(auto first, txn->NewFastAppend()); + std::shared_ptr second; + int reports = 0; + first->AppendFile(file_a_).ReportWith( + std::make_shared([&]() -> Status { + ++reports; + EXPECT_EQ(txn->state(), TransactionState::kCommitted); + EXPECT_THAT(first->Commit(), IsError(ErrorKind::kValidationFailed)); + EXPECT_THAT(second->Commit(), IsError(ErrorKind::kValidationFailed)); + EXPECT_THAT(txn->Commit(), IsError(ErrorKind::kValidationFailed)); + EXPECT_THAT(txn->NewFastAppend(), IsError(ErrorKind::kValidationFailed)); + EXPECT_THAT(txn->Abort(), IsError(ErrorKind::kValidationFailed)); + throw std::runtime_error("report failure"); + })); + ASSERT_THAT(first->Commit(), IsOk()); + ICEBERG_UNWRAP_OR_FAIL(second, txn->NewFastAppend()); + second->AppendFile(file_b_).ReportWith( + std::make_shared([&]() -> Status { + ++reports; + return IOError("another report failure"); + })); + ASSERT_THAT(second->Commit(), IsOk()); + ASSERT_THAT(txn->Commit(), IsOk()); + EXPECT_EQ(reports, 2); + EXPECT_THAT(txn->Commit(), IsError(ErrorKind::kValidationFailed)); + EXPECT_EQ(reports, 2); +} + +TEST_F(FastAppendTest, FrozenFileAliasesCannotChangeReplay) { + FailCommits(2); + ICEBERG_UNWRAP_OR_FAIL(auto txn, table_->NewTransaction()); + ICEBERG_UNWRAP_OR_FAIL(auto append, txn->NewFastAppend()); + append->AppendFile(file_a_); + const auto original_path = file_a_->file_path; + const auto original_records = file_a_->record_count; + ASSERT_THAT(append->Commit(), IsOk()); + EXPECT_THROW(append->AppendFile(file_b_), IcebergError); + EXPECT_THROW(append->DeleteWith({}), IcebergError); + file_a_->file_path = "/changed.parquet"; + file_a_->record_count = 123456; + file_a_->partition = PartitionValues({Literal::Long(42)}); + ASSERT_THAT(txn->Commit(), IsOk()); + EXPECT_THAT(table_->Refresh(), IsOk()); + ICEBERG_UNWRAP_OR_FAIL(auto manifests, CurrentDataManifests()); + ASSERT_EQ(manifests.size(), 1U); + ICEBERG_UNWRAP_OR_FAIL(auto entries, ReadEntries(manifests[0])); + ASSERT_EQ(entries.size(), 1U); + EXPECT_EQ(entries[0].data_file->file_path, original_path); + EXPECT_EQ(entries[0].data_file->record_count, original_records); + ICEBERG_UNWRAP_OR_FAIL(auto snapshot, table_->current_snapshot()); + EXPECT_EQ(snapshot->summary.at("added-records"), std::to_string(original_records)); +} + +TEST_F(FastAppendTest, ConflictThenUnknownPreservesLastAttempt) { + auto mock = std::make_shared<::testing::NiceMock>(); + EXPECT_CALL(*mock, UpdateTable(::testing::_, ::testing::_, ::testing::_)) + .WillOnce(::testing::Return(CommitFailed("conflict"))) + .WillOnce(::testing::Return(CommitStateUnknown("unknown"))); + EXPECT_CALL(*mock, LoadTable(::testing::_)).WillOnce([&](const auto& name) { + return catalog_->LoadTable(name); + }); + ICEBERG_UNWRAP_OR_FAIL( + auto table, + Table::Make(table_->name(), table_->metadata(), + std::string(table_->metadata_file_location()), file_io_, mock)); + ICEBERG_UNWRAP_OR_FAIL(auto txn, table->NewTransaction()); + ICEBERG_UNWRAP_OR_FAIL(auto append, txn->NewFastAppend()); + int deletes = 0; + append->AppendFile(file_a_).DeleteWith([&](const std::string& path) { + ++deletes; + EXPECT_THAT(txn->Abort(), IsError(ErrorKind::kValidationFailed)); + EXPECT_THAT(txn->NewFastAppend(), IsError(ErrorKind::kValidationFailed)); + return file_io_->DeleteFile(path); + }); + ASSERT_THAT(append->Commit(), IsOk()); + EXPECT_THAT(txn->Commit(), IsError(ErrorKind::kCommitStateUnknown)); + EXPECT_EQ(txn->state(), TransactionState::kCommitStateUnknown); + EXPECT_EQ(deletes, 2); + ICEBERG_UNWRAP_OR_FAIL(auto snapshot, txn->current().Snapshot()); + EXPECT_THAT(file_io_->ReadFile(snapshot->manifest_list, std::nullopt), IsOk()); + SnapshotCache cache(snapshot.get()); + ICEBERG_UNWRAP_OR_FAIL(auto manifests, cache.Manifests(file_io_)); + ASSERT_EQ(manifests.size(), 1U); + EXPECT_THAT(file_io_->ReadFile(manifests[0].manifest_path, std::nullopt), IsOk()); + EXPECT_THAT(txn->Abort(), IsError(ErrorKind::kValidationFailed)); + EXPECT_THAT(txn->Commit(), IsError(ErrorKind::kValidationFailed)); + EXPECT_THAT(append->Commit(), IsError(ErrorKind::kValidationFailed)); + EXPECT_EQ(deletes, 2); +} + +TEST_F(FastAppendTest, ApplyFailureAfterWritingCleansEveryUpdate) { + ICEBERG_UNWRAP_OR_FAIL(auto txn, table_->NewTransaction()); + int deletes = 0; + auto delete_file = [&](const std::string& path) -> Status { + ++deletes; + EXPECT_EQ(txn->state(), TransactionState::kFailed); + EXPECT_THAT(txn->Abort(), IsError(ErrorKind::kValidationFailed)); + if (deletes == 1) { + throw std::runtime_error("delete callback threw"); + } + return IOError("delete failure"); + }; + ICEBERG_UNWRAP_OR_FAIL(auto first, txn->NewFastAppend()); + first->AppendFile(file_a_).DeleteWith(delete_file); + ASSERT_THAT(first->Commit(), IsOk()); + ICEBERG_UNWRAP_OR_FAIL(auto failed, txn->NewRewriteFiles()); + failed->DeleteDataFile(file_a_).AddDataFile(file_b_).DeleteWith(delete_file); + EXPECT_THAT(failed->Commit(), HasErrorMessage("Invalid REPLACE operation")); + EXPECT_EQ(txn->state(), TransactionState::kFailed); + EXPECT_GE(deletes, 4); + const int cleanup_attempts = deletes; + EXPECT_THAT(txn->Commit(), IsError(ErrorKind::kValidationFailed)); + EXPECT_THAT(txn->Abort(), IsOk()); + EXPECT_THAT(txn->Abort(), IsOk()); + EXPECT_EQ(deletes, cleanup_attempts); +} + +TEST_F(FastAppendTest, ReplayBecomingNoopCleansStagingWithoutReporting) { + for (bool other_update : {false, true}) { + SCOPED_TRACE(other_update); + auto mock = std::make_shared<::testing::NiceMock>(); + std::shared_ptr refreshed; + std::shared_ptr refreshed_metadata; + int attempts = 0; + EXPECT_CALL(*mock, UpdateTable(::testing::_, ::testing::_, ::testing::_)) + .Times(other_update ? 2 : 1) + .WillRepeatedly([&](const auto&, const auto&, + const auto& changes) -> Result> { + const bool first_attempt = ++attempts == 1; + auto builder = TableMetadataBuilder::BuildFrom( + first_attempt ? table_->metadata().get() : refreshed_metadata.get()); + for (const auto& change : changes) { + if (first_attempt && change->kind() == TableUpdate::Kind::kSetProperties) { + continue; + } + if (!first_attempt) { + EXPECT_EQ(change->kind(), TableUpdate::Kind::kSetProperties); + } + change->ApplyTo(*builder); + } + ICEBERG_ASSIGN_OR_RAISE(std::shared_ptr metadata, + builder->Build()); + if (first_attempt) { + // Use independent files for the refreshed snapshot; generated paths are + // owned by the update that created them. + metadata->snapshots.back() = + std::make_shared(*metadata->snapshots.back()); + auto snapshot = metadata->snapshots.back(); + snapshot->manifest_list = table_location_ + "/metadata/concurrent-list.avro"; + ICEBERG_ASSIGN_OR_RAISE( + auto writer, ManifestListWriter::MakeWriter( + metadata->format_version, snapshot->snapshot_id, + snapshot->parent_snapshot_id, snapshot->manifest_list, + file_io_, snapshot->sequence_number)); + ICEBERG_RETURN_UNEXPECTED(writer->Close()); + } + // Simulate the same logical snapshot already becoming current on refresh. + refreshed_metadata = metadata; + ICEBERG_ASSIGN_OR_RAISE( + refreshed, + Table::Make(table_->name(), std::move(metadata), + std::string(table_->metadata_file_location()) + ".refreshed", + file_io_, mock)); + if (first_attempt) { + return CommitFailed("conflict"); + } + return refreshed; + }); + EXPECT_CALL(*mock, LoadTable(::testing::_)) + .WillOnce( + [&](const auto&) -> Result> { return refreshed; }); + ICEBERG_UNWRAP_OR_FAIL( + auto table, + Table::Make(table_->name(), table_->metadata(), + std::string(table_->metadata_file_location()), file_io_, mock)); + std::shared_ptr txn; + if (other_update) { + ICEBERG_UNWRAP_OR_FAIL(txn, table->NewTransaction()); + } + ICEBERG_UNWRAP_OR_FAIL(auto append, + txn ? txn->NewFastAppend() : table->NewFastAppend()); + int reports = 0; + int deletes = 0; + append->AppendFile(file_a_) + .ReportWith(std::make_shared([&]() -> Status { + ++reports; + return {}; + })) + .DeleteWith([&](const std::string& path) { + ++deletes; + return file_io_->DeleteFile(path); + }); + ASSERT_THAT(append->Commit(), IsOk()); + if (txn) { + ICEBERG_UNWRAP_OR_FAIL(auto props, txn->NewUpdateProperties()); + props->Set("effective", "value"); + ASSERT_THAT(props->Commit(), IsOk()); + ICEBERG_UNWRAP_OR_FAIL(auto committed, txn->Commit()); + EXPECT_EQ(committed->properties().configs().at("effective"), "value"); + EXPECT_EQ(txn->state(), TransactionState::kCommitted); + } + EXPECT_EQ(deletes, 4); + EXPECT_EQ(reports, 0); + EXPECT_THAT(append->Commit(), IsError(ErrorKind::kValidationFailed)); + EXPECT_EQ(deletes, 4); + } +} + +TEST_F(FastAppendTest, FrozenFilterAliasesCannotChangeReplay) { + file_a_->partition = PartitionValues({Literal::Long(1)}); + file_b_->partition = PartitionValues({Literal::Long(2)}); + ICEBERG_UNWRAP_OR_FAIL(auto first, table_->NewFastAppend()); + first->AppendFile(file_a_).AppendFile(file_b_); + ASSERT_THAT(first->Commit(), IsOk()); + ASSERT_THAT(table_->Refresh(), IsOk()); + FailCommits(2); + ICEBERG_UNWRAP_OR_FAIL(auto txn, table_->NewTransaction()); + ICEBERG_UNWRAP_OR_FAIL(auto update, txn->NewDeleteFiles()); + auto reference = Expressions::Ref("x"); + auto predicate = Expressions::Equal(reference, Literal::Long(1)); + update->DeleteFromRowFilter(predicate); + ASSERT_THAT(update->Commit(), IsOk()); + *reference = *Expressions::Ref("missing"); + ASSERT_THAT(txn->Commit(), IsOk()); + ASSERT_THAT(table_->Refresh(), IsOk()); + ICEBERG_UNWRAP_OR_FAIL(auto snapshot, table_->current_snapshot()); + EXPECT_EQ(snapshot->summary.at("deleted-data-files"), "1"); +} + +TEST_F(FastAppendTest, StagedNoopIsCleanedAndCanBecomeEffectiveOnRebase) { + for (bool rebase : {false, true}) { + SCOPED_TRACE(rebase); + auto mock = std::make_shared<::testing::NiceMock>(); + ON_CALL(*mock, LoadTable(::testing::_)).WillByDefault([&](const auto& name) { + return catalog_->LoadTable(name); + }); + ON_CALL(*mock, UpdateTable(::testing::_, ::testing::_, ::testing::_)) + .WillByDefault( + [&](const auto& name, const auto& requirements, const auto& changes) { + return catalog_->UpdateTable(name, requirements, changes); + }); + EXPECT_CALL(*mock, LoadTable(::testing::_)).Times(rebase ? 1 : 0); + EXPECT_CALL(*mock, UpdateTable(::testing::_, ::testing::_, ::testing::_)) + .Times(rebase ? 1 : 0); + ICEBERG_UNWRAP_OR_FAIL(auto ctx, + TransactionContext::Make(table_, TransactionKind::kUpdate)); + auto update = std::make_shared(ctx); + update->write_partial = true; + auto builder = TableMetadataBuilder::BuildFrom(table_->metadata().get()); + auto existing = std::make_shared(Snapshot{ + .snapshot_id = update->SnapshotId(), + .sequence_number = table_->metadata()->NextSequenceNumber(), + .timestamp_ms = CurrentTimePointMs(), + .manifest_list = table_location_ + "/metadata/existing-noop-list.avro", + .summary = {{SnapshotSummaryFields::kOperation, DataOperation::kAppend}}, + .schema_id = table_->metadata()->current_schema_id, + }); + ICEBERG_UNWRAP_OR_FAIL( + auto writer, ManifestListWriter::MakeWriter(table_->metadata()->format_version, + existing->snapshot_id, std::nullopt, + existing->manifest_list, file_io_, + existing->sequence_number)); + ASSERT_THAT(writer->Close(), IsOk()); + builder->SetBranchSnapshot(existing, std::string(SnapshotRef::kMainBranch)); + ICEBERG_UNWRAP_OR_FAIL(std::shared_ptr metadata, builder->Build()); + // Prepare a base whose current snapshot has this update's logical ID. + ICEBERG_UNWRAP_OR_FAIL( + ctx->table, + Table::Make(table_->name(), metadata, + std::string(table_->metadata_file_location()) + ".synthetic", + file_io_, mock)); + ctx->metadata_builder = TableMetadataBuilder::BuildFrom(metadata.get()); + std::vector deleted; + update->DeleteWith([&](const std::string& path) { + deleted.push_back(path); + return file_io_->DeleteFile(path); + }); + if (rebase) { + // The table refreshes before the update commits its original builder. + // Initial Apply is a no-op; internal replay sees the real empty table. + ASSERT_THAT(ctx->table->Refresh(), IsOk()); + } + ASSERT_THAT(update->Commit(), IsOk()); + EXPECT_EQ(update->applies, rebase ? 2 : 1); + EXPECT_EQ(update->finalizes, rebase ? 1 : 0); + EXPECT_EQ(update->reports, rebase ? 1 : 0); + EXPECT_THAT(deleted, ::testing::SizeIs(rebase ? 3U : 2U)); + EXPECT_THAT(file_io_->ReadFile(existing->manifest_list, std::nullopt), IsOk()); + EXPECT_THAT(update->Commit(), IsError(ErrorKind::kValidationFailed)); + } +} + +TEST_F(FastAppendTest, ReplayFailurePartwayThroughCleansAllUncommittedGenerations) { + file_a_->partition = PartitionValues({Literal::Long(1)}); + file_b_->partition = PartitionValues({Literal::Long(3)}); + ICEBERG_UNWRAP_OR_FAIL(auto initial, table_->NewFastAppend()); + initial->AppendFile(file_a_); + ASSERT_THAT(initial->Commit(), IsOk()); + ASSERT_THAT(table_->Refresh(), IsOk()); + ICEBERG_UNWRAP_OR_FAIL(auto original_snapshot, table_->current_snapshot()); + auto concurrent_file = CreateDataFile("/data/concurrent.parquet", 1, 10, 1); + auto replacement = CreateDataFile("/data/replacement.parquet", 100, 50, 1); + auto mock = std::make_shared<::testing::NiceMock>(); + EXPECT_CALL(*mock, UpdateTable(::testing::_, ::testing::_, ::testing::_)) + .Times(1) + .WillOnce( + [&](const auto&, const auto&, const auto&) -> Result> { + ICEBERG_ASSIGN_OR_RAISE(auto latest, catalog_->LoadTable(table_ident_)); + ICEBERG_ASSIGN_OR_RAISE(auto concurrent, latest->NewFastAppend()); + concurrent->AppendFile(concurrent_file); + ICEBERG_RETURN_UNEXPECTED(concurrent->Commit()); + return CommitFailed("concurrent append"); + }); + EXPECT_CALL(*mock, LoadTable(::testing::_)).Times(1).WillOnce([&](const auto& name) { + return catalog_->LoadTable(name); + }); + ICEBERG_UNWRAP_OR_FAIL( + auto table, + Table::Make(table_->name(), table_->metadata(), + std::string(table_->metadata_file_location()), file_io_, mock)); + ICEBERG_UNWRAP_OR_FAIL(auto txn, table->NewTransaction()); + std::vector deleted; + std::shared_ptr append; + auto delete_file = [&](const std::string& path) { + deleted.push_back(path); + const auto state = txn->state(); + EXPECT_THAT(append->Commit(), IsError(ErrorKind::kValidationFailed)); + EXPECT_THAT(txn->Commit(), IsError(ErrorKind::kValidationFailed)); + EXPECT_THAT(txn->NewFastAppend(), IsError(ErrorKind::kValidationFailed)); + EXPECT_THAT(txn->Abort(), IsError(ErrorKind::kValidationFailed)); + EXPECT_EQ(txn->state(), state); + return file_io_->DeleteFile(path); + }; + ICEBERG_UNWRAP_OR_FAIL(append, txn->NewFastAppend()); + append->AppendFile(file_b_).DeleteWith(delete_file); + ASSERT_THAT(append->Commit(), IsOk()); + ICEBERG_UNWRAP_OR_FAIL(auto overwrite, txn->NewOverwrite()); + overwrite->DeleteFile(file_a_) + .AddFile(replacement) + .ValidateFromSnapshot(original_snapshot->snapshot_id) + .ConflictDetectionFilter(Expressions::Equal("x", Literal::Long(1))) + .ValidateNoConflictingData() + .DeleteWith(delete_file); + ASSERT_THAT(overwrite->Commit(), IsOk()); + EXPECT_THAT(txn->Commit(), HasErrorMessage("Found conflicting files")); + EXPECT_EQ(txn->state(), TransactionState::kFailed); + EXPECT_GE(deleted.size(), 6U); + std::unordered_set unique_deletes(deleted.begin(), deleted.end()); + EXPECT_EQ(unique_deletes.size(), deleted.size()); + EXPECT_THAT(txn->Abort(), IsOk()); + EXPECT_EQ(deleted.size(), unique_deletes.size()); + + std::unordered_set committed_paths; + auto metadata = ReloadMetadata(); + for (const auto& snapshot : metadata->snapshots) { + committed_paths.insert(snapshot->manifest_list); + SnapshotCache cache(snapshot.get()); + ICEBERG_UNWRAP_OR_FAIL(auto manifests, cache.Manifests(file_io_)); + for (const auto& manifest : manifests) { + committed_paths.insert(manifest.manifest_path); + } + } + auto& io = static_cast(*file_io_); + ::arrow::fs::FileSelector selector; + selector.base_dir = table_location_ + "/metadata"; + selector.recursive = true; + auto files = io.fs()->GetFileInfo(selector); + ASSERT_TRUE(files.ok()) << files.status(); + std::unordered_set remaining; + for (const auto& file : *files) { + if (file.path().ends_with(".avro")) { + remaining.insert(file.path()); + } + } + EXPECT_EQ(remaining, committed_paths); +} + } // namespace iceberg diff --git a/src/iceberg/test/merge_append_test.cc b/src/iceberg/test/merge_append_test.cc index bd4527904..3bf0473c3 100644 --- a/src/iceberg/test/merge_append_test.cc +++ b/src/iceberg/test/merge_append_test.cc @@ -1212,8 +1212,9 @@ TEST_P(MergeAppendTest, Recovery) { ICEBERG_UNWRAP_OR_FAIL(auto second_snapshot, CurrentSnapshot()); ICEBERG_UNWRAP_OR_FAIL(auto data_manifests, CurrentDataManifests()); ASSERT_EQ(data_manifests.size(), 1U); - EXPECT_EQ(data_manifests[0].manifest_path, pending_manifest.manifest_path); - EXPECT_TRUE(FileExists(pending_manifest.manifest_path)); + EXPECT_NE(data_manifests[0].manifest_path, pending_manifest.manifest_path); + EXPECT_FALSE(FileExists(pending_manifest.manifest_path)); + EXPECT_TRUE(FileExists(data_manifests[0].manifest_path)); ExpectManifestEntries( data_manifests[0], {file_b_, file_a_}, {ManifestStatus::kAdded, ManifestStatus::kExisting}, diff --git a/src/iceberg/test/merging_snapshot_update_test.cc b/src/iceberg/test/merging_snapshot_update_test.cc index 25907c445..1526d2339 100644 --- a/src/iceberg/test/merging_snapshot_update_test.cc +++ b/src/iceberg/test/merging_snapshot_update_test.cc @@ -55,6 +55,7 @@ #include "iceberg/test/retry.h" #include "iceberg/test/update_test_base.h" #include "iceberg/transaction.h" +#include "iceberg/update/delete_files.h" #include "iceberg/update/fast_append.h" #include "iceberg/update/merge_append.h" #include "iceberg/update/row_delta.h" @@ -81,20 +82,24 @@ class MergingSnapshotCapturingReporter final : public MetricsReporter { /// \brief Concrete subclass of MergingSnapshotUpdate for testing. class TestMergeAppend : public MergingSnapshotUpdate { public: - static Result> Make(std::string table_name, + static Result> Make(std::string table_name, std::shared_ptr
table) { ICEBERG_ASSIGN_OR_RAISE( auto ctx, TransactionContext::Make(std::move(table), TransactionKind::kUpdate)); - return std::unique_ptr( + return std::shared_ptr( new TestMergeAppend(std::move(table_name), std::move(ctx))); } std::string operation() override { return "append"; } // Expose protected API for test access - using MergingSnapshotUpdate::Apply; - using MergingSnapshotUpdate::CleanUncommitted; - using MergingSnapshotUpdate::Summary; + Result> CommitManifests() { + ICEBERG_RETURN_UNEXPECTED(Commit()); + ICEBERG_ASSIGN_OR_RAISE(auto snapshot, ctx_->table->current_snapshot()); + SnapshotCache cache(snapshot.get()); + ICEBERG_ASSIGN_OR_RAISE(auto manifests, cache.Manifests(ctx_->table->io())); + return std::vector(manifests.begin(), manifests.end()); + } Status AddFile(std::shared_ptr file) { return AddDataFile(std::move(file)); } Status AddDelete(std::shared_ptr file) { @@ -125,16 +130,6 @@ class TestMergeAppend : public MergingSnapshotUpdate { Result> DataSpec() const { return MergingSnapshotUpdate::DataSpec(); } - Result> WriteDeletesForTest( - std::span> files, - const std::shared_ptr& spec) { - auto entries = files | std::views::transform([](const auto& file) { - return ContentFileWithSequenceNumber{ - .file = file, .data_sequence_number = std::nullopt}; - }) | - std::ranges::to(); - return WriteDeleteManifests(entries, spec); - } int64_t GeneratedSnapshotId() { return SnapshotId(); } void SetDataSeqNumber(int64_t seq) { SetNewDataFilesDataSequenceNumber(seq); } void SetCaseSensitive(bool case_sensitive) { CaseSensitive(case_sensitive); } @@ -231,18 +226,24 @@ class TestMergeAppend : public MergingSnapshotUpdate { class TestOverwriteUpdate : public MergingSnapshotUpdate { public: - static Result> Make(std::string table_name, + static Result> Make(std::string table_name, std::shared_ptr
table) { ICEBERG_ASSIGN_OR_RAISE( auto ctx, TransactionContext::Make(std::move(table), TransactionKind::kUpdate)); - return std::unique_ptr( + return std::shared_ptr( new TestOverwriteUpdate(std::move(table_name), std::move(ctx))); } std::string operation() override { return DataOperation::kOverwrite; } int64_t GeneratedSnapshotId() { return SnapshotId(); } - using MergingSnapshotUpdate::Apply; + Result> CommitManifests() { + ICEBERG_RETURN_UNEXPECTED(Commit()); + ICEBERG_ASSIGN_OR_RAISE(auto snapshot, ctx_->table->current_snapshot()); + SnapshotCache cache(snapshot.get()); + ICEBERG_ASSIGN_OR_RAISE(auto manifests, cache.Manifests(ctx_->table->io())); + return std::vector(manifests.begin(), manifests.end()); + } Status AddDelete(std::shared_ptr file) { return AddDeleteFile(std::move(file)); @@ -335,11 +336,11 @@ class MergingSnapshotUpdateTest : public MinimalUpdateTestBase { return f; } - Result> NewMergeAppend() { + Result> NewMergeAppend() { return TestMergeAppend::Make(TableName(), table_); } - Result> NewOverwriteUpdate() { + Result> NewOverwriteUpdate() { return TestOverwriteUpdate::Make(TableName(), table_); } @@ -892,8 +893,7 @@ TEST_F(MergingSnapshotUpdateTest, CleanUncommittedAfterSuccessfulCommitDoesNotCr EXPECT_THAT(op->AddFile(file_a_), IsOk()); EXPECT_THAT(op->Commit(), IsOk()); - // Cleanup may run from an error handler even after commit success. - EXPECT_THAT(op->CleanUncommitted({}), IsOk()); + EXPECT_THAT(op->Commit(), IsError(ErrorKind::kValidationFailed)); } TEST_F(MergingSnapshotUpdateTest, @@ -905,18 +905,15 @@ TEST_F(MergingSnapshotUpdateTest, EXPECT_THAT(table_->Refresh(), IsOk()); std::vector deleted_paths; - ICEBERG_UNWRAP_OR_FAIL(auto op, NewMergeAppend()); + ICEBERG_UNWRAP_OR_FAIL(auto txn, table_->NewTransaction()); + ICEBERG_UNWRAP_OR_FAIL(auto op, txn->NewDeleteFiles()); op->DeleteWith([&deleted_paths](const std::string& path) { deleted_paths.push_back(path); return Status{}; }); - EXPECT_THAT(op->RemoveDataFile(file_a_), IsOk()); - - ICEBERG_UNWRAP_OR_FAIL( - auto manifests, op->Apply(*table_->metadata(), table_->current_snapshot().value())); - EXPECT_THAT(manifests, ::testing::SizeIs(1)); - - EXPECT_THAT(op->CleanUncommitted({}), IsOk()); + op->DeleteFile(file_a_->file_path); + EXPECT_THAT(op->Commit(), IsOk()); + EXPECT_THAT(txn->Abort(), IsOk()); EXPECT_THAT(deleted_paths, ::testing::Contains(::testing::HasSubstr("/metadata/"))); } @@ -957,7 +954,7 @@ TEST_F(MergingSnapshotUpdateTest, AddDeleteFileWithExplicitSequenceWritesSequenc ICEBERG_UNWRAP_OR_FAIL(auto op, NewMergeAppend()); EXPECT_THAT(op->AddDelete(del_file, 17), IsOk()); - ICEBERG_UNWRAP_OR_FAIL(auto manifests, op->Apply(*table_->metadata(), nullptr)); + ICEBERG_UNWRAP_OR_FAIL(auto manifests, op->CommitManifests()); auto delete_manifest_it = std::ranges::find_if(manifests, [](const ManifestFile& manifest) { return manifest.content == ManifestContent::kDeletes; @@ -985,7 +982,10 @@ TEST_F(MergingSnapshotUpdateTest, WriteDeleteGroups) { static_cast(index % 2)); }) | std::ranges::to(); - ICEBERG_UNWRAP_OR_FAIL(auto manifests, op->WriteDeletesForTest(files, spec_)); + for (const auto& file : files) { + EXPECT_THAT(op->AddDelete(file), IsOk()); + } + ICEBERG_UNWRAP_OR_FAIL(auto manifests, op->CommitManifests()); EXPECT_EQ(executor.submit_count(), 2); ASSERT_EQ(manifests.size(), 2U); @@ -994,24 +994,17 @@ TEST_F(MergingSnapshotUpdateTest, WriteDeleteGroups) { } } -TEST_F(MergingSnapshotUpdateTest, ApplyRebuildsDeleteSummaryAfterPreparingDeletes) { +TEST_F(MergingSnapshotUpdateTest, RetryRebuildsDeleteSummary) { + FailCommits(2); auto del_file = MakeDeleteFile("/delete/del_a.parquet", 1L); - ICEBERG_UNWRAP_OR_FAIL(auto op, NewMergeAppend()); EXPECT_THAT(op->AddDelete(del_file), IsOk()); EXPECT_THAT(op->AddDelete(del_file), IsOk()); - - ICEBERG_UNWRAP_OR_FAIL(auto first_manifests, op->Apply(*table_->metadata(), nullptr)); - EXPECT_THAT(first_manifests, ::testing::Contains(::testing::Field( - &ManifestFile::content, ManifestContent::kDeletes))); - - ICEBERG_UNWRAP_OR_FAIL(auto second_manifests, op->Apply(*table_->metadata(), nullptr)); - EXPECT_THAT(second_manifests, ::testing::Contains(::testing::Field( - &ManifestFile::content, ManifestContent::kDeletes))); - - auto summary = op->Summary(); - EXPECT_EQ(summary.at(SnapshotSummaryFields::kAddedDeleteFiles), "1"); - EXPECT_EQ(summary.at(SnapshotSummaryFields::kAddedPosDeleteFiles), "1"); + EXPECT_THAT(op->Commit(), IsOk()); + EXPECT_THAT(table_->Refresh(), IsOk()); + ICEBERG_UNWRAP_OR_FAIL(auto snapshot, table_->current_snapshot()); + EXPECT_EQ(snapshot->summary.at(SnapshotSummaryFields::kAddedDeleteFiles), "1"); + EXPECT_EQ(snapshot->summary.at(SnapshotSummaryFields::kAddedPosDeleteFiles), "1"); } // Covers the bug where deleted delete files were not tracked in the snapshot summary. @@ -1101,7 +1094,7 @@ class MergingSnapshotUpdateV1Test : public UpdateTestBase { return f; } - Result> NewMergeAppend() { + Result> NewMergeAppend() { return TestMergeAppend::Make(TableName(), table_); } @@ -1179,7 +1172,7 @@ TEST_F(MergingSnapshotUpdateTest, ApplyMergesDuplicateDeletionVectors) { EXPECT_THAT(op->AddDelete(dv_a2, 7), IsOk()); EXPECT_THAT(op->AddDelete(dv_b, 8), IsOk()); - ICEBERG_UNWRAP_OR_FAIL(auto manifests, op->Apply(*table_->metadata(), nullptr)); + ICEBERG_UNWRAP_OR_FAIL(auto manifests, op->CommitManifests()); auto delete_manifest_it = std::ranges::find_if(manifests, [](const ManifestFile& manifest) { return manifest.content == ManifestContent::kDeletes; @@ -1230,7 +1223,7 @@ TEST_F(MergingSnapshotUpdateTest, ApplyMergesDuplicateDeletionVectorsWithNullPar EXPECT_THAT(op->AddDelete(dv_a1, 7), IsOk()); EXPECT_THAT(op->AddDelete(dv_a2, 7), IsOk()); - EXPECT_THAT(op->Apply(*table_->metadata(), nullptr), IsOk()); + EXPECT_THAT(op->CommitManifests(), IsOk()); } TEST_F(MergingSnapshotUpdateTest, ValidateNewDeleteFileRejectsUnsupportedVersion) { @@ -1252,9 +1245,11 @@ TEST_F(MergingSnapshotUpdateTest, ApplyRejectsV2StagedPositionDeleteAfterV3Upgra ICEBERG_UNWRAP_OR_FAIL(auto op, NewMergeAppend()); EXPECT_THAT(op->AddDelete(del_file), IsOk()); - auto metadata = std::make_shared(*table_->metadata()); - metadata->format_version = 3; - EXPECT_THAT(op->Apply(*metadata, nullptr), IsError(ErrorKind::kInvalidArgument)); + ICEBERG_UNWRAP_OR_FAIL(auto properties, table_->NewUpdateProperties()); + properties->Set("format-version", "3"); + ASSERT_THAT(properties->Commit(), IsOk()); + ASSERT_THAT(table_->Refresh(), IsOk()); + EXPECT_THAT(op->Commit(), IsError(ErrorKind::kInvalidArgument)); } // ------------------------------------------------------------------------- @@ -1314,31 +1309,22 @@ TEST_F(MergingSnapshotUpdateTest, AddManifestRetryCopiesManifestAgain) { auto path = table_location_ + "/metadata/retry-input.avro"; ICEBERG_UNWRAP_OR_FAIL(auto manifest, WriteManifest(path, {file_a_})); manifest.added_snapshot_id = 12345; - + FailCommits(2); ICEBERG_UNWRAP_OR_FAIL(auto op, NewMergeAppend()); + std::vector deleted; + op->DeleteWith([&](const std::string& path) { + deleted.push_back(path); + return file_io_->DeleteFile(path); + }); EXPECT_THAT(op->AppendManifest(manifest), IsOk()); - - ICEBERG_UNWRAP_OR_FAIL(auto first_apply, static_cast(*op).Apply()); - SnapshotCache first_snapshot_cache(first_apply.snapshot.get()); - ICEBERG_UNWRAP_OR_FAIL(auto first_manifests, - first_snapshot_cache.DataManifests(file_io_)); - ASSERT_EQ(first_manifests.size(), 1U); - EXPECT_NE(first_manifests[0].manifest_path, path); - - ICEBERG_UNWRAP_OR_FAIL(auto second_apply, static_cast(*op).Apply()); - SnapshotCache second_snapshot_cache(second_apply.snapshot.get()); - ICEBERG_UNWRAP_OR_FAIL(auto second_manifests, - second_snapshot_cache.DataManifests(file_io_)); - ASSERT_EQ(second_manifests.size(), 1U); - EXPECT_NE(second_manifests[0].manifest_path, path); - EXPECT_NE(second_manifests[0].manifest_path, first_manifests[0].manifest_path); - - std::vector second_manifest_vector(second_manifests.begin(), - second_manifests.end()); - ICEBERG_UNWRAP_OR_FAIL(auto entries, - ReadAllEntries(second_manifest_vector, *table_->metadata())); + ICEBERG_UNWRAP_OR_FAIL(auto manifests, op->CommitManifests()); + ASSERT_EQ(manifests.size(), 1U); + EXPECT_NE(manifests[0].manifest_path, path); + EXPECT_THAT(deleted, ::testing::SizeIs(4U)); + EXPECT_THAT(deleted, ::testing::Not(::testing::Contains(path))); + EXPECT_THAT(deleted, ::testing::Not(::testing::Contains(manifests[0].manifest_path))); + ICEBERG_UNWRAP_OR_FAIL(auto entries, ReadAllEntries(manifests, *table_->metadata())); ASSERT_EQ(entries.size(), 1U); - ASSERT_NE(entries[0].data_file, nullptr); EXPECT_EQ(entries[0].data_file->file_path, file_a_->file_path); } @@ -1722,7 +1708,7 @@ TEST_F(MergingSnapshotUpdateTest, ValidateDataFilesExistUsesRowFilter) { ICEBERG_UNWRAP_OR_FAIL(auto op, NewOverwriteUpdate()); EXPECT_THAT(op->RemoveDataFile(file_a_), IsOk()); const int64_t second_snapshot_id = op->GeneratedSnapshotId(); - ICEBERG_UNWRAP_OR_FAIL(auto manifests, op->Apply(*table_->metadata(), first_snapshot)); + ICEBERG_UNWRAP_OR_FAIL(auto manifests, op->CommitManifests()); ICEBERG_UNWRAP_OR_FAIL( auto second_snapshot, MakeSyntheticSnapshot(DataOperation::kOverwrite, second_snapshot_id, @@ -1756,7 +1742,7 @@ TEST_F(MergingSnapshotUpdateTest, ICEBERG_UNWRAP_OR_FAIL(auto op, NewOverwriteUpdate()); EXPECT_THAT(op->AddDelete(del_file), IsOk()); const int64_t second_snapshot_id = op->GeneratedSnapshotId(); - ICEBERG_UNWRAP_OR_FAIL(auto manifests, op->Apply(*table_->metadata(), first_snapshot)); + ICEBERG_UNWRAP_OR_FAIL(auto manifests, op->CommitManifests()); ICEBERG_UNWRAP_OR_FAIL( auto second_snapshot, MakeSyntheticSnapshot(DataOperation::kOverwrite, second_snapshot_id, @@ -1784,7 +1770,7 @@ TEST_F(MergingSnapshotUpdateTest, ValidateNoNewDeletesForDataFilesDetectsConflic ICEBERG_UNWRAP_OR_FAIL(auto op, NewOverwriteUpdate()); EXPECT_THAT(op->AddDelete(del_file), IsOk()); const int64_t second_snapshot_id = op->GeneratedSnapshotId(); - ICEBERG_UNWRAP_OR_FAIL(auto manifests, op->Apply(*table_->metadata(), first_snapshot)); + ICEBERG_UNWRAP_OR_FAIL(auto manifests, op->CommitManifests()); ICEBERG_UNWRAP_OR_FAIL( auto second_snapshot, MakeSyntheticSnapshot(DataOperation::kOverwrite, second_snapshot_id, @@ -1813,7 +1799,7 @@ TEST_F(MergingSnapshotUpdateTest, ICEBERG_UNWRAP_OR_FAIL(auto op, NewOverwriteUpdate()); EXPECT_THAT(op->AddDelete(del_file), IsOk()); const int64_t second_snapshot_id = op->GeneratedSnapshotId(); - ICEBERG_UNWRAP_OR_FAIL(auto manifests, op->Apply(*table_->metadata(), first_snapshot)); + ICEBERG_UNWRAP_OR_FAIL(auto manifests, op->CommitManifests()); ICEBERG_UNWRAP_OR_FAIL( auto second_snapshot, MakeSyntheticSnapshot(DataOperation::kOverwrite, second_snapshot_id, @@ -1848,7 +1834,7 @@ TEST_F(MergingSnapshotUpdateTest, ICEBERG_UNWRAP_OR_FAIL(auto op, NewOverwriteUpdate()); EXPECT_THAT(op->AddDelete(del_file), IsOk()); const int64_t second_snapshot_id = op->GeneratedSnapshotId(); - ICEBERG_UNWRAP_OR_FAIL(auto manifests, op->Apply(*table_->metadata(), first_snapshot)); + ICEBERG_UNWRAP_OR_FAIL(auto manifests, op->CommitManifests()); ICEBERG_UNWRAP_OR_FAIL( auto second_snapshot, MakeSyntheticSnapshot(DataOperation::kOverwrite, second_snapshot_id, @@ -1879,8 +1865,7 @@ TEST_F(MergingSnapshotUpdateTest, ICEBERG_UNWRAP_OR_FAIL(auto overwrite, NewOverwriteUpdate()); EXPECT_THAT(overwrite->AddDelete(del_file), IsOk()); const int64_t second_snapshot_id = overwrite->GeneratedSnapshotId(); - ICEBERG_UNWRAP_OR_FAIL(auto manifests, - overwrite->Apply(*table_->metadata(), first_snapshot)); + ICEBERG_UNWRAP_OR_FAIL(auto manifests, overwrite->CommitManifests()); ICEBERG_UNWRAP_OR_FAIL( auto second_snapshot, MakeSyntheticSnapshot(DataOperation::kOverwrite, second_snapshot_id, @@ -1912,7 +1897,7 @@ TEST_F(MergingSnapshotUpdateTest, ICEBERG_UNWRAP_OR_FAIL(auto op, NewOverwriteUpdate()); EXPECT_THAT(op->AddDelete(del_file), IsOk()); const int64_t second_snapshot_id = op->GeneratedSnapshotId(); - ICEBERG_UNWRAP_OR_FAIL(auto manifests, op->Apply(*table_->metadata(), first_snapshot)); + ICEBERG_UNWRAP_OR_FAIL(auto manifests, op->CommitManifests()); ICEBERG_UNWRAP_OR_FAIL( auto second_snapshot, MakeSyntheticSnapshot(DataOperation::kOverwrite, second_snapshot_id, @@ -1940,7 +1925,7 @@ TEST_F(MergingSnapshotUpdateTest, ValidateNoNewDeleteFilesWithExpressionDetectsC ICEBERG_UNWRAP_OR_FAIL(auto op, NewOverwriteUpdate()); EXPECT_THAT(op->AddDelete(del_file), IsOk()); const int64_t second_snapshot_id = op->GeneratedSnapshotId(); - ICEBERG_UNWRAP_OR_FAIL(auto manifests, op->Apply(*table_->metadata(), first_snapshot)); + ICEBERG_UNWRAP_OR_FAIL(auto manifests, op->CommitManifests()); ICEBERG_UNWRAP_OR_FAIL( auto second_snapshot, MakeSyntheticSnapshot(DataOperation::kOverwrite, second_snapshot_id, @@ -1967,7 +1952,7 @@ TEST_F(MergingSnapshotUpdateTest, ICEBERG_UNWRAP_OR_FAIL(auto op, NewOverwriteUpdate()); EXPECT_THAT(op->AddDelete(del_file), IsOk()); const int64_t second_snapshot_id = op->GeneratedSnapshotId(); - ICEBERG_UNWRAP_OR_FAIL(auto manifests, op->Apply(*table_->metadata(), first_snapshot)); + ICEBERG_UNWRAP_OR_FAIL(auto manifests, op->CommitManifests()); ICEBERG_UNWRAP_OR_FAIL( auto second_snapshot, MakeSyntheticSnapshot(DataOperation::kOverwrite, second_snapshot_id, @@ -1994,7 +1979,7 @@ TEST_F(MergingSnapshotUpdateTest, ICEBERG_UNWRAP_OR_FAIL(auto op, NewOverwriteUpdate()); EXPECT_THAT(op->AddDelete(del_file), IsOk()); const int64_t second_snapshot_id = op->GeneratedSnapshotId(); - ICEBERG_UNWRAP_OR_FAIL(auto manifests, op->Apply(*table_->metadata(), first_snapshot)); + ICEBERG_UNWRAP_OR_FAIL(auto manifests, op->CommitManifests()); ICEBERG_UNWRAP_OR_FAIL( auto second_snapshot, MakeSyntheticSnapshot(DataOperation::kOverwrite, second_snapshot_id, @@ -2138,7 +2123,7 @@ TEST_F(MergingSnapshotUpdateTest, ValidateDeletedDataFilesWithExpressionDetectsC ICEBERG_UNWRAP_OR_FAIL(auto op, NewOverwriteUpdate()); EXPECT_THAT(op->RemoveDataFile(file_a_), IsOk()); const int64_t second_snapshot_id = op->GeneratedSnapshotId(); - ICEBERG_UNWRAP_OR_FAIL(auto manifests, op->Apply(*table_->metadata(), first_snapshot)); + ICEBERG_UNWRAP_OR_FAIL(auto manifests, op->CommitManifests()); ICEBERG_UNWRAP_OR_FAIL( auto second_snapshot, MakeSyntheticSnapshot(DataOperation::kOverwrite, second_snapshot_id, @@ -2164,7 +2149,7 @@ TEST_F(MergingSnapshotUpdateTest, ICEBERG_UNWRAP_OR_FAIL(auto op, NewOverwriteUpdate()); EXPECT_THAT(op->RemoveDataFile(file_a_), IsOk()); const int64_t second_snapshot_id = op->GeneratedSnapshotId(); - ICEBERG_UNWRAP_OR_FAIL(auto manifests, op->Apply(*table_->metadata(), first_snapshot)); + ICEBERG_UNWRAP_OR_FAIL(auto manifests, op->CommitManifests()); ICEBERG_UNWRAP_OR_FAIL( auto second_snapshot, MakeSyntheticSnapshot(DataOperation::kOverwrite, second_snapshot_id, diff --git a/src/iceberg/test/transaction_test.cc b/src/iceberg/test/transaction_test.cc index 3a13b7bc5..52cfb11e4 100644 --- a/src/iceberg/test/transaction_test.cc +++ b/src/iceberg/test/transaction_test.cc @@ -19,20 +19,40 @@ #include "iceberg/transaction.h" +#include +#include +#include +#include +#include + +#include "iceberg/exception.h" #include "iceberg/expression/expressions.h" #include "iceberg/expression/term.h" +#include "iceberg/schema.h" #include "iceberg/sort_order.h" +#include "iceberg/table_metadata.h" #include "iceberg/test/matchers.h" #include "iceberg/test/mock_catalog.h" #include "iceberg/test/update_test_base.h" #include "iceberg/transform.h" #include "iceberg/type.h" +#include "iceberg/update/fast_append.h" +#include "iceberg/update/set_snapshot.h" #include "iceberg/update/update_properties.h" #include "iceberg/update/update_schema.h" #include "iceberg/update/update_sort_order.h" namespace iceberg { +class TestPendingUpdate final : public PendingUpdate { + public: + explicit TestPendingUpdate(std::shared_ptr ctx) + : PendingUpdate(std::move(ctx)) {} + + Kind kind() const override { return Kind::kUpdateProperties; } + bool IsRetryable() const override { return true; } +}; + class TransactionTest : public UpdateTestBase {}; TEST_F(TransactionTest, CreateTransaction) { @@ -46,6 +66,58 @@ TEST_F(TransactionTest, CommitEmptyTransaction) { EXPECT_THAT(txn->Commit(), IsOk()); } +TEST_F(TransactionTest, TemporaryTransactionDoesNotAttachToContext) { + ICEBERG_UNWRAP_OR_FAIL(auto ctx, + TransactionContext::Make(table_, TransactionKind::kUpdate)); + ASSERT_FALSE(ctx->transaction.has_value()); + + ICEBERG_UNWRAP_OR_FAIL(auto txn, Transaction::Make(ctx)); + + EXPECT_NE(txn, nullptr); + EXPECT_FALSE(ctx->transaction.has_value()); +} + +TEST_F(TransactionTest, StandaloneCommitRequiresSharedOwnership) { + ICEBERG_UNWRAP_OR_FAIL(auto ctx, + TransactionContext::Make(table_, TransactionKind::kUpdate)); + auto update = std::make_unique(ctx); + + EXPECT_THAT(update->Commit(), + ::testing::AllOf( + IsError(ErrorKind::kInvalidArgument), + HasErrorMessage("PendingUpdate must be owned by std::shared_ptr"))); + EXPECT_FALSE(ctx->transaction.has_value()); +} + +TEST_F(TransactionTest, StandaloneCommitClearsTemporaryTransactionBinding) { + ICEBERG_UNWRAP_OR_FAIL(auto ctx, + TransactionContext::Make(table_, TransactionKind::kUpdate)); + ICEBERG_UNWRAP_OR_FAIL(auto update, UpdateProperties::Make(ctx)); + update->Set("standalone.property", "standalone.value"); + + EXPECT_THAT(update->Commit(), IsOk()); + EXPECT_FALSE(ctx->transaction.has_value()); +} + +TEST_F(TransactionTest, CommitNoOpUpdate) { + ICEBERG_UNWRAP_OR_FAIL(auto txn, table_->NewTransaction()); + ICEBERG_UNWRAP_OR_FAIL(auto update, txn->NewSetSnapshot()); + + EXPECT_THAT(update->Commit(), IsOk()); + EXPECT_THAT(txn->Commit(), IsOk()); +} + +TEST_F(TransactionTest, ApplyFailureFinalizesTransaction) { + ICEBERG_UNWRAP_OR_FAIL(auto txn, table_->NewTransaction()); + ICEBERG_UNWRAP_OR_FAIL(auto update, txn->NewFastAppend()); + update->AppendFile(nullptr); + + EXPECT_THAT(update->Commit(), IsError(ErrorKind::kValidationFailed)); + EXPECT_THAT(txn->Commit(), + ::testing::AllOf(IsError(ErrorKind::kValidationFailed), + HasErrorMessage("Transaction is not ready"))); +} + TEST_F(TransactionTest, CommitTransactionWithPropertyUpdate) { ICEBERG_UNWRAP_OR_FAIL(auto txn, table_->NewTransaction()); ICEBERG_UNWRAP_OR_FAIL(auto update, txn->NewUpdateProperties()); @@ -151,6 +223,39 @@ TEST_F(TransactionRetryTest, CommitRetrySucceedsAfterConflict) { EXPECT_EQ(update_call_count, 2); } +TEST_F(TransactionRetryTest, StandaloneCommitRetryReappliesUpdate) { + std::vector update_counts; + ON_CALL(*mock_catalog_, UpdateTable(::testing::_, ::testing::_, ::testing::_)) + .WillByDefault( + [this, &update_counts](const TableIdentifier&, + const std::vector>&, + const std::vector>& updates) + -> Result> { + update_counts.push_back(updates.size()); + if (update_counts.size() == 1) { + return CommitFailed("conflict on first attempt"); + } + return Table::Make(mock_table_->name(), mock_table_->metadata(), + std::string(mock_table_->metadata_file_location()), + mock_table_->io(), mock_catalog_); + }); + EXPECT_CALL(*mock_catalog_, LoadTable(::testing::_)) + .WillOnce([this](const TableIdentifier&) -> Result> { + auto builder = TableMetadataBuilder::BuildFrom(mock_table_->metadata().get()); + ICEBERG_ASSIGN_OR_RAISE(auto metadata, builder->Build()); + return Table::Make( + mock_table_->name(), std::shared_ptr(std::move(metadata)), + std::format("{}.refreshed", mock_table_->metadata_file_location()), + mock_table_->io(), mock_catalog_); + }); + + ICEBERG_UNWRAP_OR_FAIL(auto update, mock_table_->NewUpdateProperties()); + update->Set("retry.test", "value"); + + EXPECT_THAT(update->Commit(), IsOk()); + EXPECT_THAT(update_counts, ::testing::ElementsAre(1U, 1U)); +} + TEST_F(TransactionRetryTest, CommitRetryExhausted) { int update_call_count = 0; ON_CALL(*mock_catalog_, UpdateTable(::testing::_, ::testing::_, ::testing::_)) @@ -195,6 +300,32 @@ TEST_F(TransactionRetryTest, CommitNonRetryableErrorStopsImmediately) { EXPECT_EQ(update_call_count, 1); // Should not retry } +TEST_F(TransactionRetryTest, CommitExceptionMakesOutcomeUnknown) { + int update_call_count = 0; + ON_CALL(*mock_catalog_, UpdateTable(::testing::_, ::testing::_, ::testing::_)) + .WillByDefault( + [&update_call_count](const TableIdentifier&, + const std::vector>&, + const std::vector>&) + -> Result> { + ++update_call_count; + throw std::runtime_error("injected catalog failure"); + }); + + ICEBERG_UNWRAP_OR_FAIL(auto txn, mock_table_->NewTransaction()); + ICEBERG_UNWRAP_OR_FAIL(auto properties, txn->NewUpdateProperties()); + properties->Set("exception.test", "value"); + EXPECT_THAT(properties->Commit(), IsOk()); + + EXPECT_THAT(txn->Commit(), IsError(ErrorKind::kCommitStateUnknown)); + EXPECT_EQ(txn->state(), TransactionState::kCommitStateUnknown); + EXPECT_THAT(txn->NewFastAppend(), IsError(ErrorKind::kValidationFailed)); + EXPECT_THAT(properties->Commit(), IsError(ErrorKind::kValidationFailed)); + EXPECT_THAT(txn->Commit(), IsError(ErrorKind::kValidationFailed)); + EXPECT_THAT(txn->Abort(), IsError(ErrorKind::kValidationFailed)); + EXPECT_EQ(update_call_count, 1); +} + TEST_F(TransactionRetryTest, CreateTransactionDoesNotRetry) { int update_call_count = 0; ON_CALL(*mock_catalog_, UpdateTable(::testing::_, ::testing::_, ::testing::_)) @@ -240,4 +371,124 @@ TEST_F(TransactionRetryTest, NonRetryableUpdatePreventsRetry) { EXPECT_EQ(update_call_count, 1); } +TEST_F(TransactionTest, AppliedUpdateCannotCompleteAnotherPendingOperation) { + ICEBERG_UNWRAP_OR_FAIL(auto txn, table_->NewTransaction()); + ICEBERG_UNWRAP_OR_FAIL(auto first, txn->NewUpdateProperties()); + first->Set("first", "1"); + ASSERT_THAT(first->Commit(), IsOk()); + EXPECT_EQ(txn->state(), TransactionState::kReady); + ICEBERG_UNWRAP_OR_FAIL(auto second, txn->NewUpdateProperties()); + EXPECT_THAT(first->Commit(), HasErrorMessage("not the current pending operation")); + EXPECT_EQ(txn->state(), TransactionState::kUpdatePending); + EXPECT_THAT(txn->Commit(), IsError(ErrorKind::kValidationFailed)); + EXPECT_THAT(txn->NewUpdateProperties(), IsError(ErrorKind::kValidationFailed)); + second->Set("second", "2"); + ASSERT_THAT(second->Commit(), IsOk()); + ASSERT_THAT(txn->Commit(), IsOk()); + EXPECT_EQ(txn->state(), TransactionState::kCommitted); + EXPECT_EQ(ReloadMetadata()->properties.configs().at("first"), "1"); + EXPECT_EQ(ReloadMetadata()->properties.configs().at("second"), "2"); + EXPECT_THAT(txn->Abort(), IsError(ErrorKind::kValidationFailed)); + EXPECT_THAT(first->Commit(), IsError(ErrorKind::kValidationFailed)); +} + +TEST_F(TransactionTest, StandaloneCommitIsTerminal) { + ICEBERG_UNWRAP_OR_FAIL(auto update, table_->NewUpdateProperties()); + update->Set("once", "value"); + ASSERT_THAT(update->Commit(), IsOk()); + EXPECT_THAT(update->Commit(), HasErrorMessage("Update is terminal")); + EXPECT_THROW(update->Set("twice", "value"), IcebergError); +} + +TEST_F(TransactionTest, AbortReadyAndPendingIsIdempotent) { + for (bool add_update : {false, true}) { + ICEBERG_UNWRAP_OR_FAIL(auto txn, table_->NewTransaction()); + std::shared_ptr update; + if (add_update) { + ICEBERG_UNWRAP_OR_FAIL(update, txn->NewUpdateProperties()); + update->Set("discard", "value"); + } + ASSERT_THAT(txn->Abort(), IsOk()); + EXPECT_EQ(txn->state(), TransactionState::kAborted); + EXPECT_THAT(txn->Abort(), IsOk()); + EXPECT_THAT(txn->Commit(), IsError(ErrorKind::kValidationFailed)); + EXPECT_THAT(txn->NewFastAppend(), IsError(ErrorKind::kValidationFailed)); + if (update) { + EXPECT_THAT(update->Commit(), IsError(ErrorKind::kValidationFailed)); + EXPECT_THROW(update->Set("reuse", "value"), IcebergError); + } + } +} + +TEST_F(TransactionTest, ApplyFailureCannotBeCorrected) { + ICEBERG_UNWRAP_OR_FAIL(auto txn, table_->NewTransaction()); + ICEBERG_UNWRAP_OR_FAIL(auto update, txn->NewUpdateProperties()); + update->Set("format-version", "100"); + EXPECT_THAT(update->Commit(), IsError(ErrorKind::kInvalidArgument)); + EXPECT_EQ(txn->state(), TransactionState::kFailed); + EXPECT_THROW(update->Set("format-version", "2"), IcebergError); + EXPECT_THAT(update->Commit(), IsError(ErrorKind::kValidationFailed)); + EXPECT_THAT(txn->Commit(), IsError(ErrorKind::kValidationFailed)); + EXPECT_THAT(txn->NewUpdateProperties(), IsError(ErrorKind::kValidationFailed)); + EXPECT_THAT(txn->Abort(), IsOk()); + EXPECT_THAT(txn->Abort(), IsOk()); +} + +TEST_F(TransactionRetryTest, NonRetryableStandaloneUpdateStopsAtFirstConflict) { + EXPECT_CALL(*mock_catalog_, UpdateTable(::testing::_, ::testing::_, ::testing::_)) + .Times(1) + .WillOnce(::testing::Return(CommitFailed("conflict"))); + EXPECT_CALL(*mock_catalog_, LoadTable(::testing::_)).Times(0); + ICEBERG_UNWRAP_OR_FAIL(auto update, mock_table_->NewUpdateSchema()); + update->AddColumn("new_column", int64()); + EXPECT_THAT(update->Commit(), IsError(ErrorKind::kCommitFailed)); + EXPECT_THAT(update->Commit(), IsError(ErrorKind::kValidationFailed)); +} + +TEST_F(TransactionRetryTest, FrozenMutationDoesNotPoisonReplay) { + EXPECT_CALL(*mock_catalog_, UpdateTable(::testing::_, ::testing::_, ::testing::_)) + .WillOnce(::testing::Return(CommitFailed("conflict"))) + .WillOnce([this](const auto& name, const auto& requirements, const auto& updates) { + return catalog_->UpdateTable(name, requirements, updates); + }); + ICEBERG_UNWRAP_OR_FAIL(auto txn, mock_table_->NewTransaction()); + ICEBERG_UNWRAP_OR_FAIL(auto update, txn->NewUpdateProperties()); + update->Set("frozen", "original"); + ASSERT_THAT(update->Commit(), IsOk()); + EXPECT_THROW(update->Set("frozen", "changed"), IcebergError); + ASSERT_THAT(txn->Commit(), IsOk()); + EXPECT_EQ(ReloadMetadata()->properties.configs().at("frozen"), "original"); +} + +TEST_F(TransactionTest, FreezeCopiesSortTransformAliases) { + FailCommits(2); + ICEBERG_UNWRAP_OR_FAIL(auto txn, table_->NewTransaction()); + ICEBERG_UNWRAP_OR_FAIL(auto update, txn->NewUpdateSortOrder()); + auto transform = Transform::Bucket(16); + ICEBERG_UNWRAP_OR_FAIL(auto term, + UnboundTransform::Make(Expressions::Ref("x"), transform)); + update->AddSortField(std::move(term), SortDirection::kAscending, NullOrder::kFirst); + ASSERT_THAT(update->Commit(), IsOk()); + *transform = *Transform::Bucket(32); + ICEBERG_UNWRAP_OR_FAIL(auto preview, update->Validate()); + *preview->fields()[0].transform() = *Transform::Bucket(64); + ASSERT_THAT(txn->Commit(), IsOk()); + ICEBERG_UNWRAP_OR_FAIL(auto order, ReloadMetadata()->SortOrder()); + EXPECT_EQ(*order->fields()[0].transform(), *Transform::Bucket(16)); +} + +TEST_F(TransactionTest, FreezeCopiesNestedSchemaTypes) { + ICEBERG_UNWRAP_OR_FAIL(auto txn, table_->NewTransaction()); + ICEBERG_UNWRAP_OR_FAIL(auto update, txn->NewUpdateSchema()); + auto type = decimal(9, 2); + update->AddColumn("amount", type); + ASSERT_THAT(update->Commit(), IsOk()); + *type = DecimalType(18, 4); + ASSERT_THAT(txn->Commit(), IsOk()); + ICEBERG_UNWRAP_OR_FAIL(auto schema, ReloadMetadata()->Schema()); + ICEBERG_UNWRAP_OR_FAIL(auto field, schema->FindFieldByName("amount")); + ASSERT_TRUE(field.has_value()); + EXPECT_EQ(*field->get().type(), *decimal(9, 2)); +} + } // namespace iceberg diff --git a/src/iceberg/test/update_location_test.cc b/src/iceberg/test/update_location_test.cc index a208209b3..403bb4af4 100644 --- a/src/iceberg/test/update_location_test.cc +++ b/src/iceberg/test/update_location_test.cc @@ -47,7 +47,7 @@ TEST_F(UpdateLocationTest, SetLocationSuccess) { ICEBERG_UNWRAP_OR_FAIL(auto update, table_->NewUpdateLocation()); update->SetLocation(new_location); - ICEBERG_UNWRAP_OR_FAIL(auto result, update->Apply()); + ICEBERG_UNWRAP_OR_FAIL(auto result, update->Validate()); EXPECT_EQ(result, new_location); // Commit and verify the location was persisted @@ -71,7 +71,7 @@ TEST_F(UpdateLocationTest, SetLocationMultipleTimes) { .SetLocation("/warehouse/second_location") .SetLocation(final_location); - ICEBERG_UNWRAP_OR_FAIL(auto result, update->Apply()); + ICEBERG_UNWRAP_OR_FAIL(auto result, update->Validate()); EXPECT_EQ(result, final_location); // Commit and verify the final location was persisted @@ -84,7 +84,7 @@ TEST_F(UpdateLocationTest, SetEmptyLocation) { ICEBERG_UNWRAP_OR_FAIL(auto update, table_->NewUpdateLocation()); update->SetLocation(""); - auto result = update->Apply(); + auto result = update->Validate(); EXPECT_THAT(result, IsError(ErrorKind::kValidationFailed)); EXPECT_THAT(result, HasErrorMessage("Location cannot be empty")); } @@ -92,7 +92,7 @@ TEST_F(UpdateLocationTest, SetEmptyLocation) { TEST_F(UpdateLocationTest, ApplyWithoutSettingLocation) { ICEBERG_UNWRAP_OR_FAIL(auto update, table_->NewUpdateLocation()); - auto result = update->Apply(); + auto result = update->Validate(); EXPECT_THAT(result, IsError(ErrorKind::kInvalidArgument)); EXPECT_THAT(result, HasErrorMessage("Location must be set before applying")); } @@ -109,7 +109,7 @@ TEST_F(UpdateLocationTest, MultipleUpdatesSequentially) { ICEBERG_UNWRAP_OR_FAIL(auto update, table_->NewUpdateLocation()); update->SetLocation(first_location); - ICEBERG_UNWRAP_OR_FAIL(auto result, update->Apply()); + ICEBERG_UNWRAP_OR_FAIL(auto result, update->Validate()); EXPECT_EQ(result, first_location); EXPECT_THAT(update->Commit(), IsOk()); @@ -123,7 +123,7 @@ TEST_F(UpdateLocationTest, MultipleUpdatesSequentially) { ICEBERG_UNWRAP_OR_FAIL(update, reloaded->NewUpdateLocation()); update->SetLocation(second_location); - ICEBERG_UNWRAP_OR_FAIL(result, update->Apply()); + ICEBERG_UNWRAP_OR_FAIL(result, update->Validate()); EXPECT_EQ(result, second_location); EXPECT_THAT(update->Commit(), IsOk()); diff --git a/src/iceberg/test/update_partition_spec_test.cc b/src/iceberg/test/update_partition_spec_test.cc index ebbb80243..ef5a49a3a 100644 --- a/src/iceberg/test/update_partition_spec_test.cc +++ b/src/iceberg/test/update_partition_spec_test.cc @@ -173,7 +173,7 @@ class UpdatePartitionSpecTest : public ::testing::TestWithParam { // Helper to apply update and get the resulting spec std::shared_ptr ApplyUpdateAndGetSpec( std::shared_ptr update) { - auto result = update->Apply(); + auto result = update->Validate(); if (!result.has_value()) { ADD_FAILURE() << "Failed to apply update: " << result.error().message; return nullptr; @@ -207,7 +207,7 @@ class UpdatePartitionSpecTest : public ::testing::TestWithParam { // Helper to expect an error with a specific message void ExpectError(std::shared_ptr update, ErrorKind expected_kind, const std::string& expected_message) { - auto result = update->Apply(); + auto result = update->Validate(); ASSERT_THAT(result, IsError(expected_kind)); ASSERT_THAT(result, HasErrorMessage(expected_message)); } @@ -529,7 +529,7 @@ TEST_P(UpdatePartitionSpecTest, TestMultipleChanges) { TEST_P(UpdatePartitionSpecTest, TestAddDeletedName) { ICEBERG_UNWRAP_OR_FAIL(auto update, partitioned_table_->NewUpdatePartitionSpec()); update->RemoveField(Expressions::Bucket("id", 16)); - ICEBERG_UNWRAP_OR_FAIL(auto result, update->Apply()); + ICEBERG_UNWRAP_OR_FAIL(auto result, update->Validate()); auto updated_spec = result.spec; if (format_version_ == 1) { @@ -608,14 +608,14 @@ TEST_P(UpdatePartitionSpecTest, TestNoEffectAddDeletedSameFieldWithSameName) { ICEBERG_UNWRAP_OR_FAIL(auto update1, partitioned_table_->NewUpdatePartitionSpec()); update1->RemoveField("shard"); update1->AddField(Expressions::Bucket("id", 16), "shard"); - ICEBERG_UNWRAP_OR_FAIL(auto result1, update1->Apply()); + ICEBERG_UNWRAP_OR_FAIL(auto result1, update1->Validate()); auto spec1 = result1.spec; AssertPartitionSpecEquals(*partitioned_spec_, *spec1); ICEBERG_UNWRAP_OR_FAIL(auto update2, partitioned_table_->NewUpdatePartitionSpec()); update2->RemoveField("shard"); update2->AddField(Expressions::Bucket("id", 16)); - ICEBERG_UNWRAP_OR_FAIL(auto result2, update2->Apply()); + ICEBERG_UNWRAP_OR_FAIL(auto result2, update2->Validate()); auto spec2 = result2.spec; AssertPartitionSpecEquals(*partitioned_spec_, *spec2); } @@ -624,7 +624,7 @@ TEST_P(UpdatePartitionSpecTest, TestGenerateNewSpecAddDeletedSameFieldWithDiffer ICEBERG_UNWRAP_OR_FAIL(auto update, partitioned_table_->NewUpdatePartitionSpec()); update->RemoveField("shard"); update->AddField(Expressions::Bucket("id", 16), "new_shard"); - ICEBERG_UNWRAP_OR_FAIL(auto result, update->Apply()); + ICEBERG_UNWRAP_OR_FAIL(auto result, update->Validate()); auto updated_spec = result.spec; ASSERT_EQ(updated_spec->fields().size(), 3); diff --git a/src/iceberg/test/update_partition_statistics_test.cc b/src/iceberg/test/update_partition_statistics_test.cc index 5ed84cc0a..23ec44bf7 100644 --- a/src/iceberg/test/update_partition_statistics_test.cc +++ b/src/iceberg/test/update_partition_statistics_test.cc @@ -57,7 +57,7 @@ class UpdatePartitionStatisticsTest : public UpdateTestBase { TEST_F(UpdatePartitionStatisticsTest, EmptyUpdate) { ICEBERG_UNWRAP_OR_FAIL(auto update, table_->NewUpdatePartitionStatistics()); - ICEBERG_UNWRAP_OR_FAIL(auto result, update->Apply()); + ICEBERG_UNWRAP_OR_FAIL(auto result, update->Validate()); EXPECT_TRUE(result.to_set.empty()); EXPECT_TRUE(result.to_remove.empty()); } @@ -68,7 +68,7 @@ TEST_F(UpdatePartitionStatisticsTest, SetPartitionStatistics) { 1, "/warehouse/test_table/metadata/partition-stats-1.parquet"); update->SetPartitionStatistics(partition_stats_file); - ICEBERG_UNWRAP_OR_FAIL(auto result, update->Apply()); + ICEBERG_UNWRAP_OR_FAIL(auto result, update->Validate()); EXPECT_EQ(result.to_set.size(), 1); EXPECT_TRUE(result.to_remove.empty()); @@ -90,7 +90,7 @@ TEST_F(UpdatePartitionStatisticsTest, SetMultiplePartitionStatistics) { update->SetPartitionStatistics(partition_stats_file1); update->SetPartitionStatistics(partition_stats_file2); - ICEBERG_UNWRAP_OR_FAIL(auto result, update->Apply()); + ICEBERG_UNWRAP_OR_FAIL(auto result, update->Validate()); EXPECT_EQ(result.to_set.size(), 2); EXPECT_TRUE(result.to_remove.empty()); @@ -115,7 +115,7 @@ TEST_F(UpdatePartitionStatisticsTest, ReplacePartitionStatistics) { update->SetPartitionStatistics(partition_stats_file1); update->SetPartitionStatistics(partition_stats_file2); - ICEBERG_UNWRAP_OR_FAIL(auto result, update->Apply()); + ICEBERG_UNWRAP_OR_FAIL(auto result, update->Validate()); EXPECT_EQ(result.to_set.size(), 1); EXPECT_TRUE(result.to_remove.empty()); @@ -130,7 +130,7 @@ TEST_F(UpdatePartitionStatisticsTest, RemovePartitionStatistics) { ICEBERG_UNWRAP_OR_FAIL(auto update, table_->NewUpdatePartitionStatistics()); update->RemovePartitionStatistics(1); - ICEBERG_UNWRAP_OR_FAIL(auto result, update->Apply()); + ICEBERG_UNWRAP_OR_FAIL(auto result, update->Validate()); EXPECT_TRUE(result.to_set.empty()); EXPECT_EQ(result.to_remove.size(), 1); EXPECT_EQ(result.to_remove[0], 1); @@ -144,7 +144,7 @@ TEST_F(UpdatePartitionStatisticsTest, SetThenRemovePartitionStatistics) { update->SetPartitionStatistics(partition_stats_file); update->RemovePartitionStatistics(1); - ICEBERG_UNWRAP_OR_FAIL(auto result, update->Apply()); + ICEBERG_UNWRAP_OR_FAIL(auto result, update->Validate()); EXPECT_TRUE(result.to_set.empty()); EXPECT_EQ(result.to_remove.size(), 1); EXPECT_EQ(result.to_remove[0], 1); @@ -155,7 +155,7 @@ TEST_F(UpdatePartitionStatisticsTest, SetNullPartitionStatistics) { update->SetPartitionStatistics(nullptr); - auto result = update->Apply(); + auto result = update->Validate(); EXPECT_THAT(result, IsError(ErrorKind::kValidationFailed)); EXPECT_THAT(result, HasErrorMessage("Statistics file cannot be null")); } @@ -172,7 +172,7 @@ TEST_F(UpdatePartitionStatisticsTest, SetAndRemoveMixed) { update->SetPartitionStatistics(partition_stats_file2); update->RemovePartitionStatistics(3); - ICEBERG_UNWRAP_OR_FAIL(auto result, update->Apply()); + ICEBERG_UNWRAP_OR_FAIL(auto result, update->Validate()); EXPECT_EQ(result.to_set.size(), 2); EXPECT_EQ(result.to_remove.size(), 1); EXPECT_EQ(result.to_remove[0], 3); diff --git a/src/iceberg/test/update_properties_test.cc b/src/iceberg/test/update_properties_test.cc index 2a22ff949..8920fb3c3 100644 --- a/src/iceberg/test/update_properties_test.cc +++ b/src/iceberg/test/update_properties_test.cc @@ -28,7 +28,7 @@ class UpdatePropertiesTest : public UpdateTestBase {}; TEST_F(UpdatePropertiesTest, EmptyUpdate) { ICEBERG_UNWRAP_OR_FAIL(auto update, table_->NewUpdateProperties()); - ICEBERG_UNWRAP_OR_FAIL(auto result, update->Apply()); + ICEBERG_UNWRAP_OR_FAIL(auto result, update->Validate()); EXPECT_THAT(result.updates.empty(), true); } @@ -36,7 +36,7 @@ TEST_F(UpdatePropertiesTest, SetProperty) { ICEBERG_UNWRAP_OR_FAIL(auto update, table_->NewUpdateProperties()); update->Set("key1", "value1").Set("key2", "value2"); - ICEBERG_UNWRAP_OR_FAIL(auto result, update->Apply()); + ICEBERG_UNWRAP_OR_FAIL(auto result, update->Validate()); EXPECT_EQ(result.updates.size(), 2); EXPECT_EQ(result.updates.at("key1"), "value1"); EXPECT_EQ(result.updates.at("key2"), "value2"); @@ -54,7 +54,7 @@ TEST_F(UpdatePropertiesTest, RemoveProperty) { ICEBERG_UNWRAP_OR_FAIL(auto update, reloaded->NewUpdateProperties()); update->Remove("key1").Remove("key2"); - ICEBERG_UNWRAP_OR_FAIL(auto result, update->Apply()); + ICEBERG_UNWRAP_OR_FAIL(auto result, update->Validate()); EXPECT_TRUE(result.updates.empty()); EXPECT_EQ(result.removals.size(), 2); EXPECT_TRUE(result.removals.contains("key1")); @@ -65,7 +65,7 @@ TEST_F(UpdatePropertiesTest, SetThenRemoveSameKey) { ICEBERG_UNWRAP_OR_FAIL(auto update, table_->NewUpdateProperties()); update->Set("key1", "value1").Remove("key1"); - auto result = update->Apply(); + auto result = update->Validate(); EXPECT_THAT(result, IsError(ErrorKind::kValidationFailed)); EXPECT_THAT(result, HasErrorMessage("already marked for update")); } @@ -74,7 +74,7 @@ TEST_F(UpdatePropertiesTest, RemoveThenSetSameKey) { ICEBERG_UNWRAP_OR_FAIL(auto update, table_->NewUpdateProperties()); update->Remove("key1").Set("key1", "value1"); - auto result = update->Apply(); + auto result = update->Validate(); EXPECT_THAT(result, IsError(ErrorKind::kValidationFailed)); EXPECT_THAT(result, HasErrorMessage("already marked for removal")); } @@ -83,7 +83,7 @@ TEST_F(UpdatePropertiesTest, SetAndRemoveDifferentKeys) { ICEBERG_UNWRAP_OR_FAIL(auto update, table_->NewUpdateProperties()); update->Set("key1", "value1").Remove("key2"); - ICEBERG_UNWRAP_OR_FAIL(auto result, update->Apply()); + ICEBERG_UNWRAP_OR_FAIL(auto result, update->Validate()); EXPECT_EQ(result.updates.size(), 1); EXPECT_EQ(result.updates.at("key1"), "value1"); EXPECT_EQ(result.removals.size(), 1); @@ -94,7 +94,7 @@ TEST_F(UpdatePropertiesTest, UpgradeFormatVersionValid) { ICEBERG_UNWRAP_OR_FAIL(auto update, table_->NewUpdateProperties()); update->Set("format-version", "3"); - ICEBERG_UNWRAP_OR_FAIL(auto result, update->Apply()); + ICEBERG_UNWRAP_OR_FAIL(auto result, update->Validate()); EXPECT_TRUE(result.updates.empty()); EXPECT_TRUE(result.removals.empty()); ASSERT_TRUE(result.format_version.has_value()); @@ -105,7 +105,7 @@ TEST_F(UpdatePropertiesTest, UpgradeFormatVersionInvalidString) { ICEBERG_UNWRAP_OR_FAIL(auto update, table_->NewUpdateProperties()); update->Set("format-version", "invalid"); - auto result = update->Apply(); + auto result = update->Validate(); EXPECT_THAT(result, IsError(ErrorKind::kInvalidArgument)); EXPECT_THAT(result, HasErrorMessage("invalid argument")); } @@ -114,7 +114,7 @@ TEST_F(UpdatePropertiesTest, UpgradeFormatVersionOutOfRange) { ICEBERG_UNWRAP_OR_FAIL(auto update, table_->NewUpdateProperties()); update->Set("format-version", "5000000000"); - auto result = update->Apply(); + auto result = update->Validate(); EXPECT_THAT(result, IsError(ErrorKind::kInvalidArgument)); EXPECT_THAT(result, HasErrorMessage("out of range")); } @@ -124,7 +124,7 @@ TEST_F(UpdatePropertiesTest, UpgradeFormatVersionUnsupported) { update->Set("format-version", std::to_string(TableMetadata::kSupportedTableFormatVersion + 1)); - auto result = update->Apply(); + auto result = update->Validate(); EXPECT_THAT(result, IsError(ErrorKind::kInvalidArgument)); EXPECT_THAT(result, HasErrorMessage("unsupported format version")); } @@ -133,7 +133,7 @@ TEST_F(UpdatePropertiesTest, SetReservedPropertyUuid) { ICEBERG_UNWRAP_OR_FAIL(auto update, table_->NewUpdateProperties()); update->Set("uuid", "some-uuid"); - auto result = update->Apply(); + auto result = update->Validate(); EXPECT_THAT(result, IsError(ErrorKind::kValidationFailed)); EXPECT_THAT(result, HasErrorMessage("Cannot set reserved property")); } @@ -142,7 +142,7 @@ TEST_F(UpdatePropertiesTest, SetReservedPropertyCurrentSchema) { ICEBERG_UNWRAP_OR_FAIL(auto update, table_->NewUpdateProperties()); update->Set("current-schema", R"({"type": "struct"})"); - auto result = update->Apply(); + auto result = update->Validate(); EXPECT_THAT(result, IsError(ErrorKind::kValidationFailed)); EXPECT_THAT(result, HasErrorMessage("Cannot set reserved property")); } @@ -151,7 +151,7 @@ TEST_F(UpdatePropertiesTest, SetReservedPropertyCurrentSnapshotId) { ICEBERG_UNWRAP_OR_FAIL(auto update, table_->NewUpdateProperties()); update->Set("current-snapshot-id", "12345"); - auto result = update->Apply(); + auto result = update->Validate(); EXPECT_THAT(result, IsError(ErrorKind::kValidationFailed)); EXPECT_THAT(result, HasErrorMessage("Cannot set reserved property")); } @@ -160,7 +160,7 @@ TEST_F(UpdatePropertiesTest, SetFormatVersionStillAllowed) { ICEBERG_UNWRAP_OR_FAIL(auto update, table_->NewUpdateProperties()); update->Set("format-version", "3"); - ICEBERG_UNWRAP_OR_FAIL(auto result, update->Apply()); + ICEBERG_UNWRAP_OR_FAIL(auto result, update->Validate()); EXPECT_TRUE(result.updates.empty()); ASSERT_TRUE(result.format_version.has_value()); EXPECT_EQ(result.format_version.value(), 3); @@ -171,7 +171,7 @@ TEST_F(UpdatePropertiesTest, SetValidAndReservedProperties) { update->Set("valid.key", "valid.value"); update->Set("snapshot-count", "10"); - auto result = update->Apply(); + auto result = update->Validate(); EXPECT_THAT(result, IsError(ErrorKind::kValidationFailed)); EXPECT_THAT(result, HasErrorMessage("Cannot set reserved property")); } diff --git a/src/iceberg/test/update_schema_test.cc b/src/iceberg/test/update_schema_test.cc index 4057c52c8..91b9f5f9a 100644 --- a/src/iceberg/test/update_schema_test.cc +++ b/src/iceberg/test/update_schema_test.cc @@ -19,10 +19,12 @@ #include "iceberg/update/update_schema.h" +#include #include #include #include #include +#include #include #include #include @@ -40,6 +42,7 @@ #include "iceberg/test/matchers.h" #include "iceberg/test/mock_io.h" #include "iceberg/test/test_resource.h" +#include "iceberg/transaction.h" #include "iceberg/type.h" #include "iceberg/util/checked_cast.h" #include "iceberg/util/uuid.h" @@ -107,7 +110,7 @@ TEST_F(UpdateSchemaTest, AddOptionalColumn) { ICEBERG_UNWRAP_OR_FAIL(auto update, table_->NewUpdateSchema()); update->AddColumn("new_col", int32(), "A new integer column"); - ICEBERG_UNWRAP_OR_FAIL(auto result, update->Apply()); + ICEBERG_UNWRAP_OR_FAIL(auto result, update->Validate()); ASSERT_TRUE(result.schema != nullptr); ICEBERG_UNWRAP_OR_FAIL(auto new_field_opt, result.schema->FindFieldByName("new_col")); @@ -115,7 +118,7 @@ TEST_F(UpdateSchemaTest, AddOptionalColumn) { const auto& new_field = new_field_opt->get(); EXPECT_EQ(new_field.name(), "new_col"); - EXPECT_EQ(new_field.type(), int32()); + EXPECT_EQ(*new_field.type(), *int32()); EXPECT_TRUE(new_field.optional()); EXPECT_EQ(new_field.doc(), "A new integer column"); } @@ -124,7 +127,7 @@ TEST_F(UpdateSchemaTest, AddRequiredColumnFails) { ICEBERG_UNWRAP_OR_FAIL(auto update, table_->NewUpdateSchema()); update->AddRequiredColumn("required_col", string(), "A required string column"); - auto result = update->Apply(); + auto result = update->Validate(); EXPECT_THAT(result, IsError(ErrorKind::kValidationFailed)); EXPECT_THAT(result, HasErrorMessage("Incompatible change")); } @@ -134,7 +137,7 @@ TEST_F(UpdateSchemaTest, AddRequiredColumnWithAllowIncompatible) { update->AllowIncompatibleChanges().AddRequiredColumn("required_col", string(), "A required string column"); - ICEBERG_UNWRAP_OR_FAIL(auto result, update->Apply()); + ICEBERG_UNWRAP_OR_FAIL(auto result, update->Validate()); ASSERT_TRUE(result.schema != nullptr); ICEBERG_UNWRAP_OR_FAIL(auto new_field_opt, @@ -143,7 +146,7 @@ TEST_F(UpdateSchemaTest, AddRequiredColumnWithAllowIncompatible) { const auto& new_field = new_field_opt->get(); EXPECT_EQ(new_field.name(), "required_col"); - EXPECT_EQ(new_field.type(), string()); + EXPECT_EQ(*new_field.type(), *string()); EXPECT_FALSE(new_field.optional()); EXPECT_EQ(new_field.doc(), "A required string column"); } @@ -169,7 +172,7 @@ TEST_F(UpdateSchemaTest, AddColumnWithDefaultValueRequiresV3) { ICEBERG_UNWRAP_OR_FAIL(auto update, table_->NewUpdateSchema()); update->AddColumn("new_col", int32(), "An integer column", Literal::Int(42)); - auto result = update->Apply(); + auto result = update->Validate(); EXPECT_THAT(result, IsError(ErrorKind::kInvalidSchema)); EXPECT_THAT(result, HasErrorMessage("is not supported until v3")); } @@ -178,7 +181,7 @@ TEST_F(UpdateSchemaDefaultValueTest, AddColumnWithDefaultValue) { ICEBERG_UNWRAP_OR_FAIL(auto update, table_->NewUpdateSchema()); update->AddColumn("new_col", int32(), "An integer column", Literal::Int(42)); - ICEBERG_UNWRAP_OR_FAIL(auto result, update->Apply()); + ICEBERG_UNWRAP_OR_FAIL(auto result, update->Validate()); ICEBERG_UNWRAP_OR_FAIL(auto new_field_opt, result.schema->FindFieldByName("new_col")); ASSERT_TRUE(new_field_opt.has_value()); @@ -194,7 +197,7 @@ TEST_F(UpdateSchemaDefaultValueTest, AddRequiredColumnWithDefaultValue) { update->AddRequiredColumn("required_col", string(), "A required string column", Literal::String("n/a")); - ICEBERG_UNWRAP_OR_FAIL(auto result, update->Apply()); + ICEBERG_UNWRAP_OR_FAIL(auto result, update->Validate()); ICEBERG_UNWRAP_OR_FAIL(auto new_field_opt, result.schema->FindFieldByName("required_col")); ASSERT_TRUE(new_field_opt.has_value()); @@ -211,7 +214,7 @@ TEST_F(UpdateSchemaDefaultValueTest, AddColumnWithMismatchedDefaultValueFails) { ICEBERG_UNWRAP_OR_FAIL(auto update, table_->NewUpdateSchema()); update->AddColumn("new_col", int32(), "An integer column", Literal::String("oops")); - auto result = update->Apply(); + auto result = update->Validate(); EXPECT_THAT(result, IsError(ErrorKind::kValidationFailed)); EXPECT_THAT(result, HasErrorMessage("Cannot cast default value")); } @@ -221,7 +224,7 @@ TEST_F(UpdateSchemaDefaultValueTest, AddColumnWithNarrowingDefaultValueFails) { update->AddColumn("new_col", int32(), "An integer column", Literal::Long(std::numeric_limits::max())); - auto result = update->Apply(); + auto result = update->Validate(); EXPECT_THAT(result, IsError(ErrorKind::kValidationFailed)); EXPECT_THAT(result, HasErrorMessage("Cannot cast default value")); } @@ -231,7 +234,7 @@ TEST_F(UpdateSchemaDefaultValueTest, UpdateColumnDefault) { update->AddColumn("new_col", int32(), "An integer column", Literal::Int(42)) .UpdateColumnDefault("new_col", Literal::Int(7)); - ICEBERG_UNWRAP_OR_FAIL(auto result, update->Apply()); + ICEBERG_UNWRAP_OR_FAIL(auto result, update->Validate()); ICEBERG_UNWRAP_OR_FAIL(auto new_field_opt, result.schema->FindFieldByName("new_col")); ASSERT_TRUE(new_field_opt.has_value()); @@ -246,7 +249,7 @@ TEST_F(UpdateSchemaDefaultValueTest, UpdateColumnDefaultOnExistingColumn) { ICEBERG_UNWRAP_OR_FAIL(auto update, table_->NewUpdateSchema()); update->UpdateColumnDefault("x", Literal::Long(0)); - ICEBERG_UNWRAP_OR_FAIL(auto result, update->Apply()); + ICEBERG_UNWRAP_OR_FAIL(auto result, update->Validate()); ICEBERG_UNWRAP_OR_FAIL(auto field_opt, result.schema->FindFieldByName("x")); ASSERT_TRUE(field_opt.has_value()); @@ -261,7 +264,7 @@ TEST_F(UpdateSchemaDefaultValueTest, UpdateColumnDefaultClearsWithNullopt) { update->AddColumn("new_col", int32(), "An integer column", Literal::Int(42)) .UpdateColumnDefault("new_col", std::nullopt); - ICEBERG_UNWRAP_OR_FAIL(auto result, update->Apply()); + ICEBERG_UNWRAP_OR_FAIL(auto result, update->Validate()); ICEBERG_UNWRAP_OR_FAIL(auto field_opt, result.schema->FindFieldByName("new_col")); ASSERT_TRUE(field_opt.has_value()); @@ -280,7 +283,7 @@ TEST_F(UpdateSchemaDefaultValueTest, AddNestedColumnPreservesNestedDefaults) { ICEBERG_UNWRAP_OR_FAIL(auto update, table_->NewUpdateSchema()); update->AddColumn("outer", nested_type, "A nested column"); - ICEBERG_UNWRAP_OR_FAIL(auto result, update->Apply()); + ICEBERG_UNWRAP_OR_FAIL(auto result, update->Validate()); ICEBERG_UNWRAP_OR_FAIL(auto outer_opt, result.schema->FindFieldByName("outer")); ASSERT_TRUE(outer_opt.has_value()); @@ -294,11 +297,91 @@ TEST_F(UpdateSchemaDefaultValueTest, AddNestedColumnPreservesNestedDefaults) { EXPECT_EQ(*inner.write_default(), Literal::Int(9)); } +TEST_F(UpdateSchemaDefaultValueTest, FrozenNestedInputsAndPreviewsAreIsolated) { + auto amount_type = decimal(9, 2); + auto key_type = fixed(4); + auto initial = std::make_shared(Literal::Decimal(1234, 9, 2)); + auto write = std::make_shared(Literal::Decimal(5678, 9, 2)); + auto element_type = std::make_shared(std::vector{ + SchemaField(100, "amount", amount_type, false, "amount doc", initial, write)}); + auto list_type = + std::make_shared(SchemaField(101, "element", element_type, true)); + auto map_type = std::make_shared(SchemaField(102, "key", key_type, false), + SchemaField(103, "value", list_type, true)); + + ICEBERG_UNWRAP_OR_FAIL(auto txn, table_->NewTransaction()); + ICEBERG_UNWRAP_OR_FAIL(auto update, txn->NewUpdateSchema()); + update->AddColumn("nested", map_type, "nested doc") + .AddRequiredColumn("copy_id", int64(), "identifier doc", Literal::Long(42)); + std::array identifier_names = {"copy_id"}; + std::span identifiers(identifier_names); + update->SetIdentifierFields(identifiers); + ICEBERG_UNWRAP_OR_FAIL(auto expected, update->Validate()); + const auto identifier_ids = expected.schema->IdentifierFieldIds(); + ASSERT_THAT(update->Commit(), IsOk()); + + auto verify = [&](const Schema& schema) { + EXPECT_THAT(schema.fields(), ::testing::ElementsAreArray(expected.schema->fields())); + EXPECT_EQ(schema.IdentifierFieldIds(), identifier_ids); + ICEBERG_UNWRAP_OR_FAIL(auto nested, schema.FindFieldByName("nested")); + ASSERT_TRUE(nested.has_value()); + EXPECT_EQ(nested->get().doc(), "nested doc"); + const auto& map = checked_cast(*nested->get().type()); + EXPECT_EQ(*map.key().type(), *fixed(4)); + const auto& list = checked_cast(*map.value().type()); + const auto& element = checked_cast(*list.element().type()); + ASSERT_EQ(element.fields().size(), 1U); + const auto& amount = element.fields()[0]; + EXPECT_EQ(*amount.type(), *decimal(9, 2)); + EXPECT_EQ(amount.doc(), "amount doc"); + ASSERT_NE(amount.initial_default(), nullptr); + EXPECT_EQ(*amount.initial_default(), Literal::Decimal(1234, 9, 2)); + ASSERT_NE(amount.write_default(), nullptr); + EXPECT_EQ(*amount.write_default(), Literal::Decimal(5678, 9, 2)); + }; + + // Mutating retained input aliases must not change the frozen operation. + *amount_type = DecimalType(18, 4); + *key_type = FixedType(8); + *initial = Literal::Decimal(9999, 18, 4); + *write = Literal::Decimal(8888, 18, 4); + ICEBERG_UNWRAP_OR_FAIL(auto preview, update->Validate()); + EXPECT_EQ(preview.schema->schema_id(), expected.schema->schema_id()); + verify(*preview.schema); + + // A public preview must also be independent of the frozen inputs and the + // metadata already staged in the transaction. + ICEBERG_UNWRAP_OR_FAIL(auto nested, preview.schema->FindFieldByName("nested")); + ASSERT_TRUE(nested.has_value()); + auto& map = checked_cast(*nested->get().type()); + checked_cast(*map.key().type()) = FixedType(16); + auto& list = checked_cast(*map.value().type()); + auto& element = checked_cast(*list.element().type()); + const auto& amount = element.fields()[0]; + checked_cast(*amount.type()) = DecimalType(20, 5); + checked_cast(*amount.initial_default()->type()) = DecimalType(21, 6); + checked_cast(*amount.write_default()->type()) = DecimalType(22, 7); + list = ListType(SchemaField(101, "element", int64(), true)); + map = MapType(SchemaField(102, "key", string(), false), + SchemaField(103, "value", int64(), false)); + + ICEBERG_UNWRAP_OR_FAIL(auto fresh_preview, update->Validate()); + EXPECT_EQ(fresh_preview.schema->schema_id(), expected.schema->schema_id()); + verify(*fresh_preview.schema); + ICEBERG_UNWRAP_OR_FAIL(auto staged, txn->current().Schema()); + verify(*staged); + ASSERT_THAT(txn->Commit(), IsOk()); + ICEBERG_UNWRAP_OR_FAIL(auto reloaded, catalog_->LoadTable(table_ident_)); + ICEBERG_UNWRAP_OR_FAIL(auto committed, reloaded->schema()); + EXPECT_EQ(committed->schema_id(), staged->schema_id()); + verify(*committed); +} + TEST_F(UpdateSchemaDefaultValueTest, UpdateColumnDefaultCastsToColumnType) { ICEBERG_UNWRAP_OR_FAIL(auto update, table_->NewUpdateSchema()); update->UpdateColumnDefault("x", Literal::Int(5)); - ICEBERG_UNWRAP_OR_FAIL(auto result, update->Apply()); + ICEBERG_UNWRAP_OR_FAIL(auto result, update->Validate()); ICEBERG_UNWRAP_OR_FAIL(auto field_opt, result.schema->FindFieldByName("x")); ASSERT_TRUE(field_opt.has_value()); @@ -312,7 +395,7 @@ TEST_F(UpdateSchemaDefaultValueTest, RequireColumnAddedWithDefault) { update->AddColumn("new_col", int32(), "An integer column", Literal::Int(42)) .RequireColumn("new_col"); - ICEBERG_UNWRAP_OR_FAIL(auto result, update->Apply()); + ICEBERG_UNWRAP_OR_FAIL(auto result, update->Validate()); ICEBERG_UNWRAP_OR_FAIL(auto new_field_opt, result.schema->FindFieldByName("new_col")); ASSERT_TRUE(new_field_opt.has_value()); EXPECT_FALSE(new_field_opt->get().optional()); @@ -345,7 +428,7 @@ TEST_F(UpdateSchemaDefaultValueTest, RequireNestedMapListColumnAddedWithDefault) .AddColumn("points", "z", int64(), "z coordinate", Literal::Long(0)) .RequireColumn("points.z"); - ICEBERG_UNWRAP_OR_FAIL(auto result, update->Apply()); + ICEBERG_UNWRAP_OR_FAIL(auto result, update->Validate()); ICEBERG_UNWRAP_OR_FAIL(auto locations_opt, result.schema->FindFieldByName("locations")); ASSERT_TRUE(locations_opt.has_value()); @@ -371,7 +454,7 @@ TEST_F(UpdateSchemaDefaultValueTest, UpdateColumnDocPreservesDefaultValues) { update->AddColumn("new_col", int32(), "An integer column", Literal::Int(42)) .UpdateColumnDoc("new_col", "updated doc"); - ICEBERG_UNWRAP_OR_FAIL(auto result, update->Apply()); + ICEBERG_UNWRAP_OR_FAIL(auto result, update->Validate()); ICEBERG_UNWRAP_OR_FAIL(auto field_opt, result.schema->FindFieldByName("new_col")); ASSERT_TRUE(field_opt.has_value()); @@ -388,12 +471,12 @@ TEST_F(UpdateSchemaDefaultValueTest, UpdateColumnTypePromotesDefaultValues) { update->AddColumn("new_col", int32(), "An integer column", Literal::Int(42)) .UpdateColumn("new_col", int64()); - ICEBERG_UNWRAP_OR_FAIL(auto result, update->Apply()); + ICEBERG_UNWRAP_OR_FAIL(auto result, update->Validate()); ICEBERG_UNWRAP_OR_FAIL(auto field_opt, result.schema->FindFieldByName("new_col")); ASSERT_TRUE(field_opt.has_value()); const auto& field = field_opt->get(); - EXPECT_EQ(field.type(), int64()); + EXPECT_EQ(*field.type(), *int64()); ASSERT_NE(field.initial_default(), nullptr); EXPECT_EQ(*field.initial_default(), Literal::Long(42)); ASSERT_NE(field.write_default(), nullptr); @@ -407,7 +490,7 @@ TEST_F(UpdateSchemaDefaultValueTest, UpdateColumnTypePromotesDecimalDefault) { Literal::Decimal(1234, 9, 2)) .UpdateColumn("new_col", decimal(18, 2)); - ICEBERG_UNWRAP_OR_FAIL(auto result, update->Apply()); + ICEBERG_UNWRAP_OR_FAIL(auto result, update->Validate()); ICEBERG_UNWRAP_OR_FAIL(auto field_opt, result.schema->FindFieldByName("new_col")); ASSERT_TRUE(field_opt.has_value()); @@ -424,7 +507,7 @@ TEST_F(UpdateSchemaDefaultValueTest, AddColumnWithWiderPrecisionDecimalDefault) update->AddColumn("new_col", decimal(18, 2), "A decimal column", Literal::Decimal(1234, 9, 2)); - ICEBERG_UNWRAP_OR_FAIL(auto result, update->Apply()); + ICEBERG_UNWRAP_OR_FAIL(auto result, update->Validate()); ICEBERG_UNWRAP_OR_FAIL(auto field_opt, result.schema->FindFieldByName("new_col")); ASSERT_TRUE(field_opt.has_value()); @@ -440,7 +523,7 @@ TEST_F(UpdateSchemaDefaultValueTest, UpdateColumnDefaultWiderPrecisionDecimal) { update->AddColumn("new_col", decimal(18, 2), "A decimal column") .UpdateColumnDefault("new_col", Literal::Decimal(1234, 9, 2)); - ICEBERG_UNWRAP_OR_FAIL(auto result, update->Apply()); + ICEBERG_UNWRAP_OR_FAIL(auto result, update->Validate()); ICEBERG_UNWRAP_OR_FAIL(auto field_opt, result.schema->FindFieldByName("new_col")); ASSERT_TRUE(field_opt.has_value()); @@ -454,7 +537,7 @@ TEST_F(UpdateSchemaDefaultValueTest, AddColumnWithDifferentScaleDecimalDefaultFa update->AddColumn("new_col", decimal(18, 2), "A decimal column", Literal::Decimal(1234, 9, 3)); - auto result = update->Apply(); + auto result = update->Validate(); EXPECT_THAT(result, IsError(ErrorKind::kValidationFailed)); EXPECT_THAT(result, HasErrorMessage("Cannot cast default value")); } @@ -464,7 +547,7 @@ TEST_F(UpdateSchemaDefaultValueTest, UpdateColumnDefaultDifferentScaleDecimalFai update->AddColumn("new_col", decimal(18, 2), "A decimal column") .UpdateColumnDefault("new_col", Literal::Decimal(1234, 9, 3)); - auto result = update->Apply(); + auto result = update->Validate(); EXPECT_THAT(result, IsError(ErrorKind::kValidationFailed)); EXPECT_THAT(result, HasErrorMessage("Cannot cast default value")); } @@ -474,7 +557,7 @@ TEST_F(UpdateSchemaDefaultValueTest, AddColumnWithOutOfPrecisionDecimalDefaultFa update->AddColumn("new_col", decimal(4, 2), "A decimal column", Literal::Decimal(1234567, 9, 2)); - auto result = update->Apply(); + auto result = update->Validate(); EXPECT_THAT(result, IsError(ErrorKind::kValidationFailed)); EXPECT_THAT(result, HasErrorMessage("Cannot cast default value")); } @@ -484,7 +567,7 @@ TEST_F(UpdateSchemaDefaultValueTest, AddColumnWithTypedNullDecimalDefaultFails) update->AddColumn("new_col", decimal(18, 2), "A decimal column", Literal::Null(decimal(18, 2))); - auto result = update->Apply(); + auto result = update->Validate(); EXPECT_THAT(result, IsError(ErrorKind::kValidationFailed)); EXPECT_THAT(result, HasErrorMessage("Cannot cast default value")); } @@ -495,7 +578,7 @@ TEST_F(UpdateSchemaTest, AddMultipleColumns) { .AddColumn("col2", string(), "Second column") .AddColumn("col3", boolean(), "Third column"); - ICEBERG_UNWRAP_OR_FAIL(auto result, update->Apply()); + ICEBERG_UNWRAP_OR_FAIL(auto result, update->Validate()); ASSERT_TRUE(result.schema != nullptr); ICEBERG_UNWRAP_OR_FAIL(auto col1_opt, result.schema->FindFieldByName("col1")); @@ -506,16 +589,16 @@ TEST_F(UpdateSchemaTest, AddMultipleColumns) { ASSERT_TRUE(col2_opt.has_value()); ASSERT_TRUE(col3_opt.has_value()); - EXPECT_EQ(col1_opt->get().type(), int32()); - EXPECT_EQ(col2_opt->get().type(), string()); - EXPECT_EQ(col3_opt->get().type(), boolean()); + EXPECT_EQ(*col1_opt->get().type(), *int32()); + EXPECT_EQ(*col2_opt->get().type(), *string()); + EXPECT_EQ(*col3_opt->get().type(), *boolean()); } TEST_F(UpdateSchemaTest, AddColumnWithDotInNameFails) { ICEBERG_UNWRAP_OR_FAIL(auto update, table_->NewUpdateSchema()); update->AddColumn("col.with.dots", int32(), "Column with dots"); - auto result = update->Apply(); + auto result = update->Validate(); EXPECT_THAT(result, IsError(ErrorKind::kValidationFailed)); EXPECT_THAT(result, HasErrorMessage("Cannot add column with ambiguous name")); } @@ -532,7 +615,7 @@ TEST_F(UpdateSchemaTest, AddColumnToNestedStruct) { ICEBERG_UNWRAP_OR_FAIL(auto update, reloaded->NewUpdateSchema()); update->AddColumn("struct_col", "new_nested_field", string(), "New nested field"); - ICEBERG_UNWRAP_OR_FAIL(auto result, update->Apply()); + ICEBERG_UNWRAP_OR_FAIL(auto result, update->Validate()); ASSERT_TRUE(result.schema != nullptr); ICEBERG_UNWRAP_OR_FAIL(auto struct_field_opt, @@ -549,14 +632,14 @@ TEST_F(UpdateSchemaTest, AddColumnToNestedStruct) { const auto& nested_field = nested_field_opt->get(); EXPECT_EQ(nested_field.name(), "new_nested_field"); - EXPECT_EQ(nested_field.type(), string()); + EXPECT_EQ(*nested_field.type(), *string()); } TEST_F(UpdateSchemaTest, AddColumnToNonExistentParentFails) { ICEBERG_UNWRAP_OR_FAIL(auto update, table_->NewUpdateSchema()); update->AddColumn("non_existent_parent", "new_field", int32(), "New field"); - auto result = update->Apply(); + auto result = update->Validate(); EXPECT_THAT(result, IsError(ErrorKind::kValidationFailed)); EXPECT_THAT(result, HasErrorMessage("Cannot find parent struct")); } @@ -570,7 +653,7 @@ TEST_F(UpdateSchemaTest, AddColumnToNonStructParentFails) { ICEBERG_UNWRAP_OR_FAIL(auto update, reloaded->NewUpdateSchema()); update->AddColumn("primitive_col", "nested_field", string(), "Should fail"); - auto result = update->Apply(); + auto result = update->Validate(); EXPECT_THAT(result, IsError(ErrorKind::kValidationFailed)); EXPECT_THAT(result, HasErrorMessage("Cannot add to non-struct column")); } @@ -580,7 +663,7 @@ TEST_F(UpdateSchemaTest, AddDuplicateColumnNameFails) { update->AddColumn("duplicate_col", int32(), "First column") .AddColumn("duplicate_col", string(), "Duplicate column"); - auto result = update->Apply(); + auto result = update->Validate(); EXPECT_THAT(result, IsError(ErrorKind::kInvalidSchema)); EXPECT_THAT(result, HasErrorMessage("Duplicate path found")); } @@ -593,7 +676,7 @@ TEST_F(UpdateSchemaTest, ColumnIdAssignment) { update->AddColumn("new_col1", int32(), "First new column") .AddColumn("new_col2", string(), "Second new column"); - ICEBERG_UNWRAP_OR_FAIL(auto result, update->Apply()); + ICEBERG_UNWRAP_OR_FAIL(auto result, update->Validate()); EXPECT_EQ(result.new_last_column_id, original_last_id + 2); @@ -615,7 +698,7 @@ TEST_F(UpdateSchemaTest, AddNestedStructColumn) { ICEBERG_UNWRAP_OR_FAIL(auto update, table_->NewUpdateSchema()); update->AddColumn("complex_struct", nested_struct, "A complex struct column"); - ICEBERG_UNWRAP_OR_FAIL(auto result, update->Apply()); + ICEBERG_UNWRAP_OR_FAIL(auto result, update->Validate()); ASSERT_TRUE(result.schema != nullptr); ICEBERG_UNWRAP_OR_FAIL(auto struct_field_opt, @@ -636,8 +719,8 @@ TEST_F(UpdateSchemaTest, AddNestedStructColumn) { ASSERT_TRUE(field1_opt.has_value()); ASSERT_TRUE(field2_opt.has_value()); - EXPECT_EQ(field1_opt->get().type(), int32()); - EXPECT_EQ(field2_opt->get().type(), string()); + EXPECT_EQ(*field1_opt->get().type(), *int32()); + EXPECT_EQ(*field2_opt->get().type(), *string()); EXPECT_TRUE(field1_opt->get().optional()); EXPECT_FALSE(field2_opt->get().optional()); } @@ -648,7 +731,7 @@ TEST_F(UpdateSchemaTest, CaseSensitiveColumnNames) { .AddColumn("Column", int32(), "Uppercase column") .AddColumn("column", string(), "Lowercase column"); - ICEBERG_UNWRAP_OR_FAIL(auto result, update->Apply()); + ICEBERG_UNWRAP_OR_FAIL(auto result, update->Validate()); ASSERT_TRUE(result.schema != nullptr); ICEBERG_UNWRAP_OR_FAIL(auto upper_opt, result.schema->FindFieldByName("Column", true)); @@ -657,8 +740,8 @@ TEST_F(UpdateSchemaTest, CaseSensitiveColumnNames) { ASSERT_TRUE(upper_opt.has_value()); ASSERT_TRUE(lower_opt.has_value()); - EXPECT_EQ(upper_opt->get().type(), int32()); - EXPECT_EQ(lower_opt->get().type(), string()); + EXPECT_EQ(*upper_opt->get().type(), *int32()); + EXPECT_EQ(*lower_opt->get().type(), *string()); } TEST_F(UpdateSchemaTest, CaseInsensitiveDuplicateDetection) { @@ -667,7 +750,7 @@ TEST_F(UpdateSchemaTest, CaseInsensitiveDuplicateDetection) { .AddColumn("Column", int32(), "First column") .AddColumn("COLUMN", string(), "Duplicate column"); - auto result = update->Apply(); + auto result = update->Validate(); EXPECT_THAT(result, IsError(ErrorKind::kInvalidSchema)); EXPECT_THAT(result, HasErrorMessage("Duplicate path found")); } @@ -676,7 +759,7 @@ TEST_F(UpdateSchemaTest, EmptyUpdate) { ICEBERG_UNWRAP_OR_FAIL(auto original_schema, table_->schema()); ICEBERG_UNWRAP_OR_FAIL(auto update, table_->NewUpdateSchema()); - ICEBERG_UNWRAP_OR_FAIL(auto result, update->Apply()); + ICEBERG_UNWRAP_OR_FAIL(auto result, update->Validate()); EXPECT_EQ(*result.schema, *original_schema); EXPECT_EQ(result.new_last_column_id, table_->metadata()->last_column_id); @@ -730,7 +813,7 @@ TEST_F(UpdateSchemaTest, AddFieldsToMapAndList) { update->AddColumn("locations", "alt", float32(), "altitude") .AddColumn("points", "z", int64(), "z coordinate"); - ICEBERG_UNWRAP_OR_FAIL(auto result, update->Apply()); + ICEBERG_UNWRAP_OR_FAIL(auto result, update->Validate()); ICEBERG_UNWRAP_OR_FAIL(auto locations_opt, result.schema->FindFieldByName("locations")); ASSERT_TRUE(locations_opt.has_value()); @@ -741,7 +824,7 @@ TEST_F(UpdateSchemaTest, AddFieldsToMapAndList) { const auto& value_struct = checked_cast(*map.value().type()); ICEBERG_UNWRAP_OR_FAIL(auto alt_opt, value_struct.GetFieldByName("alt")); ASSERT_TRUE(alt_opt.has_value()); - EXPECT_EQ(alt_opt->get().type(), float32()); + EXPECT_EQ(*alt_opt->get().type(), *float32()); ICEBERG_UNWRAP_OR_FAIL(auto points_opt, result.schema->FindFieldByName("points")); ASSERT_TRUE(points_opt.has_value()); @@ -752,7 +835,7 @@ TEST_F(UpdateSchemaTest, AddFieldsToMapAndList) { const auto& element_struct = checked_cast(*list.element().type()); ICEBERG_UNWRAP_OR_FAIL(auto z_opt, element_struct.GetFieldByName("z")); ASSERT_TRUE(z_opt.has_value()); - EXPECT_EQ(z_opt->get().type(), int64()); + EXPECT_EQ(*z_opt->get().type(), *int64()); } TEST_F(UpdateSchemaTest, AddNestedStructWithIdReassignment) { @@ -762,7 +845,7 @@ TEST_F(UpdateSchemaTest, AddNestedStructWithIdReassignment) { ICEBERG_UNWRAP_OR_FAIL(auto update, table_->NewUpdateSchema()); update->AddColumn("location", nested_struct); - ICEBERG_UNWRAP_OR_FAIL(auto result, update->Apply()); + ICEBERG_UNWRAP_OR_FAIL(auto result, update->Validate()); ICEBERG_UNWRAP_OR_FAIL(auto location_opt, result.schema->FindFieldByName("location")); ASSERT_TRUE(location_opt.has_value()); @@ -798,7 +881,7 @@ TEST_F(UpdateSchemaTest, AddNestedMapOfStructs) { ICEBERG_UNWRAP_OR_FAIL(auto update, table_->NewUpdateSchema()); update->AddColumn("locations", map_type); - ICEBERG_UNWRAP_OR_FAIL(auto result, update->Apply()); + ICEBERG_UNWRAP_OR_FAIL(auto result, update->Validate()); ICEBERG_UNWRAP_OR_FAIL(auto locations_opt, result.schema->FindFieldByName("locations")); ASSERT_TRUE(locations_opt.has_value()); @@ -831,7 +914,7 @@ TEST_F(UpdateSchemaTest, AddNestedListOfStructs) { ICEBERG_UNWRAP_OR_FAIL(auto update, table_->NewUpdateSchema()); update->AddColumn("locations", list_type); - ICEBERG_UNWRAP_OR_FAIL(auto result, update->Apply()); + ICEBERG_UNWRAP_OR_FAIL(auto result, update->Validate()); ICEBERG_UNWRAP_OR_FAIL(auto locations_opt, result.schema->FindFieldByName("locations")); ASSERT_TRUE(locations_opt.has_value()); @@ -864,7 +947,7 @@ TEST_F(UpdateSchemaTest, AddFieldWithDotsInName) { ICEBERG_UNWRAP_OR_FAIL(auto update, reloaded->NewUpdateSchema()); update->AddColumn("struct_col", "field.with.dots", int64(), "Field with dots in name"); - ICEBERG_UNWRAP_OR_FAIL(auto result, update->Apply()); + ICEBERG_UNWRAP_OR_FAIL(auto result, update->Validate()); ICEBERG_UNWRAP_OR_FAIL(auto struct_field_opt, result.schema->FindFieldByName("struct_col")); @@ -877,7 +960,7 @@ TEST_F(UpdateSchemaTest, AddFieldWithDotsInName) { nested_struct.GetFieldByName("field.with.dots")); ASSERT_TRUE(dotted_field_opt.has_value()); EXPECT_EQ(dotted_field_opt->get().name(), "field.with.dots"); - EXPECT_EQ(dotted_field_opt->get().type(), int64()); + EXPECT_EQ(*dotted_field_opt->get().type(), *int64()); } TEST_F(UpdateSchemaTest, AddFieldToMapKeyFails) { @@ -899,7 +982,7 @@ TEST_F(UpdateSchemaTest, AddFieldToMapKeyFails) { ICEBERG_UNWRAP_OR_FAIL(auto update, reloaded->NewUpdateSchema()); update->AddColumn("locations.key", "city", string(), "Should fail"); - auto result = update->Apply(); + auto result = update->Validate(); EXPECT_THAT(result, IsError(ErrorKind::kValidationFailed)); EXPECT_THAT(result, HasErrorMessage("Cannot add fields to map keys")); } @@ -913,7 +996,7 @@ TEST_F(UpdateSchemaTest, DeleteColumn) { ICEBERG_UNWRAP_OR_FAIL(auto update, reloaded->NewUpdateSchema()); update->DeleteColumn("to_delete"); - ICEBERG_UNWRAP_OR_FAIL(auto result, update->Apply()); + ICEBERG_UNWRAP_OR_FAIL(auto result, update->Validate()); ICEBERG_UNWRAP_OR_FAIL(auto field_opt, result.schema->FindFieldByName("to_delete")); EXPECT_FALSE(field_opt.has_value()); @@ -932,7 +1015,7 @@ TEST_F(UpdateSchemaTest, DeleteNestedColumn) { ICEBERG_UNWRAP_OR_FAIL(auto update, reloaded->NewUpdateSchema()); update->DeleteColumn("struct_col.field1"); - ICEBERG_UNWRAP_OR_FAIL(auto result, update->Apply()); + ICEBERG_UNWRAP_OR_FAIL(auto result, update->Validate()); ICEBERG_UNWRAP_OR_FAIL(auto struct_field_opt, result.schema->FindFieldByName("struct_col")); @@ -952,7 +1035,7 @@ TEST_F(UpdateSchemaTest, DeleteMissingColumnFails) { ICEBERG_UNWRAP_OR_FAIL(auto update, table_->NewUpdateSchema()); update->DeleteColumn("non_existent"); - auto result = update->Apply(); + auto result = update->Validate(); EXPECT_THAT(result, IsError(ErrorKind::kValidationFailed)); EXPECT_THAT(result, HasErrorMessage("Cannot delete missing column")); } @@ -967,13 +1050,13 @@ TEST_F(UpdateSchemaTest, DeleteThenAdd) { ICEBERG_UNWRAP_OR_FAIL(auto update, reloaded->NewUpdateSchema()); update->DeleteColumn("col").AddColumn("col", string(), "Now optional string"); - ICEBERG_UNWRAP_OR_FAIL(auto result, update->Apply()); + ICEBERG_UNWRAP_OR_FAIL(auto result, update->Validate()); ICEBERG_UNWRAP_OR_FAIL(auto field_opt, result.schema->FindFieldByName("col")); ASSERT_TRUE(field_opt.has_value()); const auto& field = field_opt->get(); - EXPECT_EQ(field.type(), string()); + EXPECT_EQ(*field.type(), *string()); EXPECT_TRUE(field.optional()); } @@ -990,7 +1073,7 @@ TEST_F(UpdateSchemaTest, DeleteThenAddNested) { update->DeleteColumn("struct_col.field1") .AddColumn("struct_col", "field1", int32(), "Re-added field"); - ICEBERG_UNWRAP_OR_FAIL(auto result, update->Apply()); + ICEBERG_UNWRAP_OR_FAIL(auto result, update->Validate()); ICEBERG_UNWRAP_OR_FAIL(auto struct_field_opt, result.schema->FindFieldByName("struct_col")); @@ -1001,14 +1084,14 @@ TEST_F(UpdateSchemaTest, DeleteThenAddNested) { ICEBERG_UNWRAP_OR_FAIL(auto field1_opt, nested_struct.GetFieldByName("field1")); ASSERT_TRUE(field1_opt.has_value()); - EXPECT_EQ(field1_opt->get().type(), int32()); + EXPECT_EQ(*field1_opt->get().type(), *int32()); } TEST_F(UpdateSchemaTest, AddDeleteConflict) { ICEBERG_UNWRAP_OR_FAIL(auto update, table_->NewUpdateSchema()); update->AddColumn("new_col", int32()).DeleteColumn("new_col"); - auto result = update->Apply(); + auto result = update->Validate(); EXPECT_THAT(result, IsError(ErrorKind::kValidationFailed)); EXPECT_THAT(result, HasErrorMessage("Cannot delete missing column")); } @@ -1025,7 +1108,7 @@ TEST_F(UpdateSchemaTest, DeleteColumnWithAdditionsFails) { ICEBERG_UNWRAP_OR_FAIL(auto update, reloaded->NewUpdateSchema()); update->AddColumn("struct_col", "field2", string()).DeleteColumn("struct_col"); - auto result = update->Apply(); + auto result = update->Validate(); EXPECT_THAT(result, IsError(ErrorKind::kValidationFailed)); EXPECT_THAT(result, HasErrorMessage("Cannot delete a column that has additions")); } @@ -1042,7 +1125,7 @@ TEST_F(UpdateSchemaTest, DeleteMapKeyFails) { ICEBERG_UNWRAP_OR_FAIL(auto update, reloaded->NewUpdateSchema()); update->DeleteColumn("map_col.key"); - auto result = update->Apply(); + auto result = update->Validate(); EXPECT_THAT(result, IsError(ErrorKind::kValidationFailed)); EXPECT_THAT(result, HasErrorMessage("Cannot delete map keys")); } @@ -1056,7 +1139,7 @@ TEST_F(UpdateSchemaTest, DeleteColumnCaseInsensitive) { ICEBERG_UNWRAP_OR_FAIL(auto update, reloaded->NewUpdateSchema()); update->CaseSensitive(false).DeleteColumn("mycolumn"); - ICEBERG_UNWRAP_OR_FAIL(auto result, update->Apply()); + ICEBERG_UNWRAP_OR_FAIL(auto result, update->Validate()); ICEBERG_UNWRAP_OR_FAIL(auto field_opt, result.schema->FindFieldByName("MyColumn", false)); @@ -1072,7 +1155,7 @@ TEST_F(UpdateSchemaTest, RenameColumn) { ICEBERG_UNWRAP_OR_FAIL(auto update, reloaded->NewUpdateSchema()); update->RenameColumn("old_name", "new_name"); - ICEBERG_UNWRAP_OR_FAIL(auto result, update->Apply()); + ICEBERG_UNWRAP_OR_FAIL(auto result, update->Validate()); ICEBERG_UNWRAP_OR_FAIL(auto old_field_opt, result.schema->FindFieldByName("old_name")); ICEBERG_UNWRAP_OR_FAIL(auto new_field_opt, result.schema->FindFieldByName("new_name")); @@ -1095,7 +1178,7 @@ TEST_F(UpdateSchemaTest, RenameNestedColumn) { ICEBERG_UNWRAP_OR_FAIL(auto update, reloaded->NewUpdateSchema()); update->RenameColumn("struct_col.field1", "renamed_field"); - ICEBERG_UNWRAP_OR_FAIL(auto result, update->Apply()); + ICEBERG_UNWRAP_OR_FAIL(auto result, update->Validate()); ICEBERG_UNWRAP_OR_FAIL(auto struct_field_opt, result.schema->FindFieldByName("struct_col")); @@ -1121,7 +1204,7 @@ TEST_F(UpdateSchemaTest, RenameColumnWithDotsInName) { ICEBERG_UNWRAP_OR_FAIL(auto update, reloaded->NewUpdateSchema()); update->RenameColumn("simple_name", "name.with.dots"); - ICEBERG_UNWRAP_OR_FAIL(auto result, update->Apply()); + ICEBERG_UNWRAP_OR_FAIL(auto result, update->Validate()); ICEBERG_UNWRAP_OR_FAIL(auto new_field_opt, result.schema->FindFieldByName("name.with.dots")); @@ -1133,7 +1216,7 @@ TEST_F(UpdateSchemaTest, RenameMissingColumnFails) { ICEBERG_UNWRAP_OR_FAIL(auto update, table_->NewUpdateSchema()); update->RenameColumn("non_existent", "new_name"); - auto result = update->Apply(); + auto result = update->Validate(); EXPECT_THAT(result, IsError(ErrorKind::kValidationFailed)); EXPECT_THAT(result, HasErrorMessage("Cannot rename missing column")); } @@ -1147,7 +1230,7 @@ TEST_F(UpdateSchemaTest, RenameDeletedColumnFails) { ICEBERG_UNWRAP_OR_FAIL(auto update, reloaded->NewUpdateSchema()); update->DeleteColumn("col").RenameColumn("col", "new_name"); - auto result = update->Apply(); + auto result = update->Validate(); EXPECT_THAT(result, IsError(ErrorKind::kValidationFailed)); EXPECT_THAT(result, HasErrorMessage("Cannot rename a column that will be deleted")); } @@ -1161,7 +1244,7 @@ TEST_F(UpdateSchemaTest, RenameColumnCaseInsensitive) { ICEBERG_UNWRAP_OR_FAIL(auto update, reloaded->NewUpdateSchema()); update->CaseSensitive(false).RenameColumn("mycolumn", "NewName"); - ICEBERG_UNWRAP_OR_FAIL(auto result, update->Apply()); + ICEBERG_UNWRAP_OR_FAIL(auto result, update->Validate()); ICEBERG_UNWRAP_OR_FAIL(auto old_field_opt, result.schema->FindFieldByName("MyColumn", false)); @@ -1181,7 +1264,7 @@ TEST_F(UpdateSchemaTest, RenameThenDeleteOldNameFails) { ICEBERG_UNWRAP_OR_FAIL(auto update, reloaded->NewUpdateSchema()); update->RenameColumn("old_name", "new_name").DeleteColumn("old_name"); - auto result = update->Apply(); + auto result = update->Validate(); EXPECT_THAT(result, IsError(ErrorKind::kValidationFailed)); EXPECT_THAT(result, HasErrorMessage("Cannot delete a column that has updates")); } @@ -1195,7 +1278,7 @@ TEST_F(UpdateSchemaTest, RenameThenDeleteNewNameFails) { ICEBERG_UNWRAP_OR_FAIL(auto update, reloaded->NewUpdateSchema()); update->RenameColumn("old_name", "new_name").DeleteColumn("new_name"); - auto result = update->Apply(); + auto result = update->Validate(); EXPECT_THAT(result, IsError(ErrorKind::kValidationFailed)); EXPECT_THAT(result, HasErrorMessage("Cannot delete missing column")); } @@ -1209,7 +1292,7 @@ TEST_F(UpdateSchemaTest, RenameThenAddWithOldName) { ICEBERG_UNWRAP_OR_FAIL(auto update, reloaded->NewUpdateSchema()); update->RenameColumn("old_name", "new_name").AddColumn("old_name", string()); - auto result = update->Apply(); + auto result = update->Validate(); EXPECT_THAT(result, IsError(ErrorKind::kValidationFailed)); EXPECT_THAT(result, HasErrorMessage("Cannot add column, name already exists")); } @@ -1218,7 +1301,7 @@ TEST_F(UpdateSchemaTest, AddThenRename) { ICEBERG_UNWRAP_OR_FAIL(auto update, table_->NewUpdateSchema()); update->AddColumn("temp_name", string()).RenameColumn("temp_name", "final_name"); - auto result = update->Apply(); + auto result = update->Validate(); EXPECT_THAT(result, IsError(ErrorKind::kValidationFailed)); EXPECT_THAT(result, HasErrorMessage("Cannot rename missing column")); } @@ -1234,7 +1317,7 @@ TEST_F(UpdateSchemaTest, DeleteThenAddThenRename) { .AddColumn("col", string(), "New column with same name") .RenameColumn("col", "renamed_col"); - auto result = update->Apply(); + auto result = update->Validate(); EXPECT_THAT(result, IsError(ErrorKind::kValidationFailed)); EXPECT_THAT(result, HasErrorMessage("Cannot rename a column that will be deleted")); } @@ -1248,7 +1331,7 @@ TEST_F(UpdateSchemaTest, MakeColumnOptional) { ICEBERG_UNWRAP_OR_FAIL(auto update, reloaded->NewUpdateSchema()); update->MakeColumnOptional("id"); - ICEBERG_UNWRAP_OR_FAIL(auto result, update->Apply()); + ICEBERG_UNWRAP_OR_FAIL(auto result, update->Validate()); ICEBERG_UNWRAP_OR_FAIL(auto field_opt, result.schema->FindFieldByName("id")); ASSERT_TRUE(field_opt.has_value()); @@ -1264,7 +1347,7 @@ TEST_F(UpdateSchemaTest, RequireColumn) { ICEBERG_UNWRAP_OR_FAIL(auto update, reloaded->NewUpdateSchema()); update->RequireColumn("id"); - auto result = update->Apply(); + auto result = update->Validate(); EXPECT_THAT(result, IsError(ErrorKind::kValidationFailed)); EXPECT_THAT(result, HasErrorMessage("Cannot change column nullability")); EXPECT_THAT(result, HasErrorMessage("optional -> required")); @@ -1273,7 +1356,7 @@ TEST_F(UpdateSchemaTest, RequireColumn) { ICEBERG_UNWRAP_OR_FAIL(auto update2, reloaded2->NewUpdateSchema()); update2->AllowIncompatibleChanges().RequireColumn("id"); - ICEBERG_UNWRAP_OR_FAIL(auto result2, update2->Apply()); + ICEBERG_UNWRAP_OR_FAIL(auto result2, update2->Validate()); ICEBERG_UNWRAP_OR_FAIL(auto field_opt, result2.schema->FindFieldByName("id")); ASSERT_TRUE(field_opt.has_value()); @@ -1289,7 +1372,7 @@ TEST_F(UpdateSchemaTest, RequireColumnNoop) { ICEBERG_UNWRAP_OR_FAIL(auto update, reloaded->NewUpdateSchema()); update->RequireColumn("id"); - ICEBERG_UNWRAP_OR_FAIL(auto result, update->Apply()); + ICEBERG_UNWRAP_OR_FAIL(auto result, update->Validate()); ICEBERG_UNWRAP_OR_FAIL(auto field_opt, result.schema->FindFieldByName("id")); ASSERT_TRUE(field_opt.has_value()); @@ -1305,7 +1388,7 @@ TEST_F(UpdateSchemaTest, MakeColumnOptionalNoop) { ICEBERG_UNWRAP_OR_FAIL(auto update, reloaded->NewUpdateSchema()); update->MakeColumnOptional("id"); - ICEBERG_UNWRAP_OR_FAIL(auto result, update->Apply()); + ICEBERG_UNWRAP_OR_FAIL(auto result, update->Validate()); ICEBERG_UNWRAP_OR_FAIL(auto field_opt, result.schema->FindFieldByName("id")); ASSERT_TRUE(field_opt.has_value()); @@ -1321,7 +1404,7 @@ TEST_F(UpdateSchemaTest, RequireColumnCaseInsensitive) { ICEBERG_UNWRAP_OR_FAIL(auto update, reloaded->NewUpdateSchema()); update->CaseSensitive(false).AllowIncompatibleChanges().RequireColumn("id"); - ICEBERG_UNWRAP_OR_FAIL(auto result, update->Apply()); + ICEBERG_UNWRAP_OR_FAIL(auto result, update->Validate()); ICEBERG_UNWRAP_OR_FAIL(auto field_opt, result.schema->FindFieldByName("ID", false)); ASSERT_TRUE(field_opt.has_value()); @@ -1332,7 +1415,7 @@ TEST_F(UpdateSchemaTest, MakeColumnOptionalMissingFails) { ICEBERG_UNWRAP_OR_FAIL(auto update, table_->NewUpdateSchema()); update->MakeColumnOptional("non_existent"); - auto result = update->Apply(); + auto result = update->Validate(); EXPECT_THAT(result, IsError(ErrorKind::kValidationFailed)); EXPECT_THAT(result, HasErrorMessage("Cannot update missing column")); } @@ -1341,7 +1424,7 @@ TEST_F(UpdateSchemaTest, RequireColumnMissingFails) { ICEBERG_UNWRAP_OR_FAIL(auto update, table_->NewUpdateSchema()); update->AllowIncompatibleChanges().RequireColumn("non_existent"); - auto result = update->Apply(); + auto result = update->Validate(); EXPECT_THAT(result, IsError(ErrorKind::kValidationFailed)); EXPECT_THAT(result, HasErrorMessage("Cannot update missing column")); } @@ -1355,7 +1438,7 @@ TEST_F(UpdateSchemaTest, MakeColumnOptionalDeletedFails) { ICEBERG_UNWRAP_OR_FAIL(auto update, reloaded->NewUpdateSchema()); update->DeleteColumn("col").MakeColumnOptional("col"); - auto result = update->Apply(); + auto result = update->Validate(); EXPECT_THAT(result, IsError(ErrorKind::kValidationFailed)); EXPECT_THAT(result, HasErrorMessage("Cannot update a column that will be deleted")); } @@ -1369,7 +1452,7 @@ TEST_F(UpdateSchemaTest, RequireColumnDeletedFails) { ICEBERG_UNWRAP_OR_FAIL(auto update, reloaded->NewUpdateSchema()); update->DeleteColumn("col").AllowIncompatibleChanges().RequireColumn("col"); - auto result = update->Apply(); + auto result = update->Validate(); EXPECT_THAT(result, IsError(ErrorKind::kValidationFailed)); EXPECT_THAT(result, HasErrorMessage("Cannot update a column that will be deleted")); } @@ -1379,7 +1462,7 @@ TEST_F(UpdateSchemaTest, UpdateColumnDoc) { update->AddColumn("col", int32(), "original doc"); update->UpdateColumnDoc("col", "updated doc"); - ICEBERG_UNWRAP_OR_FAIL(auto result, update->Apply()); + ICEBERG_UNWRAP_OR_FAIL(auto result, update->Validate()); ICEBERG_UNWRAP_OR_FAIL(auto field_opt, result.schema->FindFieldByName("col")); ASSERT_TRUE(field_opt.has_value()); @@ -1390,7 +1473,7 @@ TEST_F(UpdateSchemaTest, UpdateColumnDocMissingFails) { ICEBERG_UNWRAP_OR_FAIL(auto update, table_->NewUpdateSchema()); update->UpdateColumnDoc("non_existent", "some doc"); - auto result = update->Apply(); + auto result = update->Validate(); EXPECT_THAT(result, IsError(ErrorKind::kValidationFailed)); EXPECT_THAT(result, HasErrorMessage("Cannot update missing column")); } @@ -1404,7 +1487,7 @@ TEST_F(UpdateSchemaTest, UpdateColumnDocDeletedFails) { ICEBERG_UNWRAP_OR_FAIL(auto update, reloaded->NewUpdateSchema()); update->DeleteColumn("col").UpdateColumnDoc("col", "new doc"); - auto result = update->Apply(); + auto result = update->Validate(); EXPECT_THAT(result, IsError(ErrorKind::kValidationFailed)); EXPECT_THAT(result, HasErrorMessage("Cannot update a column that will be deleted")); } @@ -1418,7 +1501,7 @@ TEST_F(UpdateSchemaTest, UpdateColumnDocNoop) { ICEBERG_UNWRAP_OR_FAIL(auto update, reloaded->NewUpdateSchema()); update->UpdateColumnDoc("col", "same doc"); - ICEBERG_UNWRAP_OR_FAIL(auto result, update->Apply()); + ICEBERG_UNWRAP_OR_FAIL(auto result, update->Validate()); ICEBERG_UNWRAP_OR_FAIL(auto field_opt, result.schema->FindFieldByName("col")); ASSERT_TRUE(field_opt.has_value()); @@ -1430,7 +1513,7 @@ TEST_F(UpdateSchemaTest, UpdateColumnDocEmptyString) { update->AddColumn("col", int32(), "original doc"); update->UpdateColumnDoc("col", ""); - ICEBERG_UNWRAP_OR_FAIL(auto result, update->Apply()); + ICEBERG_UNWRAP_OR_FAIL(auto result, update->Validate()); ICEBERG_UNWRAP_OR_FAIL(auto field_opt, result.schema->FindFieldByName("col")); ASSERT_TRUE(field_opt.has_value()); @@ -1442,7 +1525,7 @@ TEST_F(UpdateSchemaTest, UpdateColumnIntToLong) { update->AddColumn("id", int32(), "An integer ID"); update->UpdateColumn("id", int64()); - ICEBERG_UNWRAP_OR_FAIL(auto result, update->Apply()); + ICEBERG_UNWRAP_OR_FAIL(auto result, update->Validate()); ICEBERG_UNWRAP_OR_FAIL(auto field_opt, result.schema->FindFieldByName("id")); ASSERT_TRUE(field_opt.has_value()); @@ -1455,7 +1538,7 @@ TEST_F(UpdateSchemaTest, UpdateColumnFloatToDouble) { update->AddColumn("value", float32(), "A float value"); update->UpdateColumn("value", float64()); - ICEBERG_UNWRAP_OR_FAIL(auto result, update->Apply()); + ICEBERG_UNWRAP_OR_FAIL(auto result, update->Validate()); ICEBERG_UNWRAP_OR_FAIL(auto field_opt, result.schema->FindFieldByName("value")); ASSERT_TRUE(field_opt.has_value()); @@ -1467,7 +1550,7 @@ TEST_F(UpdateSchemaTest, UpdateColumnUnknownToPrimitive) { update->AddColumn("mystery", unknown(), "A null-only placeholder"); update->UpdateColumn("mystery", string()); - ICEBERG_UNWRAP_OR_FAIL(auto result, update->Apply()); + ICEBERG_UNWRAP_OR_FAIL(auto result, update->Validate()); ICEBERG_UNWRAP_OR_FAIL(auto field_opt, result.schema->FindFieldByName("mystery")); ASSERT_TRUE(field_opt.has_value()); @@ -1480,7 +1563,7 @@ TEST_F(UpdateSchemaTest, AddRequiredUnknownColumnFails) { ICEBERG_UNWRAP_OR_FAIL(auto update, table_->NewUpdateSchema()); update->AllowIncompatibleChanges().AddRequiredColumn("mystery", unknown()); - auto result = update->Apply(); + auto result = update->Validate(); EXPECT_THAT(result, IsError(ErrorKind::kInvalidArgument)); EXPECT_THAT(result, HasErrorMessage("Unknown type field 'mystery' must be optional")); } @@ -1491,7 +1574,7 @@ TEST_F(UpdateSchemaTest, AddColumnWithRequiredNestedUnknownFails) { SchemaField::MakeRequired(3, "mystery", unknown()), })); - auto result = update->Apply(); + auto result = update->Validate(); EXPECT_THAT(result, IsError(ErrorKind::kInvalidArgument)); EXPECT_THAT(result, HasErrorMessage("Unknown type field 'mystery' must be optional")); } @@ -1501,7 +1584,7 @@ TEST_F(UpdateSchemaTest, UpdateColumnSameType) { update->AddColumn("id", int32()); update->UpdateColumn("id", int32()); - ICEBERG_UNWRAP_OR_FAIL(auto result, update->Apply()); + ICEBERG_UNWRAP_OR_FAIL(auto result, update->Validate()); ICEBERG_UNWRAP_OR_FAIL(auto field_opt, result.schema->FindFieldByName("id")); ASSERT_TRUE(field_opt.has_value()); @@ -1512,7 +1595,7 @@ TEST_F(UpdateSchemaTest, UpdateColumnMissingFails) { ICEBERG_UNWRAP_OR_FAIL(auto update, table_->NewUpdateSchema()); update->UpdateColumn("non_existent", int64()); - auto result = update->Apply(); + auto result = update->Validate(); EXPECT_THAT(result, IsError(ErrorKind::kValidationFailed)); EXPECT_THAT(result, HasErrorMessage("Cannot update missing column")); } @@ -1526,7 +1609,7 @@ TEST_F(UpdateSchemaTest, UpdateColumnDeletedFails) { ICEBERG_UNWRAP_OR_FAIL(auto update, reloaded->NewUpdateSchema()); update->DeleteColumn("col").UpdateColumn("col", int64()); - auto result = update->Apply(); + auto result = update->Validate(); EXPECT_THAT(result, IsError(ErrorKind::kValidationFailed)); EXPECT_THAT(result, HasErrorMessage("Cannot update a column that will be deleted")); } @@ -1536,7 +1619,7 @@ TEST_F(UpdateSchemaTest, UpdateColumnInvalidPromotionFails) { update->AddColumn("id", int64()); update->UpdateColumn("id", int32()); - auto result = update->Apply(); + auto result = update->Validate(); EXPECT_THAT(result, IsError(ErrorKind::kValidationFailed)); EXPECT_THAT(result, HasErrorMessage("Cannot change column type")); } @@ -1546,7 +1629,7 @@ TEST_F(UpdateSchemaTest, UpdateColumnInvalidPromotionDoubleToFloatFails) { update->AddColumn("value", float64()); update->UpdateColumn("value", float32()); - auto result = update->Apply(); + auto result = update->Validate(); EXPECT_THAT(result, IsError(ErrorKind::kValidationFailed)); EXPECT_THAT(result, HasErrorMessage("Cannot change column type")); } @@ -1556,7 +1639,7 @@ TEST_F(UpdateSchemaTest, UpdateColumnIncompatibleTypesFails) { update->AddColumn("id", int32()); update->UpdateColumn("id", string()); - auto result = update->Apply(); + auto result = update->Validate(); EXPECT_THAT(result, IsError(ErrorKind::kValidationFailed)); EXPECT_THAT(result, HasErrorMessage("Cannot change column type")); } @@ -1567,7 +1650,7 @@ TEST_F(UpdateSchemaTest, RenameAndUpdateColumnInSameTransaction) { update->UpdateColumn("old_name", int64()); update->RenameColumn("old_name", "new_name"); - auto result = update->Apply(); + auto result = update->Validate(); EXPECT_THAT(result, IsError(ErrorKind::kValidationFailed)); EXPECT_THAT(result, HasErrorMessage("Cannot rename missing column")); } @@ -1579,7 +1662,7 @@ TEST_F(UpdateSchemaTest, UpdateColumnDecimalPrecisionWidening) { update->AddColumn("price", decimal_10_2); update->UpdateColumn("price", decimal_20_2); - ICEBERG_UNWRAP_OR_FAIL(auto result, update->Apply()); + ICEBERG_UNWRAP_OR_FAIL(auto result, update->Validate()); ICEBERG_UNWRAP_OR_FAIL(auto field_opt, result.schema->FindFieldByName("price")); ASSERT_TRUE(field_opt.has_value()); @@ -1593,7 +1676,7 @@ TEST_F(UpdateSchemaTest, UpdateColumnDecimalDifferentScaleFails) { update->AddColumn("price", decimal_10_2); update->UpdateColumn("price", decimal_10_3); - auto result = update->Apply(); + auto result = update->Validate(); EXPECT_THAT(result, IsError(ErrorKind::kValidationFailed)); EXPECT_THAT(result, HasErrorMessage("Cannot change column type")); } @@ -1605,7 +1688,7 @@ TEST_F(UpdateSchemaTest, UpdateColumnDecimalPrecisionNarrowingFails) { update->AddColumn("price", decimal_20_2); update->UpdateColumn("price", decimal_10_2); - auto result = update->Apply(); + auto result = update->Validate(); EXPECT_THAT(result, IsError(ErrorKind::kValidationFailed)); EXPECT_THAT(result, HasErrorMessage("Cannot change column type")); } @@ -1616,7 +1699,7 @@ TEST_F(UpdateSchemaTest, UpdateTypePreservesOtherMetadata) { update->UpdateColumn("value", int64()); - ICEBERG_UNWRAP_OR_FAIL(auto result, update->Apply()); + ICEBERG_UNWRAP_OR_FAIL(auto result, update->Validate()); ICEBERG_UNWRAP_OR_FAIL(auto field_opt, result.schema->FindFieldByName("value")); ASSERT_TRUE(field_opt.has_value()); @@ -1633,7 +1716,7 @@ TEST_F(UpdateSchemaTest, UpdateDocPreservesOtherMetadata) { update->UpdateColumnDoc("id", "new doc"); - ICEBERG_UNWRAP_OR_FAIL(auto result, update->Apply()); + ICEBERG_UNWRAP_OR_FAIL(auto result, update->Validate()); ICEBERG_UNWRAP_OR_FAIL(auto field_opt, result.schema->FindFieldByName("id")); ASSERT_TRUE(field_opt.has_value()); @@ -1653,7 +1736,7 @@ TEST_F(UpdateSchemaTest, RenameDeleteConflict) { ICEBERG_UNWRAP_OR_FAIL(auto update, reloaded->NewUpdateSchema()); update->RenameColumn("col", "new_name").DeleteColumn("col"); - auto result = update->Apply(); + auto result = update->Validate(); EXPECT_THAT(result, IsError(ErrorKind::kValidationFailed)); EXPECT_THAT(result, HasErrorMessage("Cannot delete a column that has updates")); } @@ -1667,7 +1750,7 @@ TEST_F(UpdateSchemaTest, DeleteRenameConflict) { ICEBERG_UNWRAP_OR_FAIL(auto update, reloaded->NewUpdateSchema()); update->DeleteColumn("col").RenameColumn("col", "new_name"); - auto result = update->Apply(); + auto result = update->Validate(); EXPECT_THAT(result, IsError(ErrorKind::kValidationFailed)); EXPECT_THAT(result, HasErrorMessage("Cannot rename a column that will be deleted")); } @@ -1678,7 +1761,7 @@ TEST_F(UpdateSchemaTest, CaseInsensitiveAddThenUpdate) { .AddColumn("Foo", int32(), "A column with uppercase name") .UpdateColumn("foo", int64()); - ICEBERG_UNWRAP_OR_FAIL(auto result, update->Apply()); + ICEBERG_UNWRAP_OR_FAIL(auto result, update->Validate()); ICEBERG_UNWRAP_OR_FAIL(auto field_opt, result.schema->FindFieldByName("Foo", false)); ASSERT_TRUE(field_opt.has_value()); @@ -1691,7 +1774,7 @@ TEST_F(UpdateSchemaTest, CaseInsensitiveAddThenUpdateDoc) { .AddColumn("Foo", int32(), "original doc") .UpdateColumnDoc("foo", "updated doc"); - ICEBERG_UNWRAP_OR_FAIL(auto result, update->Apply()); + ICEBERG_UNWRAP_OR_FAIL(auto result, update->Validate()); ICEBERG_UNWRAP_OR_FAIL(auto field_opt, result.schema->FindFieldByName("Foo", false)); ASSERT_TRUE(field_opt.has_value()); @@ -1705,7 +1788,7 @@ TEST_F(UpdateSchemaTest, CaseInsensitiveAddThenMakeOptional) { .AddRequiredColumn("Foo", int32(), "required column") .MakeColumnOptional("foo"); - ICEBERG_UNWRAP_OR_FAIL(auto result, update->Apply()); + ICEBERG_UNWRAP_OR_FAIL(auto result, update->Validate()); ICEBERG_UNWRAP_OR_FAIL(auto field_opt, result.schema->FindFieldByName("Foo", false)); ASSERT_TRUE(field_opt.has_value()); @@ -1719,7 +1802,7 @@ TEST_F(UpdateSchemaTest, CaseInsensitiveAddThenRequire) { .AddColumn("Foo", int32(), "optional column") .RequireColumn("foo"); // Require using lowercase - ICEBERG_UNWRAP_OR_FAIL(auto result, update->Apply()); + ICEBERG_UNWRAP_OR_FAIL(auto result, update->Validate()); ICEBERG_UNWRAP_OR_FAIL(auto field_opt, result.schema->FindFieldByName("Foo", false)); ASSERT_TRUE(field_opt.has_value()); @@ -1791,7 +1874,7 @@ TEST_F(UpdateSchemaTest, MixedChanges) { .RequireColumn("data") .AddRequiredColumn("locations", "description", string(), "Location description"); - ICEBERG_UNWRAP_OR_FAIL(auto result, update->Apply()); + ICEBERG_UNWRAP_OR_FAIL(auto result, update->Validate()); ICEBERG_UNWRAP_OR_FAIL(auto id_opt, result.schema->FindFieldByName("id")); ASSERT_TRUE(id_opt.has_value()); @@ -1913,7 +1996,7 @@ TEST_F(UpdateSchemaTest, TestMultipleMoves) { update->MoveFirst("w").MoveFirst("z").MoveAfter("y", "w").MoveBefore("w", "x"); - ICEBERG_UNWRAP_OR_FAIL(auto result, update->Apply()); + ICEBERG_UNWRAP_OR_FAIL(auto result, update->Validate()); ASSERT_TRUE(result.schema != nullptr); const auto& fields = result.schema->fields(); @@ -1935,7 +2018,7 @@ TEST_F(UpdateSchemaTest, TestMoveTopLevelColumnFirst) { ICEBERG_UNWRAP_OR_FAIL(auto update, table_->NewUpdateSchema()); update->MoveFirst("y"); - ICEBERG_UNWRAP_OR_FAIL(auto result, update->Apply()); + ICEBERG_UNWRAP_OR_FAIL(auto result, update->Validate()); ASSERT_TRUE(result.schema != nullptr); const auto& fields = result.schema->fields(); @@ -1946,7 +2029,7 @@ TEST_F(UpdateSchemaTest, TestMoveTopLevelColumnBeforeFirst) { ICEBERG_UNWRAP_OR_FAIL(auto update, table_->NewUpdateSchema()); update->MoveBefore("y", "x"); - ICEBERG_UNWRAP_OR_FAIL(auto result, update->Apply()); + ICEBERG_UNWRAP_OR_FAIL(auto result, update->Validate()); ASSERT_TRUE(result.schema != nullptr); const auto& fields = result.schema->fields(); @@ -1965,7 +2048,7 @@ TEST_F(UpdateSchemaTest, TestMoveTopLevelColumnAfterLast) { ICEBERG_UNWRAP_OR_FAIL(auto update, table_->NewUpdateSchema()); update->MoveAfter("x", "z"); - ICEBERG_UNWRAP_OR_FAIL(auto result, update->Apply()); + ICEBERG_UNWRAP_OR_FAIL(auto result, update->Validate()); ASSERT_TRUE(result.schema != nullptr); const auto& fields = result.schema->fields(); @@ -1981,7 +2064,7 @@ TEST_F(UpdateSchemaTest, TestMoveTopLevelColumnAfter) { ICEBERG_UNWRAP_OR_FAIL(auto update, reloaded->NewUpdateSchema()); update->MoveAfter("w", "x"); - ICEBERG_UNWRAP_OR_FAIL(auto result, update->Apply()); + ICEBERG_UNWRAP_OR_FAIL(auto result, update->Validate()); ASSERT_TRUE(result.schema != nullptr); const auto& fields = result.schema->fields(); @@ -2005,7 +2088,7 @@ TEST_F(UpdateSchemaTest, TestMoveTopLevelColumnBefore) { ICEBERG_UNWRAP_OR_FAIL(auto update, reloaded->NewUpdateSchema()); update->MoveBefore("w", "z"); - ICEBERG_UNWRAP_OR_FAIL(auto result, update->Apply()); + ICEBERG_UNWRAP_OR_FAIL(auto result, update->Validate()); ASSERT_TRUE(result.schema != nullptr); const auto& fields = result.schema->fields(); @@ -2031,7 +2114,7 @@ TEST_F(UpdateSchemaTest, TestMoveNestedFieldFirst) { ICEBERG_UNWRAP_OR_FAIL(auto update, reloaded->NewUpdateSchema()); update->MoveFirst("s.b"); - ICEBERG_UNWRAP_OR_FAIL(auto result, update->Apply()); + ICEBERG_UNWRAP_OR_FAIL(auto result, update->Validate()); ASSERT_TRUE(result.schema != nullptr); ICEBERG_UNWRAP_OR_FAIL(auto s_opt, result.schema->FindFieldByName("s")); @@ -2053,7 +2136,7 @@ TEST_F(UpdateSchemaTest, TestMoveNestedFieldBeforeFirst) { ICEBERG_UNWRAP_OR_FAIL(auto update, reloaded->NewUpdateSchema()); update->MoveBefore("s.b", "s.a"); - ICEBERG_UNWRAP_OR_FAIL(auto result, update->Apply()); + ICEBERG_UNWRAP_OR_FAIL(auto result, update->Validate()); ASSERT_TRUE(result.schema != nullptr); ICEBERG_UNWRAP_OR_FAIL(auto s_opt, result.schema->FindFieldByName("s")); @@ -2083,7 +2166,7 @@ TEST_F(UpdateSchemaTest, TestMoveNestedFieldAfterLast) { ICEBERG_UNWRAP_OR_FAIL(auto update, reloaded->NewUpdateSchema()); update->MoveAfter("s.a", "s.b"); - ICEBERG_UNWRAP_OR_FAIL(auto result, update->Apply()); + ICEBERG_UNWRAP_OR_FAIL(auto result, update->Validate()); ASSERT_TRUE(result.schema != nullptr); ICEBERG_UNWRAP_OR_FAIL(auto s_opt, result.schema->FindFieldByName("s")); @@ -2106,7 +2189,7 @@ TEST_F(UpdateSchemaTest, TestMoveNestedFieldAfter) { ICEBERG_UNWRAP_OR_FAIL(auto update, reloaded->NewUpdateSchema()); update->MoveAfter("s.c", "s.a"); - ICEBERG_UNWRAP_OR_FAIL(auto result, update->Apply()); + ICEBERG_UNWRAP_OR_FAIL(auto result, update->Validate()); ASSERT_TRUE(result.schema != nullptr); ICEBERG_UNWRAP_OR_FAIL(auto s_opt, result.schema->FindFieldByName("s")); @@ -2137,7 +2220,7 @@ TEST_F(UpdateSchemaTest, TestMoveNestedFieldBefore) { ICEBERG_UNWRAP_OR_FAIL(auto update, reloaded->NewUpdateSchema()); update->MoveBefore("s.c", "s.b"); - ICEBERG_UNWRAP_OR_FAIL(auto result, update->Apply()); + ICEBERG_UNWRAP_OR_FAIL(auto result, update->Validate()); ASSERT_TRUE(result.schema != nullptr); ICEBERG_UNWRAP_OR_FAIL(auto s_opt, result.schema->FindFieldByName("s")); @@ -2169,7 +2252,7 @@ TEST_F(UpdateSchemaTest, TestMoveListElementField) { ICEBERG_UNWRAP_OR_FAIL(auto update, reloaded->NewUpdateSchema()); update->MoveAfter("list.a", "list.b"); - ICEBERG_UNWRAP_OR_FAIL(auto result, update->Apply()); + ICEBERG_UNWRAP_OR_FAIL(auto result, update->Validate()); ASSERT_TRUE(result.schema != nullptr); ICEBERG_UNWRAP_OR_FAIL(auto list_opt, result.schema->FindFieldByName("list")); @@ -2205,7 +2288,7 @@ TEST_F(UpdateSchemaTest, TestMoveMapValueStructField) { ICEBERG_UNWRAP_OR_FAIL(auto update, reloaded->NewUpdateSchema()); update->MoveAfter("locations.lat", "locations.long"); - ICEBERG_UNWRAP_OR_FAIL(auto result, update->Apply()); + ICEBERG_UNWRAP_OR_FAIL(auto result, update->Validate()); ASSERT_TRUE(result.schema != nullptr); ICEBERG_UNWRAP_OR_FAIL(auto locs_opt, result.schema->FindFieldByName("locations")); @@ -2230,7 +2313,7 @@ TEST_F(UpdateSchemaTest, TestMoveAddedTopLevelColumn) { ICEBERG_UNWRAP_OR_FAIL(auto update, table_->NewUpdateSchema()); update->AddColumn("ts", timestamp_tz()).MoveAfter("ts", "x"); - ICEBERG_UNWRAP_OR_FAIL(auto result, update->Apply()); + ICEBERG_UNWRAP_OR_FAIL(auto result, update->Validate()); ASSERT_TRUE(result.schema != nullptr); const auto& fields = result.schema->fields(); @@ -2252,7 +2335,7 @@ TEST_F(UpdateSchemaTest, TestMoveAddedTopLevelColumnAfterAddedColumn) { .MoveAfter("ts", "x") .MoveAfter("count", "ts"); - ICEBERG_UNWRAP_OR_FAIL(auto result, update->Apply()); + ICEBERG_UNWRAP_OR_FAIL(auto result, update->Validate()); ASSERT_TRUE(result.schema != nullptr); const auto& fields = result.schema->fields(); @@ -2284,7 +2367,7 @@ TEST_F(UpdateSchemaTest, TestMoveAddedNestedStructField) { update->AddColumn("preferences", "ts", timestamp_tz()) .MoveBefore("preferences.ts", "preferences.feature1"); - ICEBERG_UNWRAP_OR_FAIL(auto result, update->Apply()); + ICEBERG_UNWRAP_OR_FAIL(auto result, update->Validate()); ASSERT_TRUE(result.schema != nullptr); ICEBERG_UNWRAP_OR_FAIL(auto prefs_opt, result.schema->FindFieldByName("preferences")); @@ -2310,7 +2393,7 @@ TEST_F(UpdateSchemaTest, TestMoveAddedNestedStructFieldBeforeAddedColumn) { .MoveBefore("preferences.ts", "preferences.feature1") .MoveBefore("preferences.size", "preferences.ts"); - ICEBERG_UNWRAP_OR_FAIL(auto result, update->Apply()); + ICEBERG_UNWRAP_OR_FAIL(auto result, update->Validate()); ASSERT_TRUE(result.schema != nullptr); ICEBERG_UNWRAP_OR_FAIL(auto prefs_opt, result.schema->FindFieldByName("preferences")); @@ -2325,13 +2408,13 @@ TEST_F(UpdateSchemaTest, TestMoveAddedNestedStructFieldBeforeAddedColumn) { TEST_F(UpdateSchemaTest, TestMoveSelfReferenceFails) { ICEBERG_UNWRAP_OR_FAIL(auto update1, table_->NewUpdateSchema()); update1->MoveBefore("x", "x"); - auto result1 = update1->Apply(); + auto result1 = update1->Validate(); EXPECT_THAT(result1, IsError(ErrorKind::kValidationFailed)); EXPECT_THAT(result1, HasErrorMessage("Cannot move x before itself")); ICEBERG_UNWRAP_OR_FAIL(auto update2, table_->NewUpdateSchema()); update2->MoveAfter("x", "x"); - auto result2 = update2->Apply(); + auto result2 = update2->Validate(); EXPECT_THAT(result2, IsError(ErrorKind::kValidationFailed)); EXPECT_THAT(result2, HasErrorMessage("Cannot move x after itself")); } @@ -2339,19 +2422,19 @@ TEST_F(UpdateSchemaTest, TestMoveSelfReferenceFails) { TEST_F(UpdateSchemaTest, TestMoveMissingColumnFails) { ICEBERG_UNWRAP_OR_FAIL(auto update1, table_->NewUpdateSchema()); update1->MoveFirst("items"); - auto result1 = update1->Apply(); + auto result1 = update1->Validate(); EXPECT_THAT(result1, IsError(ErrorKind::kValidationFailed)); EXPECT_THAT(result1, HasErrorMessage("Cannot move missing column: items")); ICEBERG_UNWRAP_OR_FAIL(auto update2, table_->NewUpdateSchema()); update2->MoveBefore("items", "x"); - auto result2 = update2->Apply(); + auto result2 = update2->Validate(); EXPECT_THAT(result2, IsError(ErrorKind::kValidationFailed)); EXPECT_THAT(result2, HasErrorMessage("Cannot move missing column: items")); ICEBERG_UNWRAP_OR_FAIL(auto update3, table_->NewUpdateSchema()); update3->MoveAfter("items", "y"); - auto result3 = update3->Apply(); + auto result3 = update3->Validate(); EXPECT_THAT(result3, IsError(ErrorKind::kValidationFailed)); EXPECT_THAT(result3, HasErrorMessage("Cannot move missing column: items")); } @@ -2359,7 +2442,7 @@ TEST_F(UpdateSchemaTest, TestMoveMissingColumnFails) { TEST_F(UpdateSchemaTest, TestMoveBeforeAddFails) { ICEBERG_UNWRAP_OR_FAIL(auto update, table_->NewUpdateSchema()); update->MoveBefore("ts", "x").AddColumn("ts", timestamp_tz()); - auto result = update->Apply(); + auto result = update->Validate(); EXPECT_THAT(result, IsError(ErrorKind::kValidationFailed)); EXPECT_THAT(result, HasErrorMessage("Cannot move missing column: ts")); } @@ -2367,13 +2450,13 @@ TEST_F(UpdateSchemaTest, TestMoveBeforeAddFails) { TEST_F(UpdateSchemaTest, TestMoveMissingReferenceColumnFails) { ICEBERG_UNWRAP_OR_FAIL(auto update1, table_->NewUpdateSchema()); update1->MoveBefore("x", "items"); - auto result1 = update1->Apply(); + auto result1 = update1->Validate(); EXPECT_THAT(result1, IsError(ErrorKind::kValidationFailed)); EXPECT_THAT(result1, HasErrorMessage("Cannot move x before missing column: items")); ICEBERG_UNWRAP_OR_FAIL(auto update2, table_->NewUpdateSchema()); update2->MoveAfter("y", "items"); - auto result2 = update2->Apply(); + auto result2 = update2->Validate(); EXPECT_THAT(result2, IsError(ErrorKind::kValidationFailed)); EXPECT_THAT(result2, HasErrorMessage("Cannot move y after missing column: items")); } @@ -2389,7 +2472,7 @@ TEST_F(UpdateSchemaTest, TestMovePrimitiveMapKeyFails) { ICEBERG_UNWRAP_OR_FAIL(auto reloaded, catalog_->LoadTable(table_ident_)); ICEBERG_UNWRAP_OR_FAIL(auto update, reloaded->NewUpdateSchema()); update->MoveBefore("properties.key", "properties.value"); - auto result = update->Apply(); + auto result = update->Validate(); EXPECT_THAT(result, IsError(ErrorKind::kValidationFailed)); EXPECT_THAT(result, HasErrorMessage("Cannot move fields in non-struct type")); } @@ -2405,7 +2488,7 @@ TEST_F(UpdateSchemaTest, TestMovePrimitiveMapValueFails) { ICEBERG_UNWRAP_OR_FAIL(auto reloaded, catalog_->LoadTable(table_ident_)); ICEBERG_UNWRAP_OR_FAIL(auto update, reloaded->NewUpdateSchema()); update->MoveBefore("properties.value", "properties.key"); - auto result = update->Apply(); + auto result = update->Validate(); EXPECT_THAT(result, IsError(ErrorKind::kValidationFailed)); EXPECT_THAT(result, HasErrorMessage("Cannot move fields in non-struct type")); } @@ -2421,7 +2504,7 @@ TEST_F(UpdateSchemaTest, TestMovePrimitiveListElementFails) { ICEBERG_UNWRAP_OR_FAIL(auto reloaded, catalog_->LoadTable(table_ident_)); ICEBERG_UNWRAP_OR_FAIL(auto update, reloaded->NewUpdateSchema()); update->MoveBefore("doubles.element", "doubles"); - auto result = update->Apply(); + auto result = update->Validate(); EXPECT_THAT(result, IsError(ErrorKind::kValidationFailed)); EXPECT_THAT(result, HasErrorMessage("Cannot move fields in non-struct type")); } @@ -2436,7 +2519,7 @@ TEST_F(UpdateSchemaTest, TestMoveTopLevelBetweenStructsFails) { ICEBERG_UNWRAP_OR_FAIL(auto reloaded, catalog_->LoadTable(table_ident_)); ICEBERG_UNWRAP_OR_FAIL(auto update, reloaded->NewUpdateSchema()); update->MoveBefore("x", "preferences.feature1"); - auto result = update->Apply(); + auto result = update->Validate(); EXPECT_THAT(result, IsError(ErrorKind::kValidationFailed)); EXPECT_THAT(result, HasErrorMessage("Cannot move field x to a different struct")); } @@ -2454,7 +2537,7 @@ TEST_F(UpdateSchemaTest, TestMoveBetweenStructsFails) { ICEBERG_UNWRAP_OR_FAIL(auto reloaded, catalog_->LoadTable(table_ident_)); ICEBERG_UNWRAP_OR_FAIL(auto update, reloaded->NewUpdateSchema()); update->MoveBefore("points.a", "preferences.feature1"); - auto result = update->Apply(); + auto result = update->Validate(); EXPECT_THAT(result, IsError(ErrorKind::kValidationFailed)); EXPECT_THAT(result, HasErrorMessage("Cannot move field points.a to a different struct")); @@ -2467,7 +2550,7 @@ TEST_F(UpdateSchemaTest, TestMoveTopDeletedColumnAfterAnotherColumn) { .AddRequiredColumn("z", int32()) .MoveAfter("z", "y"); - ICEBERG_UNWRAP_OR_FAIL(auto result, update->Apply()); + ICEBERG_UNWRAP_OR_FAIL(auto result, update->Validate()); ASSERT_TRUE(result.schema != nullptr); const auto& fields = result.schema->fields(); @@ -2489,7 +2572,7 @@ TEST_F(UpdateSchemaTest, TestMoveTopDeletedColumnBeforeAnotherColumn) { .AddRequiredColumn("z", int32()) .MoveBefore("z", "x"); - ICEBERG_UNWRAP_OR_FAIL(auto result, update->Apply()); + ICEBERG_UNWRAP_OR_FAIL(auto result, update->Validate()); ASSERT_TRUE(result.schema != nullptr); const auto& fields = result.schema->fields(); @@ -2511,7 +2594,7 @@ TEST_F(UpdateSchemaTest, TestMoveTopDeletedColumnToFirst) { .AddRequiredColumn("z", int32()) .MoveFirst("z"); - ICEBERG_UNWRAP_OR_FAIL(auto result, update->Apply()); + ICEBERG_UNWRAP_OR_FAIL(auto result, update->Validate()); ASSERT_TRUE(result.schema != nullptr); const auto& fields = result.schema->fields(); @@ -2533,7 +2616,7 @@ TEST_F(UpdateSchemaTest, TestMoveDeletedNestedStructFieldAfterAnotherColumn) { .AddRequiredColumn("preferences", "feature1", boolean()) .MoveAfter("preferences.feature1", "preferences.feature2"); - ICEBERG_UNWRAP_OR_FAIL(auto result, update->Apply()); + ICEBERG_UNWRAP_OR_FAIL(auto result, update->Validate()); ASSERT_TRUE(result.schema != nullptr); ICEBERG_UNWRAP_OR_FAIL(auto prefs_opt, result.schema->FindFieldByName("preferences")); @@ -2559,7 +2642,7 @@ TEST_F(UpdateSchemaTest, TestMoveDeletedNestedStructFieldBeforeAnotherColumn) { .AddColumn("preferences", "feature2", boolean()) .MoveBefore("preferences.feature2", "preferences.feature1"); - ICEBERG_UNWRAP_OR_FAIL(auto result, update->Apply()); + ICEBERG_UNWRAP_OR_FAIL(auto result, update->Validate()); ASSERT_TRUE(result.schema != nullptr); ICEBERG_UNWRAP_OR_FAIL(auto prefs_opt, result.schema->FindFieldByName("preferences")); @@ -2585,7 +2668,7 @@ TEST_F(UpdateSchemaTest, TestMoveDeletedNestedStructFieldToFirst) { .AddColumn("preferences", "feature2", boolean()) .MoveFirst("preferences.feature2"); - ICEBERG_UNWRAP_OR_FAIL(auto result, update->Apply()); + ICEBERG_UNWRAP_OR_FAIL(auto result, update->Validate()); ASSERT_TRUE(result.schema != nullptr); ICEBERG_UNWRAP_OR_FAIL(auto prefs_opt, result.schema->FindFieldByName("preferences")); @@ -2600,7 +2683,7 @@ TEST_F(UpdateSchemaTest, TestCaseInsensitiveAddTopLevelAndMove) { ICEBERG_UNWRAP_OR_FAIL(auto update, table_->NewUpdateSchema()); update->CaseSensitive(false).AddColumn("TS", timestamp_tz()).MoveAfter("ts", "X"); - ICEBERG_UNWRAP_OR_FAIL(auto result, update->Apply()); + ICEBERG_UNWRAP_OR_FAIL(auto result, update->Validate()); ASSERT_TRUE(result.schema != nullptr); const auto& fields = result.schema->fields(); @@ -2628,7 +2711,7 @@ TEST_F(UpdateSchemaTest, TestCaseInsensitiveAddNestedAndMove) { .AddColumn("Preferences", "TS", timestamp_tz()) .MoveBefore("preferences.ts", "PREFERENCES.Feature1"); - ICEBERG_UNWRAP_OR_FAIL(auto result, update->Apply()); + ICEBERG_UNWRAP_OR_FAIL(auto result, update->Validate()); ASSERT_TRUE(result.schema != nullptr); ICEBERG_UNWRAP_OR_FAIL(auto prefs_opt, result.schema->FindFieldByName("Preferences")); @@ -2647,7 +2730,7 @@ TEST_F(UpdateSchemaTest, TestCaseInsensitiveMoveAfterNewlyAddedField) { .MoveAfter("ts", "X") .MoveAfter("count", "TS"); - ICEBERG_UNWRAP_OR_FAIL(auto result, update->Apply()); + ICEBERG_UNWRAP_OR_FAIL(auto result, update->Validate()); ASSERT_TRUE(result.schema != nullptr); const auto& fields = result.schema->fields(); diff --git a/src/iceberg/test/update_sort_order_test.cc b/src/iceberg/test/update_sort_order_test.cc index 4d83d381d..30f5d3c9a 100644 --- a/src/iceberg/test/update_sort_order_test.cc +++ b/src/iceberg/test/update_sort_order_test.cc @@ -41,7 +41,7 @@ class UpdateSortOrderTest : public UpdateTestBase { // Helper function to apply update and verify the resulting sort order void ApplyAndExpectSortOrder(UpdateSortOrder* update, std::vector expected_fields) { - ICEBERG_UNWRAP_OR_FAIL(auto result, update->Apply()); + ICEBERG_UNWRAP_OR_FAIL(auto result, update->Validate()); ICEBERG_UNWRAP_OR_FAIL( auto expected_sort_order, SortOrder::Make(result->order_id(), std::move(expected_fields))); @@ -51,7 +51,7 @@ class UpdateSortOrderTest : public UpdateTestBase { TEST_F(UpdateSortOrderTest, EmptySortOrder) { ICEBERG_UNWRAP_OR_FAIL(auto update, table_->NewUpdateSortOrder()); - ICEBERG_UNWRAP_OR_FAIL(auto result, update->Apply()); + ICEBERG_UNWRAP_OR_FAIL(auto result, update->Validate()); // Should succeed with an unsorted order EXPECT_TRUE(result->fields().empty()); } @@ -144,7 +144,7 @@ TEST_F(UpdateSortOrderTest, AddSortFieldNullTerm) { update->AddSortField(nullptr, SortDirection::kAscending, NullOrder::kFirst); - auto result = update->Apply(); + auto result = update->Validate(); EXPECT_THAT(result, IsError(ErrorKind::kValidationFailed)); EXPECT_THAT(result, HasErrorMessage("Term cannot be null")); } @@ -158,7 +158,7 @@ TEST_F(UpdateSortOrderTest, AddSortFieldInvalidTransform) { update->AddSortField(std::move(term), SortDirection::kAscending, NullOrder::kFirst); - auto result = update->Apply(); + auto result = update->Validate(); EXPECT_THAT(result, IsError(ErrorKind::kValidationFailed)); EXPECT_THAT(result, HasErrorMessage("not a valid input type")); } @@ -171,7 +171,7 @@ TEST_F(UpdateSortOrderTest, AddSortFieldNonExistentField) { update->AddSortField(std::move(term), SortDirection::kAscending, NullOrder::kFirst); - auto result = update->Apply(); + auto result = update->Validate(); EXPECT_THAT(result, IsError(ErrorKind::kValidationFailed)); EXPECT_THAT(result, HasErrorMessage("Cannot find")); } @@ -185,7 +185,7 @@ TEST_F(UpdateSortOrderTest, CaseSensitiveTrue) { update->CaseSensitive(true).AddSortField(std::move(term), SortDirection::kAscending, NullOrder::kFirst); - auto result = update->Apply(); + auto result = update->Validate(); // Should fail because schema has "x" (lowercase) EXPECT_THAT(result, IsError(ErrorKind::kValidationFailed)); } diff --git a/src/iceberg/test/update_statistics_test.cc b/src/iceberg/test/update_statistics_test.cc index d6721e5bd..3010fb9fe 100644 --- a/src/iceberg/test/update_statistics_test.cc +++ b/src/iceberg/test/update_statistics_test.cc @@ -31,6 +31,7 @@ #include "iceberg/test/matchers.h" #include "iceberg/test/mock_catalog.h" #include "iceberg/test/update_test_base.h" +#include "iceberg/transaction.h" namespace iceberg { @@ -99,7 +100,7 @@ class UpdateStatisticsRetryTest : public UpdateStatisticsTest { TEST_F(UpdateStatisticsTest, EmptyUpdate) { ICEBERG_UNWRAP_OR_FAIL(auto update, table_->NewUpdateStatistics()); - ICEBERG_UNWRAP_OR_FAIL(auto result, update->Apply()); + ICEBERG_UNWRAP_OR_FAIL(auto result, update->Validate()); EXPECT_TRUE(result.to_set.empty()); EXPECT_TRUE(result.to_remove.empty()); } @@ -110,10 +111,11 @@ TEST_F(UpdateStatisticsTest, SetStatistics) { MakeStatisticsFile(1, "/warehouse/test_table/metadata/stats-1.puffin"); update->SetStatistics(stats_file); - ICEBERG_UNWRAP_OR_FAIL(auto result, update->Apply()); + ICEBERG_UNWRAP_OR_FAIL(auto result, update->Validate()); EXPECT_EQ(result.to_set.size(), 1); EXPECT_TRUE(result.to_remove.empty()); - EXPECT_EQ(FindStatistics(result.to_set, 1), stats_file); + EXPECT_THAT(FindStatistics(result.to_set, 1), + ::testing::Pointee(::testing::Eq(*stats_file))); } TEST_F(UpdateStatisticsTest, SetMultipleStatistics) { @@ -125,18 +127,20 @@ TEST_F(UpdateStatisticsTest, SetMultipleStatistics) { update->SetStatistics(stats_file_1).SetStatistics(stats_file_2); - ICEBERG_UNWRAP_OR_FAIL(auto result, update->Apply()); + ICEBERG_UNWRAP_OR_FAIL(auto result, update->Validate()); EXPECT_EQ(result.to_set.size(), 2); EXPECT_TRUE(result.to_remove.empty()); - EXPECT_EQ(FindStatistics(result.to_set, 1), stats_file_1); - EXPECT_EQ(FindStatistics(result.to_set, 2), stats_file_2); + EXPECT_THAT(FindStatistics(result.to_set, 1), + ::testing::Pointee(::testing::Eq(*stats_file_1))); + EXPECT_THAT(FindStatistics(result.to_set, 2), + ::testing::Pointee(::testing::Eq(*stats_file_2))); } TEST_F(UpdateStatisticsTest, RemoveStatistics) { ICEBERG_UNWRAP_OR_FAIL(auto update, table_->NewUpdateStatistics()); update->RemoveStatistics(1); - ICEBERG_UNWRAP_OR_FAIL(auto result, update->Apply()); + ICEBERG_UNWRAP_OR_FAIL(auto result, update->Validate()); EXPECT_TRUE(result.to_set.empty()); EXPECT_EQ(result.to_remove.size(), 1); EXPECT_THAT(result.to_remove, ::testing::Contains(1)); @@ -146,7 +150,7 @@ TEST_F(UpdateStatisticsTest, RemoveMultipleStatistics) { ICEBERG_UNWRAP_OR_FAIL(auto update, table_->NewUpdateStatistics()); update->RemoveStatistics(1).RemoveStatistics(2).RemoveStatistics(3); - ICEBERG_UNWRAP_OR_FAIL(auto result, update->Apply()); + ICEBERG_UNWRAP_OR_FAIL(auto result, update->Validate()); EXPECT_TRUE(result.to_set.empty()); EXPECT_EQ(result.to_remove.size(), 3); EXPECT_THAT(result.to_remove, ::testing::UnorderedElementsAre(1, 2, 3)); @@ -159,9 +163,10 @@ TEST_F(UpdateStatisticsTest, SetAndRemoveDifferentSnapshots) { update->SetStatistics(stats_file).RemoveStatistics(2); - ICEBERG_UNWRAP_OR_FAIL(auto result, update->Apply()); + ICEBERG_UNWRAP_OR_FAIL(auto result, update->Validate()); EXPECT_EQ(result.to_set.size(), 1); - EXPECT_EQ(FindStatistics(result.to_set, 1), stats_file); + EXPECT_THAT(FindStatistics(result.to_set, 1), + ::testing::Pointee(::testing::Eq(*stats_file))); EXPECT_EQ(result.to_remove.size(), 1); EXPECT_THAT(result.to_remove, ::testing::Contains(2)); } @@ -176,11 +181,12 @@ TEST_F(UpdateStatisticsTest, ReplaceStatistics) { // Set statistics for snapshot 1, then replace it update->SetStatistics(stats_file_1).SetStatistics(stats_file_2); - ICEBERG_UNWRAP_OR_FAIL(auto result, update->Apply()); + ICEBERG_UNWRAP_OR_FAIL(auto result, update->Validate()); EXPECT_EQ(result.to_set.size(), 1); EXPECT_TRUE(result.to_remove.empty()); // Should have the second one (replacement) - EXPECT_EQ(FindStatistics(result.to_set, 1), stats_file_2); + EXPECT_THAT(FindStatistics(result.to_set, 1), + ::testing::Pointee(::testing::Eq(*stats_file_2))); EXPECT_NE(FindStatistics(result.to_set, 1), stats_file_1); } @@ -192,7 +198,7 @@ TEST_F(UpdateStatisticsTest, SetThenRemoveSameSnapshot) { // Set statistics for snapshot 1, then remove it update->SetStatistics(stats_file).RemoveStatistics(1); - ICEBERG_UNWRAP_OR_FAIL(auto result, update->Apply()); + ICEBERG_UNWRAP_OR_FAIL(auto result, update->Validate()); EXPECT_TRUE(result.to_set.empty()); EXPECT_EQ(result.to_remove.size(), 1); EXPECT_THAT(result.to_remove, ::testing::Contains(1)); @@ -206,10 +212,11 @@ TEST_F(UpdateStatisticsTest, RemoveThenSetSameSnapshot) { // Remove statistics for snapshot 1, then set new ones update->RemoveStatistics(1).SetStatistics(stats_file); - ICEBERG_UNWRAP_OR_FAIL(auto result, update->Apply()); + ICEBERG_UNWRAP_OR_FAIL(auto result, update->Validate()); EXPECT_EQ(result.to_set.size(), 1); EXPECT_TRUE(result.to_remove.empty()); - EXPECT_EQ(FindStatistics(result.to_set, 1), stats_file); + EXPECT_THAT(FindStatistics(result.to_set, 1), + ::testing::Pointee(::testing::Eq(*stats_file))); } TEST_F(UpdateStatisticsTest, SetNullStatistics) { @@ -217,7 +224,7 @@ TEST_F(UpdateStatisticsTest, SetNullStatistics) { update->SetStatistics(nullptr); - auto result = update->Apply(); + auto result = update->Validate(); EXPECT_THAT(result, IsError(ErrorKind::kValidationFailed)); EXPECT_THAT(result, HasErrorMessage("Statistics file cannot be null")); } @@ -281,4 +288,29 @@ TEST_F(UpdateStatisticsRetryTest, StandaloneCommitRetriesAfterConflict) { EXPECT_EQ(load_table_count_, 1); } +TEST_F(UpdateStatisticsTest, FreezeCopiesInputsAndPreviewResults) { + FailCommits(2); + ICEBERG_UNWRAP_OR_FAIL(auto snapshot, table_->current_snapshot()); + const auto snapshot_id = snapshot->snapshot_id; + ICEBERG_UNWRAP_OR_FAIL(auto txn, table_->NewTransaction()); + ICEBERG_UNWRAP_OR_FAIL(auto update, txn->NewUpdateStatistics()); + auto statistics = + MakeStatisticsFile(snapshot_id, table_location_ + "/metadata/frozen.puffin"); + const auto expected = *statistics; + update->SetStatistics(statistics); + ASSERT_THAT(update->Commit(), IsOk()); + statistics->path = "/changed.puffin"; + statistics->blob_metadata[0].properties["ndv"] = "999"; + ICEBERG_UNWRAP_OR_FAIL(auto preview, update->Validate()); + ASSERT_EQ(preview.to_set.size(), 1U); + preview.to_set[0].second->path = "/changed-preview.puffin"; + ASSERT_THAT(txn->Commit(), IsOk()); + auto metadata = ReloadMetadata(); + auto it = std::ranges::find_if(metadata->statistics, [&](const auto& file) { + return file->snapshot_id == snapshot_id; + }); + ASSERT_NE(it, metadata->statistics.end()); + EXPECT_EQ(**it, expected); +} + } // namespace iceberg diff --git a/src/iceberg/test/update_test_base.h b/src/iceberg/test/update_test_base.h index 232060d2e..e0ea53c3b 100644 --- a/src/iceberg/test/update_test_base.h +++ b/src/iceberg/test/update_test_base.h @@ -20,6 +20,7 @@ #pragma once #include +#include #include #include @@ -34,6 +35,7 @@ #include "iceberg/table_identifier.h" #include "iceberg/table_metadata.h" #include "iceberg/test/matchers.h" +#include "iceberg/test/mock_catalog.h" #include "iceberg/test/test_resource.h" #include "iceberg/util/uuid.h" @@ -85,6 +87,41 @@ class UpdateTestBase : public ::testing::Test { catalog_->RegisterTable(table_ident_, metadata_location)); } + // Exercise the public commit retry path against real catalog metadata. + void FailCommits(int conflicts, std::function before_attempt = {}) { + auto mock = std::make_shared<::testing::NiceMock>(); + EXPECT_CALL(*mock, LoadTable(::testing::_)) + .Times(::testing::AtLeast(conflicts)) + .WillRepeatedly([catalog = catalog_](const TableIdentifier& name) { + return catalog->LoadTable(name); + }); + EXPECT_CALL(*mock, UpdateTable(::testing::_, ::testing::_, ::testing::_)) + .Times(conflicts + 1) + .WillRepeatedly( + [catalog = catalog_, conflicts, before_attempt, + attempt = std::make_shared(0)]( + const auto& name, const auto& requirements, + const auto& updates) mutable -> Result> { + if (before_attempt) { + before_attempt(*attempt); + } + if ((*attempt)++ < conflicts) { + return CommitFailed("injected conflict"); + } + auto result = catalog->UpdateTable(name, requirements, updates); + if (!result) { + ADD_FAILURE() << result.error().message; + return ValidationFailed("Test catalog failed: {}", + result.error().message); + } + return result; + }); + ICEBERG_UNWRAP_OR_FAIL( + table_, Table::Make(table_->name(), table_->metadata(), + std::string(table_->metadata_file_location()), file_io_, mock, + table_->full_name(), table_->reporter())); + } + /// \brief Reload the table from catalog and return its metadata. std::shared_ptr ReloadMetadata() { auto result = catalog_->LoadTable(table_ident_); diff --git a/src/iceberg/transaction.cc b/src/iceberg/transaction.cc index bacd880de..a067449ff 100644 --- a/src/iceberg/transaction.cc +++ b/src/iceberg/transaction.cc @@ -18,11 +18,14 @@ */ #include "iceberg/transaction.h" +#include #include #include +#include #include "iceberg/catalog.h" #include "iceberg/location_provider.h" +#include "iceberg/logging/log_macros.h" #include "iceberg/schema.h" #include "iceberg/snapshot.h" #include "iceberg/statistics_file.h" @@ -58,6 +61,21 @@ #include "iceberg/util/retry_util.h" namespace iceberg { +namespace { + +class ScopedTrue { + public: + explicit ScopedTrue(bool& value) : value_(value) { value_ = true; } + ~ScopedTrue() { value_ = false; } + + ScopedTrue(const ScopedTrue&) = delete; + ScopedTrue& operator=(const ScopedTrue&) = delete; + + private: + bool& value_; +}; + +} // namespace // --------------------------------------------------------------------------- // TransactionContext @@ -72,6 +90,7 @@ Result> TransactionContext::Make( auto ctx = std::make_shared(); ctx->kind = kind; ctx->table = std::move(table); + ctx->base_metadata_ = ctx->table->metadata(); if (kind == TransactionKind::kCreate) { ctx->metadata_builder = TableMetadataBuilder::BuildFromEmpty(); std::ignore = ctx->metadata_builder->ApplyChangesForCreate(*ctx->table->metadata()); @@ -123,9 +142,7 @@ Result> Transaction::Make(std::shared_ptr
ta Result> Transaction::Make( std::shared_ptr ctx) { ICEBERG_PRECHECK(ctx != nullptr, "TransactionContext cannot be null"); - auto txn = std::shared_ptr(new Transaction(ctx)); - ctx->transaction = std::weak_ptr(txn); - return txn; + return std::shared_ptr(new Transaction(std::move(ctx))); } const std::shared_ptr
& Transaction::table() const { return ctx_->table; } @@ -138,16 +155,77 @@ std::string Transaction::MetadataFileLocation(std::string_view filename) const { return ctx_->MetadataFileLocation(filename); } -Status Transaction::AddUpdate(const std::shared_ptr& update) { - ICEBERG_CHECK(last_update_committed_, - "Cannot add update when previous update is not committed"); +Status Transaction::CheckActive() const { + ICEBERG_CHECK(!ctx_->in_progress_, "Cannot reenter a transaction operation"); + ICEBERG_CHECK( + state_ == TransactionState::kReady || state_ == TransactionState::kUpdatePending, + "Transaction is terminal"); + return {}; +} + +Status Transaction::CheckReady() const { + ICEBERG_CHECK(!ctx_->in_progress_, "Cannot reenter a transaction operation"); + ICEBERG_CHECK(state_ == TransactionState::kReady, "Transaction is not ready (state {})", + static_cast(state_)); + return {}; +} +Status Transaction::AddUpdate(const std::shared_ptr& update) { + ICEBERG_RETURN_UNEXPECTED(CheckReady()); + ICEBERG_PRECHECK(update && update->ctx_.get() == ctx_.get(), + "Update must belong to this transaction context"); + ICEBERG_CHECK(update->phase_ == PendingUpdate::Phase::kMutable, + "Update has already been used"); pending_updates_.push_back(update); - last_update_committed_ = false; + state_ = TransactionState::kUpdatePending; return {}; } Status Transaction::Apply(PendingUpdate& update) { + ICEBERG_CHECK(!ctx_->in_progress_, "Cannot reenter a transaction operation"); + ICEBERG_CHECK(state_ == TransactionState::kUpdatePending, + "Transaction has no pending operation (state {})", + static_cast(state_)); + ICEBERG_CHECK(!pending_updates_.empty() && pending_updates_.back().get() == &update, + "Update is not the current pending operation"); + ScopedTrue running(ctx_->in_progress_); + Status status; + try { + update.phase_ = PendingUpdate::Phase::kFrozen; + update.staged_ = true; + status = update.Freeze(); + if (status) { + status = ApplyRegistered(update); + } + } catch (const std::exception& e) { + status = ValidationFailed("Update Apply threw: {}", e.what()); + } catch (...) { + status = ValidationFailed("Update Apply threw an unknown exception"); + } + if (!status) { + SetTerminalState(TransactionState::kFailed); + CleanupUpdates(); + return status; + } + state_ = TransactionState::kReady; + return {}; +} + +Status Transaction::ReplayApply(PendingUpdate& update) { + ICEBERG_CHECK(state_ == TransactionState::kReady && ctx_->in_progress_, + "Replay requires an active transaction commit"); + ICEBERG_CHECK(std::ranges::any_of(pending_updates_, + [&update](const auto& registered) { + return registered.get() == &update; + }), + "Cannot replay an unregistered update"); + ICEBERG_CHECK(update.phase_ == PendingUpdate::Phase::kFrozen, + "Cannot replay this update"); + update.staged_ = true; + return ApplyRegistered(update); +} + +Status Transaction::ApplyRegistered(PendingUpdate& update) { switch (update.kind()) { case PendingUpdate::Kind::kExpireSnapshots: ICEBERG_RETURN_UNEXPECTED( @@ -198,9 +276,7 @@ Status Transaction::Apply(PendingUpdate& update) { static_cast(update.kind())); } - last_update_committed_ = true; - - return {}; + return ctx_->metadata_builder->CheckErrors(); } Status Transaction::ApplyExpireSnapshots(ExpireSnapshots& update) { @@ -225,7 +301,7 @@ Status Transaction::ApplyExpireSnapshots(ExpireSnapshots& update) { } Status Transaction::ApplySetSnapshot(SetSnapshot& update) { - ICEBERG_ASSIGN_OR_RAISE(auto snapshot_id, update.Apply()); + ICEBERG_ASSIGN_OR_RAISE(auto snapshot_id, update.Validate()); ctx_->metadata_builder->SetBranchSnapshot(snapshot_id, std::string(SnapshotRef::kMainBranch)); ICEBERG_RETURN_UNEXPECTED(ctx_->metadata_builder->CheckErrors()); @@ -233,13 +309,13 @@ Status Transaction::ApplySetSnapshot(SetSnapshot& update) { } Status Transaction::ApplyUpdateLocation(UpdateLocation& update) { - ICEBERG_ASSIGN_OR_RAISE(auto location, update.Apply()); + ICEBERG_ASSIGN_OR_RAISE(auto location, update.Validate()); ctx_->metadata_builder->SetLocation(location); return {}; } Status Transaction::ApplyUpdatePartitionSpec(UpdatePartitionSpec& update) { - ICEBERG_ASSIGN_OR_RAISE(auto result, update.Apply()); + ICEBERG_ASSIGN_OR_RAISE(auto result, update.Validate()); if (result.set_as_default) { ctx_->metadata_builder->SetDefaultPartitionSpec(std::move(result.spec)); } else { @@ -250,7 +326,7 @@ Status Transaction::ApplyUpdatePartitionSpec(UpdatePartitionSpec& update) { } Status Transaction::ApplyUpdateProperties(UpdateProperties& update) { - ICEBERG_ASSIGN_OR_RAISE(auto result, update.Apply()); + ICEBERG_ASSIGN_OR_RAISE(auto result, update.Validate()); if (!result.updates.empty()) { ctx_->metadata_builder->SetProperties(std::move(result.updates)); } @@ -265,7 +341,7 @@ Status Transaction::ApplyUpdateProperties(UpdateProperties& update) { } Status Transaction::ApplyUpdateSchema(UpdateSchema& update) { - ICEBERG_ASSIGN_OR_RAISE(auto result, update.Apply()); + ICEBERG_ASSIGN_OR_RAISE(auto result, update.Validate()); ctx_->metadata_builder->SetCurrentSchema(std::move(result.schema), result.new_last_column_id); if (!result.updated_props.empty()) { @@ -294,9 +370,9 @@ Status Transaction::ApplyUpdateSnapshot(SnapshotUpdate& update) { ICEBERG_RETURN_UNEXPECTED(temp_update->CheckErrors()); if (temp_update->changes().empty()) { - // Do not commit if the metadata has not changed. for example, this may happen - // when setting the current snapshot to an ID that is already current. note that - // this check uses identity. + // Apply may already have written files. Consume this generation immediately, + // while retaining its frozen intent for a possible replay against new metadata. + update.Cleanup(); return {}; } @@ -315,7 +391,7 @@ Status Transaction::ApplyUpdateSnapshot(SnapshotUpdate& update) { } Status Transaction::ApplyUpdateSnapshotReference(UpdateSnapshotReference& update) { - ICEBERG_ASSIGN_OR_RAISE(auto result, update.Apply()); + ICEBERG_ASSIGN_OR_RAISE(auto result, update.Validate()); for (const auto& name : result.to_remove) { ctx_->metadata_builder->RemoveRef(name); } @@ -327,14 +403,14 @@ Status Transaction::ApplyUpdateSnapshotReference(UpdateSnapshotReference& update } Status Transaction::ApplyUpdateSortOrder(UpdateSortOrder& update) { - ICEBERG_ASSIGN_OR_RAISE(auto sort_order, update.Apply()); + ICEBERG_ASSIGN_OR_RAISE(auto sort_order, update.Validate()); ctx_->metadata_builder->SetDefaultSortOrder(std::move(sort_order)); ICEBERG_RETURN_UNEXPECTED(ctx_->metadata_builder->CheckErrors()); return {}; } Status Transaction::ApplyUpdateStatistics(UpdateStatistics& update) { - ICEBERG_ASSIGN_OR_RAISE(auto result, update.Apply()); + ICEBERG_ASSIGN_OR_RAISE(auto result, update.Validate()); for (auto&& [_, stat_file] : result.to_set) { ctx_->metadata_builder->SetStatistics(std::move(stat_file)); } @@ -346,7 +422,7 @@ Status Transaction::ApplyUpdateStatistics(UpdateStatistics& update) { } Status Transaction::ApplyUpdatePartitionStatistics(UpdatePartitionStatistics& update) { - ICEBERG_ASSIGN_OR_RAISE(auto result, update.Apply()); + ICEBERG_ASSIGN_OR_RAISE(auto result, update.Validate()); for (auto&& [_, partition_stat_file] : result.to_set) { ctx_->metadata_builder->SetPartitionStatistics(std::move(partition_stat_file)); } @@ -358,78 +434,170 @@ Status Transaction::ApplyUpdatePartitionStatistics(UpdatePartitionStatistics& up } Result> Transaction::Commit() { - ICEBERG_CHECK(!committed_, "Transaction already committed"); - ICEBERG_CHECK(last_update_committed_, - "Cannot commit transaction when previous update is not committed"); - - const auto& updates = ctx_->metadata_builder->changes(); - if (updates.empty()) { - committed_ = true; - return ctx_->table; - } - - const auto& props = ctx_->table->properties(); - int32_t num_retries = - CanRetry() ? static_cast(props.Get(TableProperties::kCommitNumRetries)) - : 0; - int32_t min_wait_ms = props.Get(TableProperties::kCommitMinRetryWaitMs); - int32_t max_wait_ms = props.Get(TableProperties::kCommitMaxRetryWaitMs); - int32_t total_timeout_ms = props.Get(TableProperties::kCommitTotalRetryTimeMs); - - bool is_first_attempt = true; - auto commit_result = - MakeCommitRetryRunner(num_retries, min_wait_ms, max_wait_ms, total_timeout_ms) - .Run([this, &is_first_attempt]() -> Result> { - auto result = CommitOnce(is_first_attempt); - is_first_attempt = false; - return result; - }); - - Result finalize_result = - commit_result.has_value() - ? Result(commit_result.value()->metadata().get()) - : std::unexpected(commit_result.error()); + ICEBERG_RETURN_UNEXPECTED(CheckReady()); + ScopedTrue running(ctx_->in_progress_); + Result> commit_result = ctx_->table; + bool catalog_state_unknown = false; + try { + ConfigureExpirationCleanup(); + auto builder_status = ctx_->metadata_builder->CheckErrors(); + if (!builder_status) { + commit_result = std::unexpected(builder_status.error()); + } else { + const auto& props = ctx_->table->properties(); + const int32_t num_retries = + CanRetry() ? static_cast(props.Get(TableProperties::kCommitNumRetries)) + : 0; + bool is_first_attempt = true; + std::optional replay_error; + commit_result = + MakeCommitRetryRunner(num_retries, + props.Get(TableProperties::kCommitMinRetryWaitMs), + props.Get(TableProperties::kCommitMaxRetryWaitMs), + props.Get(TableProperties::kCommitTotalRetryTimeMs)) + .Run([this, &is_first_attempt, &replay_error, + &catalog_state_unknown]() -> Result> { + auto result = + CommitOnce(is_first_attempt, replay_error, catalog_state_unknown); + is_first_attempt = false; + // A replay failure is not another catalog conflict. Stop the + // runner, then restore the original Apply error for the + // caller below. + if (replay_error) { + return ValidationFailed("Transaction replay failed"); + } + return result; + }); + if (replay_error) { + commit_result = std::unexpected(std::move(*replay_error)); + } + } + } catch (const std::exception& e) { + // CommitOnce catches exceptions at the catalog boundary separately. + commit_result = ValidationFailed("Transaction preparation threw: {}", e.what()); + } catch (...) { + commit_result = + ValidationFailed("Transaction preparation threw an unknown exception"); + } + + if (!commit_result) { + if (catalog_state_unknown) { + SetTerminalState(TransactionState::kCommitStateUnknown); + } else { + SetTerminalState(TransactionState::kFailed); + CleanupUpdates(); + } + return commit_result; + } + + ctx_->table = std::move(commit_result.value()); + SetTerminalState(TransactionState::kCommitted); + FinalizeUpdates(*ctx_->table->metadata()); + return ctx_->table; +} +Status Transaction::Abort() { + ICEBERG_CHECK(!ctx_->in_progress_, "Cannot reenter a transaction operation"); + if (state_ == TransactionState::kAborted) { + return {}; + } + ICEBERG_CHECK(state_ == TransactionState::kReady || + state_ == TransactionState::kUpdatePending || + state_ == TransactionState::kFailed, + "Cannot abort a committed or unknown transaction"); + ScopedTrue running(ctx_->in_progress_); + SetTerminalState(TransactionState::kAborted); + CleanupUpdates(); + return {}; +} + +void Transaction::SetTerminalState(TransactionState state) { + state_ = state; + // Publish every marker before invoking even the first user callback. for (const auto& update : pending_updates_) { - std::ignore = update->Finalize(finalize_result); + update->phase_ = PendingUpdate::Phase::kTerminal; } +} - ICEBERG_RETURN_UNEXPECTED(commit_result); +void Transaction::CleanupUpdates() noexcept { + for (const auto& update : pending_updates_) { + update->Cleanup(); + } +} - // Mark as committed and update table reference - committed_ = true; - ctx_->table = std::move(commit_result.value()); +void Transaction::FinalizeUpdates(const TableMetadata& committed) noexcept { + for (const auto& update : pending_updates_) { + update->FinalizeOnce(committed); + } +} - return ctx_->table; +void Transaction::ConfigureExpirationCleanup() { + bool may_add_references = false; + for (const auto& update : pending_updates_ | std::views::reverse) { + if (update->kind() == PendingUpdate::Kind::kExpireSnapshots) { + internal::checked_cast(*update).skip_physical_cleanup_ = + may_add_references; + } + may_add_references |= update->MayAddFileReferences(); + } } -Result> Transaction::CommitOnce(bool is_first_attempt) { +Result> Transaction::CommitOnce(bool is_first_attempt, + std::optional& replay_error, + bool& catalog_state_unknown) { std::vector> requirements; - - switch (ctx_->kind) { - case TransactionKind::kCreate: { - ICEBERG_ASSIGN_OR_RAISE(requirements, TableRequirements::ForCreateTable( - ctx_->metadata_builder->changes())); - } break; - case TransactionKind::kUpdate: { - if (!is_first_attempt) { - ICEBERG_RETURN_UNEXPECTED(ctx_->table->Refresh()); - } - if (ctx_->metadata_builder->base() != ctx_->table->metadata().get()) { - ctx_->metadata_builder = - TableMetadataBuilder::BuildFrom(ctx_->table->metadata().get()); - for (const auto& update : pending_updates_) { - ICEBERG_RETURN_UNEXPECTED(update->Commit()); + if (ctx_->kind == TransactionKind::kUpdate) { + if (!is_first_attempt) { + ICEBERG_RETURN_UNEXPECTED(ctx_->table->Refresh()); + } + if (!is_first_attempt || + ctx_->metadata_builder->base() != ctx_->table->metadata().get()) { + ICEBERG_CHECK(CanRetry(), + "Cannot rebase a transaction containing a non-retryable update"); + CleanupUpdates(); + ctx_->metadata_builder = + TableMetadataBuilder::BuildFrom(ctx_->table->metadata().get()); + ctx_->base_metadata_ = ctx_->table->metadata(); + for (const auto& update : pending_updates_) { + Status applied; + try { + applied = ReplayApply(*update); + } catch (const std::exception& e) { + applied = ValidationFailed("Replay Apply threw: {}", e.what()); + } catch (...) { + applied = ValidationFailed("Replay Apply threw an unknown exception"); + } + if (!applied) { + replay_error = applied.error(); + return std::unexpected(applied.error()); } } - ICEBERG_ASSIGN_OR_RAISE(requirements, TableRequirements::ForUpdateTable( - *ctx_->metadata_builder->base(), - ctx_->metadata_builder->changes())); - } break; + } + if (ctx_->metadata_builder->changes().empty()) { + return ctx_->table; + } + ICEBERG_ASSIGN_OR_RAISE(requirements, TableRequirements::ForUpdateTable( + *ctx_->metadata_builder->base(), + ctx_->metadata_builder->changes())); + } else { + ICEBERG_ASSIGN_OR_RAISE(requirements, TableRequirements::ForCreateTable( + ctx_->metadata_builder->changes())); } - return ctx_->table->catalog()->UpdateTable(ctx_->table->name(), requirements, - ctx_->metadata_builder->changes()); + // Only this boundary can turn an uncaught exception into an unknown commit. + try { + auto result = ctx_->table->catalog()->UpdateTable(ctx_->table->name(), requirements, + ctx_->metadata_builder->changes()); + catalog_state_unknown = + !result && result.error().kind == ErrorKind::kCommitStateUnknown; + return result; + } catch (const std::exception& e) { + catalog_state_unknown = true; + return CommitStateUnknown("Catalog commit threw: {}", e.what()); + } catch (...) { + catalog_state_unknown = true; + return CommitStateUnknown("Catalog commit threw an unknown exception"); + } } bool Transaction::CanRetry() const { @@ -445,6 +613,7 @@ bool Transaction::CanRetry() const { } Result> Transaction::NewUpdatePartitionSpec() { + ICEBERG_RETURN_UNEXPECTED(CheckReady()); ICEBERG_ASSIGN_OR_RAISE(std::shared_ptr update_spec, UpdatePartitionSpec::Make(ctx_)); ICEBERG_RETURN_UNEXPECTED(AddUpdate(update_spec)); @@ -452,6 +621,7 @@ Result> Transaction::NewUpdatePartitionSpec } Result> Transaction::NewUpdateProperties() { + ICEBERG_RETURN_UNEXPECTED(CheckReady()); ICEBERG_ASSIGN_OR_RAISE(std::shared_ptr update_properties, UpdateProperties::Make(ctx_)); ICEBERG_RETURN_UNEXPECTED(AddUpdate(update_properties)); @@ -459,6 +629,7 @@ Result> Transaction::NewUpdateProperties() { } Result> Transaction::NewUpdateSortOrder() { + ICEBERG_RETURN_UNEXPECTED(CheckReady()); ICEBERG_ASSIGN_OR_RAISE(std::shared_ptr update_sort_order, UpdateSortOrder::Make(ctx_)); ICEBERG_RETURN_UNEXPECTED(AddUpdate(update_sort_order)); @@ -466,6 +637,7 @@ Result> Transaction::NewUpdateSortOrder() { } Result> Transaction::NewUpdateSchema() { + ICEBERG_RETURN_UNEXPECTED(CheckReady()); ICEBERG_ASSIGN_OR_RAISE(std::shared_ptr update_schema, UpdateSchema::Make(ctx_)); ICEBERG_RETURN_UNEXPECTED(AddUpdate(update_schema)); @@ -473,6 +645,7 @@ Result> Transaction::NewUpdateSchema() { } Result> Transaction::NewExpireSnapshots() { + ICEBERG_RETURN_UNEXPECTED(CheckReady()); ICEBERG_ASSIGN_OR_RAISE(std::shared_ptr expire_snapshots, ExpireSnapshots::Make(ctx_)); ICEBERG_RETURN_UNEXPECTED(AddUpdate(expire_snapshots)); @@ -480,6 +653,7 @@ Result> Transaction::NewExpireSnapshots() { } Result> Transaction::NewUpdateLocation() { + ICEBERG_RETURN_UNEXPECTED(CheckReady()); ICEBERG_ASSIGN_OR_RAISE(std::shared_ptr update_location, UpdateLocation::Make(ctx_)); ICEBERG_RETURN_UNEXPECTED(AddUpdate(update_location)); @@ -487,6 +661,7 @@ Result> Transaction::NewUpdateLocation() { } Result> Transaction::NewSetSnapshot() { + ICEBERG_RETURN_UNEXPECTED(CheckReady()); ICEBERG_ASSIGN_OR_RAISE(std::shared_ptr set_snapshot, SetSnapshot::Make(ctx_)); ICEBERG_RETURN_UNEXPECTED(AddUpdate(set_snapshot)); @@ -494,6 +669,7 @@ Result> Transaction::NewSetSnapshot() { } Result> Transaction::NewFastAppend() { + ICEBERG_RETURN_UNEXPECTED(CheckReady()); ICEBERG_ASSIGN_OR_RAISE(std::shared_ptr fast_append, FastAppend::Make(ctx_->table->name().name, ctx_)); ICEBERG_RETURN_UNEXPECTED(AddUpdate(fast_append)); @@ -501,6 +677,7 @@ Result> Transaction::NewFastAppend() { } Result> Transaction::NewMergeAppend() { + ICEBERG_RETURN_UNEXPECTED(CheckReady()); ICEBERG_ASSIGN_OR_RAISE(std::shared_ptr merge_append, MergeAppend::Make(ctx_->table->name().name, ctx_)); ICEBERG_RETURN_UNEXPECTED(AddUpdate(merge_append)); @@ -508,6 +685,7 @@ Result> Transaction::NewMergeAppend() { } Result> Transaction::NewDeleteFiles() { + ICEBERG_RETURN_UNEXPECTED(CheckReady()); ICEBERG_ASSIGN_OR_RAISE(std::shared_ptr delete_files, DeleteFiles::Make(ctx_->table->name().name, ctx_)); ICEBERG_RETURN_UNEXPECTED(AddUpdate(delete_files)); @@ -515,6 +693,7 @@ Result> Transaction::NewDeleteFiles() { } Result> Transaction::NewRowDelta() { + ICEBERG_RETURN_UNEXPECTED(CheckReady()); ICEBERG_ASSIGN_OR_RAISE(std::shared_ptr row_delta, RowDelta::Make(ctx_->table->name().name, ctx_)); ICEBERG_RETURN_UNEXPECTED(AddUpdate(row_delta)); @@ -522,6 +701,7 @@ Result> Transaction::NewRowDelta() { } Result> Transaction::NewOverwrite() { + ICEBERG_RETURN_UNEXPECTED(CheckReady()); ICEBERG_ASSIGN_OR_RAISE(std::shared_ptr overwrite, OverwriteFiles::Make(ctx_->table->name().name, ctx_)); ICEBERG_RETURN_UNEXPECTED(AddUpdate(overwrite)); @@ -529,6 +709,7 @@ Result> Transaction::NewOverwrite() { } Result> Transaction::NewRewriteFiles() { + ICEBERG_RETURN_UNEXPECTED(CheckReady()); ICEBERG_ASSIGN_OR_RAISE(std::shared_ptr rewrite_files, RewriteFiles::Make(ctx_->table->name().name, ctx_)); ICEBERG_RETURN_UNEXPECTED(AddUpdate(rewrite_files)); @@ -536,6 +717,7 @@ Result> Transaction::NewRewriteFiles() { } Result> Transaction::NewReplacePartitions() { + ICEBERG_RETURN_UNEXPECTED(CheckReady()); ICEBERG_ASSIGN_OR_RAISE(std::shared_ptr replace_partitions, ReplacePartitions::Make(ctx_->table->name().name, ctx_)); ICEBERG_RETURN_UNEXPECTED(AddUpdate(replace_partitions)); @@ -543,6 +725,7 @@ Result> Transaction::NewReplacePartitions() { } Result> Transaction::NewUpdateStatistics() { + ICEBERG_RETURN_UNEXPECTED(CheckReady()); ICEBERG_ASSIGN_OR_RAISE(std::shared_ptr update_statistics, UpdateStatistics::Make(ctx_)); ICEBERG_RETURN_UNEXPECTED(AddUpdate(update_statistics)); @@ -551,6 +734,7 @@ Result> Transaction::NewUpdateStatistics() { Result> Transaction::NewUpdatePartitionStatistics() { + ICEBERG_RETURN_UNEXPECTED(CheckReady()); ICEBERG_ASSIGN_OR_RAISE( std::shared_ptr update_partition_statistics, UpdatePartitionStatistics::Make(ctx_)); @@ -560,6 +744,7 @@ Transaction::NewUpdatePartitionStatistics() { Result> Transaction::NewUpdateSnapshotReference() { + ICEBERG_RETURN_UNEXPECTED(CheckReady()); ICEBERG_ASSIGN_OR_RAISE(std::shared_ptr update_ref, UpdateSnapshotReference::Make(ctx_)); ICEBERG_RETURN_UNEXPECTED(AddUpdate(update_ref)); @@ -567,6 +752,7 @@ Transaction::NewUpdateSnapshotReference() { } Result> Transaction::NewSnapshotManager() { + ICEBERG_RETURN_UNEXPECTED(CheckReady()); // SnapshotManager has its own commit logic, so it is not added to the pending updates. return SnapshotManager::Make(shared_from_this()); } diff --git a/src/iceberg/transaction.h b/src/iceberg/transaction.h index 3ee0372ad..cfe61e957 100644 --- a/src/iceberg/transaction.h +++ b/src/iceberg/transaction.h @@ -39,6 +39,16 @@ namespace iceberg { /// \brief Whether a transaction creates a new table or updates an existing one. enum class TransactionKind : uint8_t { kCreate, kUpdate }; +/// \brief Lifecycle outcomes of a transaction. Failed transactions may only be aborted. +enum class TransactionState : uint8_t { + kReady, + kUpdatePending, + kCommitted, + kFailed, + kAborted, + kCommitStateUnknown, +}; + /// \brief A transaction for performing multiple updates to a table class ICEBERG_EXPORT Transaction : public std::enable_shared_from_this { public: @@ -50,8 +60,8 @@ class ICEBERG_EXPORT Transaction : public std::enable_shared_from_this> Make( std::shared_ptr ctx); @@ -77,6 +87,13 @@ class ICEBERG_EXPORT Transaction : public std::enable_shared_from_this> Commit(); + /// \brief Discard staged changes and clean owned files best effort. + /// Repeated Abort succeeds without repeating cleanup. Committed and unknown + /// transactions cannot be aborted. Destructors never perform cleanup. + Status Abort(); + + TransactionState state() const { return state_; } + /// \brief Create a new UpdatePartitionSpec to update the partition spec of this table /// and commit the changes. Result> NewUpdatePartitionSpec(); @@ -146,10 +163,14 @@ class ICEBERG_EXPORT Transaction : public std::enable_shared_from_this ctx); + Status CheckReady() const; + Status CheckActive() const; Status AddUpdate(const std::shared_ptr& update); /// \brief Apply the pending changes to current table. - Status Apply(PendingUpdate& updates); + Status Apply(PendingUpdate& update); + Status ApplyRegistered(PendingUpdate& update); + Status ReplayApply(PendingUpdate& update); // Helper methods for applying different types of updates Status ApplyExpireSnapshots(ExpireSnapshots& update); @@ -165,23 +186,28 @@ class ICEBERG_EXPORT Transaction : public std::enable_shared_from_this> CommitOnce(bool is_first_attempt); + Result> CommitOnce(bool is_first_attempt, + std::optional& replay_error, + bool& catalog_state_unknown); /// \brief Whether this transaction can retry after a commit conflict. bool CanRetry() const; + void SetTerminalState(TransactionState state); + void CleanupUpdates() noexcept; + void FinalizeUpdates(const TableMetadata& committed) noexcept; + void ConfigureExpirationCleanup(); + private: friend class PendingUpdate; + friend class SnapshotManager; // Shared context owning the table, metadata builder, and kind. std::shared_ptr ctx_; // Keep track of all created pending updates. std::vector> pending_updates_; - // To make the state simple, we require updates are added and committed in order. - bool last_update_committed_ = true; - // Tracks if transaction has been committed to prevent double-commit - bool committed_ = false; + TransactionState state_ = TransactionState::kReady; }; /// \brief Shared context between Transaction and PendingUpdate instances. @@ -204,6 +230,14 @@ class ICEBERG_EXPORT TransactionContext { // If PendingUpdate is created directly from Table, this is nullopt; // otherwise, it holds a weak pointer to the Transaction that created it. std::optional> transaction; + + private: + friend class Transaction; + friend class PendingUpdate; + + // Keep the builder's base alive when Table::Refresh replaces table metadata. + std::shared_ptr base_metadata_; + bool in_progress_ = false; }; } // namespace iceberg diff --git a/src/iceberg/update/delete_files.cc b/src/iceberg/update/delete_files.cc index 9759e3eb9..d2bbac71e 100644 --- a/src/iceberg/update/delete_files.cc +++ b/src/iceberg/update/delete_files.cc @@ -30,11 +30,11 @@ namespace iceberg { -Result> DeleteFiles::Make( +Result> DeleteFiles::Make( std::string table_name, std::shared_ptr ctx) { ICEBERG_PRECHECK(!table_name.empty(), "Table name cannot be empty"); ICEBERG_PRECHECK(ctx != nullptr, "Cannot create DeleteFiles without a context"); - return std::unique_ptr( + return std::shared_ptr( new DeleteFiles(std::move(table_name), std::move(ctx))); } @@ -42,27 +42,32 @@ DeleteFiles::DeleteFiles(std::string table_name, std::shared_ptr& file) { + EnsureMutable(); ICEBERG_BUILDER_RETURN_IF_ERROR(DeleteDataFile(file)); return *this; } DeleteFiles& DeleteFiles::DeleteFromRowFilter(std::shared_ptr expr) { + EnsureMutable(); ICEBERG_BUILDER_RETURN_IF_ERROR(DeleteByRowFilter(std::move(expr))); return *this; } DeleteFiles& DeleteFiles::CaseSensitive(bool case_sensitive) { + EnsureMutable(); MergingSnapshotUpdate::CaseSensitive(case_sensitive); return *this; } DeleteFiles& DeleteFiles::ValidateFilesExist() { + EnsureMutable(); validate_files_to_delete_exist_ = true; return *this; } diff --git a/src/iceberg/update/delete_files.h b/src/iceberg/update/delete_files.h index 7e567830e..18756ff48 100644 --- a/src/iceberg/update/delete_files.h +++ b/src/iceberg/update/delete_files.h @@ -43,7 +43,7 @@ namespace iceberg { /// differently-normalized URIs are not considered matches. class ICEBERG_EXPORT DeleteFiles : public MergingSnapshotUpdate { public: - static Result> Make( + static Result> Make( std::string table_name, std::shared_ptr ctx); /// \brief Delete a file by path from the underlying table. diff --git a/src/iceberg/update/expire_snapshots.cc b/src/iceberg/update/expire_snapshots.cc index 5573efa77..fbd21b16c 100644 --- a/src/iceberg/update/expire_snapshots.cc +++ b/src/iceberg/update/expire_snapshots.cc @@ -31,6 +31,7 @@ #include #include "iceberg/file_io.h" +#include "iceberg/logging/log_macros.h" #include "iceberg/manifest/manifest_entry.h" #include "iceberg/manifest/manifest_reader.h" #include "iceberg/result.h" @@ -108,24 +109,18 @@ class FileCleanupStrategy { return; } - if (!delete_func_) { - std::vector path_list(paths.begin(), paths.end()); - - TaskGroup> group(kDeleteRetryConfig); - group.Submit([this, paths = std::move(path_list)]() -> Status { - return file_io_->DeleteFiles(paths); - }); - std::ignore = std::move(group).Run(); - return; - } - TaskGroup> group(kDeleteRetryConfig); - group.SetExecutor(delete_executor_); + if (delete_func_) { + group.SetExecutor(delete_executor_); + } for (const auto& path : paths) { group.Submit([this, path]() -> Status { try { - delete_func_(path); - return {}; + if (delete_func_) { + delete_func_(path); + return {}; + } + return file_io_->DeleteFile(path); } catch (const std::exception& e) { return IOError("Delete callback failed for {}: {}", path, e.what()); } catch (...) { @@ -133,7 +128,9 @@ class FileCleanupStrategy { } }); } - std::ignore = std::move(group).Run(); + if (auto status = std::move(group).Run(); !status) { + ICEBERG_LOG_WARN("Expiration deletion failed: {}", status.error().message); + } } bool HasAnyStatisticsFiles(const TableMetadata& metadata) const { @@ -754,17 +751,20 @@ ExpireSnapshots::ExpireSnapshots(std::shared_ptr ctx) ExpireSnapshots::~ExpireSnapshots() = default; ExpireSnapshots& ExpireSnapshots::ExpireSnapshotId(int64_t snapshot_id) { + EnsureMutable(); snapshot_ids_to_expire_.push_back(snapshot_id); specified_snapshot_id_ = true; return *this; } ExpireSnapshots& ExpireSnapshots::ExpireOlderThan(int64_t timestamp_millis) { + EnsureMutable(); default_expire_older_than_ = TimePointMsFromUnixMs(timestamp_millis); return *this; } ExpireSnapshots& ExpireSnapshots::RetainLast(int num_snapshots) { + EnsureMutable(); ICEBERG_BUILDER_CHECK(num_snapshots > 0, "Number of snapshots to retain must be positive: {}", num_snapshots); @@ -774,26 +774,31 @@ ExpireSnapshots& ExpireSnapshots::RetainLast(int num_snapshots) { ExpireSnapshots& ExpireSnapshots::DeleteWith( std::function delete_func) { + EnsureMutable(); delete_func_ = std::move(delete_func); return *this; } ExpireSnapshots& ExpireSnapshots::PlanWith(Executor& executor) { + EnsureMutable(); plan_executor_ = std::ref(executor); return *this; } ExpireSnapshots& ExpireSnapshots::CleanupLevel(enum CleanupLevel level) { + EnsureMutable(); cleanup_level_ = level; return *this; } ExpireSnapshots& ExpireSnapshots::CleanExpiredMetadata(bool clean) { + EnsureMutable(); clean_expired_metadata_ = clean; return *this; } ExpireSnapshots& ExpireSnapshots::ExecuteDeleteWith(Executor& executor) { + EnsureMutable(); delete_executor_ = std::ref(executor); return *this; } @@ -908,7 +913,7 @@ Result ExpireSnapshots::ComputeRetainedRefs( return retained_refs; } -Result ExpireSnapshots::Apply() { +Result ExpireSnapshots::Validate() const { ICEBERG_RETURN_UNEXPECTED(CheckErrors()); const TableMetadata& base = this->base(); @@ -999,33 +1004,35 @@ Result ExpireSnapshots::Apply() { std::ranges::to>(); } - // Cache the result for use during Finalize() - apply_result_ = result; - return result; } -Status ExpireSnapshots::Finalize(Result commit_result) { - if (!commit_result.has_value()) { - return {}; - } +Result ExpireSnapshots::Apply() { + ICEBERG_ASSIGN_OR_RAISE(auto result, Validate()); + apply_result_ = result; + return result; +} - if (cleanup_level_ == CleanupLevel::kNone) { +Status ExpireSnapshots::Finalize(const TableMetadata& metadata_after_expiration) { + // The cached Apply result belongs to this generation only; consume it regardless + // of whether any physical cleanup happens. + auto apply_result = std::exchange(apply_result_, std::nullopt); + if (cleanup_level_ == CleanupLevel::kNone || !apply_result.has_value() || + apply_result->snapshot_ids_to_remove.empty()) { return {}; } - if (!apply_result_.has_value() || apply_result_->snapshot_ids_to_remove.empty()) { + if (skip_physical_cleanup_) { + ICEBERG_LOG_WARN( + "Skipping expiration file deletion because a later update may add file " + "references"); return {}; } - ICEBERG_PRECHECK(apply_result_->metadata_before_expiration != nullptr, + ICEBERG_PRECHECK(apply_result->metadata_before_expiration != nullptr, "Missing pre-expiration table metadata for cleanup"); - ICEBERG_PRECHECK(commit_result.value() != nullptr, - "Missing committed table metadata for cleanup"); - auto metadata_before_expiration_ptr = apply_result_->metadata_before_expiration; - const TableMetadata& metadata_before_expiration = *metadata_before_expiration_ptr; - const TableMetadata& metadata_after_expiration = *commit_result.value(); - apply_result_.reset(); + const TableMetadata& metadata_before_expiration = + *apply_result->metadata_before_expiration; // Pick incremental cleanup when the expiration is a simple linear-ancestry walk: // no explicit snapshot IDs, no removed snapshots outside main ancestry, and no diff --git a/src/iceberg/update/expire_snapshots.h b/src/iceberg/update/expire_snapshots.h index 215ad85fd..cba08672c 100644 --- a/src/iceberg/update/expire_snapshots.h +++ b/src/iceberg/update/expire_snapshots.h @@ -64,7 +64,7 @@ enum class CleanupLevel : uint8_t { /// that were deleted by snapshots that are expired will be deleted. DeleteWith() can be /// used to pass an alternative deletion method. /// -/// Apply() returns a list of the snapshots that will be removed. +/// Validate() previews the snapshots that will be removed without staging cleanup. class ICEBERG_EXPORT ExpireSnapshots : public PendingUpdate { public: static Result> Make( @@ -158,22 +158,21 @@ class ICEBERG_EXPORT ExpireSnapshots : public PendingUpdate { Kind kind() const final { return Kind::kExpireSnapshots; } bool IsRetryable() const override { return true; } - /// \brief Apply the pending changes and return the results - /// \return The results of changes - Result Apply(); - - /// \brief Finalize the expire snapshots update, cleaning up expired files. - /// - /// After a successful commit, this method deletes manifest files, manifest lists, - /// data files, and statistics files that are no longer referenced by any valid - /// snapshot. The cleanup behavior is controlled by the CleanupLevel setting. - /// - /// \param commit_result The committed table metadata when the commit succeeds, or the - /// commit error when it fails. - /// \return Status indicating success or failure - Status Finalize(Result commit_result) override; + /// \brief Validate and preview changes without staging or modifying this update. + Result Validate() const; private: + friend class Transaction; + + bool MayAddFileReferences() const override { return false; } + Status Finalize(const TableMetadata& committed) override; + Status CleanStaged() override { + apply_result_.reset(); + return {}; + } + bool skip_physical_cleanup_ = false; + Result Apply(); + explicit ExpireSnapshots(std::shared_ptr ctx); using SnapshotToRef = std::unordered_map>; diff --git a/src/iceberg/update/fast_append.cc b/src/iceberg/update/fast_append.cc index 4167387c7..688edd034 100644 --- a/src/iceberg/update/fast_append.cc +++ b/src/iceberg/update/fast_append.cc @@ -30,16 +30,17 @@ #include "iceberg/table_metadata.h" #include "iceberg/table_properties.h" #include "iceberg/transaction.h" +#include "iceberg/update/update_util_internal.h" #include "iceberg/util/error_collector.h" #include "iceberg/util/macros.h" namespace iceberg { -Result> FastAppend::Make( +Result> FastAppend::Make( std::string table_name, std::shared_ptr ctx) { ICEBERG_PRECHECK(!table_name.empty(), "Table name cannot be empty"); ICEBERG_PRECHECK(ctx != nullptr, "Cannot create FastAppend without a context"); - return std::unique_ptr( + return std::shared_ptr( new FastAppend(std::move(table_name), std::move(ctx))); } @@ -47,6 +48,7 @@ FastAppend::FastAppend(std::string table_name, std::shared_ptr& file) { + EnsureMutable(); ICEBERG_BUILDER_CHECK(file != nullptr, "Invalid data file: null"); ICEBERG_BUILDER_CHECK(file->partition_spec_id.has_value(), "Data file must have partition spec ID"); @@ -65,6 +67,7 @@ FastAppend& FastAppend::AppendFile(const std::shared_ptr& file) { } FastAppend& FastAppend::AppendManifest(const ManifestFile& manifest) { + EnsureMutable(); ICEBERG_BUILDER_CHECK(!manifest.has_existing_files(), "Cannot append manifest with existing files"); ICEBERG_BUILDER_CHECK(!manifest.has_deleted_files(), @@ -79,20 +82,38 @@ FastAppend& FastAppend::AppendManifest(const ManifestFile& manifest) { append_manifests_.push_back(manifest); } else { // The manifest must be rewritten with this update's snapshot ID - ICEBERG_BUILDER_ASSIGN_OR_RETURN(auto copied_manifest, - CopyManifest(manifest, /*update_summary=*/true)); append_manifests_to_copy_.push_back(manifest); - rewritten_append_manifests_.push_back(std::move(copied_manifest)); } return *this; } +Status FastAppend::Freeze() { + std::unordered_map frozen; + added_data_files_summary_.Clear(); + for (const auto& [_, files] : new_data_files_by_spec_) { + for (const auto& file : files) { + ICEBERG_ASSIGN_OR_RAISE(auto copy, internal::CopyUpdateDataFile(*file)); + ICEBERG_PRECHECK(copy->partition_spec_id.has_value(), + "Data file must have partition spec ID"); + ICEBERG_ASSIGN_OR_RAISE(auto spec, Spec(*copy->partition_spec_id)); + frozen[*copy->partition_spec_id].insert(copy); + ICEBERG_RETURN_UNEXPECTED(added_data_files_summary_.AddedFile(*spec, *copy)); + } + } + new_data_files_by_spec_ = std::move(frozen); + return {}; +} + std::string FastAppend::operation() { return DataOperation::kAppend; } Result> FastAppend::Apply( const TableMetadata& metadata_to_update, const std::shared_ptr& snapshot) { std::vector manifests; + appended_manifests_summary_.Clear(); + for (const auto& manifest : append_manifests_) { + appended_manifests_summary_.AddedManifest(manifest); + } ICEBERG_ASSIGN_OR_RAISE(auto new_written_manifests, WriteNewManifests()); // A retry cleanup deletes copied append manifests and clears the rewritten @@ -100,7 +121,7 @@ Result> FastAppend::Apply( if (rewritten_append_manifests_.empty() && !append_manifests_to_copy_.empty()) { for (const auto& manifest : append_manifests_to_copy_) { ICEBERG_ASSIGN_OR_RAISE(auto copied_manifest, - CopyManifest(manifest, /*update_summary=*/false)); + CopyManifest(manifest, /*update_summary=*/true)); rewritten_append_manifests_.push_back(std::move(copied_manifest)); } } @@ -184,15 +205,6 @@ Status FastAppend::CleanUncommitted(const std::unordered_set& commi return {}; } -bool FastAppend::CleanupAfterCommit() const { - // Cleanup after committing is disabled for FastAppend unless append manifests - // were copied or need to be copied on retry because: - // 1.) Directly appended manifests are never rewritten - // 2.) Manifests which are written out as part of AppendFile are already cleaned - // up between commit attempts in WriteNewManifests - return !rewritten_append_manifests_.empty() || !append_manifests_to_copy_.empty(); -} - Result> FastAppend::Spec(int32_t spec_id) { return base().PartitionSpecById(spec_id); } diff --git a/src/iceberg/update/fast_append.h b/src/iceberg/update/fast_append.h index f5e1ceaba..eeb2e374e 100644 --- a/src/iceberg/update/fast_append.h +++ b/src/iceberg/update/fast_append.h @@ -51,7 +51,7 @@ class ICEBERG_EXPORT FastAppend : public SnapshotUpdate { /// \param table_name The name of the table /// \param ctx The transaction context to use for this update /// \return A Result containing the FastAppend instance or an error - static Result> Make( + static Result> Make( std::string table_name, std::shared_ptr ctx); /// \brief Append a DataFile to the table. @@ -73,6 +73,8 @@ class ICEBERG_EXPORT FastAppend : public SnapshotUpdate { /// \return This FastAppend for method chaining. FastAppend& AppendManifest(const ManifestFile& manifest); + protected: + Status Freeze() override; std::string operation() override; Result> Apply( @@ -81,7 +83,6 @@ class ICEBERG_EXPORT FastAppend : public SnapshotUpdate { std::unordered_map Summary() override; void SetSummaryProperty(const std::string& property, const std::string& value) override; Status CleanUncommitted(const std::unordered_set& committed) override; - bool CleanupAfterCommit() const override; private: explicit FastAppend(std::string table_name, std::shared_ptr ctx); diff --git a/src/iceberg/update/merge_append.cc b/src/iceberg/update/merge_append.cc index 70cd7b8e7..85ec48590 100644 --- a/src/iceberg/update/merge_append.cc +++ b/src/iceberg/update/merge_append.cc @@ -29,11 +29,11 @@ namespace iceberg { -Result> MergeAppend::Make( +Result> MergeAppend::Make( std::string table_name, std::shared_ptr ctx) { ICEBERG_PRECHECK(!table_name.empty(), "Table name cannot be empty"); ICEBERG_PRECHECK(ctx != nullptr, "Cannot create MergeAppend without a context"); - return std::unique_ptr( + return std::shared_ptr( new MergeAppend(std::move(table_name), std::move(ctx))); } @@ -41,11 +41,13 @@ MergeAppend::MergeAppend(std::string table_name, std::shared_ptr& file) { + EnsureMutable(); ICEBERG_BUILDER_RETURN_IF_ERROR(AddDataFile(file)); return *this; } MergeAppend& MergeAppend::AppendManifest(const ManifestFile& manifest) { + EnsureMutable(); ICEBERG_BUILDER_CHECK(!manifest.has_existing_files(), "Cannot append manifest with existing files"); ICEBERG_BUILDER_CHECK(!manifest.has_deleted_files(), diff --git a/src/iceberg/update/merge_append.h b/src/iceberg/update/merge_append.h index cd4f4acbb..18503b3c1 100644 --- a/src/iceberg/update/merge_append.h +++ b/src/iceberg/update/merge_append.h @@ -47,7 +47,7 @@ class ICEBERG_EXPORT MergeAppend : public MergingSnapshotUpdate { /// \param table_name The name of the table /// \param ctx The transaction context to use for this update /// \return A Result containing the MergeAppend instance or an error - static Result> Make( + static Result> Make( std::string table_name, std::shared_ptr ctx); /// \brief Append a DataFile to the table. diff --git a/src/iceberg/update/merging_snapshot_update.cc b/src/iceberg/update/merging_snapshot_update.cc index 7ce577076..54994c28f 100644 --- a/src/iceberg/update/merging_snapshot_update.cc +++ b/src/iceberg/update/merging_snapshot_update.cc @@ -51,6 +51,7 @@ #include "iceberg/table_metadata.h" #include "iceberg/table_properties.h" #include "iceberg/transaction.h" +#include "iceberg/update/update_util_internal.h" #include "iceberg/util/content_file_util.h" #include "iceberg/util/macros.h" #include "iceberg/util/snapshot_util_internal.h" @@ -476,7 +477,7 @@ Status MergingSnapshotUpdate::AddDataFile(std::shared_ptr file) { // Suppress first_row_id in the staged copy. The commit assigns row IDs for newly // added files and must not mutate the caller-owned file object. - auto staged_file = std::make_shared(*file); + ICEBERG_ASSIGN_OR_RAISE(auto staged_file, internal::CopyUpdateDataFile(*file)); staged_file->first_row_id = std::nullopt; auto& data_files = new_data_files_by_spec_[spec_id]; @@ -552,7 +553,7 @@ Status MergingSnapshotUpdate::AddDeleteFile(std::shared_ptr file, } ICEBERG_RETURN_UNEXPECTED(base().PartitionSpecById(file->partition_spec_id.value())); - auto staged_file = std::make_shared(*file); + ICEBERG_ASSIGN_OR_RAISE(auto staged_file, internal::CopyUpdateDataFile(*file)); has_new_delete_files_ = true; PendingDeleteFile pending_file{.file = std::move(staged_file), .data_sequence_number = std::move(data_sequence_number)}; @@ -572,7 +573,7 @@ Status MergingSnapshotUpdate::DeleteDataFile(std::shared_ptr file) { if (!file) { return InvalidArgument("Cannot delete a null data file"); } - auto staged_file = std::make_shared(*file); + ICEBERG_ASSIGN_OR_RAISE(auto staged_file, internal::CopyUpdateDataFile(*file)); return data_filter_manager_->DeleteFile(std::move(staged_file)); } @@ -580,7 +581,7 @@ Status MergingSnapshotUpdate::DeleteDeleteFile(std::shared_ptr file) { if (!file) { return InvalidArgument("Cannot delete a null delete file"); } - auto staged_file = std::make_shared(*file); + ICEBERG_ASSIGN_OR_RAISE(auto staged_file, internal::CopyUpdateDataFile(*file)); return delete_filter_manager_->DeleteFile(std::move(staged_file)); } @@ -589,6 +590,7 @@ Status MergingSnapshotUpdate::DeleteByPath(std::string_view path) { } Status MergingSnapshotUpdate::DeleteByRowFilter(std::shared_ptr expr) { + ICEBERG_ASSIGN_OR_RAISE(expr, internal::CopyUpdateExpression(expr)); // If a delete file matches the row filter, it can also be removed because the rows // it references will also be deleted. Both filter managers receive the expression. delete_expression_ = expr; @@ -665,9 +667,7 @@ Status MergingSnapshotUpdate::AddManifest(ManifestFile manifest) { appended_manifests_summary_.AddedManifest(manifest); append_manifests_.push_back(std::move(manifest)); } else { - ICEBERG_ASSIGN_OR_RAISE(auto copied, CopyManifest(manifest, /*update_summary=*/true)); append_manifests_to_copy_.push_back(std::move(manifest)); - rewritten_append_manifests_.push_back(std::move(copied)); } return {}; } @@ -821,9 +821,10 @@ MergingSnapshotUpdate::MergeDVs() { } ICEBERG_ASSIGN_OR_RAISE(auto location_provider, ctx_->NewLocationProvider()); - auto output_path = location_provider->NewDataLocation( - std::format("merged-dvs-{}-{}.puffin", SnapshotId(), ++dv_merge_attempt_)); + auto output_path = location_provider->NewDataLocation(std::format( + "merged-dvs-{}-{}-{}.puffin", SnapshotId(), commit_uuid(), ++dv_merge_attempt_)); + RegisterStagedFile(output_path, /*data_file=*/true); auto merged_files = DVUtil::MergeAndWriteDVs(groups, output_path, ctx_->table->io()); if (!merged_files) { std::ignore = DeleteFile(output_path); @@ -924,6 +925,11 @@ Result> MergingSnapshotUpdate::WriteNewDeleteManifests Result> MergingSnapshotUpdate::Apply( const TableMetadata& metadata_to_update, const std::shared_ptr& snapshot) { + appended_manifests_summary_.Clear(); + for (const auto& manifest : append_manifests_) { + appended_manifests_summary_.AddedManifest(manifest); + } + ICEBERG_RETURN_UNEXPECTED(ManagersReady()); // Re-validate buffered delete files against the current format version. A format @@ -989,7 +995,7 @@ Result> MergingSnapshotUpdate::Apply( if (rewritten_append_manifests_.empty() && !append_manifests_to_copy_.empty()) { for (const auto& manifest : append_manifests_to_copy_) { ICEBERG_ASSIGN_OR_RAISE(auto copied, CopyManifest(manifest, - /*update_summary=*/false)); + /*update_summary=*/true)); rewritten_append_manifests_.push_back(std::move(copied)); } } diff --git a/src/iceberg/update/overwrite_files.cc b/src/iceberg/update/overwrite_files.cc index cba6e028c..e35a725bf 100644 --- a/src/iceberg/update/overwrite_files.cc +++ b/src/iceberg/update/overwrite_files.cc @@ -34,6 +34,7 @@ #include "iceberg/table_metadata.h" #include "iceberg/transaction.h" #include "iceberg/type.h" +#include "iceberg/update/update_util_internal.h" #include "iceberg/util/error_collector.h" #include "iceberg/util/macros.h" @@ -54,6 +55,7 @@ OverwriteFiles::OverwriteFiles(std::string table_name, OverwriteFiles::~OverwriteFiles() = default; OverwriteFiles& OverwriteFiles::AddFile(const std::shared_ptr& file) { + EnsureMutable(); ICEBERG_BUILDER_CHECK(file != nullptr, "Invalid data file: null"); ICEBERG_BUILDER_CHECK(file->content == DataFile::Content::kData, "Invalid data file to add: {} has delete-file content", @@ -63,17 +65,20 @@ OverwriteFiles& OverwriteFiles::AddFile(const std::shared_ptr& file) { } OverwriteFiles& OverwriteFiles::DeleteFile(const std::shared_ptr& file) { + EnsureMutable(); ICEBERG_BUILDER_CHECK(file != nullptr, "Invalid data file: null"); ICEBERG_BUILDER_CHECK(file->content == DataFile::Content::kData, "Invalid data file to delete: {} has delete-file content", file->file_path); - deleted_data_files_.insert(file); + ICEBERG_BUILDER_ASSIGN_OR_RETURN(auto frozen, internal::CopyUpdateDataFile(*file)); + deleted_data_files_.insert(std::move(frozen)); ICEBERG_BUILDER_RETURN_IF_ERROR(DeleteDataFile(file)); return *this; } OverwriteFiles& OverwriteFiles::DeleteFiles(const DataFileSet& data_files_to_delete, const DeleteFileSet& delete_files_to_delete) { + EnsureMutable(); // Both sets use DataFile pointers, so validate content before forwarding to the // data-file and delete-file removal paths. for (const auto& file : data_files_to_delete) { @@ -81,7 +86,8 @@ OverwriteFiles& OverwriteFiles::DeleteFiles(const DataFileSet& data_files_to_del ICEBERG_BUILDER_CHECK(file->content == DataFile::Content::kData, "Invalid data file to delete: {} has delete-file content", file->file_path); - deleted_data_files_.insert(file); + ICEBERG_BUILDER_ASSIGN_OR_RETURN(auto frozen, internal::CopyUpdateDataFile(*file)); + deleted_data_files_.insert(std::move(frozen)); ICEBERG_BUILDER_RETURN_IF_ERROR(DeleteDataFile(file)); } for (const auto& file : delete_files_to_delete) { @@ -95,12 +101,14 @@ OverwriteFiles& OverwriteFiles::DeleteFiles(const DataFileSet& data_files_to_del } OverwriteFiles& OverwriteFiles::OverwriteByRowFilter(std::shared_ptr expr) { + EnsureMutable(); ICEBERG_BUILDER_CHECK(expr != nullptr, "Invalid row filter expression: null"); ICEBERG_BUILDER_RETURN_IF_ERROR(DeleteByRowFilter(std::move(expr))); return *this; } OverwriteFiles& OverwriteFiles::ValidateFromSnapshot(int64_t snapshot_id) { + EnsureMutable(); ICEBERG_BUILDER_CHECK(snapshot_id >= 0, "Invalid snapshot id: {}", snapshot_id); starting_snapshot_id_ = snapshot_id; return *this; @@ -108,29 +116,35 @@ OverwriteFiles& OverwriteFiles::ValidateFromSnapshot(int64_t snapshot_id) { OverwriteFiles& OverwriteFiles::ConflictDetectionFilter( std::shared_ptr expr) { + EnsureMutable(); ICEBERG_BUILDER_CHECK(expr != nullptr, "Invalid conflict detection filter: null"); - conflict_detection_filter_ = std::move(expr); + ICEBERG_BUILDER_ASSIGN_OR_RETURN(conflict_detection_filter_, + internal::CopyUpdateExpression(expr)); return *this; } OverwriteFiles& OverwriteFiles::CaseSensitive(bool case_sensitive) { + EnsureMutable(); MergingSnapshotUpdate::CaseSensitive(case_sensitive); return *this; } OverwriteFiles& OverwriteFiles::ValidateNoConflictingData() { + EnsureMutable(); validate_new_data_files_ = true; FailMissingDeletePaths(); return *this; } OverwriteFiles& OverwriteFiles::ValidateNoConflictingDeletes() { + EnsureMutable(); validate_new_deletes_ = true; FailMissingDeletePaths(); return *this; } OverwriteFiles& OverwriteFiles::ValidateAddedFilesMatchOverwriteFilter() { + EnsureMutable(); validate_added_files_match_overwrite_filter_ = true; return *this; } diff --git a/src/iceberg/update/pending_update.cc b/src/iceberg/update/pending_update.cc index 4b3000652..b0eb60f0a 100644 --- a/src/iceberg/update/pending_update.cc +++ b/src/iceberg/update/pending_update.cc @@ -19,49 +19,97 @@ #include "iceberg/update/pending_update.h" +#include "iceberg/exception.h" +#include "iceberg/logging/log_macros.h" #include "iceberg/result.h" #include "iceberg/table.h" #include "iceberg/transaction.h" #include "iceberg/util/macros.h" namespace iceberg { +namespace { + +template +void BestEffort(std::string_view name, Hook&& hook) noexcept { + try { + if (auto result = hook(); !result) { + ICEBERG_LOG_WARN("Update {} failed: {}", name, result.error().message); + } + } catch (const std::exception& e) { + ICEBERG_LOG_WARN("Update {} threw: {}", name, e.what()); + } catch (...) { + ICEBERG_LOG_WARN("Update {} threw an unknown exception", name); + } +} + +} // namespace PendingUpdate::PendingUpdate(std::shared_ptr ctx) : ctx_(std::move(ctx)) {} PendingUpdate::~PendingUpdate() = default; -Status PendingUpdate::Commit() { - if (!ctx_->transaction) { - // Table-created path: no transaction exists yet, create a temporary one. - ICEBERG_ASSIGN_OR_RAISE(auto txn, Transaction::Make(ctx_)); - auto apply_status = txn->Apply(*this); - if (!apply_status.has_value()) { - std::ignore = Finalize(std::unexpected(apply_status.error())); - return apply_status; - } +void PendingUpdate::EnsureMutable() const { + ICEBERG_CHECK_OR_DIE(phase_ == Phase::kMutable, + "Update configuration is frozen or terminal"); + ICEBERG_CHECK_OR_DIE(!ctx_->in_progress_, + "Cannot mutate an update during an operation"); + if (ctx_->transaction) { + auto txn = ctx_->transaction->lock(); + ICEBERG_CHECK_OR_DIE(txn != nullptr, "Transaction has been destroyed"); + ICEBERG_CHECK_OR_DIE(txn->state() == TransactionState::kReady || + txn->state() == TransactionState::kUpdatePending, + "Transaction is terminal"); + } +} - auto commit_result = txn->Commit(); - if (!commit_result.has_value()) { - std::ignore = Finalize(std::unexpected(commit_result.error())); - return std::unexpected(commit_result.error()); - } +Status PendingUpdate::CheckCommitAllowed() const { + ICEBERG_CHECK(phase_ != Phase::kTerminal, "Update is terminal"); + ICEBERG_CHECK(!ctx_->in_progress_, "Cannot reenter an update or transaction operation"); + return {}; +} - std::ignore = Finalize(commit_result.value()->metadata().get()); - return {}; - } - auto txn = ctx_->transaction->lock(); - if (!txn) { - return CommitFailed("Transaction has been destroyed"); +Status PendingUpdate::Commit() { + ICEBERG_RETURN_UNEXPECTED(CheckCommitAllowed()); + if (ctx_->transaction) { + auto txn = ctx_->transaction->lock(); + ICEBERG_CHECK(txn != nullptr, "Transaction has been destroyed"); + return txn->Apply(*this); } - return txn->Apply(*this); + + auto self = weak_from_this().lock(); + ICEBERG_PRECHECK(self != nullptr, "PendingUpdate must be owned by std::shared_ptr"); + ICEBERG_ASSIGN_OR_RAISE(auto txn, Transaction::Make(ctx_)); + ICEBERG_RETURN_UNEXPECTED(txn->AddUpdate(self)); + ICEBERG_RETURN_UNEXPECTED(txn->Apply(*this)); + ICEBERG_RETURN_UNEXPECTED(txn->Commit()); + return {}; } -Status PendingUpdate::Finalize( - [[maybe_unused]] Result commit_result) { +Status PendingUpdate::Finalize([[maybe_unused]] const TableMetadata& committed) { return {}; } +void PendingUpdate::Cleanup() noexcept { + if (!staged_) { + return; + } + // Consume the generation before any callback can throw or reenter. + staged_ = false; + BestEffort("staging cleanup", [this] { return CleanStaged(); }); +} + +void PendingUpdate::FinalizeOnce(const TableMetadata& committed) noexcept { + // A generation already consumed by Cleanup (a staged snapshot that produced no + // metadata change) has nothing to finalize or report. + if (!staged_) { + return; + } + staged_ = false; + BestEffort("finalization", [this, &committed] { return Finalize(committed); }); + BestEffort("reporting", [this] { return ReportCommitted(); }); +} + const TableMetadata& PendingUpdate::base() const { return ctx_->current(); } } // namespace iceberg diff --git a/src/iceberg/update/pending_update.h b/src/iceberg/update/pending_update.h index 19998ddb3..becc33625 100644 --- a/src/iceberg/update/pending_update.h +++ b/src/iceberg/update/pending_update.h @@ -31,15 +31,34 @@ namespace iceberg { +enum class TransactionState : uint8_t; + /// \brief Base class for all kinds of table metadata updates. /// /// Any created `PendingUpdate` instance is tracked by the `Transaction` instance /// and commit is also delegated to the `Transaction` instance. /// +/// Lifecycle: an update is configured through its fluent mutators, then committed +/// exactly once. `Commit()` freezes the configuration before the first Apply so a +/// commit retry replays the same intent against refreshed metadata. After that point +/// the update is either frozen (explicit transaction, awaiting `Transaction::Commit()`) +/// or terminal (standalone commit finished, or the owning transaction reached a +/// terminal state). +/// /// \note Implementations are expected to use builder pattern and errors -/// should be handled by the ErrorCollector base class. -class ICEBERG_EXPORT PendingUpdate : public ErrorCollector { +/// should be handled by the ErrorCollector base class. Configuration errors are +/// collected and surfaced by `Commit()`. Lifecycle misuse is different: calling a +/// mutator on a frozen or terminal update, or while an operation is in progress, +/// throws `IcebergError` rather than adding a collected error, so the frozen replay +/// intent can never be poisoned. +class ICEBERG_EXPORT PendingUpdate : protected ErrorCollector, + public std::enable_shared_from_this { public: + using ErrorCollector::CheckErrors; + using ErrorCollector::error_count; + using ErrorCollector::errors; + using ErrorCollector::has_errors; + enum class Kind : uint8_t { kExpireSnapshots, kSetSnapshot, @@ -66,23 +85,15 @@ class ICEBERG_EXPORT PendingUpdate : public ErrorCollector { /// - ValidationFailed: if it cannot be applied to the current table metadata. /// - CommitFailed: if it cannot be committed due to conflicts. /// - CommitStateUnknown: unknown status, no cleanup should be done. + /// \note The update must be owned by a `std::shared_ptr` before calling Commit(). + /// Commit freezes its configuration. Later mutators throw IcebergError; an Apply + /// failure is terminal and cannot be corrected within the same transaction. virtual Status Commit(); - /// \brief Finalize the pending update. - /// - /// This method is called after the update is committed. - /// Implementations should override this method to clean up any resources. - /// - /// \param commit_result The committed table metadata when the commit succeeds, or the - /// commit error when it fails. - /// \return Status indicating success or failure - virtual Status Finalize(Result commit_result); - - // Non-copyable, movable PendingUpdate(const PendingUpdate&) = delete; PendingUpdate& operator=(const PendingUpdate&) = delete; - PendingUpdate(PendingUpdate&&) noexcept = default; - PendingUpdate& operator=(PendingUpdate&&) noexcept = default; + PendingUpdate(PendingUpdate&&) = delete; + PendingUpdate& operator=(PendingUpdate&&) = delete; ~PendingUpdate() override; @@ -91,7 +102,50 @@ class ICEBERG_EXPORT PendingUpdate : public ErrorCollector { const TableMetadata& base() const; + /// \brief Reject lifecycle misuse without poisoning the builder's collected errors. + /// Fluent mutators throw IcebergError when frozen, terminal, or reentered. + void EnsureMutable() const; + Status CheckCommitAllowed() const; + + /// \brief Capture mutable input values before the first Apply. + virtual Status Freeze() { return {}; } + /// \brief Discard this generation's staging, retaining frozen operation intent. + virtual Status CleanStaged() { return {}; } + /// \brief Complete a known successful commit. Called only by Transaction with the + /// final committed table metadata. + virtual Status Finalize(const TableMetadata& committed); + virtual Status ReportCommitted() { return {}; } + virtual bool MayAddFileReferences() const { return true; } + + using ErrorCollector::AddError; + using ErrorCollector::ClearErrors; + std::shared_ptr ctx_; + + private: + friend class Transaction; + // ErrorCollector's explicit-object helpers access the derived builder's errors. + friend class ErrorCollector; + + void Cleanup() noexcept; + void FinalizeOnce(const TableMetadata& committed) noexcept; + + /// \brief Lifecycle of this update, advanced only by Transaction. + /// + /// kMutable: configurable, not yet applied. + /// kFrozen: public configuration is closed. Entered before Freeze and the first + /// Apply; reaching this phase does not imply either step has succeeded. + /// kTerminal: the owning transaction reached a terminal state (an update that was + /// never applied can also become terminal when its transaction is aborted). + enum class Phase : uint8_t { kMutable, kFrozen, kTerminal }; + Phase phase_ = Phase::kMutable; + + // True while the current Apply generation has not been consumed by Cleanup or + // FinalizeOnce. Set before Freeze/Apply, even before any output exists, so a + // failure can clean partially staged resources. Cleared before cleanup or + // finalization hooks run to make them idempotent per generation and to skip + // finalization/reporting for a snapshot generation already cleaned as a no-op. + bool staged_ = false; }; } // namespace iceberg diff --git a/src/iceberg/update/replace_partitions.cc b/src/iceberg/update/replace_partitions.cc index 3d224375c..22dc2909e 100644 --- a/src/iceberg/update/replace_partitions.cc +++ b/src/iceberg/update/replace_partitions.cc @@ -32,11 +32,11 @@ namespace iceberg { -Result> ReplacePartitions::Make( +Result> ReplacePartitions::Make( std::string table_name, std::shared_ptr ctx) { ICEBERG_PRECHECK(!table_name.empty(), "Table name cannot be empty"); ICEBERG_PRECHECK(ctx != nullptr, "Cannot create ReplacePartitions without a context"); - return std::unique_ptr( + return std::shared_ptr( new ReplacePartitions(std::move(table_name), std::move(ctx))); } @@ -47,6 +47,7 @@ ReplacePartitions::ReplacePartitions(std::string table_name, } ReplacePartitions& ReplacePartitions::AddFile(const std::shared_ptr& file) { + EnsureMutable(); ICEBERG_BUILDER_CHECK(file != nullptr, "Invalid data file: null"); ICEBERG_BUILDER_CHECK(file->partition_spec_id.has_value(), "Data file must have partition spec ID"); @@ -59,21 +60,25 @@ ReplacePartitions& ReplacePartitions::AddFile(const std::shared_ptr& f } ReplacePartitions& ReplacePartitions::ValidateAppendOnly() { + EnsureMutable(); FailAnyDelete(); return *this; } ReplacePartitions& ReplacePartitions::ValidateFromSnapshot(int64_t snapshot_id) { + EnsureMutable(); starting_snapshot_id_ = snapshot_id; return *this; } ReplacePartitions& ReplacePartitions::ValidateNoConflictingData() { + EnsureMutable(); validate_conflicting_data_ = true; return *this; } ReplacePartitions& ReplacePartitions::ValidateNoConflictingDeletes() { + EnsureMutable(); validate_conflicting_deletes_ = true; return *this; } diff --git a/src/iceberg/update/replace_partitions.h b/src/iceberg/update/replace_partitions.h index 40048ba5c..ee57b6124 100644 --- a/src/iceberg/update/replace_partitions.h +++ b/src/iceberg/update/replace_partitions.h @@ -62,7 +62,7 @@ class ICEBERG_EXPORT ReplacePartitions : public MergingSnapshotUpdate { /// \param table_name The name of the table /// \param ctx The transaction context /// \return A Result containing the ReplacePartitions instance or an error - static Result> Make( + static Result> Make( std::string table_name, std::shared_ptr ctx); /// \brief Add a data file to the table. diff --git a/src/iceberg/update/rewrite_files.cc b/src/iceberg/update/rewrite_files.cc index b7fc048d3..a3c9a2395 100644 --- a/src/iceberg/update/rewrite_files.cc +++ b/src/iceberg/update/rewrite_files.cc @@ -28,6 +28,7 @@ #include "iceberg/snapshot.h" #include "iceberg/table.h" #include "iceberg/transaction.h" +#include "iceberg/update/update_util_internal.h" #include "iceberg/util/macros.h" namespace iceberg { @@ -39,26 +40,29 @@ RewriteFiles::RewriteFiles(std::string table_name, FailMissingDeletePaths(); } -Result> RewriteFiles::Make( +Result> RewriteFiles::Make( std::string table_name, std::shared_ptr ctx) { ICEBERG_PRECHECK(!table_name.empty(), "Table name cannot be empty"); ICEBERG_PRECHECK(ctx != nullptr, "Cannot create RewriteFiles without a context"); - return std::unique_ptr( + return std::shared_ptr( new RewriteFiles(std::move(table_name), std::move(ctx))); } RewriteFiles& RewriteFiles::DeleteDataFile(const std::shared_ptr& data_file) { + EnsureMutable(); ICEBERG_BUILDER_CHECK(data_file != nullptr, "Invalid data file: null"); ICEBERG_BUILDER_CHECK(data_file->content == DataFile::Content::kData, "Invalid data file to delete: {} has delete-file content", data_file->file_path); ICEBERG_BUILDER_RETURN_IF_ERROR(MergingSnapshotUpdate::DeleteDataFile(data_file)); - replaced_data_files_.insert(std::make_shared(*data_file)); + ICEBERG_BUILDER_ASSIGN_OR_RETURN(auto frozen, internal::CopyUpdateDataFile(*data_file)); + replaced_data_files_.insert(std::move(frozen)); return *this; } RewriteFiles& RewriteFiles::DeleteDeleteFile( const std::shared_ptr& delete_file) { + EnsureMutable(); ICEBERG_BUILDER_CHECK(delete_file != nullptr, "Invalid delete file: null"); ICEBERG_BUILDER_CHECK(delete_file->content != DataFile::Content::kData, "Invalid delete file to delete: {} has data-file content", @@ -68,6 +72,7 @@ RewriteFiles& RewriteFiles::DeleteDeleteFile( } RewriteFiles& RewriteFiles::AddDataFile(const std::shared_ptr& file) { + EnsureMutable(); ICEBERG_BUILDER_CHECK(file != nullptr, "Invalid data file: null"); ICEBERG_BUILDER_CHECK(file->content == DataFile::Content::kData, "Invalid data file to add: {} has delete-file content", @@ -77,6 +82,7 @@ RewriteFiles& RewriteFiles::AddDataFile(const std::shared_ptr& file) { } RewriteFiles& RewriteFiles::AddDeleteFile(const std::shared_ptr& delete_file) { + EnsureMutable(); ICEBERG_BUILDER_CHECK(delete_file != nullptr, "Invalid delete file: null"); ICEBERG_BUILDER_CHECK(delete_file->content != DataFile::Content::kData, "Invalid delete file to add: {} has data-file content", @@ -87,6 +93,7 @@ RewriteFiles& RewriteFiles::AddDeleteFile(const std::shared_ptr& delet RewriteFiles& RewriteFiles::AddDeleteFile(const std::shared_ptr& delete_file, int64_t data_sequence_number) { + EnsureMutable(); ICEBERG_BUILDER_CHECK(delete_file != nullptr, "Invalid delete file: null"); ICEBERG_BUILDER_CHECK(delete_file->content != DataFile::Content::kData, "Invalid delete file to add: {} has data-file content", @@ -97,6 +104,7 @@ RewriteFiles& RewriteFiles::AddDeleteFile(const std::shared_ptr& delet } RewriteFiles& RewriteFiles::SetDataSequenceNumber(int64_t sequence_number) { + EnsureMutable(); SetNewDataFilesDataSequenceNumber(sequence_number); return *this; } @@ -104,6 +112,7 @@ RewriteFiles& RewriteFiles::SetDataSequenceNumber(int64_t sequence_number) { RewriteFiles& RewriteFiles::RewriteDataFiles( const std::vector>& files_to_delete, const std::vector>& files_to_add, int64_t sequence_number) { + EnsureMutable(); SetNewDataFilesDataSequenceNumber(sequence_number); Rewrite(files_to_delete, {}, files_to_add, {}); return *this; @@ -114,6 +123,7 @@ RewriteFiles& RewriteFiles::Rewrite( const std::vector>& delete_files_to_replace, const std::vector>& data_files_to_add, const std::vector>& delete_files_to_add) { + EnsureMutable(); for (const auto& data_file : data_files_to_replace) { DeleteDataFile(data_file); } @@ -134,6 +144,7 @@ RewriteFiles& RewriteFiles::Rewrite( } RewriteFiles& RewriteFiles::ValidateFromSnapshot(int64_t snapshot_id) { + EnsureMutable(); starting_snapshot_id_ = snapshot_id; return *this; } diff --git a/src/iceberg/update/rewrite_files.h b/src/iceberg/update/rewrite_files.h index ce219c3ae..283091c13 100644 --- a/src/iceberg/update/rewrite_files.h +++ b/src/iceberg/update/rewrite_files.h @@ -54,8 +54,8 @@ class ICEBERG_EXPORT RewriteFiles : public MergingSnapshotUpdate { /// /// \param table_name The name of the table /// \param ctx The transaction context - /// \return A unique pointer to the new RewriteFiles operation - static Result> Make( + /// \return A shared pointer to the new RewriteFiles operation + static Result> Make( std::string table_name, std::shared_ptr ctx); ~RewriteFiles() override = default; diff --git a/src/iceberg/update/row_delta.cc b/src/iceberg/update/row_delta.cc index dd3f50c58..64fe583e9 100644 --- a/src/iceberg/update/row_delta.cc +++ b/src/iceberg/update/row_delta.cc @@ -31,6 +31,7 @@ #include "iceberg/table.h" #include "iceberg/table_metadata.h" #include "iceberg/transaction.h" +#include "iceberg/update/update_util_internal.h" #include "iceberg/util/error_collector.h" #include "iceberg/util/formatter_internal.h" #include "iceberg/util/macros.h" @@ -38,11 +39,11 @@ namespace iceberg { -Result> RowDelta::Make( +Result> RowDelta::Make( std::string table_name, std::shared_ptr ctx) { ICEBERG_PRECHECK(!table_name.empty(), "Table name cannot be empty"); ICEBERG_PRECHECK(ctx != nullptr, "Cannot create RowDelta without a context"); - return std::unique_ptr(new RowDelta(std::move(table_name), std::move(ctx))); + return std::shared_ptr(new RowDelta(std::move(table_name), std::move(ctx))); } RowDelta::RowDelta(std::string table_name, std::shared_ptr ctx) @@ -50,38 +51,46 @@ RowDelta::RowDelta(std::string table_name, std::shared_ptr c conflict_detection_filter_(Expressions::AlwaysTrue()) {} RowDelta& RowDelta::AddRows(const std::shared_ptr& inserts) { + EnsureMutable(); ICEBERG_BUILDER_RETURN_IF_ERROR(AddDataFile(inserts)); return *this; } RowDelta& RowDelta::AddDeletes(const std::shared_ptr& deletes) { + EnsureMutable(); ICEBERG_BUILDER_RETURN_IF_ERROR(AddDeleteFile(deletes)); return *this; } RowDelta& RowDelta::RemoveRows(const std::shared_ptr& file) { + EnsureMutable(); ICEBERG_BUILDER_RETURN_IF_ERROR(DeleteDataFile(file)); - removed_data_files_.insert(file); + ICEBERG_BUILDER_ASSIGN_OR_RETURN(auto frozen, internal::CopyUpdateDataFile(*file)); + removed_data_files_.insert(std::move(frozen)); return *this; } RowDelta& RowDelta::RemoveDeletes(const std::shared_ptr& deletes) { + EnsureMutable(); ICEBERG_BUILDER_RETURN_IF_ERROR(DeleteDeleteFile(deletes)); return *this; } RowDelta& RowDelta::ValidateFromSnapshot(int64_t snapshot_id) { + EnsureMutable(); starting_snapshot_id_ = snapshot_id; return *this; } RowDelta& RowDelta::CaseSensitive(bool case_sensitive) { + EnsureMutable(); MergingSnapshotUpdate::CaseSensitive(case_sensitive); return *this; } RowDelta& RowDelta::ValidateDataFilesExist( std::span referenced_files) { + EnsureMutable(); for (const auto& file : referenced_files) { referenced_data_files_.insert(file); } @@ -89,22 +98,27 @@ RowDelta& RowDelta::ValidateDataFilesExist( } RowDelta& RowDelta::ValidateDeletedFiles() { + EnsureMutable(); validate_deletes_ = true; return *this; } RowDelta& RowDelta::ConflictDetectionFilter(std::shared_ptr filter) { + EnsureMutable(); ICEBERG_BUILDER_CHECK(filter != nullptr, "Conflict detection filter cannot be null"); - conflict_detection_filter_ = std::move(filter); + ICEBERG_BUILDER_ASSIGN_OR_RETURN(conflict_detection_filter_, + internal::CopyUpdateExpression(filter)); return *this; } RowDelta& RowDelta::ValidateNoConflictingDataFiles() { + EnsureMutable(); validate_new_data_files_ = true; return *this; } RowDelta& RowDelta::ValidateNoConflictingDeleteFiles() { + EnsureMutable(); validate_new_delete_files_ = true; return *this; } diff --git a/src/iceberg/update/row_delta.h b/src/iceberg/update/row_delta.h index bf58a048c..f6faf8c92 100644 --- a/src/iceberg/update/row_delta.h +++ b/src/iceberg/update/row_delta.h @@ -47,7 +47,7 @@ namespace iceberg { class ICEBERG_EXPORT RowDelta : public MergingSnapshotUpdate { public: /// \brief Create a new RowDelta instance. - static Result> Make(std::string table_name, + static Result> Make(std::string table_name, std::shared_ptr ctx); /// \brief Add a DataFile to the table. diff --git a/src/iceberg/update/set_snapshot.cc b/src/iceberg/update/set_snapshot.cc index 79662890b..966842850 100644 --- a/src/iceberg/update/set_snapshot.cc +++ b/src/iceberg/update/set_snapshot.cc @@ -45,6 +45,7 @@ SetSnapshot::SetSnapshot(std::shared_ptr ctx) SetSnapshot::~SetSnapshot() = default; SetSnapshot& SetSnapshot::SetCurrentSnapshot(int64_t snapshot_id) { + EnsureMutable(); // Validate that the snapshot exists ICEBERG_BUILDER_ASSIGN_OR_RETURN(auto snapshot, base().SnapshotById(snapshot_id)); ICEBERG_BUILDER_CHECK(snapshot != nullptr, @@ -54,6 +55,7 @@ SetSnapshot& SetSnapshot::SetCurrentSnapshot(int64_t snapshot_id) { } SetSnapshot& SetSnapshot::RollbackToTime(int64_t timestamp_ms) { + EnsureMutable(); // Find the latest snapshot by timestamp older than timestamp_ms ICEBERG_BUILDER_ASSIGN_OR_RETURN(auto snapshot_id_opt, FindLatestAncestorOlderThan(timestamp_ms)); @@ -69,6 +71,7 @@ SetSnapshot& SetSnapshot::RollbackToTime(int64_t timestamp_ms) { } SetSnapshot& SetSnapshot::RollbackTo(int64_t snapshot_id) { + EnsureMutable(); // Validate that the snapshot exists auto snapshot_result = base().SnapshotById(snapshot_id); ICEBERG_BUILDER_CHECK(snapshot_result.has_value(), @@ -85,7 +88,7 @@ SetSnapshot& SetSnapshot::RollbackTo(int64_t snapshot_id) { return SetCurrentSnapshot(snapshot_id); } -Result SetSnapshot::Apply() { +Result SetSnapshot::Validate() const { ICEBERG_RETURN_UNEXPECTED(CheckErrors()); const TableMetadata& base_metadata = ctx_->current(); diff --git a/src/iceberg/update/set_snapshot.h b/src/iceberg/update/set_snapshot.h index 431e636b2..9dcf33433 100644 --- a/src/iceberg/update/set_snapshot.h +++ b/src/iceberg/update/set_snapshot.h @@ -53,8 +53,8 @@ class ICEBERG_EXPORT SetSnapshot : public PendingUpdate { Kind kind() const final { return Kind::kSetSnapshot; } bool IsRetryable() const override { return true; } - /// \brief Apply the pending changes and return the target snapshot ID. - Result Apply(); + /// \brief Validate and preview changes without staging or modifying this update. + Result Validate() const; private: explicit SetSnapshot(std::shared_ptr ctx); diff --git a/src/iceberg/update/snapshot_manager.cc b/src/iceberg/update/snapshot_manager.cc index 5473f3033..81bee7161 100644 --- a/src/iceberg/update/snapshot_manager.cc +++ b/src/iceberg/update/snapshot_manager.cc @@ -51,9 +51,16 @@ SnapshotManager::SnapshotManager(std::shared_ptr transaction, : transaction_(std::move(transaction)), is_external_transaction_(is_external_transaction) {} +void SnapshotManager::EnsureMutable() const { + const auto status = transaction_->CheckActive(); + ICEBERG_CHECK_OR_DIE(status.has_value(), "Cannot mutate snapshot manager: {}", + status.error().message); +} + SnapshotManager::~SnapshotManager() = default; SnapshotManager& SnapshotManager::Cherrypick(int64_t snapshot_id) { + EnsureMutable(); ICEBERG_BUILDER_RETURN_IF_ERROR(CommitIfRefUpdatesExist()); // TODO(anyone): Implement cherrypick operation ICEBERG_BUILDER_CHECK(false, "Cherrypick operation not yet implemented"); @@ -61,6 +68,7 @@ SnapshotManager& SnapshotManager::Cherrypick(int64_t snapshot_id) { } SnapshotManager& SnapshotManager::SetCurrentSnapshot(int64_t snapshot_id) { + EnsureMutable(); ICEBERG_BUILDER_RETURN_IF_ERROR(CommitIfRefUpdatesExist()); ICEBERG_BUILDER_ASSIGN_OR_RETURN(auto set_snapshot, transaction_->NewSetSnapshot()); set_snapshot->SetCurrentSnapshot(snapshot_id); @@ -69,6 +77,7 @@ SnapshotManager& SnapshotManager::SetCurrentSnapshot(int64_t snapshot_id) { } SnapshotManager& SnapshotManager::RollbackToTime(int64_t timestamp_ms) { + EnsureMutable(); ICEBERG_BUILDER_RETURN_IF_ERROR(CommitIfRefUpdatesExist()); ICEBERG_BUILDER_ASSIGN_OR_RETURN(auto set_snapshot, transaction_->NewSetSnapshot()); set_snapshot->RollbackToTime(timestamp_ms); @@ -77,6 +86,7 @@ SnapshotManager& SnapshotManager::RollbackToTime(int64_t timestamp_ms) { } SnapshotManager& SnapshotManager::RollbackTo(int64_t snapshot_id) { + EnsureMutable(); ICEBERG_BUILDER_RETURN_IF_ERROR(CommitIfRefUpdatesExist()); ICEBERG_BUILDER_ASSIGN_OR_RETURN(auto set_snapshot, transaction_->NewSetSnapshot()); set_snapshot->RollbackTo(snapshot_id); @@ -85,6 +95,7 @@ SnapshotManager& SnapshotManager::RollbackTo(int64_t snapshot_id) { } SnapshotManager& SnapshotManager::CreateBranch(const std::string& name) { + EnsureMutable(); const auto& base = transaction_->current(); if (base.current_snapshot_id != kInvalidSnapshotId) { ICEBERG_BUILDER_ASSIGN_OR_RETURN(auto current_snapshot, base.Snapshot()); @@ -99,6 +110,7 @@ SnapshotManager& SnapshotManager::CreateBranch(const std::string& name) { SnapshotManager& SnapshotManager::CreateBranch(const std::string& name, int64_t snapshot_id) { + EnsureMutable(); ICEBERG_BUILDER_ASSIGN_OR_RETURN(auto update_ref, UpdateSnapshotReferencesOperation()); update_ref->CreateBranch(name, snapshot_id); return *this; @@ -106,18 +118,21 @@ SnapshotManager& SnapshotManager::CreateBranch(const std::string& name, SnapshotManager& SnapshotManager::CreateTag(const std::string& name, int64_t snapshot_id) { + EnsureMutable(); ICEBERG_BUILDER_ASSIGN_OR_RETURN(auto update_ref, UpdateSnapshotReferencesOperation()); update_ref->CreateTag(name, snapshot_id); return *this; } SnapshotManager& SnapshotManager::RemoveBranch(const std::string& name) { + EnsureMutable(); ICEBERG_BUILDER_ASSIGN_OR_RETURN(auto update_ref, UpdateSnapshotReferencesOperation()); update_ref->RemoveBranch(name); return *this; } SnapshotManager& SnapshotManager::RemoveTag(const std::string& name) { + EnsureMutable(); ICEBERG_BUILDER_ASSIGN_OR_RETURN(auto update_ref, UpdateSnapshotReferencesOperation()); update_ref->RemoveTag(name); return *this; @@ -125,6 +140,7 @@ SnapshotManager& SnapshotManager::RemoveTag(const std::string& name) { SnapshotManager& SnapshotManager::ReplaceTag(const std::string& name, int64_t snapshot_id) { + EnsureMutable(); ICEBERG_BUILDER_ASSIGN_OR_RETURN(auto update_ref, UpdateSnapshotReferencesOperation()); update_ref->ReplaceTag(name, snapshot_id); return *this; @@ -132,6 +148,7 @@ SnapshotManager& SnapshotManager::ReplaceTag(const std::string& name, SnapshotManager& SnapshotManager::ReplaceBranch(const std::string& name, int64_t snapshot_id) { + EnsureMutable(); ICEBERG_BUILDER_ASSIGN_OR_RETURN(auto update_ref, UpdateSnapshotReferencesOperation()); update_ref->ReplaceBranch(name, snapshot_id); return *this; @@ -139,6 +156,7 @@ SnapshotManager& SnapshotManager::ReplaceBranch(const std::string& name, SnapshotManager& SnapshotManager::ReplaceBranch(const std::string& from, const std::string& to) { + EnsureMutable(); ICEBERG_BUILDER_ASSIGN_OR_RETURN(auto update_ref, UpdateSnapshotReferencesOperation()); update_ref->ReplaceBranch(from, to); return *this; @@ -146,6 +164,7 @@ SnapshotManager& SnapshotManager::ReplaceBranch(const std::string& from, SnapshotManager& SnapshotManager::FastForwardBranch(const std::string& from, const std::string& to) { + EnsureMutable(); ICEBERG_BUILDER_ASSIGN_OR_RETURN(auto update_ref, UpdateSnapshotReferencesOperation()); update_ref->FastForward(from, to); return *this; @@ -153,6 +172,7 @@ SnapshotManager& SnapshotManager::FastForwardBranch(const std::string& from, SnapshotManager& SnapshotManager::RenameBranch(const std::string& name, const std::string& new_name) { + EnsureMutable(); ICEBERG_BUILDER_ASSIGN_OR_RETURN(auto update_ref, UpdateSnapshotReferencesOperation()); update_ref->RenameBranch(name, new_name); return *this; @@ -160,6 +180,7 @@ SnapshotManager& SnapshotManager::RenameBranch(const std::string& name, SnapshotManager& SnapshotManager::SetMinSnapshotsToKeep(const std::string& branch_name, int32_t min_snapshots_to_keep) { + EnsureMutable(); ICEBERG_BUILDER_ASSIGN_OR_RETURN(auto update_ref, UpdateSnapshotReferencesOperation()); update_ref->SetMinSnapshotsToKeep(branch_name, min_snapshots_to_keep); return *this; @@ -167,6 +188,7 @@ SnapshotManager& SnapshotManager::SetMinSnapshotsToKeep(const std::string& branc SnapshotManager& SnapshotManager::SetMaxSnapshotAgeMs(const std::string& branch_name, int64_t max_snapshot_age_ms) { + EnsureMutable(); ICEBERG_BUILDER_ASSIGN_OR_RETURN(auto update_ref, UpdateSnapshotReferencesOperation()); update_ref->SetMaxSnapshotAgeMs(branch_name, max_snapshot_age_ms); return *this; @@ -174,6 +196,7 @@ SnapshotManager& SnapshotManager::SetMaxSnapshotAgeMs(const std::string& branch_ SnapshotManager& SnapshotManager::SetMaxRefAgeMs(const std::string& name, int64_t max_ref_age_ms) { + EnsureMutable(); ICEBERG_BUILDER_ASSIGN_OR_RETURN(auto update_ref, UpdateSnapshotReferencesOperation()); update_ref->SetMaxRefAgeMs(name, max_ref_age_ms); return *this; @@ -181,6 +204,7 @@ SnapshotManager& SnapshotManager::SetMaxRefAgeMs(const std::string& name, Status SnapshotManager::Commit() { ICEBERG_RETURN_UNEXPECTED(CheckErrors()); + ICEBERG_RETURN_UNEXPECTED(transaction_->CheckActive()); ICEBERG_RETURN_UNEXPECTED(CommitIfRefUpdatesExist()); if (!is_external_transaction_) { ICEBERG_RETURN_UNEXPECTED(transaction_->Commit()); diff --git a/src/iceberg/update/snapshot_manager.h b/src/iceberg/update/snapshot_manager.h index fd81f8339..9e7509afa 100644 --- a/src/iceberg/update/snapshot_manager.h +++ b/src/iceberg/update/snapshot_manager.h @@ -187,6 +187,7 @@ class ICEBERG_EXPORT SnapshotManager : public ErrorCollector { Status Commit(); private: + void EnsureMutable() const; SnapshotManager(std::shared_ptr transaction, bool is_external_transaction); /// \brief Get or create the UpdateSnapshotReference operation. diff --git a/src/iceberg/update/snapshot_update.cc b/src/iceberg/update/snapshot_update.cc index 6e8d28bd5..0095d843c 100644 --- a/src/iceberg/update/snapshot_update.cc +++ b/src/iceberg/update/snapshot_update.cc @@ -26,6 +26,7 @@ #include "iceberg/constants.h" #include "iceberg/file_io.h" +#include "iceberg/logging/log_macros.h" #include "iceberg/manifest/manifest_entry.h" #include "iceberg/manifest/manifest_list.h" #include "iceberg/manifest/manifest_reader.h" @@ -202,17 +203,17 @@ SnapshotUpdate::SnapshotUpdate(std::shared_ptr ctx) reporter_(ctx_->table->reporter()) {} Status SnapshotUpdate::Commit() { - commit_metrics_->attempts->Increment(); + ICEBERG_RETURN_UNEXPECTED(CheckCommitAllowed()); [[maybe_unused]] auto commit_timer = commit_metrics_->total_duration->Start(); return PendingUpdate::Commit(); } -void SnapshotUpdate::ReportCommit() const { +Status SnapshotUpdate::ReportCommitted() { ICEBERG_DCHECK(staged_snapshot_ != nullptr, "Staged snapshot is null after a successful commit"); if (!reporter_) { - return; + return {}; } const auto operation = staged_snapshot_->Operation(); @@ -225,7 +226,7 @@ void SnapshotUpdate::ReportCommit() const { CommitMetricsResult::From(*commit_metrics_, staged_snapshot_->summary), .metadata = {}, }; - std::ignore = reporter_->Report(report); + return reporter_->Report(report); } void SnapshotUpdate::SetSummaryProperty(const std::string& property, @@ -302,17 +303,11 @@ int64_t SnapshotUpdate::SnapshotId() { } Result SnapshotUpdate::Apply() { + commit_metrics_->attempts->Increment(); ICEBERG_RETURN_UNEXPECTED(CheckErrors()); - - if (staged_snapshot_ != nullptr) { - for (const auto& manifest_list : manifest_lists_) { - std::ignore = DeleteFile(manifest_list); - } - manifest_lists_.clear(); - ICEBERG_RETURN_UNEXPECTED(CleanUncommitted(std::unordered_set{})); - - staged_snapshot_ = nullptr; - summary_.Clear(); + { + std::lock_guard lock(staging_mutex_); + attempted_deletes_.clear(); } ICEBERG_ASSIGN_OR_RAISE(auto parent_snapshot, @@ -338,7 +333,6 @@ Result SnapshotUpdate::Apply() { ICEBERG_RETURN_UNEXPECTED(std::move(metadata_tasks).Run()); std::string manifest_list_path = ManifestListPath(); - manifest_lists_.push_back(manifest_list_path); ICEBERG_ASSIGN_OR_RAISE( auto writer, ManifestListWriter::MakeWriter(base().format_version, SnapshotId(), parent_snapshot_id, manifest_list_path, @@ -388,38 +382,42 @@ Result SnapshotUpdate::Apply() { .stage_only = stage_only_}; } -Status SnapshotUpdate::Finalize(Result commit_result) { - if (!commit_result.has_value()) { - if (commit_result.error().kind == ErrorKind::kCommitStateUnknown) { - return {}; +Status SnapshotUpdate::Finalize([[maybe_unused]] const TableMetadata& metadata) { + ICEBERG_CHECK(staged_snapshot_ != nullptr, "Missing staged snapshot after commit"); + auto cached_snapshot = SnapshotCache(staged_snapshot_.get()); + ICEBERG_ASSIGN_OR_RAISE(auto manifests, cached_snapshot.Manifests(ctx_->table->io())); + auto committed = + manifests | + std::views::transform([](const auto& manifest) { return manifest.manifest_path; }) | + std::ranges::to>(); + // Let derived updates release their caches and clean operation-specific files. + try { + if (auto status = CleanUncommitted(committed); !status) { + ICEBERG_LOG_WARN("Snapshot cleanup failed: {}", status.error().message); } - std::ignore = CleanAll(); - return {}; + } catch (const std::exception& e) { + ICEBERG_LOG_WARN("Snapshot cleanup threw: {}", e.what()); + } catch (...) { + ICEBERG_LOG_WARN("Snapshot cleanup threw an unknown exception"); } - - if (CleanupAfterCommit()) { - ICEBERG_CHECK(staged_snapshot_ != nullptr, - "Staged snapshot is null during finalize after commit"); - auto cached_snapshot = SnapshotCache(staged_snapshot_.get()); - if (auto manifests = cached_snapshot.Manifests(ctx_->table->io()); - manifests.has_value()) { - std::ignore = CleanUncommitted(manifests.value() | - std::views::transform([](const auto& manifest) { - return manifest.manifest_path; - }) | - std::ranges::to>()); + committed.insert(staged_snapshot_->manifest_list); + std::vector unused; + { + std::lock_guard lock(staging_mutex_); + for (const auto& path : staged_files_) { + if (!committed.contains(path) && !staged_data_files_.contains(path)) { + unused.push_back(path); + } } } - - // Also clean up unused manifest lists created by multiple attempts - for (const auto& manifest_list : manifest_lists_) { - if (manifest_list != staged_snapshot_->manifest_list) { - std::ignore = DeleteFile(manifest_list); - } + for (const auto& path : unused) { + std::ignore = DeleteFile(path); + } + { + std::lock_guard lock(staging_mutex_); + staged_files_.clear(); + staged_data_files_.clear(); } - - ReportCommit(); - return {}; } @@ -473,20 +471,59 @@ Result> SnapshotUpdate::ComputeSumm return summary; } -Status SnapshotUpdate::CleanAll() { - for (const auto& manifest_list : manifest_lists_) { - std::ignore = DeleteFile(manifest_list); +Status SnapshotUpdate::CleanStaged() { + try { + if (auto status = CleanUncommitted({}); !status) { + ICEBERG_LOG_WARN("Snapshot staging cleanup failed: {}", status.error().message); + } + } catch (const std::exception& e) { + ICEBERG_LOG_WARN("Snapshot staging cleanup threw: {}", e.what()); + } catch (...) { + ICEBERG_LOG_WARN("Snapshot staging cleanup threw an unknown exception"); + } + // Include paths whose writes failed before a derived cache recorded a manifest. + std::unordered_set paths; + { + std::lock_guard lock(staging_mutex_); + paths.swap(staged_files_); + staged_data_files_.clear(); } - manifest_lists_.clear(); - std::ignore = CleanUncommitted(std::unordered_set{}); + for (const auto& path : paths) { + std::ignore = DeleteFile(path); + } + staged_snapshot_.reset(); + summary_.Clear(); return {}; } -Status SnapshotUpdate::DeleteFile(const std::string& path) { - if (delete_func_) { - return delete_func_(path); +void SnapshotUpdate::RegisterStagedFile(const std::string& path, bool data_file) { + std::lock_guard lock(staging_mutex_); + staged_files_.insert(path); + if (data_file) { + staged_data_files_.insert(path); } - return ctx_->table->io()->DeleteFile(path); +} + +Status SnapshotUpdate::DeleteFile(const std::string& path) noexcept { + try { + { + std::lock_guard lock(staging_mutex_); + if (!attempted_deletes_.insert(path).second) { + return {}; + } + staged_files_.erase(path); + staged_data_files_.erase(path); + } + auto result = delete_func_ ? delete_func_(path) : ctx_->table->io()->DeleteFile(path); + if (!result) { + ICEBERG_LOG_WARN("Cannot clean staged file {}: {}", path, result.error().message); + } + } catch (const std::exception& e) { + ICEBERG_LOG_WARN("Cannot clean staged file {}: {}", path, e.what()); + } catch (...) { + ICEBERG_LOG_WARN("Cannot clean staged file {}: unknown exception", path); + } + return {}; } std::string SnapshotUpdate::ManifestListPath() { @@ -496,7 +533,9 @@ std::string SnapshotUpdate::ManifestListPath() { auto attempt = attempt_.fetch_add(1, std::memory_order_relaxed) + 1; std::string filename = std::format("snap-{}-{}-{}.avro", snapshot_id, attempt, commit_uuid_); - return ctx_->MetadataFileLocation(filename); + auto path = ctx_->MetadataFileLocation(filename); + RegisterStagedFile(path); + return path; } SnapshotSummaryBuilder SnapshotUpdate::BuildManifestCountSummary( @@ -526,7 +565,9 @@ std::string SnapshotUpdate::ManifestPath() { // Format: {metadata_location}/{uuid}-m{manifest_count}.avro auto manifest_count = manifest_count_.fetch_add(1, std::memory_order_relaxed); std::string filename = std::format("{}-m{}.avro", commit_uuid_, manifest_count); - return ctx_->MetadataFileLocation(filename); + auto path = ctx_->MetadataFileLocation(filename); + RegisterStagedFile(path); + return path; } } // namespace iceberg diff --git a/src/iceberg/update/snapshot_update.h b/src/iceberg/update/snapshot_update.h index 1812b7072..52b5bf6a9 100644 --- a/src/iceberg/update/snapshot_update.h +++ b/src/iceberg/update/snapshot_update.h @@ -25,6 +25,7 @@ #include #include #include +#include #include #include #include @@ -66,6 +67,7 @@ class ICEBERG_EXPORT SnapshotUpdate : public PendingUpdate { /// \param reporter The metrics reporter to use. /// \return Reference to this for method chaining. auto& ReportWith(this auto& self, std::shared_ptr reporter) { + self.EnsureMutable(); static_cast(self).reporter_ = std::move(reporter); return self; } @@ -77,6 +79,7 @@ class ICEBERG_EXPORT SnapshotUpdate : public PendingUpdate { /// \note Cannot be called more than once. auto& DeleteWith(this auto& self, std::function delete_func) { + self.EnsureMutable(); if (self.delete_func_) { return self.AddError(ErrorKind::kInvalidArgument, "Cannot set delete callback more than once"); @@ -92,6 +95,7 @@ class ICEBERG_EXPORT SnapshotUpdate : public PendingUpdate { /// /// \return This update for method chaining. auto& StageOnly(this auto& self) { + self.EnsureMutable(); self.stage_only_ = true; return self; } @@ -101,6 +105,7 @@ class ICEBERG_EXPORT SnapshotUpdate : public PendingUpdate { /// \param executor Executor to use while planning manifests. /// \return Reference to this for method chaining. auto& ScanManifestsWith(this auto& self, Executor& executor) { + self.EnsureMutable(); self.plan_executor_ = std::ref(executor); return self; } @@ -110,6 +115,7 @@ class ICEBERG_EXPORT SnapshotUpdate : public PendingUpdate { /// \param branch The name of a SnapshotRef of type branch. /// \return This update for method chaining. auto& ToBranch(this auto& self, const std::string& branch) { + self.EnsureMutable(); if (branch.empty()) [[unlikely]] { return self.AddError(ErrorKind::kInvalidArgument, "Branch name cannot be empty"); } @@ -133,6 +139,7 @@ class ICEBERG_EXPORT SnapshotUpdate : public PendingUpdate { /// \param value A String property value. /// \return This update for method chaining. auto& Set(this auto& self, const std::string& property, const std::string& value) { + self.EnsureMutable(); static_cast(self).SetSummaryProperty(property, value); return self; } @@ -145,6 +152,7 @@ class ICEBERG_EXPORT SnapshotUpdate : public PendingUpdate { /// \note Custom FileIO implementations and registered writer factories used for /// manifest writes must support concurrent calls when an executor is configured. auto& WriteManifestsWith(this auto& self, Executor& executor, int32_t parallelism) { + self.EnsureMutable(); if (parallelism <= 0) [[unlikely]] { return self.AddError( ErrorKind::kInvalidArgument, @@ -156,20 +164,14 @@ class ICEBERG_EXPORT SnapshotUpdate : public PendingUpdate { return self; } - /// \brief Apply the update's changes to create a new snapshot. - /// - /// This method validates the changes, applies them to the current base - /// metadata, and creates a new snapshot without committing it. Commit retries - /// call Apply() again with refreshed metadata so the same changes can be - /// applied to the new latest snapshot. - /// - /// \return A result containing the new snapshot, or an error. - Result Apply(); + protected: + friend class Transaction; - /// \brief Finalize the snapshot update, cleaning up any uncommitted files. - Status Finalize(Result commit_result) override; + Result Apply(); + Status Finalize(const TableMetadata& committed) override; + Status CleanStaged() override; + Status ReportCommitted() override; - protected: struct ContentFileWithSequenceNumber { std::shared_ptr file; std::optional data_sequence_number; @@ -249,11 +251,6 @@ class ICEBERG_EXPORT SnapshotUpdate : public PendingUpdate { /// retry-safe summary rebuilds. virtual void SetSummaryProperty(const std::string& property, const std::string& value); - /// \brief Check if cleanup should happen after commit - /// - /// \return True if cleanup should happen after commit - virtual bool CleanupAfterCommit() const { return true; } - /// \brief Get or generate the snapshot ID for the new snapshot. int64_t SnapshotId(); @@ -261,7 +258,8 @@ class ICEBERG_EXPORT SnapshotUpdate : public PendingUpdate { /// /// \param path The path of the file to delete /// \return A status indicating the result of the deletion - Status DeleteFile(const std::string& path); + Status DeleteFile(const std::string& path) noexcept; + void RegisterStagedFile(const std::string& path, bool data_file = false); std::string ManifestPath(); std::string ManifestListPath(); @@ -274,12 +272,6 @@ class ICEBERG_EXPORT SnapshotUpdate : public PendingUpdate { Result> ComputeSummary( const TableMetadata& previous); - /// \brief Clean up all uncommitted files - Status CleanAll(); - - /// \brief Report metrics for the most recently staged snapshot. - void ReportCommit() const; - protected: SnapshotSummaryBuilder summary_; @@ -290,7 +282,11 @@ class ICEBERG_EXPORT SnapshotUpdate : public PendingUpdate { int32_t write_manifest_parallelism_{1}; std::atomic manifest_count_{0}; std::atomic attempt_{0}; - std::vector manifest_lists_; + // All paths are registered before writes, including partial/failed writes. + std::mutex staging_mutex_; + std::unordered_set staged_files_; + std::unordered_set staged_data_files_; + std::unordered_set attempted_deletes_; const int64_t target_manifest_size_bytes_; std::optional snapshot_id_; OptionalExecutor plan_executor_; diff --git a/src/iceberg/update/update_location.cc b/src/iceberg/update/update_location.cc index 064bebc8c..7008dd89a 100644 --- a/src/iceberg/update/update_location.cc +++ b/src/iceberg/update/update_location.cc @@ -41,12 +41,13 @@ UpdateLocation::UpdateLocation(std::shared_ptr ctx) UpdateLocation::~UpdateLocation() = default; UpdateLocation& UpdateLocation::SetLocation(std::string_view location) { + EnsureMutable(); ICEBERG_BUILDER_CHECK(!location.empty(), "Location cannot be empty"); location_ = std::string(location); return *this; } -Result UpdateLocation::Apply() { +Result UpdateLocation::Validate() const { ICEBERG_RETURN_UNEXPECTED(CheckErrors()); if (location_.empty()) { return InvalidArgument("Location must be set before applying"); diff --git a/src/iceberg/update/update_location.h b/src/iceberg/update/update_location.h index 48fb84c21..2fb0df846 100644 --- a/src/iceberg/update/update_location.h +++ b/src/iceberg/update/update_location.h @@ -47,10 +47,12 @@ class ICEBERG_EXPORT UpdateLocation : public PendingUpdate { Kind kind() const final { return Kind::kUpdateLocation; } bool IsRetryable() const override { return true; } - /// \brief Apply the pending changes and return the new location. - Result Apply(); + /// \brief Validate and preview changes without staging or modifying this update. + Result Validate() const; private: + bool MayAddFileReferences() const override { return false; } + explicit UpdateLocation(std::shared_ptr ctx); std::string location_; diff --git a/src/iceberg/update/update_partition_spec.cc b/src/iceberg/update/update_partition_spec.cc index 2be6a59a5..a615a6388 100644 --- a/src/iceberg/update/update_partition_spec.cc +++ b/src/iceberg/update/update_partition_spec.cc @@ -86,16 +86,19 @@ UpdatePartitionSpec::UpdatePartitionSpec(std::shared_ptr ctx UpdatePartitionSpec::~UpdatePartitionSpec() = default; UpdatePartitionSpec& UpdatePartitionSpec::CaseSensitive(bool is_case_sensitive) { + EnsureMutable(); case_sensitive_ = is_case_sensitive; return *this; } UpdatePartitionSpec& UpdatePartitionSpec::AddNonDefaultSpec() { + EnsureMutable(); set_as_default_ = false; return *this; } UpdatePartitionSpec& UpdatePartitionSpec::AddField(std::string_view source_name) { + EnsureMutable(); // Find the source field in the schema ICEBERG_BUILDER_ASSIGN_OR_RETURN( auto field_opt, schema_->FindFieldByName(source_name, case_sensitive_)); @@ -108,6 +111,7 @@ UpdatePartitionSpec& UpdatePartitionSpec::AddField(std::string_view source_name) UpdatePartitionSpec& UpdatePartitionSpec::AddField(const std::shared_ptr& term, std::string_view part_name) { + EnsureMutable(); ICEBERG_BUILDER_CHECK(term->is_unbound(), "Cannot add bound term to partition spec"); // Bind the term to get the source id if (term->kind() == Term::Kind::kReference) { @@ -130,6 +134,7 @@ UpdatePartitionSpec& UpdatePartitionSpec::AddField(const std::shared_ptr& UpdatePartitionSpec& UpdatePartitionSpec::AddFieldInternal( std::string_view name, int32_t source_id, const std::shared_ptr& transform) { + EnsureMutable(); // Check for duplicate name in added fields ICEBERG_BUILDER_CHECK(name.empty() || !added_field_names_.contains(name), "Cannot add duplicate partition field: {}", name); @@ -212,6 +217,7 @@ UpdatePartitionSpec& UpdatePartitionSpec::AddFieldInternal( UpdatePartitionSpec& UpdatePartitionSpec::RewriteDeleteAndAddField( const PartitionField& existing, std::string_view name) { + EnsureMutable(); deletes_.erase(existing.field_id()); if (name.empty() || std::string(existing.name()) == name) { return *this; @@ -220,6 +226,7 @@ UpdatePartitionSpec& UpdatePartitionSpec::RewriteDeleteAndAddField( } UpdatePartitionSpec& UpdatePartitionSpec::RemoveField(std::string_view name) { + EnsureMutable(); // Cannot delete newly added fields ICEBERG_BUILDER_CHECK(!added_field_names_.contains(name), "Cannot delete newly added field: {}", name); @@ -237,6 +244,7 @@ UpdatePartitionSpec& UpdatePartitionSpec::RemoveField(std::string_view name) { } UpdatePartitionSpec& UpdatePartitionSpec::RemoveField(const std::shared_ptr& term) { + EnsureMutable(); ICEBERG_BUILDER_CHECK(term->is_unbound(), "Cannot remove bound term from partition spec"); // Bind the term to get the source id @@ -263,6 +271,7 @@ UpdatePartitionSpec& UpdatePartitionSpec::RemoveField(const std::shared_ptrsecond)) { @@ -306,7 +316,7 @@ UpdatePartitionSpec& UpdatePartitionSpec::RenameField(std::string_view name, return *this; } -Result UpdatePartitionSpec::Apply() { +Result UpdatePartitionSpec::Validate() const { ICEBERG_RETURN_UNEXPECTED(CheckErrors()); std::vector new_fields; @@ -341,6 +351,12 @@ Result UpdatePartitionSpec::Apply() { // Add new fields new_fields.insert(new_fields.end(), adds_.begin(), adds_.end()); + // Keep preview results independent of this operation's frozen transforms. + for (auto& field : new_fields) { + field = PartitionField(field.source_id(), field.field_id(), std::string(field.name()), + std::make_shared(*field.transform())); + } + // Use -1 as a placeholder for the spec id, the actual spec id will be assigned by // TableMetadataBuilder when the AddPartitionSpec update is applied. ICEBERG_ASSIGN_OR_RAISE(auto new_spec, @@ -446,4 +462,12 @@ void UpdatePartitionSpec::BuildHistoricalFieldsIndex() { } } +Status UpdatePartitionSpec::Freeze() { + for (auto& field : adds_) { + field = PartitionField(field.source_id(), field.field_id(), std::string(field.name()), + std::make_shared(*field.transform())); + } + return {}; +} + } // namespace iceberg diff --git a/src/iceberg/update/update_partition_spec.h b/src/iceberg/update/update_partition_spec.h index 67dcb1413..ef9e1f965 100644 --- a/src/iceberg/update/update_partition_spec.h +++ b/src/iceberg/update/update_partition_spec.h @@ -111,9 +111,14 @@ class ICEBERG_EXPORT UpdatePartitionSpec : public PendingUpdate { std::shared_ptr spec; bool set_as_default; }; - Result Apply(); + /// \brief Validate and preview changes without staging or modifying this update. + Result Validate() const; private: + Status Freeze() override; + + bool MayAddFileReferences() const override { return false; } + explicit UpdatePartitionSpec(std::shared_ptr ctx); /// \brief Pair of source ID and transform string for indexing. diff --git a/src/iceberg/update/update_partition_statistics.cc b/src/iceberg/update/update_partition_statistics.cc index 3a5ab4f8a..0fd989a79 100644 --- a/src/iceberg/update/update_partition_statistics.cc +++ b/src/iceberg/update/update_partition_statistics.cc @@ -47,6 +47,7 @@ UpdatePartitionStatistics::~UpdatePartitionStatistics() = default; UpdatePartitionStatistics& UpdatePartitionStatistics::SetPartitionStatistics( std::shared_ptr partition_statistics_file) { + EnsureMutable(); ICEBERG_BUILDER_CHECK(partition_statistics_file != nullptr, "Statistics file cannot be null"); @@ -57,17 +58,20 @@ UpdatePartitionStatistics& UpdatePartitionStatistics::SetPartitionStatistics( UpdatePartitionStatistics& UpdatePartitionStatistics::RemovePartitionStatistics( int64_t snapshot_id) { + EnsureMutable(); partition_statistics_to_set_[snapshot_id] = nullptr; return *this; } -Result UpdatePartitionStatistics::Apply() { +Result UpdatePartitionStatistics::Validate() + const { ICEBERG_RETURN_UNEXPECTED(CheckErrors()); ApplyResult result; for (const auto& [snapshot_id, partition_stats] : partition_statistics_to_set_) { if (partition_stats) { - result.to_set.emplace_back(snapshot_id, partition_stats); + result.to_set.emplace_back( + snapshot_id, std::make_shared(*partition_stats)); } else { result.to_remove.push_back(snapshot_id); } @@ -75,4 +79,13 @@ Result UpdatePartitionStatistics::Apply( return result; } +Status UpdatePartitionStatistics::Freeze() { + for (auto& [_, value] : partition_statistics_to_set_) { + if (value) { + value = std::make_shared(*value); + } + } + return {}; +} + } // namespace iceberg diff --git a/src/iceberg/update/update_partition_statistics.h b/src/iceberg/update/update_partition_statistics.h index 982b1bd39..5633c4da0 100644 --- a/src/iceberg/update/update_partition_statistics.h +++ b/src/iceberg/update/update_partition_statistics.h @@ -75,9 +75,12 @@ class ICEBERG_EXPORT UpdatePartitionStatistics : public PendingUpdate { std::vector to_remove; }; - Result Apply(); + /// \brief Validate and preview changes without staging or modifying this update. + Result Validate() const; private: + Status Freeze() override; + explicit UpdatePartitionStatistics(std::shared_ptr ctx); std::unordered_map> diff --git a/src/iceberg/update/update_properties.cc b/src/iceberg/update/update_properties.cc index d2e75c301..091bc047f 100644 --- a/src/iceberg/update/update_properties.cc +++ b/src/iceberg/update/update_properties.cc @@ -46,6 +46,7 @@ UpdateProperties::~UpdateProperties() = default; UpdateProperties& UpdateProperties::Set(const std::string& key, const std::string& value) { + EnsureMutable(); ICEBERG_BUILDER_CHECK(!removals_.contains(key), "Cannot set property '{}' that is already marked for removal", key); @@ -59,6 +60,7 @@ UpdateProperties& UpdateProperties::Set(const std::string& key, } UpdateProperties& UpdateProperties::Remove(const std::string& key) { + EnsureMutable(); ICEBERG_BUILDER_CHECK(!updates_.contains(key), "Cannot remove property '{}' that is already marked for update", key); @@ -66,8 +68,10 @@ UpdateProperties& UpdateProperties::Remove(const std::string& key) { return *this; } -Result UpdateProperties::Apply() { +Result UpdateProperties::Validate() const { ICEBERG_RETURN_UNEXPECTED(CheckErrors()); + auto updates = updates_; + std::optional format_version; const auto& current_props = base().properties.configs(); std::unordered_map new_properties; std::vector removals; @@ -91,17 +95,18 @@ Result UpdateProperties::Apply() { "Cannot upgrade table to unsupported format version: v{} (supported: v{})", parsed_version, TableMetadata::kSupportedTableFormatVersion); } - format_version_ = static_cast(parsed_version); + format_version = static_cast(parsed_version); - updates_.erase(TableProperties::kFormatVersion.key()); + updates.erase(TableProperties::kFormatVersion.key()); } if (auto schema = base().Schema(); schema.has_value()) { ICEBERG_RETURN_UNEXPECTED( MetricsConfig::VerifyReferencedColumns(new_properties, *schema.value())); } - return ApplyResult{ - .updates = updates_, .removals = removals_, .format_version = format_version_}; + return ApplyResult{.updates = std::move(updates), + .removals = removals_, + .format_version = format_version}; } } // namespace iceberg diff --git a/src/iceberg/update/update_properties.h b/src/iceberg/update/update_properties.h index 18eba427b..8ab41e318 100644 --- a/src/iceberg/update/update_properties.h +++ b/src/iceberg/update/update_properties.h @@ -53,7 +53,7 @@ class ICEBERG_EXPORT UpdateProperties : public PendingUpdate { /// /// The key must not have been previously marked for removal and must not be a /// reserved property key (except `format-version`). Setting a reserved property - /// will result in a validation error at Apply() time. + /// will result in an error during validation. /// /// \param key The property key to set /// \param value The property value to set @@ -69,15 +69,16 @@ class ICEBERG_EXPORT UpdateProperties : public PendingUpdate { Kind kind() const final { return Kind::kUpdateProperties; } bool IsRetryable() const override { return true; } - /// \brief Apply the pending changes and return the updates and removals. - Result Apply(); + /// \brief Validate and preview changes without staging or modifying this update. + Result Validate() const; private: + bool MayAddFileReferences() const override { return false; } + explicit UpdateProperties(std::shared_ptr ctx); std::unordered_map updates_; std::unordered_set removals_; - std::optional format_version_; }; } // namespace iceberg diff --git a/src/iceberg/update/update_schema.cc b/src/iceberg/update/update_schema.cc index 56e167e10..77d4b4ddd 100644 --- a/src/iceberg/update/update_schema.cc +++ b/src/iceberg/update/update_schema.cc @@ -38,6 +38,7 @@ #include "iceberg/table_properties.h" #include "iceberg/transaction.h" #include "iceberg/type.h" +#include "iceberg/update/update_util_internal.h" #include "iceberg/util/checked_cast.h" #include "iceberg/util/error_collector.h" #include "iceberg/util/formatter.h" // IWYU pragma: keep @@ -339,11 +340,13 @@ UpdateSchema::Move UpdateSchema::Move::After(int32_t field_id, } UpdateSchema& UpdateSchema::AllowIncompatibleChanges() { + EnsureMutable(); allow_incompatible_changes_ = true; return *this; } UpdateSchema& UpdateSchema::CaseSensitive(bool case_sensitive) { + EnsureMutable(); case_sensitive_ = case_sensitive; return *this; } @@ -351,6 +354,7 @@ UpdateSchema& UpdateSchema::CaseSensitive(bool case_sensitive) { UpdateSchema& UpdateSchema::AddColumn(std::string_view name, std::shared_ptr type, std::string_view doc, std::optional default_value) { + EnsureMutable(); ICEBERG_BUILDER_CHECK(!name.contains('.'), "Cannot add column with ambiguous name: {}, use " "AddColumn(parent, name, type, doc)", @@ -363,6 +367,7 @@ UpdateSchema& UpdateSchema::AddColumn(std::optional parent, std::string_view name, std::shared_ptr type, std::string_view doc, std::optional default_value) { + EnsureMutable(); return AddColumnInternal(std::move(parent), name, /*is_optional=*/true, std::move(type), doc, std::move(default_value)); } @@ -371,6 +376,7 @@ UpdateSchema& UpdateSchema::AddRequiredColumn(std::string_view name, std::shared_ptr type, std::string_view doc, std::optional default_value) { + EnsureMutable(); ICEBERG_BUILDER_CHECK(!name.contains('.'), "Cannot add column with ambiguous name: {}, use " "AddRequiredColumn(parent, name, type, doc)", @@ -384,12 +390,14 @@ UpdateSchema& UpdateSchema::AddRequiredColumn(std::optional pa std::shared_ptr type, std::string_view doc, std::optional default_value) { + EnsureMutable(); return AddColumnInternal(std::move(parent), name, /*is_optional=*/false, std::move(type), doc, std::move(default_value)); } UpdateSchema& UpdateSchema::UpdateColumn(std::string_view name, std::shared_ptr new_type) { + EnsureMutable(); ICEBERG_BUILDER_ASSIGN_OR_RETURN(auto field_opt, FindFieldForUpdate(name)); ICEBERG_BUILDER_CHECK(field_opt.has_value(), "Cannot update missing column: {}", name); @@ -432,6 +440,7 @@ UpdateSchema& UpdateSchema::UpdateColumn(std::string_view name, UpdateSchema& UpdateSchema::UpdateColumnDoc(std::string_view name, std::string_view new_doc) { + EnsureMutable(); ICEBERG_BUILDER_ASSIGN_OR_RETURN(auto field_opt, FindFieldForUpdate(name)); ICEBERG_BUILDER_CHECK(field_opt.has_value(), "Cannot update missing column: {}", name); @@ -452,6 +461,7 @@ UpdateSchema& UpdateSchema::UpdateColumnDoc(std::string_view name, UpdateSchema& UpdateSchema::UpdateColumnDefault(std::string_view name, std::optional new_default) { + EnsureMutable(); ICEBERG_BUILDER_ASSIGN_OR_RETURN(auto field_opt, FindFieldForUpdate(name)); ICEBERG_BUILDER_CHECK(field_opt.has_value(), "Cannot update missing column: {}", name); @@ -489,6 +499,7 @@ UpdateSchema& UpdateSchema::UpdateColumnDefault(std::string_view name, UpdateSchema& UpdateSchema::RenameColumn(std::string_view name, std::string_view new_name) { + EnsureMutable(); ICEBERG_BUILDER_ASSIGN_OR_RETURN(auto field_opt, FindField(name)); ICEBERG_BUILDER_CHECK(field_opt.has_value(), "Cannot rename missing column: {}", name); ICEBERG_BUILDER_CHECK(!new_name.empty(), "Cannot rename a column to null"); @@ -514,15 +525,18 @@ UpdateSchema& UpdateSchema::RenameColumn(std::string_view name, } UpdateSchema& UpdateSchema::MakeColumnOptional(std::string_view name) { + EnsureMutable(); return UpdateColumnRequirementInternal(name, /*is_optional=*/true); } UpdateSchema& UpdateSchema::RequireColumn(std::string_view name) { + EnsureMutable(); return UpdateColumnRequirementInternal(name, /*is_optional=*/false); } UpdateSchema& UpdateSchema::UpdateColumnRequirementInternal(std::string_view name, bool is_optional) { + EnsureMutable(); ICEBERG_BUILDER_ASSIGN_OR_RETURN(auto field_opt, FindFieldForUpdate(name)); ICEBERG_BUILDER_CHECK(field_opt.has_value(), "Cannot update missing column: {}", name); @@ -552,6 +566,7 @@ UpdateSchema& UpdateSchema::UpdateColumnRequirementInternal(std::string_view nam } UpdateSchema& UpdateSchema::DeleteColumn(std::string_view name) { + EnsureMutable(); ICEBERG_BUILDER_ASSIGN_OR_RETURN(auto field_opt, FindField(name)); ICEBERG_BUILDER_CHECK(field_opt.has_value(), "Cannot delete missing column: {}", name); @@ -569,6 +584,7 @@ UpdateSchema& UpdateSchema::DeleteColumn(std::string_view name) { } UpdateSchema& UpdateSchema::MoveFirst(std::string_view name) { + EnsureMutable(); ICEBERG_BUILDER_ASSIGN_OR_RETURN(auto field_id, FindFieldIdForMove(name)); return MoveInternal(name, Move::First(field_id)); @@ -576,6 +592,7 @@ UpdateSchema& UpdateSchema::MoveFirst(std::string_view name) { UpdateSchema& UpdateSchema::MoveBefore(std::string_view name, std::string_view before_name) { + EnsureMutable(); ICEBERG_BUILDER_ASSIGN_OR_RETURN(auto field_id, FindFieldIdForMove(name)); ICEBERG_BUILDER_ASSIGN_OR_RETURN_WITH_ERROR( auto before_id, FindFieldIdForMove(before_name), @@ -588,6 +605,7 @@ UpdateSchema& UpdateSchema::MoveBefore(std::string_view name, UpdateSchema& UpdateSchema::MoveAfter(std::string_view name, std::string_view after_name) { + EnsureMutable(); ICEBERG_BUILDER_ASSIGN_OR_RETURN(auto field_id, FindFieldIdForMove(name)); ICEBERG_BUILDER_ASSIGN_OR_RETURN_WITH_ERROR( auto after_id, FindFieldIdForMove(after_name), @@ -599,6 +617,7 @@ UpdateSchema& UpdateSchema::MoveAfter(std::string_view name, } UpdateSchema& UpdateSchema::UnionByNameWith(std::shared_ptr new_schema) { + EnsureMutable(); // TODO(Guotao Yu): Implement UnionByNameWith AddError(NotImplemented("UpdateSchema::UnionByNameWith not implemented")); return *this; @@ -606,11 +625,12 @@ UpdateSchema& UpdateSchema::UnionByNameWith(std::shared_ptr new_schema) UpdateSchema& UpdateSchema::SetIdentifierFields( const std::span& names) { + EnsureMutable(); identifier_field_names_ = names | std::ranges::to>(); return *this; } -Result UpdateSchema::Apply() { +Result UpdateSchema::Validate() const { ICEBERG_RETURN_UNEXPECTED(CheckErrors()); for (const auto& name : identifier_field_names_) { @@ -653,10 +673,17 @@ Result UpdateSchema::Apply() { fresh_identifier_ids.push_back(field_opt->get().field_id()); } - auto new_fields = temp_schema->fields() | std::ranges::to>(); - ICEBERG_ASSIGN_OR_RAISE( - auto new_schema, - Schema::Make(std::move(new_fields), schema_->schema_id(), fresh_identifier_ids)); + // The result must not alias the frozen replay parameters (or the base schema), so + // deep-copy the fields before building the schema that is returned. + std::vector new_fields; + new_fields.reserve(temp_schema->fields().size()); + for (const auto& field : temp_schema->fields()) { + ICEBERG_ASSIGN_OR_RAISE(auto copy, internal::CopyUpdateField(field)); + new_fields.push_back(std::move(copy)); + } + ICEBERG_ASSIGN_OR_RAISE(auto new_schema, + Schema::Make(std::move(new_fields), schema_->schema_id(), + std::move(fresh_identifier_ids))); ICEBERG_RETURN_UNEXPECTED(new_schema->Validate(base().format_version)); std::unordered_map updated_props; @@ -688,6 +715,7 @@ UpdateSchema& UpdateSchema::AddColumnInternal(std::optional pa std::shared_ptr type, std::string_view doc, std::optional default_value) { + EnsureMutable(); int32_t parent_id = kTableRootId; std::string full_name; // For map/list adds, this omits synthetic value/element path segments. @@ -847,6 +875,7 @@ Result UpdateSchema::FindFieldIdForMove(std::string_view name) const { } UpdateSchema& UpdateSchema::MoveInternal(std::string_view name, const Move& move) { + EnsureMutable(); auto parent_it = id_to_parent_.find(move.field_id); if (parent_it != id_to_parent_.end()) { @@ -881,4 +910,13 @@ UpdateSchema& UpdateSchema::MoveInternal(std::string_view name, const Move& move return *this; } +Status UpdateSchema::Freeze() { + // Field values may contain nested types and mutable aliases to defaults. + for (auto& [_, field] : updates_) { + ICEBERG_ASSIGN_OR_RAISE(auto copy, internal::CopyUpdateField(*field)); + field = std::make_shared(std::move(copy)); + } + return {}; +} + } // namespace iceberg diff --git a/src/iceberg/update/update_schema.h b/src/iceberg/update/update_schema.h index d0dd944ef..9e4ae29f7 100644 --- a/src/iceberg/update/update_schema.h +++ b/src/iceberg/update/update_schema.h @@ -364,14 +364,14 @@ class ICEBERG_EXPORT UpdateSchema : public PendingUpdate { std::unordered_map updated_props; }; - /// \brief Apply the pending changes to the original schema and return the result. - /// - /// This does not result in a permanent update. - /// - /// \return The result Schema and last column id when all pending updates are applied. - Result Apply(); + /// \brief Validate and preview changes without staging or modifying this update. + Result Validate() const; private: + Status Freeze() override; + + bool MayAddFileReferences() const override { return false; } + explicit UpdateSchema(std::shared_ptr ctx); /// \brief Internal implementation for adding a column with full control. diff --git a/src/iceberg/update/update_snapshot_reference.cc b/src/iceberg/update/update_snapshot_reference.cc index 908962ecd..e11e3acb3 100644 --- a/src/iceberg/update/update_snapshot_reference.cc +++ b/src/iceberg/update/update_snapshot_reference.cc @@ -47,6 +47,7 @@ UpdateSnapshotReference::~UpdateSnapshotReference() = default; UpdateSnapshotReference& UpdateSnapshotReference::CreateBranch(const std::string& name, int64_t snapshot_id) { + EnsureMutable(); ICEBERG_BUILDER_CHECK(!name.empty(), "Branch name cannot be empty"); ICEBERG_BUILDER_ASSIGN_OR_RETURN(auto branch, SnapshotRef::MakeBranch(snapshot_id)); auto [_, inserted] = updated_refs_.emplace(name, std::move(branch)); @@ -56,6 +57,7 @@ UpdateSnapshotReference& UpdateSnapshotReference::CreateBranch(const std::string UpdateSnapshotReference& UpdateSnapshotReference::CreateTag(const std::string& name, int64_t snapshot_id) { + EnsureMutable(); ICEBERG_BUILDER_CHECK(!name.empty(), "Tag name cannot be empty"); ICEBERG_BUILDER_ASSIGN_OR_RETURN(auto tag, SnapshotRef::MakeTag(snapshot_id)); auto [_, inserted] = updated_refs_.emplace(name, std::move(tag)); @@ -64,6 +66,7 @@ UpdateSnapshotReference& UpdateSnapshotReference::CreateTag(const std::string& n } UpdateSnapshotReference& UpdateSnapshotReference::RemoveBranch(const std::string& name) { + EnsureMutable(); ICEBERG_BUILDER_CHECK(!name.empty(), "Branch name cannot be empty"); ICEBERG_BUILDER_CHECK(name != SnapshotRef::kMainBranch, "Cannot remove main branch"); auto it = updated_refs_.find(name); @@ -75,6 +78,7 @@ UpdateSnapshotReference& UpdateSnapshotReference::RemoveBranch(const std::string } UpdateSnapshotReference& UpdateSnapshotReference::RemoveTag(const std::string& name) { + EnsureMutable(); ICEBERG_BUILDER_CHECK(!name.empty(), "Tag name cannot be empty"); auto it = updated_refs_.find(name); ICEBERG_BUILDER_CHECK(it != updated_refs_.end(), "Tag does not exist: {}", name); @@ -86,6 +90,7 @@ UpdateSnapshotReference& UpdateSnapshotReference::RemoveTag(const std::string& n UpdateSnapshotReference& UpdateSnapshotReference::RenameBranch( const std::string& name, const std::string& new_name) { + EnsureMutable(); ICEBERG_BUILDER_CHECK(!name.empty(), "Branch to rename cannot be empty"); ICEBERG_BUILDER_CHECK(!new_name.empty(), "New branch name cannot be empty"); ICEBERG_BUILDER_CHECK(name != SnapshotRef::kMainBranch, "Cannot rename main branch"); @@ -101,6 +106,7 @@ UpdateSnapshotReference& UpdateSnapshotReference::RenameBranch( UpdateSnapshotReference& UpdateSnapshotReference::ReplaceBranch(const std::string& name, int64_t snapshot_id) { + EnsureMutable(); ICEBERG_BUILDER_CHECK(!name.empty(), "Branch name cannot be empty"); auto it = updated_refs_.find(name); ICEBERG_BUILDER_CHECK(it != updated_refs_.end(), "Branch does not exist: {}", name); @@ -112,16 +118,19 @@ UpdateSnapshotReference& UpdateSnapshotReference::ReplaceBranch(const std::strin UpdateSnapshotReference& UpdateSnapshotReference::ReplaceBranch(const std::string& from, const std::string& to) { + EnsureMutable(); return ReplaceBranchInternal(from, to, /*fast_forward=*/false); } UpdateSnapshotReference& UpdateSnapshotReference::FastForward(const std::string& from, const std::string& to) { + EnsureMutable(); return ReplaceBranchInternal(from, to, /*fast_forward=*/true); } UpdateSnapshotReference& UpdateSnapshotReference::ReplaceBranchInternal( const std::string& from, const std::string& to, bool fast_forward) { + EnsureMutable(); ICEBERG_BUILDER_CHECK(!from.empty(), "Branch to update cannot be empty"); ICEBERG_BUILDER_CHECK(!to.empty(), "Destination ref cannot be empty"); auto to_it = updated_refs_.find(to); @@ -160,6 +169,7 @@ UpdateSnapshotReference& UpdateSnapshotReference::ReplaceBranchInternal( UpdateSnapshotReference& UpdateSnapshotReference::ReplaceTag(const std::string& name, int64_t snapshot_id) { + EnsureMutable(); ICEBERG_BUILDER_CHECK(!name.empty(), "Tag name cannot be empty"); auto it = updated_refs_.find(name); ICEBERG_BUILDER_CHECK(it != updated_refs_.end(), "Tag does not exist: {}", name); @@ -171,6 +181,7 @@ UpdateSnapshotReference& UpdateSnapshotReference::ReplaceTag(const std::string& UpdateSnapshotReference& UpdateSnapshotReference::SetMinSnapshotsToKeep( const std::string& name, int32_t min_snapshots_to_keep) { + EnsureMutable(); ICEBERG_BUILDER_CHECK(!name.empty(), "Branch name cannot be empty"); auto it = updated_refs_.find(name); ICEBERG_BUILDER_CHECK(it != updated_refs_.end(), "Branch does not exist: {}", name); @@ -187,6 +198,7 @@ UpdateSnapshotReference& UpdateSnapshotReference::SetMinSnapshotsToKeep( UpdateSnapshotReference& UpdateSnapshotReference::SetMaxSnapshotAgeMs( const std::string& name, int64_t max_snapshot_age_ms) { + EnsureMutable(); ICEBERG_BUILDER_CHECK(!name.empty(), "Branch name cannot be empty"); auto it = updated_refs_.find(name); ICEBERG_BUILDER_CHECK(it != updated_refs_.end(), "Branch does not exist: {}", name); @@ -203,6 +215,7 @@ UpdateSnapshotReference& UpdateSnapshotReference::SetMaxSnapshotAgeMs( UpdateSnapshotReference& UpdateSnapshotReference::SetMaxRefAgeMs(const std::string& name, int64_t max_ref_age_ms) { + EnsureMutable(); ICEBERG_BUILDER_CHECK(!name.empty(), "Reference name cannot be empty"); auto it = updated_refs_.find(name); ICEBERG_BUILDER_CHECK(it != updated_refs_.end(), "Ref does not exist: {}", name); @@ -217,7 +230,7 @@ UpdateSnapshotReference& UpdateSnapshotReference::SetMaxRefAgeMs(const std::stri return *this; } -Result UpdateSnapshotReference::Apply() { +Result UpdateSnapshotReference::Validate() const { ICEBERG_RETURN_UNEXPECTED(CheckErrors()); ApplyResult result; @@ -234,11 +247,20 @@ Result UpdateSnapshotReference::Apply() { for (const auto& [name, ref] : updated_refs_) { if (auto iter = current_refs.find(name); iter == current_refs.end() || *iter->second != *ref) { - result.to_set.emplace_back(name, ref); + result.to_set.emplace_back(name, std::make_shared(*ref)); } } return result; } +Status UpdateSnapshotReference::Freeze() { + for (auto& [_, value] : updated_refs_) { + if (value) { + value = std::make_shared(*value); + } + } + return {}; +} + } // namespace iceberg diff --git a/src/iceberg/update/update_snapshot_reference.h b/src/iceberg/update/update_snapshot_reference.h index 9ff0a5083..bd40938d4 100644 --- a/src/iceberg/update/update_snapshot_reference.h +++ b/src/iceberg/update/update_snapshot_reference.h @@ -148,10 +148,12 @@ class ICEBERG_EXPORT UpdateSnapshotReference : public PendingUpdate { std::vector to_remove; }; - /// \brief Apply the pending changes and return the updated and removed references. - Result Apply(); + /// \brief Validate and preview changes without staging or modifying this update. + Result Validate() const; private: + Status Freeze() override; + explicit UpdateSnapshotReference(std::shared_ptr ctx); UpdateSnapshotReference& ReplaceBranchInternal(const std::string& from, diff --git a/src/iceberg/update/update_sort_order.cc b/src/iceberg/update/update_sort_order.cc index 8086b903f..6c79ef5ae 100644 --- a/src/iceberg/update/update_sort_order.cc +++ b/src/iceberg/update/update_sort_order.cc @@ -47,6 +47,7 @@ UpdateSortOrder::~UpdateSortOrder() = default; UpdateSortOrder& UpdateSortOrder::AddSortField(const std::shared_ptr& term, SortDirection direction, NullOrder null_order) { + EnsureMutable(); ICEBERG_BUILDER_CHECK(term != nullptr, "Term cannot be null"); ICEBERG_BUILDER_CHECK(term->is_unbound(), "Term must be unbound"); @@ -75,17 +76,19 @@ UpdateSortOrder& UpdateSortOrder::AddSortField(const std::shared_ptr& term UpdateSortOrder& UpdateSortOrder::AddSortFieldByName(std::string_view name, SortDirection direction, NullOrder null_order) { + EnsureMutable(); ICEBERG_BUILDER_ASSIGN_OR_RETURN(auto named_ref, NamedReference::Make(std::string(name))); return AddSortField(std::move(named_ref), direction, null_order); } UpdateSortOrder& UpdateSortOrder::CaseSensitive(bool case_sensitive) { + EnsureMutable(); case_sensitive_ = case_sensitive; return *this; } -Result> UpdateSortOrder::Apply() { +Result> UpdateSortOrder::Validate() const { ICEBERG_RETURN_UNEXPECTED(CheckErrors()); // If no sort fields are specified, return an unsorted order (ID = 0). @@ -96,11 +99,25 @@ Result> UpdateSortOrder::Apply() { // Use -1 as a placeholder for non-empty sort orders. // The actual sort order ID will be assigned by TableMetadataBuilder when // the AddSortOrder update is applied. - ICEBERG_ASSIGN_OR_RAISE(order, SortOrder::Make(/*sort_id=*/-1, sort_fields_)); + std::vector fields; + for (const auto& field : sort_fields_) { + fields.emplace_back(field.source_id(), + std::make_shared(*field.transform()), + field.direction(), field.null_order()); + } + ICEBERG_ASSIGN_OR_RAISE(order, SortOrder::Make(/*sort_id=*/-1, std::move(fields))); ICEBERG_ASSIGN_OR_RAISE(auto schema, base().Schema()); ICEBERG_RETURN_UNEXPECTED(order->Validate(*schema)); } return order; } +Status UpdateSortOrder::Freeze() { + for (auto& field : sort_fields_) { + field = SortField(field.source_id(), std::make_shared(*field.transform()), + field.direction(), field.null_order()); + } + return {}; +} + } // namespace iceberg diff --git a/src/iceberg/update/update_sort_order.h b/src/iceberg/update/update_sort_order.h index 4696ae72d..11cf88387 100644 --- a/src/iceberg/update/update_sort_order.h +++ b/src/iceberg/update/update_sort_order.h @@ -68,10 +68,14 @@ class ICEBERG_EXPORT UpdateSortOrder : public PendingUpdate { Kind kind() const final { return Kind::kUpdateSortOrder; } bool IsRetryable() const override { return true; } - /// \brief Apply the pending changes and return the new SortOrder. - Result> Apply(); + /// \brief Validate and preview changes without staging or modifying this update. + Result> Validate() const; private: + Status Freeze() override; + + bool MayAddFileReferences() const override { return false; } + explicit UpdateSortOrder(std::shared_ptr ctx); std::vector sort_fields_; diff --git a/src/iceberg/update/update_statistics.cc b/src/iceberg/update/update_statistics.cc index afa6ea0c6..1e7208b0f 100644 --- a/src/iceberg/update/update_statistics.cc +++ b/src/iceberg/update/update_statistics.cc @@ -44,23 +44,25 @@ UpdateStatistics::~UpdateStatistics() = default; UpdateStatistics& UpdateStatistics::SetStatistics( std::shared_ptr statistics_file) { + EnsureMutable(); ICEBERG_BUILDER_CHECK(statistics_file != nullptr, "Statistics file cannot be null"); statistics_to_set_[statistics_file->snapshot_id] = std::move(statistics_file); return *this; } UpdateStatistics& UpdateStatistics::RemoveStatistics(int64_t snapshot_id) { + EnsureMutable(); statistics_to_set_[snapshot_id] = nullptr; return *this; } -Result UpdateStatistics::Apply() { +Result UpdateStatistics::Validate() const { ICEBERG_RETURN_UNEXPECTED(CheckErrors()); ApplyResult result; for (const auto& [snapshot_id, stats] : statistics_to_set_) { if (stats) { - result.to_set.emplace_back(snapshot_id, stats); + result.to_set.emplace_back(snapshot_id, std::make_shared(*stats)); } else { result.to_remove.push_back(snapshot_id); } @@ -68,4 +70,13 @@ Result UpdateStatistics::Apply() { return result; } +Status UpdateStatistics::Freeze() { + for (auto& [_, value] : statistics_to_set_) { + if (value) { + value = std::make_shared(*value); + } + } + return {}; +} + } // namespace iceberg diff --git a/src/iceberg/update/update_statistics.h b/src/iceberg/update/update_statistics.h index 4a6c12fc4..28ec30bb4 100644 --- a/src/iceberg/update/update_statistics.h +++ b/src/iceberg/update/update_statistics.h @@ -73,9 +73,12 @@ class ICEBERG_EXPORT UpdateStatistics : public PendingUpdate { std::vector to_remove; }; - Result Apply(); + /// \brief Validate and preview changes without staging or modifying this update. + Result Validate() const; private: + Status Freeze() override; + explicit UpdateStatistics(std::shared_ptr ctx); std::unordered_map> statistics_to_set_; diff --git a/src/iceberg/update/update_util_internal.h b/src/iceberg/update/update_util_internal.h new file mode 100644 index 000000000..cd7d1fe51 --- /dev/null +++ b/src/iceberg/update/update_util_internal.h @@ -0,0 +1,193 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#pragma once + +#include "iceberg/expression/expression.h" +#include "iceberg/expression/literal.h" +#include "iceberg/expression/predicate.h" +#include "iceberg/expression/term.h" +#include "iceberg/manifest/manifest_entry.h" +#include "iceberg/schema_field.h" +#include "iceberg/transform.h" +#include "iceberg/type.h" +#include "iceberg/util/checked_cast.h" +#include "iceberg/util/macros.h" + +namespace iceberg::internal { + +// Most primitive types have no mutable state. Parameterized types must be copied +// even when they are reached through a const Literal or shared_ptr alias. +// Returns nullptr for stateless primitives, which are safe to share. +inline std::shared_ptr CopyParameterizedPrimitive(const Type& type) { + switch (type.type_id()) { + case TypeId::kDecimal: + return std::make_shared(checked_cast(type)); + case TypeId::kFixed: + return std::make_shared(checked_cast(type)); + case TypeId::kGeometry: + return std::make_shared(checked_cast(type)); + case TypeId::kGeography: + return std::make_shared(checked_cast(type)); + default: + return nullptr; + } +} + +inline Result CopyUpdateLiteral(const Literal& literal) { + auto type = CopyParameterizedPrimitive(*literal.type()); + if (!type) { + return literal; + } + if (literal.IsNull()) { + return Literal::Null(std::move(type)); + } + ICEBERG_ASSIGN_OR_RAISE(auto bytes, literal.Serialize()); + return Literal::Deserialize(bytes, std::move(type)); +} + +inline Result> CopyUpdateType(const std::shared_ptr& type); + +// Deep-copies a field, including nested types and default literals, so a caller +// keeping an alias to any of those objects cannot change frozen replay intent. +inline Result CopyUpdateField(const SchemaField& field) { + ICEBERG_ASSIGN_OR_RAISE(auto type, CopyUpdateType(field.type())); + std::shared_ptr initial_default; + if (field.initial_default()) { + ICEBERG_ASSIGN_OR_RAISE(auto copy, CopyUpdateLiteral(*field.initial_default())); + initial_default = std::make_shared(std::move(copy)); + } + std::shared_ptr write_default; + if (field.write_default()) { + ICEBERG_ASSIGN_OR_RAISE(auto copy, CopyUpdateLiteral(*field.write_default())); + write_default = std::make_shared(std::move(copy)); + } + return SchemaField(field.field_id(), field.name(), std::move(type), field.optional(), + field.doc(), std::move(initial_default), std::move(write_default)); +} + +inline Result> CopyUpdateType(const std::shared_ptr& type) { + if (!type) { + return nullptr; + } + switch (type->type_id()) { + case TypeId::kStruct: { + const auto source_fields = checked_cast(*type).fields(); + std::vector fields; + fields.reserve(source_fields.size()); + for (const auto& field : source_fields) { + ICEBERG_ASSIGN_OR_RAISE(auto copy, CopyUpdateField(field)); + fields.push_back(std::move(copy)); + } + return std::make_shared(std::move(fields)); + } + case TypeId::kList: { + ICEBERG_ASSIGN_OR_RAISE( + auto element, CopyUpdateField(checked_cast(*type).element())); + return std::make_shared(std::move(element)); + } + case TypeId::kMap: { + const auto& map = checked_cast(*type); + ICEBERG_ASSIGN_OR_RAISE(auto key, CopyUpdateField(map.key())); + ICEBERG_ASSIGN_OR_RAISE(auto value, CopyUpdateField(map.value())); + return std::make_shared(std::move(key), std::move(value)); + } + default: + if (auto copy = CopyParameterizedPrimitive(*type)) { + return copy; + } + return type; + } +} + +inline Result> CopyUpdateDataFile(const DataFile& file) { + auto copy = std::make_shared(file); + std::vector values; + values.reserve(file.partition.num_fields()); + for (const auto& value : file.partition.values()) { + ICEBERG_ASSIGN_OR_RAISE(auto frozen, CopyUpdateLiteral(value)); + values.push_back(std::move(frozen)); + } + copy->partition.Reset(std::move(values)); + return copy; +} + +inline Result> CopyUpdateExpression( + const std::shared_ptr& expression) { + if (!expression) { + return nullptr; + } + using Op = Expression::Operation; + switch (expression->op()) { + case Op::kTrue: + return True::Instance(); + case Op::kFalse: + return False::Instance(); + case Op::kAnd: { + const auto& node = checked_cast(*expression); + ICEBERG_ASSIGN_OR_RAISE(auto left, CopyUpdateExpression(node.left())); + ICEBERG_ASSIGN_OR_RAISE(auto right, CopyUpdateExpression(node.right())); + return And::Make(std::move(left), std::move(right)); + } + case Op::kOr: { + const auto& node = checked_cast(*expression); + ICEBERG_ASSIGN_OR_RAISE(auto left, CopyUpdateExpression(node.left())); + ICEBERG_ASSIGN_OR_RAISE(auto right, CopyUpdateExpression(node.right())); + return Or::Make(std::move(left), std::move(right)); + } + case Op::kNot: { + const auto& node = checked_cast(*expression); + ICEBERG_ASSIGN_OR_RAISE(auto child, CopyUpdateExpression(node.child())); + return Not::Make(std::move(child)); + } + default: + break; + } + if (!expression->is_unbound_predicate()) { + // Preserve the normal validation path for already-bound/invalid filters. + return expression; + } + // Expression is a virtual base of UnboundPredicate, so this cast must stay + // dynamic in release builds too. + const auto& predicate = dynamic_cast(*expression); + std::vector values; + for (const auto& value : predicate.literals()) { + ICEBERG_ASSIGN_OR_RAISE(auto copy, CopyUpdateLiteral(value)); + values.push_back(std::move(copy)); + } + const auto& term = predicate.unbound_term(); + if (term.kind() == Term::Kind::kReference) { + const auto& reference = checked_cast(term); + ICEBERG_ASSIGN_OR_RAISE(auto copy, + NamedReference::Make(std::string(reference.name()))); + return UnboundPredicateImpl::Make(expression->op(), std::move(copy), + std::move(values)); + } + const auto& transform = checked_cast(term); + ICEBERG_ASSIGN_OR_RAISE( + auto reference, NamedReference::Make(std::string(transform.reference()->name()))); + ICEBERG_ASSIGN_OR_RAISE( + auto copy, + UnboundTransform::Make(std::move(reference), + std::make_shared(*transform.transform()))); + return UnboundPredicateImpl::Make(expression->op(), std::move(copy), + std::move(values)); +} + +} // namespace iceberg::internal