From 5c7e64bea4ba5081cfea52788e4333d219dbec72 Mon Sep 17 00:00:00 2001 From: mats Date: Thu, 23 Apr 2026 16:56:44 +0900 Subject: [PATCH 01/25] build: add ZeroBus output plugin build configuration Signed-off-by: mats --- CMakeLists.txt | 6 ++ cmake/plugins_options.cmake | 1 + cmake/zerobus-ffi.cmake | 122 ++++++++++++++++++++++++++++++++++++ plugins/CMakeLists.txt | 1 + 4 files changed, 130 insertions(+) create mode 100644 cmake/zerobus-ffi.cmake diff --git a/CMakeLists.txt b/CMakeLists.txt index 5ffc6271e3c..626336ca0eb 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -1370,6 +1370,12 @@ if(FLB_OUT_PGSQL AND (NOT PostgreSQL_FOUND)) FLB_OPTION(FLB_OUT_PGSQL OFF) endif() +# ZeroBus FFI +# =========== +if(FLB_OUT_ZEROBUS) + include(cmake/zerobus-ffi.cmake) +endif() + # Arrow GLib # ========== find_package(PkgConfig) diff --git a/cmake/plugins_options.cmake b/cmake/plugins_options.cmake index aa8bfd05bda..25cf6de7dd0 100644 --- a/cmake/plugins_options.cmake +++ b/cmake/plugins_options.cmake @@ -156,3 +156,4 @@ DEFINE_OPTION(FLB_OUT_TCP "Enable TCP output plugin" DEFINE_OPTION(FLB_OUT_UDP "Enable UDP output plugin" ON) DEFINE_OPTION(FLB_OUT_VIVO_EXPORTER "Enable Vivo exporter output plugin" ON) DEFINE_OPTION(FLB_OUT_WEBSOCKET "Enable Websocket output plugin" ON) +DEFINE_OPTION(FLB_OUT_ZEROBUS "Enable Databricks ZeroBus output plugin" ON) diff --git a/cmake/zerobus-ffi.cmake b/cmake/zerobus-ffi.cmake new file mode 100644 index 00000000000..442e184a0a0 --- /dev/null +++ b/cmake/zerobus-ffi.cmake @@ -0,0 +1,122 @@ +# Set up the ZeroBus FFI prebuilt static library. +# +# If ZEROBUS_LIB_DIR is already set by the user, that path is used as-is. +# Otherwise the official release tarball is downloaded and the correct +# platform subdirectory is selected automatically. +# +# On unsupported platforms or when the download fails, the plugin is +# disabled automatically (FLB_OUT_ZEROBUS is set to OFF). +# +# After this module runs: +# ZEROBUS_LIB_DIR — directory containing the static library +# ZEROBUS_LIB_FILE — full path to the static library + +set(_ZEROBUS_LIB_FILENAME + "${CMAKE_STATIC_LIBRARY_PREFIX}zerobus_ffi${CMAKE_STATIC_LIBRARY_SUFFIX}") + +if(ZEROBUS_LIB_DIR) + set(ZEROBUS_LIB_FILE + "${ZEROBUS_LIB_DIR}/${_ZEROBUS_LIB_FILENAME}" + CACHE FILEPATH "Full path to ZeroBus FFI static library" FORCE) + if(NOT EXISTS "${ZEROBUS_LIB_FILE}") + message(STATUS + "ZeroBus FFI: library not found at ${ZEROBUS_LIB_FILE}, " + "disabling out_zerobus.") + unset(ZEROBUS_LIB_FILE CACHE) + FLB_OPTION(FLB_OUT_ZEROBUS OFF) + endif() + return() +endif() + +set(_ZEROBUS_URL + "https://github.com/databricks/zerobus-sdk/releases/download/ffi-v1.0.0/zerobus-ffi-1.0.0.tar.gz") +set(_ZEROBUS_SHA256 + "c38609f5bddc160b43b35f9047919b35f66375308be69a0d0d6cd20bc01cee5a") + +# Determine the platform subdirectory inside the tarball +if(CMAKE_SYSTEM_NAME STREQUAL "Linux") + if(CMAKE_SYSTEM_PROCESSOR MATCHES "^(aarch64|arm64|ARM64|AARCH64)$") + set(_ZEROBUS_PLATFORM "linux-aarch64") + elseif(CMAKE_SYSTEM_PROCESSOR MATCHES "^(x86_64|AMD64)$") + set(_ZEROBUS_PLATFORM "linux-x86-64") + else() + message(STATUS + "ZeroBus FFI: unsupported Linux architecture '${CMAKE_SYSTEM_PROCESSOR}', " + "disabling out_zerobus. " + "To build manually, set -DZEROBUS_LIB_DIR=/path/to/lib.") + FLB_OPTION(FLB_OUT_ZEROBUS OFF) + return() + endif() +elseif(CMAKE_SYSTEM_NAME STREQUAL "Windows") + if(CMAKE_SYSTEM_PROCESSOR MATCHES "^(x86_64|AMD64)$") + set(_ZEROBUS_PLATFORM "windows-x86-64") + else() + message(STATUS + "ZeroBus FFI: unsupported Windows architecture '${CMAKE_SYSTEM_PROCESSOR}', " + "disabling out_zerobus. " + "To build manually, set -DZEROBUS_LIB_DIR=/path/to/lib.") + FLB_OPTION(FLB_OUT_ZEROBUS OFF) + return() + endif() +else() + message(STATUS + "ZeroBus FFI: no prebuilt library available for ${CMAKE_SYSTEM_NAME}, " + "disabling out_zerobus. " + "To build manually, set -DZEROBUS_LIB_DIR=/path/to/lib.") + FLB_OPTION(FLB_OUT_ZEROBUS OFF) + return() +endif() + +# Download the tarball if not already cached +set(_ZEROBUS_TARBALL "${CMAKE_BINARY_DIR}/zerobus-ffi-1.0.0.tar.gz") +if(NOT EXISTS "${_ZEROBUS_TARBALL}") + message(STATUS "ZeroBus FFI: downloading ${_ZEROBUS_URL}") + file(DOWNLOAD + "${_ZEROBUS_URL}" + "${_ZEROBUS_TARBALL}" + EXPECTED_HASH "SHA256=${_ZEROBUS_SHA256}" + SHOW_PROGRESS + STATUS _DOWNLOAD_STATUS + ) + list(GET _DOWNLOAD_STATUS 0 _DOWNLOAD_ERROR) + if(_DOWNLOAD_ERROR) + message(STATUS + "ZeroBus FFI: download failed (${_DOWNLOAD_STATUS}), " + "disabling out_zerobus. " + "To build manually, set -DZEROBUS_LIB_DIR=/path/to/lib.") + file(REMOVE "${_ZEROBUS_TARBALL}") + FLB_OPTION(FLB_OUT_ZEROBUS OFF) + return() + endif() +endif() + +# Extract the tarball +set(_ZEROBUS_EXTRACT_DIR "${CMAKE_BINARY_DIR}") +if(NOT EXISTS "${_ZEROBUS_EXTRACT_DIR}/native/${_ZEROBUS_PLATFORM}/${_ZEROBUS_LIB_FILENAME}") + execute_process( + COMMAND ${CMAKE_COMMAND} -E tar xzf "${_ZEROBUS_TARBALL}" + WORKING_DIRECTORY "${CMAKE_BINARY_DIR}" + RESULT_VARIABLE _EXTRACT_RESULT + ) + if(_EXTRACT_RESULT) + message(STATUS + "ZeroBus FFI: extraction failed, disabling out_zerobus.") + FLB_OPTION(FLB_OUT_ZEROBUS OFF) + return() + endif() +endif() + +set(ZEROBUS_LIB_DIR "${_ZEROBUS_EXTRACT_DIR}/native/${_ZEROBUS_PLATFORM}" + CACHE PATH "Path to ZeroBus FFI library directory" FORCE) +set(ZEROBUS_LIB_FILE "${ZEROBUS_LIB_DIR}/${_ZEROBUS_LIB_FILENAME}" + CACHE FILEPATH "Full path to ZeroBus FFI static library" FORCE) + +if(NOT EXISTS "${ZEROBUS_LIB_FILE}") + message(STATUS + "ZeroBus FFI: ${_ZEROBUS_LIB_FILENAME} not found at ${ZEROBUS_LIB_DIR}, " + "disabling out_zerobus.") + FLB_OPTION(FLB_OUT_ZEROBUS OFF) + return() +endif() + +message(STATUS "ZeroBus FFI library: ${ZEROBUS_LIB_FILE}") diff --git a/plugins/CMakeLists.txt b/plugins/CMakeLists.txt index c885f51fda3..01133ec5709 100644 --- a/plugins/CMakeLists.txt +++ b/plugins/CMakeLists.txt @@ -414,6 +414,7 @@ REGISTER_OUT_PLUGIN("out_prometheus_remote_write") REGISTER_OUT_PLUGIN("out_s3") REGISTER_OUT_PLUGIN("out_vivo_exporter") REGISTER_OUT_PLUGIN("out_chronicle") +REGISTER_OUT_PLUGIN("out_zerobus") if(FLB_ZIG) REGISTER_OUT_PLUGIN("out_zig_demo" "zig") From ff1a4dd7d135bd518b49c176cbf3f56642dc7726 Mon Sep 17 00:00:00 2001 From: mats Date: Thu, 23 Apr 2026 16:56:50 +0900 Subject: [PATCH 02/25] out_zerobus: add ZeroBus output plugin implementation Signed-off-by: mats --- plugins/out_zerobus/CMakeLists.txt | 28 ++ plugins/out_zerobus/zerobus.c | 623 +++++++++++++++++++++++++++++ plugins/out_zerobus/zerobus.h | 132 ++++++ 3 files changed, 783 insertions(+) create mode 100644 plugins/out_zerobus/CMakeLists.txt create mode 100644 plugins/out_zerobus/zerobus.c create mode 100644 plugins/out_zerobus/zerobus.h diff --git a/plugins/out_zerobus/CMakeLists.txt b/plugins/out_zerobus/CMakeLists.txt new file mode 100644 index 00000000000..c10a95a9fd0 --- /dev/null +++ b/plugins/out_zerobus/CMakeLists.txt @@ -0,0 +1,28 @@ +set(src + zerobus.c) + +FLB_PLUGIN(out_zerobus "${src}" "") + +# ZEROBUS_LIB_FILE is set automatically by cmake/zerobus-ffi.cmake or +# can be overridden by the user via -DZEROBUS_LIB_DIR=/path/to/lib. +if(NOT ZEROBUS_LIB_FILE) + message(FATAL_ERROR + "ZEROBUS_LIB_FILE is not set. This should not happen when " + "FLB_OUT_ZEROBUS is ON — check that cmake/zerobus-ffi.cmake is included.") +endif() + +target_link_libraries(flb-plugin-out_zerobus "${ZEROBUS_LIB_FILE}") + +# Platform-specific linker flags required by the Rust FFI static library +if(WIN32) + target_link_libraries(flb-plugin-out_zerobus + ws2_32 ntdll userenv advapi32 bcrypt) +elseif(APPLE) + target_link_libraries(flb-plugin-out_zerobus + "-framework CoreFoundation" + "-framework Security" + -liconv) +elseif(UNIX) + target_link_libraries(flb-plugin-out_zerobus + -ldl -lpthread -lm -lresolv -lgcc_s) +endif() diff --git a/plugins/out_zerobus/zerobus.c b/plugins/out_zerobus/zerobus.c new file mode 100644 index 00000000000..3ea5ba2de5d --- /dev/null +++ b/plugins/out_zerobus/zerobus.c @@ -0,0 +1,623 @@ +/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ + +/* Fluent Bit + * ========== + * Copyright (C) 2015-2026 The Fluent Bit Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include + +#include "zerobus.h" + +/* + * Prepend "https://" to url if no scheme is present. + * Returns a newly allocated sds string; caller must flb_sds_destroy it. + * Returns NULL on allocation failure. + */ +static flb_sds_t ensure_url_scheme(const char *url) +{ + size_t url_len; + flb_sds_t out; + flb_sds_t tmp; + + if (strncmp(url, "https://", 8) == 0 || + strncmp(url, "http://", 7) == 0) { + return flb_sds_create(url); + } + + url_len = strlen(url); + out = flb_sds_create_size(url_len + 9); + if (!out) { + return NULL; + } + tmp = flb_sds_cat(out, "https://", 8); + if (!tmp) { + flb_sds_destroy(out); + return NULL; + } + out = tmp; + tmp = flb_sds_cat(out, url, url_len); + if (!tmp) { + flb_sds_destroy(out); + return NULL; + } + return tmp; +} + +/* + * Format tm as an RFC 3339 timestamp with nanosecond precision into buf. + * Returns the number of characters written (excluding the null terminator), + * or -1 if gmtime_r fails. + */ +static int format_timestamp_rfc3339nano(struct flb_time *tm, + char *buf, size_t size) +{ + struct tm gmt; + time_t sec = (time_t) tm->tm.tv_sec; + + if (!gmtime_r(&sec, &gmt)) { + return -1; + } + return snprintf(buf, size, + "%04d-%02d-%02dT%02d:%02d:%02d.%09luZ", + gmt.tm_year + 1900, gmt.tm_mon + 1, gmt.tm_mday, + gmt.tm_hour, gmt.tm_min, gmt.tm_sec, + (unsigned long) tm->tm.tv_nsec); +} + +/* + * Return 1 if key (with length key_len) matches any entry in log_keys, + * 0 otherwise. + */ +static int key_in_log_keys(const char *key, int key_len, + struct mk_list *log_keys) +{ + struct mk_list *head; + struct flb_slist_entry *entry; + + mk_list_foreach(head, log_keys) { + entry = mk_list_entry(head, struct flb_slist_entry, _head); + if ((int) flb_sds_len(entry->str) == key_len && + memcmp(entry->str, key, key_len) == 0) { + return 1; + } + } + return 0; +} + +/* + * Return 1 if msgpack object k is a string of length name_len equal to name, + * 0 otherwise. + */ +static inline int str_key_equals(const msgpack_object *k, + const char *name, int name_len) +{ + return k->type == MSGPACK_OBJECT_STR && + (int) k->via.str.size == name_len && + memcmp(k->via.str.ptr, name, name_len) == 0; +} + +/* + * Log an error from a failed CResult and free its error_message. + * Returns FLB_RETRY if is_retryable, FLB_ERROR otherwise. + */ +static int log_cresult_error(struct flb_output_instance *ins, + CResult *r, const char *context) +{ + int ret = r->is_retryable ? FLB_RETRY : FLB_ERROR; + + flb_plg_error(ins, "%s: %s", + context, + r->error_message ? r->error_message : "unknown"); + if (r->error_message) { + zerobus_free_error_message(r->error_message); + r->error_message = NULL; + } + return ret; +} + +/* + * Convert a log event body to a JSON string for ZeroBus ingestion. + * + * Matches the Go plugin's recordToJSON: applies log_keys filter, then + * injects raw_log_key (full pre-filter record), time_key, and _tag + * without overwriting existing keys. + * + * Uses flb_mp_map_header for single-pass packing (no pre-counting). + * The caller-owned msgpack_sbuffer is reused across records. + */ +static flb_sds_t record_to_json(struct flb_out_zerobus *ctx, + msgpack_object *body, + struct flb_time *tm, + const char *tag, int tag_len, + msgpack_sbuffer *sbuf, + int escape_unicode) +{ + int i; + int has_time_key = 0; + int has_tag_key = 0; + int has_raw_key = 0; + int time_key_len; + int raw_key_len; + int include; + char *raw_json = NULL; + char time_buf[64]; + msgpack_packer pk; + struct flb_mp_map_header mh; + flb_sds_t json; + + if (body->type != MSGPACK_OBJECT_MAP) { + return NULL; + } + + msgpack_object_map *map = &body->via.map; + time_key_len = (ctx->time_key) ? (int) flb_sds_len(ctx->time_key) : 0; + raw_key_len = (ctx->raw_log_key) ? (int) flb_sds_len(ctx->raw_log_key) : 0; + + msgpack_sbuffer_clear(sbuf); + msgpack_packer_init(&pk, sbuf, msgpack_sbuffer_write); + flb_mp_map_header_init(&mh, &pk); + + /* Single pass: pack included body keys, track collision flags */ + for (i = 0; i < (int) map->size; i++) { + msgpack_object *k = &map->ptr[i].key; + + if (ctx->log_keys) { + if (k->type != MSGPACK_OBJECT_STR) { + continue; + } + include = key_in_log_keys(k->via.str.ptr, + (int) k->via.str.size, + ctx->log_keys); + if (!include) { + continue; + } + } + + flb_mp_map_header_append(&mh); + msgpack_pack_object(&pk, map->ptr[i].key); + msgpack_pack_object(&pk, map->ptr[i].val); + + if (k->type == MSGPACK_OBJECT_STR) { + if (time_key_len > 0 && + str_key_equals(k, ctx->time_key, time_key_len)) { + has_time_key = 1; + } + if (ctx->add_tag && str_key_equals(k, "_tag", 4)) { + has_tag_key = 1; + } + if (raw_key_len > 0 && + str_key_equals(k, ctx->raw_log_key, raw_key_len)) { + has_raw_key = 1; + } + } + } + + if (raw_key_len > 0 && !has_raw_key) { + size_t rj_len; + + /* + * Serialize the original (pre-filter) body only when the key is + * absent. Deferring to here avoids a full serialize+discard on + * every record that already carries the field. + * body is unchanged by the loop above, so the result is identical + * to capturing it before filtering (matching Go's json.Marshal(m)). + */ + raw_json = flb_msgpack_to_json_str(0, body, escape_unicode); + if (!raw_json) { + return NULL; + } + rj_len = strlen(raw_json); + + flb_mp_map_header_append(&mh); + msgpack_pack_str(&pk, raw_key_len); + msgpack_pack_str_body(&pk, ctx->raw_log_key, raw_key_len); + msgpack_pack_str(&pk, rj_len); + msgpack_pack_str_body(&pk, raw_json, rj_len); + } + + if (time_key_len > 0 && !has_time_key) { + int time_len = format_timestamp_rfc3339nano(tm, time_buf, sizeof(time_buf)); + if (time_len > 0) { + flb_mp_map_header_append(&mh); + msgpack_pack_str(&pk, time_key_len); + msgpack_pack_str_body(&pk, ctx->time_key, time_key_len); + msgpack_pack_str(&pk, time_len); + msgpack_pack_str_body(&pk, time_buf, time_len); + } + } + + if (ctx->add_tag && tag_len > 0 && !has_tag_key) { + flb_mp_map_header_append(&mh); + msgpack_pack_str(&pk, 4); + msgpack_pack_str_body(&pk, "_tag", 4); + msgpack_pack_str(&pk, tag_len); + msgpack_pack_str_body(&pk, tag, tag_len); + } + + flb_mp_map_header_end(&mh); + + json = flb_msgpack_raw_to_json_sds(sbuf->data, sbuf->size, escape_unicode); + + if (raw_json) { + flb_free(raw_json); + } + + return json; +} + +/* + * Plugin init callback: validate required config, then create the ZeroBus + * SDK handle and stream. Returns 0 on success, -1 on failure. + */ +static int cb_zerobus_init(struct flb_output_instance *ins, + struct flb_config *config, + void *data) +{ + int ret; + const char *tmp; + struct flb_out_zerobus *ctx; + CResult result; + CStreamConfigurationOptions opts; + + (void) config; + (void) data; + + ctx = flb_calloc(1, sizeof(struct flb_out_zerobus)); + if (!ctx) { + flb_errno(); + return -1; + } + ctx->ins = ins; + + ret = flb_output_config_map_set(ins, (void *) ctx); + if (ret == -1) { + flb_free(ctx); + return -1; + } + + /* + * Both endpoint and workspace_url get https:// prepended when no + * scheme is present. + */ + tmp = flb_output_get_property("endpoint", ins); + if (!tmp || strlen(tmp) == 0) { + flb_plg_error(ins, "'endpoint' is required"); + goto init_error; + } + ctx->endpoint = ensure_url_scheme(tmp); + if (!ctx->endpoint) { + goto init_error; + } + + tmp = flb_output_get_property("workspace_url", ins); + if (!tmp || strlen(tmp) == 0) { + flb_plg_error(ins, "'workspace_url' is required"); + goto init_error; + } + ctx->workspace_url = ensure_url_scheme(tmp); + if (!ctx->workspace_url) { + goto init_error; + } + + if (!ctx->table_name || flb_sds_len(ctx->table_name) == 0) { + flb_plg_error(ins, "'table_name' is required"); + goto init_error; + } + if (!ctx->client_id || flb_sds_len(ctx->client_id) == 0) { + flb_plg_error(ins, "'client_id' is required"); + goto init_error; + } + if (!ctx->client_secret || flb_sds_len(ctx->client_secret) == 0) { + flb_plg_error(ins, "'client_secret' is required"); + goto init_error; + } + + memset(&result, 0, sizeof(result)); + ctx->sdk = zerobus_sdk_new(ctx->endpoint, + ctx->workspace_url, + &result); + if (!ctx->sdk || !result.success) { + log_cresult_error(ins, &result, "failed to create ZeroBus SDK"); + goto init_error; + } + + if (strncmp(ctx->endpoint, "http://", 7) == 0) { + zerobus_sdk_set_use_tls(ctx->sdk, false); + } + + opts = zerobus_get_default_config(); + opts.record_type = ZEROBUS_RECORD_TYPE_JSON; + + memset(&result, 0, sizeof(result)); + ctx->stream = zerobus_sdk_create_stream(ctx->sdk, + ctx->table_name, + NULL, 0, + ctx->client_id, + ctx->client_secret, + &opts, + &result); + if (!ctx->stream || !result.success) { + log_cresult_error(ins, &result, "failed to create ZeroBus stream"); + zerobus_sdk_free(ctx->sdk); + ctx->sdk = NULL; + goto init_error; + } + + flb_plg_info(ins, "connected to %s, table: %s", + ctx->endpoint, ctx->table_name); + + flb_output_set_context(ins, ctx); + return 0; + +init_error: + if (ctx->endpoint) { + flb_sds_destroy(ctx->endpoint); + } + if (ctx->workspace_url) { + flb_sds_destroy(ctx->workspace_url); + } + flb_free(ctx); + return -1; +} + +/* + * Plugin flush callback: decode incoming log events, convert each to JSON, + * and ingest the batch via ZeroBus. Waits for server-side acknowledgment + * before returning. Returns FLB_OK, FLB_RETRY, or FLB_ERROR via + * FLB_OUTPUT_RETURN. + */ +static void cb_zerobus_flush(struct flb_event_chunk *event_chunk, + struct flb_output_flush *out_flush, + struct flb_input_instance *i_ins, + void *out_context, + struct flb_config *config) +{ + int ret; + size_t capacity; + size_t num_records = 0; + int convert_errors = 0; + struct flb_out_zerobus *ctx = out_context; + struct flb_log_event_decoder log_decoder; + struct flb_log_event log_event; + flb_sds_t *json_records = NULL; + flb_sds_t json; + msgpack_sbuffer sbuf; + CResult result; + int64_t offset; + size_t i; + int tag_len; + + (void) i_ins; + + tag_len = event_chunk->tag ? (int) flb_sds_len(event_chunk->tag) : 0; + + ret = flb_log_event_decoder_init(&log_decoder, + (char *) event_chunk->data, + event_chunk->size); + if (ret != FLB_EVENT_DECODER_SUCCESS) { + flb_plg_error(ctx->ins, + "log event decoder initialization error: %d", ret); + FLB_OUTPUT_RETURN(FLB_RETRY); + } + + capacity = event_chunk->total_events > 0 ? event_chunk->total_events : 64; + json_records = flb_malloc(sizeof(flb_sds_t) * capacity); + if (!json_records) { + flb_log_event_decoder_destroy(&log_decoder); + FLB_OUTPUT_RETURN(FLB_RETRY); + } + + /* Reuse a single sbuffer across all record conversions */ + msgpack_sbuffer_init(&sbuf); + + while (flb_log_event_decoder_next(&log_decoder, + &log_event) == FLB_EVENT_DECODER_SUCCESS) { + json = record_to_json(ctx, + log_event.body, + &log_event.timestamp, + event_chunk->tag, tag_len, + &sbuf, + config->json_escape_unicode); + if (!json) { + convert_errors++; + flb_plg_warn(ctx->ins, "failed to convert record to JSON"); + continue; + } + + if (num_records == capacity) { + size_t new_cap = capacity * 2; + flb_sds_t *tmp = flb_realloc(json_records, + sizeof(flb_sds_t) * new_cap); + if (!tmp) { + flb_plg_error(ctx->ins, + "realloc failed, retrying entire batch"); + flb_sds_destroy(json); + for (i = 0; i < num_records; i++) { + flb_sds_destroy(json_records[i]); + } + flb_free(json_records); + msgpack_sbuffer_destroy(&sbuf); + flb_log_event_decoder_destroy(&log_decoder); + FLB_OUTPUT_RETURN(FLB_RETRY); + } + json_records = tmp; + capacity = new_cap; + } + + json_records[num_records] = json; + num_records++; + } + + msgpack_sbuffer_destroy(&sbuf); + flb_log_event_decoder_destroy(&log_decoder); + + if (num_records == 0) { + flb_free(json_records); + if (convert_errors > 0) { + flb_plg_error(ctx->ins, + "all %d records failed conversion", convert_errors); + FLB_OUTPUT_RETURN(FLB_ERROR); + } + FLB_OUTPUT_RETURN(FLB_OK); + } + + if (convert_errors > 0) { + flb_plg_warn(ctx->ins, + "skipped %d records due to conversion errors", + convert_errors); + } + + /* flb_sds_t is char*, so the cast to const char** is safe */ + memset(&result, 0, sizeof(result)); + offset = zerobus_stream_ingest_json_records(ctx->stream, + (const char **) json_records, + num_records, + &result); + if (!result.success) { + ret = log_cresult_error(ctx->ins, &result, "ingestion error"); + goto flush_cleanup; + } + + memset(&result, 0, sizeof(result)); + zerobus_stream_wait_for_offset(ctx->stream, offset, &result); + if (!result.success) { + ret = log_cresult_error(ctx->ins, &result, "wait_for_offset error"); + goto flush_cleanup; + } + + ret = FLB_OK; + +flush_cleanup: + for (i = 0; i < num_records; i++) { + flb_sds_destroy(json_records[i]); + } + flb_free(json_records); + + FLB_OUTPUT_RETURN(ret); +} + +/* + * Plugin exit callback: close the ZeroBus stream, free the SDK handle, + * and release the plugin context. Returns 0. + */ +static int cb_zerobus_exit(void *data, struct flb_config *config) +{ + struct flb_out_zerobus *ctx = data; + CResult result; + + (void) config; + + if (!ctx) { + return 0; + } + + if (ctx->stream) { + memset(&result, 0, sizeof(result)); + zerobus_stream_close(ctx->stream, &result); + if (!result.success && result.error_message) { + flb_plg_error(ctx->ins, "stream close error: %s", + result.error_message); + zerobus_free_error_message(result.error_message); + } + zerobus_stream_free(ctx->stream); + } + + if (ctx->sdk) { + zerobus_sdk_free(ctx->sdk); + } + + if (ctx->endpoint) { + flb_sds_destroy(ctx->endpoint); + } + if (ctx->workspace_url) { + flb_sds_destroy(ctx->workspace_url); + } + + flb_free(ctx); + return 0; +} + +static struct flb_config_map config_map[] = { + { + FLB_CONFIG_MAP_STR, "endpoint", NULL, + 0, FLB_FALSE, 0, + "ZeroBus gRPC endpoint URL (https:// prepended if no scheme)" + }, + { + FLB_CONFIG_MAP_STR, "workspace_url", NULL, + 0, FLB_FALSE, 0, + "Databricks workspace URL" + }, + { + FLB_CONFIG_MAP_STR, "table_name", NULL, + 0, FLB_TRUE, offsetof(struct flb_out_zerobus, table_name), + "Fully qualified table name (catalog.schema.table)" + }, + { + FLB_CONFIG_MAP_STR, "client_id", NULL, + 0, FLB_TRUE, offsetof(struct flb_out_zerobus, client_id), + "OAuth2 client ID for authentication" + }, + { + FLB_CONFIG_MAP_STR, "client_secret", NULL, + 0, FLB_TRUE, offsetof(struct flb_out_zerobus, client_secret), + "OAuth2 client secret for authentication" + }, + { + FLB_CONFIG_MAP_BOOL, "add_tag", "true", + 0, FLB_TRUE, offsetof(struct flb_out_zerobus, add_tag), + "Add Fluent Bit tag as _tag field in each record" + }, + { + FLB_CONFIG_MAP_STR, "time_key", "_time", + 0, FLB_TRUE, offsetof(struct flb_out_zerobus, time_key), + "Key name for the injected timestamp (RFC 3339 with nanoseconds)" + }, + { + FLB_CONFIG_MAP_CLIST, "log_key", NULL, + 0, FLB_TRUE, offsetof(struct flb_out_zerobus, log_keys), + "Comma-separated list of record keys to include (all if unset)" + }, + { + FLB_CONFIG_MAP_STR, "raw_log_key", NULL, + 0, FLB_TRUE, offsetof(struct flb_out_zerobus, raw_log_key), + "If set, store the full original record as a JSON string under this key" + }, + {0} +}; + +struct flb_output_plugin out_zerobus_plugin = { + .name = "zerobus", + .description = "Send logs to Databricks ZeroBus", + .cb_init = cb_zerobus_init, + .cb_flush = cb_zerobus_flush, + .cb_exit = cb_zerobus_exit, + .config_map = config_map, + .event_type = FLB_OUTPUT_LOGS, + .flags = 0, + .workers = 1, +}; diff --git a/plugins/out_zerobus/zerobus.h b/plugins/out_zerobus/zerobus.h new file mode 100644 index 00000000000..bc67b81fdcb --- /dev/null +++ b/plugins/out_zerobus/zerobus.h @@ -0,0 +1,132 @@ +/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ + +/* Fluent Bit + * ========== + * Copyright (C) 2015-2026 The Fluent Bit Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef FLB_OUT_ZEROBUS_H +#define FLB_OUT_ZEROBUS_H + +#include +#include +#include + +#include +#include + +/* + * ZeroBus FFI declarations + * + * These types and functions are provided by the prebuilt Rust FFI static + * library (libzerobus_ffi.a). The declarations below are extracted from + * the Go SDK CGO preamble at: + * github.com/databricks/zerobus-sdk/go@v1.0.0/ffi.go + */ + +/* Opaque SDK / stream handles */ +typedef struct CZerobusSdk CZerobusSdk; +typedef struct CZerobusStream CZerobusStream; + +/* Result returned by every fallible FFI call */ +typedef struct CResult { + bool success; + char *error_message; + bool is_retryable; +} CResult; + +/* Stream configuration passed to create_stream */ +typedef struct CStreamConfigurationOptions { + uintptr_t max_inflight_requests; + bool recovery; + uint64_t recovery_timeout_ms; + uint64_t recovery_backoff_ms; + uint32_t recovery_retries; + uint64_t server_lack_of_ack_timeout_ms; + uint64_t flush_timeout_ms; + int32_t record_type; + uint64_t stream_paused_max_wait_time_ms; + bool has_stream_paused_max_wait_time_ms; + uint64_t callback_max_wait_time_ms; + bool has_callback_max_wait_time_ms; +} CStreamConfigurationOptions; + +/* Record type enum values */ +#define ZEROBUS_RECORD_TYPE_JSON 2 + +/* --- SDK lifecycle --- */ +extern CZerobusSdk *zerobus_sdk_new(const char *endpoint, + const char *unity_catalog_url, + CResult *result); +extern void zerobus_sdk_free(CZerobusSdk *sdk); +extern void zerobus_sdk_set_use_tls(CZerobusSdk *sdk, bool use_tls); + +/* --- Stream lifecycle --- */ +extern CZerobusStream *zerobus_sdk_create_stream( + CZerobusSdk *sdk, + const char *table_name, + const uint8_t *descriptor_proto_bytes, + uintptr_t descriptor_proto_len, + const char *client_id, + const char *client_secret, + const CStreamConfigurationOptions *options, + CResult *result); + +extern bool zerobus_stream_close(CZerobusStream *stream, CResult *result); +extern void zerobus_stream_free(CZerobusStream *stream); + +/* --- Ingestion --- */ +extern int64_t zerobus_stream_ingest_json_records( + CZerobusStream *stream, + const char **json_records, + uintptr_t num_records, + CResult *result); + +extern bool zerobus_stream_wait_for_offset(CZerobusStream *stream, + int64_t offset, + CResult *result); + +/* --- Utilities --- */ +extern void zerobus_free_error_message(char *error_message); +extern CStreamConfigurationOptions zerobus_get_default_config(void); + +/* ------------------------------------------------------------------ */ + +/* Plugin context */ +struct flb_out_zerobus { + /* ZeroBus handles */ + CZerobusSdk *sdk; + CZerobusStream *stream; + + /* Required config -- URL fields are read manually */ + flb_sds_t endpoint; /* https:// auto-prepended if missing */ + flb_sds_t workspace_url; /* https:// auto-prepended if missing */ + + /* Required config -- auto-populated by config_map */ + flb_sds_t table_name; + flb_sds_t client_id; + flb_sds_t client_secret; + + /* Optional config -- auto-populated by config_map */ + int add_tag; /* FLB_TRUE / FLB_FALSE */ + flb_sds_t time_key; /* default "_time" */ + struct mk_list *log_keys; /* CLIST, NULL when unset */ + flb_sds_t raw_log_key; /* NULL when unset */ + + /* Fluent Bit instance reference (used for logging macros) */ + struct flb_output_instance *ins; +}; + +#endif /* FLB_OUT_ZEROBUS_H */ From cb9465c37bd44e010ef8287176738649fd5e6794 Mon Sep 17 00:00:00 2001 From: mats Date: Thu, 23 Apr 2026 17:04:47 +0900 Subject: [PATCH 03/25] build: update Windows architecture check for ZeroBus FFI Signed-off-by: mats --- cmake/zerobus-ffi.cmake | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/cmake/zerobus-ffi.cmake b/cmake/zerobus-ffi.cmake index 442e184a0a0..d7b961c6aac 100644 --- a/cmake/zerobus-ffi.cmake +++ b/cmake/zerobus-ffi.cmake @@ -48,11 +48,11 @@ if(CMAKE_SYSTEM_NAME STREQUAL "Linux") return() endif() elseif(CMAKE_SYSTEM_NAME STREQUAL "Windows") - if(CMAKE_SYSTEM_PROCESSOR MATCHES "^(x86_64|AMD64)$") + if(CMAKE_SIZEOF_VOID_P EQUAL 8) set(_ZEROBUS_PLATFORM "windows-x86-64") else() message(STATUS - "ZeroBus FFI: unsupported Windows architecture '${CMAKE_SYSTEM_PROCESSOR}', " + "ZeroBus FFI: no prebuilt library for 32-bit Windows, " "disabling out_zerobus. " "To build manually, set -DZEROBUS_LIB_DIR=/path/to/lib.") FLB_OPTION(FLB_OUT_ZEROBUS OFF) From 115c66c9e6fa82036c224c29cbf89860950309b3 Mon Sep 17 00:00:00 2001 From: Kazuki Matsuda Date: Fri, 24 Apr 2026 14:49:53 +0900 Subject: [PATCH 04/25] build: add ZeroBus output plugin build configuration (#1) * build: disable out_zerobus on Windows ARM64 The previous pointer-size only gate falsely selected the windows-x86-64 prebuilt library when cross-compiling for Windows ARM64, causing the ARM64 package job to fail at link time. Reject ARM/ARM64 by processor name first, and keep the pointer-size check to pick the x86-64 prebuilt on 64-bit x86 and disable the plugin on 32-bit Windows. * build: align ZeroBus Windows ARM regex with Linux branch The ARM rejection regex included lowercase spellings and bare ARM/arm tokens that Windows never reports for CMAKE_SYSTEM_PROCESSOR. Use the same alternation as the sibling Linux branch and quote the processor value in the status message to match the existing style. Signed-off-by: mats --- cmake/zerobus-ffi.cmake | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/cmake/zerobus-ffi.cmake b/cmake/zerobus-ffi.cmake index d7b961c6aac..68a2459db39 100644 --- a/cmake/zerobus-ffi.cmake +++ b/cmake/zerobus-ffi.cmake @@ -48,7 +48,14 @@ if(CMAKE_SYSTEM_NAME STREQUAL "Linux") return() endif() elseif(CMAKE_SYSTEM_NAME STREQUAL "Windows") - if(CMAKE_SIZEOF_VOID_P EQUAL 8) + if(CMAKE_SYSTEM_PROCESSOR MATCHES "^(aarch64|arm64|ARM64|AARCH64)$") + message(STATUS + "ZeroBus FFI: no prebuilt library for Windows '${CMAKE_SYSTEM_PROCESSOR}', " + "disabling out_zerobus. " + "To build manually, set -DZEROBUS_LIB_DIR=/path/to/lib.") + FLB_OPTION(FLB_OUT_ZEROBUS OFF) + return() + elseif(CMAKE_SIZEOF_VOID_P EQUAL 8) set(_ZEROBUS_PLATFORM "windows-x86-64") else() message(STATUS From 7cb7552b00ea52dc834f43b94b6c8040234e6b59 Mon Sep 17 00:00:00 2001 From: mats Date: Fri, 24 Apr 2026 18:20:48 +0900 Subject: [PATCH 05/25] build: refine ZeroBus output plugin configuration Updated the ZeroBus output plugin's CMake configuration by correcting the spelling of "Zerobus" and simplifying the library search logic. The plugin now checks for the library in user-defined paths and standard system paths, improving the build process and error messaging for missing dependencies. Signed-off-by: mats --- cmake/plugins_options.cmake | 2 +- cmake/zerobus-ffi.cmake | 139 +++++------------------------ plugins/out_zerobus/CMakeLists.txt | 9 -- 3 files changed, 24 insertions(+), 126 deletions(-) diff --git a/cmake/plugins_options.cmake b/cmake/plugins_options.cmake index 25cf6de7dd0..4d11ee32186 100644 --- a/cmake/plugins_options.cmake +++ b/cmake/plugins_options.cmake @@ -156,4 +156,4 @@ DEFINE_OPTION(FLB_OUT_TCP "Enable TCP output plugin" DEFINE_OPTION(FLB_OUT_UDP "Enable UDP output plugin" ON) DEFINE_OPTION(FLB_OUT_VIVO_EXPORTER "Enable Vivo exporter output plugin" ON) DEFINE_OPTION(FLB_OUT_WEBSOCKET "Enable Websocket output plugin" ON) -DEFINE_OPTION(FLB_OUT_ZEROBUS "Enable Databricks ZeroBus output plugin" ON) +DEFINE_OPTION(FLB_OUT_ZEROBUS "Enable Databricks Zerobus output plugin" ON) diff --git a/cmake/zerobus-ffi.cmake b/cmake/zerobus-ffi.cmake index 68a2459db39..e29f6a25966 100644 --- a/cmake/zerobus-ffi.cmake +++ b/cmake/zerobus-ffi.cmake @@ -1,129 +1,36 @@ -# Set up the ZeroBus FFI prebuilt static library. +# Locate the pre-installed ZeroBus FFI static library. # -# If ZEROBUS_LIB_DIR is already set by the user, that path is used as-is. -# Otherwise the official release tarball is downloaded and the correct -# platform subdirectory is selected automatically. +# The out_zerobus plugin requires the ZeroBus FFI library to be installed +# on the build system before configuring. Fluent Bit does not download +# third-party dependencies at configure/build time. # -# On unsupported platforms or when the download fails, the plugin is -# disabled automatically (FLB_OUT_ZEROBUS is set to OFF). +# If the library is not found the plugin is silently disabled. # -# After this module runs: -# ZEROBUS_LIB_DIR — directory containing the static library +# Use -DZEROBUS_LIB_DIR=/path/to/dir to point to a custom location. +# +# After this module runs successfully: # ZEROBUS_LIB_FILE — full path to the static library -set(_ZEROBUS_LIB_FILENAME - "${CMAKE_STATIC_LIBRARY_PREFIX}zerobus_ffi${CMAKE_STATIC_LIBRARY_SUFFIX}") +set(_ZEROBUS_LIB_NAME "zerobus_ffi") if(ZEROBUS_LIB_DIR) - set(ZEROBUS_LIB_FILE - "${ZEROBUS_LIB_DIR}/${_ZEROBUS_LIB_FILENAME}" - CACHE FILEPATH "Full path to ZeroBus FFI static library" FORCE) - if(NOT EXISTS "${ZEROBUS_LIB_FILE}") - message(STATUS - "ZeroBus FFI: library not found at ${ZEROBUS_LIB_FILE}, " - "disabling out_zerobus.") - unset(ZEROBUS_LIB_FILE CACHE) - FLB_OPTION(FLB_OUT_ZEROBUS OFF) - endif() - return() -endif() - -set(_ZEROBUS_URL - "https://github.com/databricks/zerobus-sdk/releases/download/ffi-v1.0.0/zerobus-ffi-1.0.0.tar.gz") -set(_ZEROBUS_SHA256 - "c38609f5bddc160b43b35f9047919b35f66375308be69a0d0d6cd20bc01cee5a") - -# Determine the platform subdirectory inside the tarball -if(CMAKE_SYSTEM_NAME STREQUAL "Linux") - if(CMAKE_SYSTEM_PROCESSOR MATCHES "^(aarch64|arm64|ARM64|AARCH64)$") - set(_ZEROBUS_PLATFORM "linux-aarch64") - elseif(CMAKE_SYSTEM_PROCESSOR MATCHES "^(x86_64|AMD64)$") - set(_ZEROBUS_PLATFORM "linux-x86-64") - else() - message(STATUS - "ZeroBus FFI: unsupported Linux architecture '${CMAKE_SYSTEM_PROCESSOR}', " - "disabling out_zerobus. " - "To build manually, set -DZEROBUS_LIB_DIR=/path/to/lib.") - FLB_OPTION(FLB_OUT_ZEROBUS OFF) - return() - endif() -elseif(CMAKE_SYSTEM_NAME STREQUAL "Windows") - if(CMAKE_SYSTEM_PROCESSOR MATCHES "^(aarch64|arm64|ARM64|AARCH64)$") - message(STATUS - "ZeroBus FFI: no prebuilt library for Windows '${CMAKE_SYSTEM_PROCESSOR}', " - "disabling out_zerobus. " - "To build manually, set -DZEROBUS_LIB_DIR=/path/to/lib.") - FLB_OPTION(FLB_OUT_ZEROBUS OFF) - return() - elseif(CMAKE_SIZEOF_VOID_P EQUAL 8) - set(_ZEROBUS_PLATFORM "windows-x86-64") - else() - message(STATUS - "ZeroBus FFI: no prebuilt library for 32-bit Windows, " - "disabling out_zerobus. " - "To build manually, set -DZEROBUS_LIB_DIR=/path/to/lib.") - FLB_OPTION(FLB_OUT_ZEROBUS OFF) - return() - endif() -else() - message(STATUS - "ZeroBus FFI: no prebuilt library available for ${CMAKE_SYSTEM_NAME}, " - "disabling out_zerobus. " - "To build manually, set -DZEROBUS_LIB_DIR=/path/to/lib.") - FLB_OPTION(FLB_OUT_ZEROBUS OFF) - return() -endif() - -# Download the tarball if not already cached -set(_ZEROBUS_TARBALL "${CMAKE_BINARY_DIR}/zerobus-ffi-1.0.0.tar.gz") -if(NOT EXISTS "${_ZEROBUS_TARBALL}") - message(STATUS "ZeroBus FFI: downloading ${_ZEROBUS_URL}") - file(DOWNLOAD - "${_ZEROBUS_URL}" - "${_ZEROBUS_TARBALL}" - EXPECTED_HASH "SHA256=${_ZEROBUS_SHA256}" - SHOW_PROGRESS - STATUS _DOWNLOAD_STATUS - ) - list(GET _DOWNLOAD_STATUS 0 _DOWNLOAD_ERROR) - if(_DOWNLOAD_ERROR) - message(STATUS - "ZeroBus FFI: download failed (${_DOWNLOAD_STATUS}), " - "disabling out_zerobus. " - "To build manually, set -DZEROBUS_LIB_DIR=/path/to/lib.") - file(REMOVE "${_ZEROBUS_TARBALL}") - FLB_OPTION(FLB_OUT_ZEROBUS OFF) - return() - endif() -endif() - -# Extract the tarball -set(_ZEROBUS_EXTRACT_DIR "${CMAKE_BINARY_DIR}") -if(NOT EXISTS "${_ZEROBUS_EXTRACT_DIR}/native/${_ZEROBUS_PLATFORM}/${_ZEROBUS_LIB_FILENAME}") - execute_process( - COMMAND ${CMAKE_COMMAND} -E tar xzf "${_ZEROBUS_TARBALL}" - WORKING_DIRECTORY "${CMAKE_BINARY_DIR}" - RESULT_VARIABLE _EXTRACT_RESULT + # User provided an explicit directory — search only there. + find_library(ZEROBUS_LIB_FILE + NAMES ${_ZEROBUS_LIB_NAME} + PATHS "${ZEROBUS_LIB_DIR}" + NO_DEFAULT_PATH + NO_CMAKE_FIND_ROOT_PATH ) - if(_EXTRACT_RESULT) - message(STATUS - "ZeroBus FFI: extraction failed, disabling out_zerobus.") - FLB_OPTION(FLB_OUT_ZEROBUS OFF) - return() - endif() +else() + # Search standard system paths. + find_library(ZEROBUS_LIB_FILE NAMES ${_ZEROBUS_LIB_NAME}) endif() -set(ZEROBUS_LIB_DIR "${_ZEROBUS_EXTRACT_DIR}/native/${_ZEROBUS_PLATFORM}" - CACHE PATH "Path to ZeroBus FFI library directory" FORCE) -set(ZEROBUS_LIB_FILE "${ZEROBUS_LIB_DIR}/${_ZEROBUS_LIB_FILENAME}" - CACHE FILEPATH "Full path to ZeroBus FFI static library" FORCE) - -if(NOT EXISTS "${ZEROBUS_LIB_FILE}") +if(ZEROBUS_LIB_FILE) + message(STATUS "ZeroBus FFI library: ${ZEROBUS_LIB_FILE}") +else() message(STATUS - "ZeroBus FFI: ${_ZEROBUS_LIB_FILENAME} not found at ${ZEROBUS_LIB_DIR}, " - "disabling out_zerobus.") + "ZeroBus FFI: library not found, disabling out_zerobus. " + "To enable, install libzerobus_ffi or set -DZEROBUS_LIB_DIR=/path/to/lib.") FLB_OPTION(FLB_OUT_ZEROBUS OFF) - return() endif() - -message(STATUS "ZeroBus FFI library: ${ZEROBUS_LIB_FILE}") diff --git a/plugins/out_zerobus/CMakeLists.txt b/plugins/out_zerobus/CMakeLists.txt index c10a95a9fd0..a331887d418 100644 --- a/plugins/out_zerobus/CMakeLists.txt +++ b/plugins/out_zerobus/CMakeLists.txt @@ -2,15 +2,6 @@ set(src zerobus.c) FLB_PLUGIN(out_zerobus "${src}" "") - -# ZEROBUS_LIB_FILE is set automatically by cmake/zerobus-ffi.cmake or -# can be overridden by the user via -DZEROBUS_LIB_DIR=/path/to/lib. -if(NOT ZEROBUS_LIB_FILE) - message(FATAL_ERROR - "ZEROBUS_LIB_FILE is not set. This should not happen when " - "FLB_OUT_ZEROBUS is ON — check that cmake/zerobus-ffi.cmake is included.") -endif() - target_link_libraries(flb-plugin-out_zerobus "${ZEROBUS_LIB_FILE}") # Platform-specific linker flags required by the Rust FFI static library From bab0f281736f3177fa627f2a3f83e64f99f9c79e Mon Sep 17 00:00:00 2001 From: mats Date: Thu, 21 May 2026 16:12:49 +0900 Subject: [PATCH 06/25] build: make ZeroBus output plugin opt-in Signed-off-by: mats --- cmake/plugins_options.cmake | 2 +- cmake/zerobus-ffi.cmake | 10 +++++----- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/cmake/plugins_options.cmake b/cmake/plugins_options.cmake index 4d11ee32186..a9db923095f 100644 --- a/cmake/plugins_options.cmake +++ b/cmake/plugins_options.cmake @@ -156,4 +156,4 @@ DEFINE_OPTION(FLB_OUT_TCP "Enable TCP output plugin" DEFINE_OPTION(FLB_OUT_UDP "Enable UDP output plugin" ON) DEFINE_OPTION(FLB_OUT_VIVO_EXPORTER "Enable Vivo exporter output plugin" ON) DEFINE_OPTION(FLB_OUT_WEBSOCKET "Enable Websocket output plugin" ON) -DEFINE_OPTION(FLB_OUT_ZEROBUS "Enable Databricks Zerobus output plugin" ON) +DEFINE_OPTION(FLB_OUT_ZEROBUS "Enable Databricks Zerobus output plugin" OFF) diff --git a/cmake/zerobus-ffi.cmake b/cmake/zerobus-ffi.cmake index e29f6a25966..6507adceaaa 100644 --- a/cmake/zerobus-ffi.cmake +++ b/cmake/zerobus-ffi.cmake @@ -4,7 +4,7 @@ # on the build system before configuring. Fluent Bit does not download # third-party dependencies at configure/build time. # -# If the library is not found the plugin is silently disabled. +# If the plugin is enabled and the library is not found, configuration fails. # # Use -DZEROBUS_LIB_DIR=/path/to/dir to point to a custom location. # @@ -29,8 +29,8 @@ endif() if(ZEROBUS_LIB_FILE) message(STATUS "ZeroBus FFI library: ${ZEROBUS_LIB_FILE}") else() - message(STATUS - "ZeroBus FFI: library not found, disabling out_zerobus. " - "To enable, install libzerobus_ffi or set -DZEROBUS_LIB_DIR=/path/to/lib.") - FLB_OPTION(FLB_OUT_ZEROBUS OFF) + message(FATAL_ERROR + "ZeroBus FFI library not found. Install libzerobus_ffi or set " + "-DZEROBUS_LIB_DIR=/path/to/lib, or disable the plugin with " + "-DFLB_OUT_ZEROBUS=OFF.") endif() From 690001d2382007b7f34988b563f8a90b41db2b5d Mon Sep 17 00:00:00 2001 From: mats Date: Thu, 21 May 2026 18:15:17 +0900 Subject: [PATCH 07/25] build: standardize Zerobus naming Signed-off-by: mats --- CMakeLists.txt | 2 +- cmake/zerobus-ffi.cmake | 8 ++++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 626336ca0eb..9cabc9a9e69 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -1370,7 +1370,7 @@ if(FLB_OUT_PGSQL AND (NOT PostgreSQL_FOUND)) FLB_OPTION(FLB_OUT_PGSQL OFF) endif() -# ZeroBus FFI +# Zerobus FFI # =========== if(FLB_OUT_ZEROBUS) include(cmake/zerobus-ffi.cmake) diff --git a/cmake/zerobus-ffi.cmake b/cmake/zerobus-ffi.cmake index 6507adceaaa..db45e755c60 100644 --- a/cmake/zerobus-ffi.cmake +++ b/cmake/zerobus-ffi.cmake @@ -1,6 +1,6 @@ -# Locate the pre-installed ZeroBus FFI static library. +# Locate the pre-installed Zerobus FFI static library. # -# The out_zerobus plugin requires the ZeroBus FFI library to be installed +# The out_zerobus plugin requires the Zerobus FFI library to be installed # on the build system before configuring. Fluent Bit does not download # third-party dependencies at configure/build time. # @@ -27,10 +27,10 @@ else() endif() if(ZEROBUS_LIB_FILE) - message(STATUS "ZeroBus FFI library: ${ZEROBUS_LIB_FILE}") + message(STATUS "Zerobus FFI library: ${ZEROBUS_LIB_FILE}") else() message(FATAL_ERROR - "ZeroBus FFI library not found. Install libzerobus_ffi or set " + "Zerobus FFI library not found. Install libzerobus_ffi or set " "-DZEROBUS_LIB_DIR=/path/to/lib, or disable the plugin with " "-DFLB_OUT_ZEROBUS=OFF.") endif() From a8f0fde1546d362b17e2805e059262df8855f200 Mon Sep 17 00:00:00 2001 From: mats Date: Thu, 21 May 2026 18:16:19 +0900 Subject: [PATCH 08/25] out_zerobus: standardize Zerobus naming Signed-off-by: mats --- plugins/out_zerobus/zerobus.c | 16 ++++++++-------- plugins/out_zerobus/zerobus.h | 4 ++-- 2 files changed, 10 insertions(+), 10 deletions(-) diff --git a/plugins/out_zerobus/zerobus.c b/plugins/out_zerobus/zerobus.c index 3ea5ba2de5d..bc14df21f65 100644 --- a/plugins/out_zerobus/zerobus.c +++ b/plugins/out_zerobus/zerobus.c @@ -140,7 +140,7 @@ static int log_cresult_error(struct flb_output_instance *ins, } /* - * Convert a log event body to a JSON string for ZeroBus ingestion. + * Convert a log event body to a JSON string for Zerobus ingestion. * * Matches the Go plugin's recordToJSON: applies log_keys filter, then * injects raw_log_key (full pre-filter record), time_key, and _tag @@ -270,7 +270,7 @@ static flb_sds_t record_to_json(struct flb_out_zerobus *ctx, } /* - * Plugin init callback: validate required config, then create the ZeroBus + * Plugin init callback: validate required config, then create the Zerobus * SDK handle and stream. Returns 0 on success, -1 on failure. */ static int cb_zerobus_init(struct flb_output_instance *ins, @@ -341,7 +341,7 @@ static int cb_zerobus_init(struct flb_output_instance *ins, ctx->workspace_url, &result); if (!ctx->sdk || !result.success) { - log_cresult_error(ins, &result, "failed to create ZeroBus SDK"); + log_cresult_error(ins, &result, "failed to create Zerobus SDK"); goto init_error; } @@ -361,7 +361,7 @@ static int cb_zerobus_init(struct flb_output_instance *ins, &opts, &result); if (!ctx->stream || !result.success) { - log_cresult_error(ins, &result, "failed to create ZeroBus stream"); + log_cresult_error(ins, &result, "failed to create Zerobus stream"); zerobus_sdk_free(ctx->sdk); ctx->sdk = NULL; goto init_error; @@ -386,7 +386,7 @@ static int cb_zerobus_init(struct flb_output_instance *ins, /* * Plugin flush callback: decode incoming log events, convert each to JSON, - * and ingest the batch via ZeroBus. Waits for server-side acknowledgment + * and ingest the batch via Zerobus. Waits for server-side acknowledgment * before returning. Returns FLB_OK, FLB_RETRY, or FLB_ERROR via * FLB_OUTPUT_RETURN. */ @@ -521,7 +521,7 @@ static void cb_zerobus_flush(struct flb_event_chunk *event_chunk, } /* - * Plugin exit callback: close the ZeroBus stream, free the SDK handle, + * Plugin exit callback: close the Zerobus stream, free the SDK handle, * and release the plugin context. Returns 0. */ static int cb_zerobus_exit(void *data, struct flb_config *config) @@ -565,7 +565,7 @@ static struct flb_config_map config_map[] = { { FLB_CONFIG_MAP_STR, "endpoint", NULL, 0, FLB_FALSE, 0, - "ZeroBus gRPC endpoint URL (https:// prepended if no scheme)" + "Zerobus gRPC endpoint URL (https:// prepended if no scheme)" }, { FLB_CONFIG_MAP_STR, "workspace_url", NULL, @@ -612,7 +612,7 @@ static struct flb_config_map config_map[] = { struct flb_output_plugin out_zerobus_plugin = { .name = "zerobus", - .description = "Send logs to Databricks ZeroBus", + .description = "Send logs to Databricks Zerobus", .cb_init = cb_zerobus_init, .cb_flush = cb_zerobus_flush, .cb_exit = cb_zerobus_exit, diff --git a/plugins/out_zerobus/zerobus.h b/plugins/out_zerobus/zerobus.h index bc67b81fdcb..6353d1951ea 100644 --- a/plugins/out_zerobus/zerobus.h +++ b/plugins/out_zerobus/zerobus.h @@ -28,7 +28,7 @@ #include /* - * ZeroBus FFI declarations + * Zerobus FFI declarations * * These types and functions are provided by the prebuilt Rust FFI static * library (libzerobus_ffi.a). The declarations below are extracted from @@ -106,7 +106,7 @@ extern CStreamConfigurationOptions zerobus_get_default_config(void); /* Plugin context */ struct flb_out_zerobus { - /* ZeroBus handles */ + /* Zerobus handles */ CZerobusSdk *sdk; CZerobusStream *stream; From 03675a25ddc673bb35872b8e04a6121f0db39116 Mon Sep 17 00:00:00 2001 From: mats Date: Wed, 1 Jul 2026 02:40:31 +0900 Subject: [PATCH 09/25] lib: vendor zerobus ffi 1.3.0 Signed-off-by: mats --- .gitignore | 1 + lib/update_zerobus_ffi.sh | 252 ++ lib/zerobus-ffi-1.3.0/LICENSE | 201 + lib/zerobus-ffi-1.3.0/README.fluent-bit.md | 54 + lib/zerobus-ffi-1.3.0/rust/Cargo.lock | 3621 +++++++++++++++++ lib/zerobus-ffi-1.3.0/rust/Cargo.toml | 62 + lib/zerobus-ffi-1.3.0/rust/LICENSE | 201 + lib/zerobus-ffi-1.3.0/rust/NOTICE | 43 + lib/zerobus-ffi-1.3.0/rust/README.md | 1094 +++++ lib/zerobus-ffi-1.3.0/rust/ffi/CHANGELOG.md | 104 + lib/zerobus-ffi-1.3.0/rust/ffi/Cargo.toml | 37 + .../rust/ffi/NEXT_CHANGELOG.md | 21 + lib/zerobus-ffi-1.3.0/rust/ffi/README.md | 140 + lib/zerobus-ffi-1.3.0/rust/ffi/build.rs | 19 + lib/zerobus-ffi-1.3.0/rust/ffi/cbindgen.toml | 13 + lib/zerobus-ffi-1.3.0/rust/ffi/src/lib.rs | 2250 ++++++++++ lib/zerobus-ffi-1.3.0/rust/ffi/src/tests.rs | 1068 +++++ lib/zerobus-ffi-1.3.0/rust/ffi/zerobus.h | 574 +++ lib/zerobus-ffi-1.3.0/rust/sdk/Cargo.toml | 86 + lib/zerobus-ffi-1.3.0/rust/sdk/LICENSE | 1 + lib/zerobus-ffi-1.3.0/rust/sdk/build.rs | 14 + .../rust/sdk/src/arrow_configuration.rs | 133 + .../rust/sdk/src/arrow_metadata.rs | 189 + .../rust/sdk/src/arrow_stream.rs | 1711 ++++++++ .../rust/sdk/src/builder/mod.rs | 36 + .../rust/sdk/src/builder/sdk_builder.rs | 408 ++ .../rust/sdk/src/builder/stream_builder.rs | 672 +++ .../rust/sdk/src/callbacks.rs | 117 + .../rust/sdk/src/client_warnings.rs | 292 ++ .../rust/sdk/src/default_token_factory.rs | 423 ++ lib/zerobus-ffi-1.3.0/rust/sdk/src/errors.rs | 151 + .../rust/sdk/src/headers_provider.rs | 153 + .../rust/sdk/src/landing_zone.rs | 420 ++ lib/zerobus-ffi-1.3.0/rust/sdk/src/lib.rs | 1913 +++++++++ .../rust/sdk/src/offset_generator.rs | 172 + lib/zerobus-ffi-1.3.0/rust/sdk/src/proxy.rs | 196 + .../rust/sdk/src/record_types.rs | 946 +++++ lib/zerobus-ffi-1.3.0/rust/sdk/src/schema.rs | 1448 +++++++ .../rust/sdk/src/stream_configuration.rs | 178 + .../rust/sdk/src/stream_options.rs | 25 + .../rust/sdk/src/tls_config.rs | 100 + .../rust/sdk/src/token_cache.rs | 540 +++ .../rust/sdk/src/zeroparser/README.md | 141 + .../rust/sdk/src/zeroparser/benches/README.md | 45 + .../sdk/src/zeroparser/benches/bench_plot.rs | 364 ++ .../sdk/src/zeroparser/benches/bench_plot.svg | 315 ++ .../zeroparser/benches/bench_sample_data.json | 117 + .../sdk/src/zeroparser/benches/common/mod.rs | 463 +++ .../src/zeroparser/benches/parser_bench.rs | 125 + .../benches/proto/air_quality.proto | 9 + .../proto/supported_nullable_types.proto | 19 + .../benches/proto/wide_schema.proto | 110 + .../rust/sdk/src/zeroparser/errors.rs | 157 + .../rust/sdk/src/zeroparser/mod.rs | 32 + .../rust/sdk/src/zeroparser/owned.rs | 152 + .../rust/sdk/src/zeroparser/parser.rs | 2733 +++++++++++++ .../rust/sdk/src/zeroparser/proto_build.rs | 38 + .../rust/sdk/src/zeroparser/registry.rs | 358 ++ .../sdk/src/zeroparser/sparse_field_map.rs | 60 + .../sdk/src/zeroparser/tests/common/mod.rs | 250 ++ .../rust/sdk/src/zeroparser/tests/e2e.rs | 3379 +++++++++++++++ .../proto/google/protobuf/duration.proto | 117 + .../proto/google/protobuf/timestamp.proto | 135 + .../proto/google/protobuf/wrappers.proto | 118 + .../zeroparser/tests/proto/test_proto2.proto | 192 + .../zeroparser/tests/proto/test_proto3.proto | 187 + .../rust/sdk/src/zeroparser/types.rs | 814 ++++ .../rust/sdk/src/zeroparser/wire.rs | 676 +++ .../rust/sdk/zerobus_service.proto | 223 + 69 files changed, 31408 insertions(+) create mode 100755 lib/update_zerobus_ffi.sh create mode 100644 lib/zerobus-ffi-1.3.0/LICENSE create mode 100644 lib/zerobus-ffi-1.3.0/README.fluent-bit.md create mode 100644 lib/zerobus-ffi-1.3.0/rust/Cargo.lock create mode 100644 lib/zerobus-ffi-1.3.0/rust/Cargo.toml create mode 100644 lib/zerobus-ffi-1.3.0/rust/LICENSE create mode 100644 lib/zerobus-ffi-1.3.0/rust/NOTICE create mode 100644 lib/zerobus-ffi-1.3.0/rust/README.md create mode 100644 lib/zerobus-ffi-1.3.0/rust/ffi/CHANGELOG.md create mode 100644 lib/zerobus-ffi-1.3.0/rust/ffi/Cargo.toml create mode 100644 lib/zerobus-ffi-1.3.0/rust/ffi/NEXT_CHANGELOG.md create mode 100644 lib/zerobus-ffi-1.3.0/rust/ffi/README.md create mode 100644 lib/zerobus-ffi-1.3.0/rust/ffi/build.rs create mode 100644 lib/zerobus-ffi-1.3.0/rust/ffi/cbindgen.toml create mode 100644 lib/zerobus-ffi-1.3.0/rust/ffi/src/lib.rs create mode 100644 lib/zerobus-ffi-1.3.0/rust/ffi/src/tests.rs create mode 100644 lib/zerobus-ffi-1.3.0/rust/ffi/zerobus.h create mode 100644 lib/zerobus-ffi-1.3.0/rust/sdk/Cargo.toml create mode 120000 lib/zerobus-ffi-1.3.0/rust/sdk/LICENSE create mode 100644 lib/zerobus-ffi-1.3.0/rust/sdk/build.rs create mode 100644 lib/zerobus-ffi-1.3.0/rust/sdk/src/arrow_configuration.rs create mode 100644 lib/zerobus-ffi-1.3.0/rust/sdk/src/arrow_metadata.rs create mode 100644 lib/zerobus-ffi-1.3.0/rust/sdk/src/arrow_stream.rs create mode 100644 lib/zerobus-ffi-1.3.0/rust/sdk/src/builder/mod.rs create mode 100644 lib/zerobus-ffi-1.3.0/rust/sdk/src/builder/sdk_builder.rs create mode 100644 lib/zerobus-ffi-1.3.0/rust/sdk/src/builder/stream_builder.rs create mode 100644 lib/zerobus-ffi-1.3.0/rust/sdk/src/callbacks.rs create mode 100644 lib/zerobus-ffi-1.3.0/rust/sdk/src/client_warnings.rs create mode 100644 lib/zerobus-ffi-1.3.0/rust/sdk/src/default_token_factory.rs create mode 100644 lib/zerobus-ffi-1.3.0/rust/sdk/src/errors.rs create mode 100644 lib/zerobus-ffi-1.3.0/rust/sdk/src/headers_provider.rs create mode 100644 lib/zerobus-ffi-1.3.0/rust/sdk/src/landing_zone.rs create mode 100644 lib/zerobus-ffi-1.3.0/rust/sdk/src/lib.rs create mode 100644 lib/zerobus-ffi-1.3.0/rust/sdk/src/offset_generator.rs create mode 100644 lib/zerobus-ffi-1.3.0/rust/sdk/src/proxy.rs create mode 100644 lib/zerobus-ffi-1.3.0/rust/sdk/src/record_types.rs create mode 100644 lib/zerobus-ffi-1.3.0/rust/sdk/src/schema.rs create mode 100644 lib/zerobus-ffi-1.3.0/rust/sdk/src/stream_configuration.rs create mode 100644 lib/zerobus-ffi-1.3.0/rust/sdk/src/stream_options.rs create mode 100644 lib/zerobus-ffi-1.3.0/rust/sdk/src/tls_config.rs create mode 100644 lib/zerobus-ffi-1.3.0/rust/sdk/src/token_cache.rs create mode 100644 lib/zerobus-ffi-1.3.0/rust/sdk/src/zeroparser/README.md create mode 100644 lib/zerobus-ffi-1.3.0/rust/sdk/src/zeroparser/benches/README.md create mode 100644 lib/zerobus-ffi-1.3.0/rust/sdk/src/zeroparser/benches/bench_plot.rs create mode 100644 lib/zerobus-ffi-1.3.0/rust/sdk/src/zeroparser/benches/bench_plot.svg create mode 100644 lib/zerobus-ffi-1.3.0/rust/sdk/src/zeroparser/benches/bench_sample_data.json create mode 100644 lib/zerobus-ffi-1.3.0/rust/sdk/src/zeroparser/benches/common/mod.rs create mode 100644 lib/zerobus-ffi-1.3.0/rust/sdk/src/zeroparser/benches/parser_bench.rs create mode 100644 lib/zerobus-ffi-1.3.0/rust/sdk/src/zeroparser/benches/proto/air_quality.proto create mode 100644 lib/zerobus-ffi-1.3.0/rust/sdk/src/zeroparser/benches/proto/supported_nullable_types.proto create mode 100644 lib/zerobus-ffi-1.3.0/rust/sdk/src/zeroparser/benches/proto/wide_schema.proto create mode 100644 lib/zerobus-ffi-1.3.0/rust/sdk/src/zeroparser/errors.rs create mode 100644 lib/zerobus-ffi-1.3.0/rust/sdk/src/zeroparser/mod.rs create mode 100644 lib/zerobus-ffi-1.3.0/rust/sdk/src/zeroparser/owned.rs create mode 100644 lib/zerobus-ffi-1.3.0/rust/sdk/src/zeroparser/parser.rs create mode 100644 lib/zerobus-ffi-1.3.0/rust/sdk/src/zeroparser/proto_build.rs create mode 100644 lib/zerobus-ffi-1.3.0/rust/sdk/src/zeroparser/registry.rs create mode 100644 lib/zerobus-ffi-1.3.0/rust/sdk/src/zeroparser/sparse_field_map.rs create mode 100644 lib/zerobus-ffi-1.3.0/rust/sdk/src/zeroparser/tests/common/mod.rs create mode 100644 lib/zerobus-ffi-1.3.0/rust/sdk/src/zeroparser/tests/e2e.rs create mode 100644 lib/zerobus-ffi-1.3.0/rust/sdk/src/zeroparser/tests/proto/google/protobuf/duration.proto create mode 100644 lib/zerobus-ffi-1.3.0/rust/sdk/src/zeroparser/tests/proto/google/protobuf/timestamp.proto create mode 100644 lib/zerobus-ffi-1.3.0/rust/sdk/src/zeroparser/tests/proto/google/protobuf/wrappers.proto create mode 100644 lib/zerobus-ffi-1.3.0/rust/sdk/src/zeroparser/tests/proto/test_proto2.proto create mode 100644 lib/zerobus-ffi-1.3.0/rust/sdk/src/zeroparser/tests/proto/test_proto3.proto create mode 100644 lib/zerobus-ffi-1.3.0/rust/sdk/src/zeroparser/types.rs create mode 100644 lib/zerobus-ffi-1.3.0/rust/sdk/src/zeroparser/wire.rs create mode 100644 lib/zerobus-ffi-1.3.0/rust/sdk/zerobus_service.proto diff --git a/.gitignore b/.gitignore index aa5b6f9871b..63ab3774c5f 100644 --- a/.gitignore +++ b/.gitignore @@ -39,6 +39,7 @@ workflow/ .vagrant/ # examples examples/wasi_serde_json/target/ +lib/zerobus-ffi-1.3.0/rust/target/ # WASM test data tests/runtime/wasm/go/*.wasm tests/integration/.venv/ diff --git a/lib/update_zerobus_ffi.sh b/lib/update_zerobus_ffi.sh new file mode 100755 index 00000000000..c275212f0d3 --- /dev/null +++ b/lib/update_zerobus_ffi.sh @@ -0,0 +1,252 @@ +#!/usr/bin/env bash + +set -euo pipefail + +usage() +{ + cat <&2 + exit 1 +} + +need_command() +{ + command -v "$1" >/dev/null 2>&1 || die "missing required command: $1" +} + +prune_workspace_members() +{ + local cargo_toml="$1" + local cargo_toml_tmp="${cargo_toml}.tmp" + + awk ' + BEGIN { in_members = 0 } + /^[[:space:]]*members[[:space:]]*=[[:space:]]*\[/ { + print "members = [" + print " \"sdk\"," + print " \"ffi\"," + print "]" + in_members = 1 + next + } + in_members { + if ($0 ~ /^[[:space:]]*\]/) { + in_members = 0 + } + next + } + { print } + ' "$cargo_toml" > "$cargo_toml_tmp" + + mv "$cargo_toml_tmp" "$cargo_toml" +} + +patch_ffi_build_rs() +{ + local build_rs="$1" + + perl -0pi -e \ + 's/let output_file = PathBuf::from\(&crate_dir\)\.join\("zerobus\.h"\);/let output_file = PathBuf::from(env::var("OUT_DIR").unwrap()).join("zerobus.h");/' \ + "$build_rs" + + perl -0pi -e \ + 's/\.write_to_file\(output_file\);/.write_to_file(\&output_file);/' \ + "$build_rs" + + grep -q 'env::var("OUT_DIR")' "$build_rs" || + die "failed to patch $build_rs to write cbindgen output under OUT_DIR" + + grep -q 'write_to_file(&output_file)' "$build_rs" || + die "failed to patch $build_rs to pass output_file by reference" +} + +write_fluent_bit_notes() +{ + local output_file="$1" + local version="$2" + + cat > "$output_file" </dev/null + cargo metadata --locked --no-deps --format-version 1 >/dev/null +) + +rm -rf "$dest_dir" +mv "$stage_dir" "$dest_dir" + +echo "Vendored Zerobus FFI ${version} into ${dest_dir}" diff --git a/lib/zerobus-ffi-1.3.0/LICENSE b/lib/zerobus-ffi-1.3.0/LICENSE new file mode 100644 index 00000000000..253d123149d --- /dev/null +++ b/lib/zerobus-ffi-1.3.0/LICENSE @@ -0,0 +1,201 @@ + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright (2025) Databricks, Inc. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/lib/zerobus-ffi-1.3.0/README.fluent-bit.md b/lib/zerobus-ffi-1.3.0/README.fluent-bit.md new file mode 100644 index 00000000000..3438aad2e67 --- /dev/null +++ b/lib/zerobus-ffi-1.3.0/README.fluent-bit.md @@ -0,0 +1,54 @@ +# Zerobus FFI vendoring notes + +This directory vendors the Zerobus Rust FFI source used by the Fluent Bit +`out_zerobus` plugin. + +Upstream: https://github.com/databricks/zerobus-sdk +Version: `ffi/v1.3.0` + +Only the Rust crates needed to build the C FFI are included: + +- `LICENSE` +- `rust/Cargo.toml` +- `rust/Cargo.lock` +- `rust/LICENSE` +- `rust/NOTICE` +- `rust/README.md` +- `rust/ffi/` +- `rust/sdk/` + +The upstream repository also contains language bindings, examples, tests, and +prebuilt archives under `go/lib/`; those are intentionally not vendored. + +## Update + +Run this from the Fluent Bit repository root: + +```console +lib/update_zerobus_ffi.sh 1.3.0 +``` + +The script downloads the Databricks Zerobus SDK release archive, copies only +the FFI build inputs listed above, narrows the Rust workspace to `sdk` and +`ffi`, and refreshes `Cargo.lock` for that narrowed workspace. + +## Fluent Bit changes + +The vendored `rust/Cargo.toml` workspace is narrowed to `sdk` and `ffi`. + +The vendored `rust/ffi/build.rs` writes the cbindgen output to Cargo's +`OUT_DIR` instead of rewriting `rust/ffi/zerobus.h` during every build. The +checked-in `rust/ffi/zerobus.h` is the header used by the C plugin. + +## Build behavior + +Fluent Bit builds the bundled library with: + +```console +cargo build --locked --release -p zerobus-ffi +``` + +Crates.io dependencies are not vendored in this repository. `Cargo.lock` is +checked in to keep dependency resolution stable. In the Ubuntu packaging image, +the package build runs from `CMD`, so Cargo registry configuration should be +provided when the container is run if a registry proxy is required. diff --git a/lib/zerobus-ffi-1.3.0/rust/Cargo.lock b/lib/zerobus-ffi-1.3.0/rust/Cargo.lock new file mode 100644 index 00000000000..c718c6dccac --- /dev/null +++ b/lib/zerobus-ffi-1.3.0/rust/Cargo.lock @@ -0,0 +1,3621 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 3 + +[[package]] +name = "ahash" +version = "0.8.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5a15f179cd60c4584b8a8c596927aadc462e27f2ca70c04e0071964a73ba7a75" +dependencies = [ + "cfg-if", + "const-random", + "getrandom 0.3.4", + "once_cell", + "version_check", + "zerocopy", +] + +[[package]] +name = "aho-corasick" +version = "1.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ddd31a130427c27518df266943a5308ed92d4b226cc639f5a8f1002816174301" +dependencies = [ + "memchr", +] + +[[package]] +name = "android_system_properties" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "819e7219dbd41043ac279b19830f2efc897156490d7fd6ea916720117ee66311" +dependencies = [ + "libc", +] + +[[package]] +name = "anes" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4b46cbb362ab8752921c97e041f5e366ee6297bd428a31275b9fcf1e380f7299" + +[[package]] +name = "anstream" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "824a212faf96e9acacdbd09febd34438f8f711fb84e09a8916013cd7815ca28d" +dependencies = [ + "anstyle", + "anstyle-parse", + "anstyle-query", + "anstyle-wincon", + "colorchoice", + "is_terminal_polyfill", + "utf8parse", +] + +[[package]] +name = "anstyle" +version = "1.0.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5192cca8006f1fd4f7237516f40fa183bb07f8fbdfedaa0036de5ea9b0b45e78" + +[[package]] +name = "anstyle-parse" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "52ce7f38b242319f7cabaa6813055467063ecdc9d355bbb4ce0c68908cd8130e" +dependencies = [ + "utf8parse", +] + +[[package]] +name = "anstyle-query" +version = "1.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "40c48f72fd53cd289104fc64099abca73db4166ad86ea0b4341abe65af83dadc" +dependencies = [ + "windows-sys 0.61.2", +] + +[[package]] +name = "anstyle-wincon" +version = "3.0.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "291e6a250ff86cd4a820112fb8898808a366d8f9f58ce16d1f538353ad55747d" +dependencies = [ + "anstyle", + "once_cell_polyfill", + "windows-sys 0.61.2", +] + +[[package]] +name = "anyhow" +version = "1.0.102" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7f202df86484c868dbad7eaa557ef785d5c66295e41b460ef922eca0723b842c" + +[[package]] +name = "arrow-array" +version = "58.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "841321891f247aa86c6112c80d83d89cb36e0addd020fa2425085b8eb6c3f579" +dependencies = [ + "ahash", + "arrow-buffer", + "arrow-data", + "arrow-schema", + "chrono", + "half", + "hashbrown 0.17.0", + "num-complex", + "num-integer", + "num-traits", +] + +[[package]] +name = "arrow-buffer" +version = "58.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f955dfb73fae000425f49c8226d2044dab60fb7ad4af1e24f961756354d996c9" +dependencies = [ + "bytes", + "half", + "num-bigint", + "num-traits", +] + +[[package]] +name = "arrow-cast" +version = "58.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ca5e686972523798f76bef355145bc1ae25a84c731e650268d31ab763c701663" +dependencies = [ + "arrow-array", + "arrow-buffer", + "arrow-data", + "arrow-ord", + "arrow-schema", + "arrow-select", + "atoi", + "base64", + "chrono", + "half", + "lexical-core", + "num-traits", + "ryu", +] + +[[package]] +name = "arrow-data" +version = "58.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "db3b5846209775b6dc8056d77ff9a032b27043383dd5488abd0b663e265b9373" +dependencies = [ + "arrow-buffer", + "arrow-schema", + "half", + "num-integer", + "num-traits", +] + +[[package]] +name = "arrow-flight" +version = "58.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e7c379738a9e41eda5873bb3256c89164f87c65b16b0b153cccc74d8a462b7e8" +dependencies = [ + "arrow-array", + "arrow-buffer", + "arrow-cast", + "arrow-ipc", + "arrow-schema", + "base64", + "bytes", + "futures", + "prost", + "prost-types", + "tonic", + "tonic-prost", +] + +[[package]] +name = "arrow-ipc" +version = "58.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fd8907ddd8f9fbabf91ec2c85c1d81fe2874e336d2443eb36373595e28b98dd5" +dependencies = [ + "arrow-array", + "arrow-buffer", + "arrow-data", + "arrow-schema", + "arrow-select", + "flatbuffers", + "lz4_flex", + "zstd", +] + +[[package]] +name = "arrow-ord" +version = "58.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "efa70d9d6b1356f1fb9f1f651b84a725b7e0abb93f188cf7d31f14abfa2f2e6f" +dependencies = [ + "arrow-array", + "arrow-buffer", + "arrow-data", + "arrow-schema", + "arrow-select", +] + +[[package]] +name = "arrow-schema" +version = "58.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "18aa020f6bc8e5201dcd2d4b7f98c68f8a410ef37128263243e6ff2a47a67d4f" + +[[package]] +name = "arrow-select" +version = "58.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a657ab5132e9c8ca3b24eb15a823d0ced38017fe3930ff50167466b02e2d592c" +dependencies = [ + "ahash", + "arrow-array", + "arrow-buffer", + "arrow-data", + "arrow-schema", + "num-traits", +] + +[[package]] +name = "async-trait" +version = "0.1.89" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9035ad2d096bed7955a320ee7e2230574d28fd3c3a0f186cbea1ff3c7eed5dbb" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "atoi" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f28d99ec8bfea296261ca1af174f24225171fea9664ba9003cbebee704810528" +dependencies = [ + "num-traits", +] + +[[package]] +name = "atomic-waker" +version = "1.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1505bd5d3d116872e7271a6d4e16d81d0c8570876c8de68093a09ac269d8aac0" + +[[package]] +name = "autocfg" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c08606f8c3cbf4ce6ec8e28fb0014a2c086708fe954eaa885384a6165172e7e8" + +[[package]] +name = "axum" +version = "0.8.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b52af3cb4058c895d37317bb27508dccc8e5f2d39454016b297bf4a400597b8" +dependencies = [ + "axum-core", + "bytes", + "futures-util", + "http", + "http-body", + "http-body-util", + "itoa", + "matchit", + "memchr", + "mime", + "percent-encoding", + "pin-project-lite", + "serde_core", + "sync_wrapper", + "tower", + "tower-layer", + "tower-service", +] + +[[package]] +name = "axum-core" +version = "0.5.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "08c78f31d7b1291f7ee735c1c6780ccde7785daae9a9206026862dab7d8792d1" +dependencies = [ + "bytes", + "futures-core", + "http", + "http-body", + "http-body-util", + "mime", + "pin-project-lite", + "sync_wrapper", + "tower-layer", + "tower-service", +] + +[[package]] +name = "base64" +version = "0.22.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "72b3254f16251a8381aa12e40e3c4d2f0199f8c6508fbecb9d91f575e0fbb8c6" + +[[package]] +name = "bitflags" +version = "2.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "843867be96c8daad0d758b57df9392b6d8d271134fce549de6ce169ff98a92af" + +[[package]] +name = "block-buffer" +version = "0.10.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3078c7629b62d3f0439517fa394996acacc5cbc91c5a20d8c658e77abd503a71" +dependencies = [ + "generic-array", +] + +[[package]] +name = "bumpalo" +version = "3.20.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5d20789868f4b01b2f2caec9f5c4e0213b41e3e5702a50157d699ae31ced2fcb" + +[[package]] +name = "bytes" +version = "1.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e748733b7cbc798e1434b6ac524f0c1ff2ab456fe201501e6497c8417a4fc33" + +[[package]] +name = "cast" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "37b2a672a2cb129a2e41c10b1224bb368f9f37a2b16b612598138befd7b37eb5" + +[[package]] +name = "cbindgen" +version = "0.27.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3fce8dd7fcfcbf3a0a87d8f515194b49d6135acab73e18bd380d1d93bb1a15eb" +dependencies = [ + "clap", + "heck 0.4.1", + "indexmap", + "log", + "proc-macro2", + "quote", + "serde", + "serde_json", + "syn", + "tempfile", + "toml", +] + +[[package]] +name = "cc" +version = "1.2.56" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "aebf35691d1bfb0ac386a69bac2fde4dd276fb618cf8bf4f5318fe285e821bb2" +dependencies = [ + "find-msvc-tools", + "jobserver", + "libc", + "shlex", +] + +[[package]] +name = "cfg-if" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9330f8b2ff13f34540b44e946ef35111825727b38d33286ef986142615121801" + +[[package]] +name = "cfg_aliases" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "613afe47fcd5fac7ccf1db93babcb082c5994d996f20b8b159f2ad1658eb5724" + +[[package]] +name = "chrono" +version = "0.4.44" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c673075a2e0e5f4a1dde27ce9dee1ea4558c7ffe648f576438a20ca1d2acc4b0" +dependencies = [ + "iana-time-zone", + "num-traits", + "windows-link", +] + +[[package]] +name = "ciborium" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "42e69ffd6f0917f5c029256a24d0161db17cea3997d185db0d35926308770f0e" +dependencies = [ + "ciborium-io", + "ciborium-ll", + "serde", +] + +[[package]] +name = "ciborium-io" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "05afea1e0a06c9be33d539b876f1ce3692f4afea2cb41f740e7743225ed1c757" + +[[package]] +name = "ciborium-ll" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "57663b653d948a338bfb3eeba9bb2fd5fcfaecb9e199e87e1eda4d9e8b240fd9" +dependencies = [ + "ciborium-io", + "half", +] + +[[package]] +name = "clap" +version = "4.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ddb117e43bbf7dacf0a4190fef4d345b9bad68dfc649cb349e7d17d28428e51" +dependencies = [ + "clap_builder", +] + +[[package]] +name = "clap_builder" +version = "4.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "714a53001bf66416adb0e2ef5ac857140e7dc3a0c48fb28b2f10762fc4b5069f" +dependencies = [ + "anstream", + "anstyle", + "clap_lex", + "strsim", +] + +[[package]] +name = "clap_lex" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3a822ea5bc7590f9d40f1ba12c0dc3c2760f3482c6984db1573ad11031420831" + +[[package]] +name = "colorchoice" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b05b61dc5112cbb17e4b6cd61790d9845d13888356391624cbe7e41efeac1e75" + +[[package]] +name = "const-random" +version = "0.1.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "87e00182fe74b066627d63b85fd550ac2998d4b0bd86bfed477a0ae4c7c71359" +dependencies = [ + "const-random-macro", +] + +[[package]] +name = "const-random-macro" +version = "0.1.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f9d839f2a20b0aee515dc581a6172f2321f96cab76c1a38a4c584a194955390e" +dependencies = [ + "getrandom 0.2.17", + "once_cell", + "tiny-keccak", +] + +[[package]] +name = "core-foundation" +version = "0.9.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "91e195e091a93c46f7102ec7818a2aa394e1e1771c3ab4825963fa03e45afb8f" +dependencies = [ + "core-foundation-sys", + "libc", +] + +[[package]] +name = "core-foundation" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b2a6cd9ae233e7f62ba4e9353e81a88df7fc8a5987b8d445b4d90c879bd156f6" +dependencies = [ + "core-foundation-sys", + "libc", +] + +[[package]] +name = "core-foundation-sys" +version = "0.8.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "773648b94d0e5d620f64f280777445740e61fe701025087ec8b57f45c791888b" + +[[package]] +name = "cpufeatures" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "59ed5838eebb26a2bb2e58f6d5b5316989ae9d08bab10e0e6d103e656d1b0280" +dependencies = [ + "libc", +] + +[[package]] +name = "criterion" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f2b12d017a929603d80db1831cd3a24082f8137ce19c69e6447f54f5fc8d692f" +dependencies = [ + "anes", + "cast", + "ciborium", + "clap", + "criterion-plot", + "is-terminal", + "itertools 0.10.5", + "num-traits", + "once_cell", + "oorandom", + "regex", + "serde", + "serde_derive", + "serde_json", + "tinytemplate", + "walkdir", +] + +[[package]] +name = "criterion-plot" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6b50826342786a51a89e2da3a28f1c32b06e387201bc2d19791f622c673706b1" +dependencies = [ + "cast", + "itertools 0.10.5", +] + +[[package]] +name = "crunchy" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "460fbee9c2c2f33933d720630a6a0bac33ba7053db5344fac858d4b8952d77d5" + +[[package]] +name = "crypto-common" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "78c8292055d1c1df0cce5d180393dc8cce0abec0a7102adb6c7b1eef6016d60a" +dependencies = [ + "generic-array", + "typenum", +] + +[[package]] +name = "databricks-zerobus-ingest-sdk" +version = "2.2.2" +dependencies = [ + "arrow-array", + "arrow-flight", + "arrow-ipc", + "arrow-schema", + "async-trait", + "bytes", + "criterion", + "futures", + "hyper-http-proxy", + "hyper-util", + "plotters", + "prost", + "prost-build", + "prost-reflect", + "prost-types", + "protoc-bin-vendored", + "reqwest", + "rstest", + "self_cell", + "serde", + "serde_json", + "sha2", + "smallvec", + "strum", + "thiserror 1.0.69", + "tokio", + "tokio-retry", + "tokio-stream", + "tokio-util", + "tonic", + "tonic-prost", + "tonic-prost-build", + "tracing", + "tracing-subscriber", +] + +[[package]] +name = "digest" +version = "0.10.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9ed9a281f7bc9b7576e61468ba615a66a5c8cfdff42420a70aa82701a3b1e292" +dependencies = [ + "block-buffer", + "crypto-common", +] + +[[package]] +name = "displaydoc" +version = "0.2.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "97369cbbc041bc366949bc74d34658d6cda5621039731c6310521892a3a20ae0" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "either" +version = "1.15.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "48c757948c5ede0e46177b7add2e67155f70e33c07fea8284df6576da70b3719" + +[[package]] +name = "equivalent" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "877a4ace8713b0bcf2a4e7eec82529c029f1d0619886d18145fea96c3ffe5c0f" + +[[package]] +name = "errno" +version = "0.3.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb" +dependencies = [ + "libc", + "windows-sys 0.61.2", +] + +[[package]] +name = "fastrand" +version = "2.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "37909eebbb50d72f9059c3b6d82c0463f2ff062c9e95845c43a6c9c0355411be" + +[[package]] +name = "find-msvc-tools" +version = "0.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5baebc0774151f905a1a2cc41989300b1e6fbb29aff0ceffa1064fdd3088d582" + +[[package]] +name = "fixedbitset" +version = "0.5.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1d674e81391d1e1ab681a28d99df07927c6d4aa5b027d7da16ba32d1d21ecd99" + +[[package]] +name = "flatbuffers" +version = "25.12.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "35f6839d7b3b98adde531effaf34f0c2badc6f4735d26fe74709d8e513a96ef3" +dependencies = [ + "bitflags", + "rustc_version", +] + +[[package]] +name = "fnv" +version = "1.0.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1" + +[[package]] +name = "foldhash" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d9c4f5dac5e15c24eb999c26181a6ca40b39fe946cbe4c263c7209467bc83af2" + +[[package]] +name = "form_urlencoded" +version = "1.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cb4cb245038516f5f85277875cdaa4f7d2c9a0fa0468de06ed190163b1581fcf" +dependencies = [ + "percent-encoding", +] + +[[package]] +name = "futures" +version = "0.3.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b147ee9d1f6d097cef9ce628cd2ee62288d963e16fb287bd9286455b241382d" +dependencies = [ + "futures-channel", + "futures-core", + "futures-executor", + "futures-io", + "futures-sink", + "futures-task", + "futures-util", +] + +[[package]] +name = "futures-channel" +version = "0.3.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "07bbe89c50d7a535e539b8c17bc0b49bdb77747034daa8087407d655f3f7cc1d" +dependencies = [ + "futures-core", + "futures-sink", +] + +[[package]] +name = "futures-core" +version = "0.3.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7e3450815272ef58cec6d564423f6e755e25379b217b0bc688e295ba24df6b1d" + +[[package]] +name = "futures-executor" +version = "0.3.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "baf29c38818342a3b26b5b923639e7b1f4a61fc5e76102d4b1981c6dc7a7579d" +dependencies = [ + "futures-core", + "futures-task", + "futures-util", +] + +[[package]] +name = "futures-io" +version = "0.3.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cecba35d7ad927e23624b22ad55235f2239cfa44fd10428eecbeba6d6a717718" + +[[package]] +name = "futures-macro" +version = "0.3.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e835b70203e41293343137df5c0664546da5745f82ec9b84d40be8336958447b" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "futures-sink" +version = "0.3.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c39754e157331b013978ec91992bde1ac089843443c49cbc7f46150b0fad0893" + +[[package]] +name = "futures-task" +version = "0.3.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "037711b3d59c33004d3856fbdc83b99d4ff37a24768fa1be9ce3538a1cde4393" + +[[package]] +name = "futures-timer" +version = "3.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "af43fadb8a98512d547e37b4e92e0ced13e205c061b87b4623eff01d918d6968" + +[[package]] +name = "futures-util" +version = "0.3.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "389ca41296e6190b48053de0321d02a77f32f8a5d2461dd38762c0593805c6d6" +dependencies = [ + "futures-channel", + "futures-core", + "futures-io", + "futures-macro", + "futures-sink", + "futures-task", + "memchr", + "pin-project-lite", + "slab", +] + +[[package]] +name = "generic-array" +version = "0.14.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85649ca51fd72272d7821adaf274ad91c288277713d9c18820d8499a7ff69e9a" +dependencies = [ + "typenum", + "version_check", +] + +[[package]] +name = "getrandom" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ff2abc00be7fca6ebc474524697ae276ad847ad0a6b3faa4bcb027e9a4614ad0" +dependencies = [ + "cfg-if", + "js-sys", + "libc", + "wasi", + "wasm-bindgen", +] + +[[package]] +name = "getrandom" +version = "0.3.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "899def5c37c4fd7b2664648c28120ecec138e4d395b459e5ca34f9cce2dd77fd" +dependencies = [ + "cfg-if", + "js-sys", + "libc", + "r-efi 5.3.0", + "wasip2", + "wasm-bindgen", +] + +[[package]] +name = "getrandom" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0de51e6874e94e7bf76d726fc5d13ba782deca734ff60d5bb2fb2607c7406555" +dependencies = [ + "cfg-if", + "libc", + "r-efi 6.0.0", + "wasip2", + "wasip3", +] + +[[package]] +name = "glob" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0cc23270f6e1808e30a928bdc84dea0b9b4136a8bc82338574f23baf47bbd280" + +[[package]] +name = "h2" +version = "0.4.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2f44da3a8150a6703ed5d34e164b875fd14c2cdab9af1252a9a1020bde2bdc54" +dependencies = [ + "atomic-waker", + "bytes", + "fnv", + "futures-core", + "futures-sink", + "http", + "indexmap", + "slab", + "tokio", + "tokio-util", + "tracing", +] + +[[package]] +name = "half" +version = "2.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6ea2d84b969582b4b1864a92dc5d27cd2b77b622a8d79306834f1be5ba20d84b" +dependencies = [ + "cfg-if", + "crunchy", + "num-traits", + "zerocopy", +] + +[[package]] +name = "hashbrown" +version = "0.15.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9229cfe53dfd69f0609a49f65461bd93001ea1ef889cd5529dd176593f5338a1" +dependencies = [ + "foldhash", +] + +[[package]] +name = "hashbrown" +version = "0.16.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "841d1cc9bed7f9236f321df977030373f4a4163ae1a7dbfe1a51a2c1a51d9100" + +[[package]] +name = "hashbrown" +version = "0.17.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4f467dd6dccf739c208452f8014c75c18bb8301b050ad1cfb27153803edb0f51" + +[[package]] +name = "headers" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b3314d5adb5d94bcdf56771f2e50dbbc80bb4bdf88967526706205ac9eff24eb" +dependencies = [ + "base64", + "bytes", + "headers-core", + "http", + "httpdate", + "mime", + "sha1", +] + +[[package]] +name = "headers-core" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "54b4a22553d4242c49fddb9ba998a99962b5cc6f22cb5a3482bec22522403ce4" +dependencies = [ + "http", +] + +[[package]] +name = "heck" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "95505c38b4572b2d910cecb0281560f54b440a19336cbbcb27bf6ce6adc6f5a8" + +[[package]] +name = "heck" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea" + +[[package]] +name = "hermit-abi" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fc0fef456e4baa96da950455cd02c081ca953b141298e41db3fc7e36b1da849c" + +[[package]] +name = "http" +version = "1.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e3ba2a386d7f85a81f119ad7498ebe444d2e22c2af0b86b069416ace48b3311a" +dependencies = [ + "bytes", + "itoa", +] + +[[package]] +name = "http-body" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1efedce1fb8e6913f23e0c92de8e62cd5b772a67e7b3946df930a62566c93184" +dependencies = [ + "bytes", + "http", +] + +[[package]] +name = "http-body-util" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b021d93e26becf5dc7e1b75b1bed1fd93124b374ceb73f43d4d4eafec896a64a" +dependencies = [ + "bytes", + "futures-core", + "http", + "http-body", + "pin-project-lite", +] + +[[package]] +name = "httparse" +version = "1.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6dbf3de79e51f3d586ab4cb9d5c3e2c14aa28ed23d180cf89b4df0454a69cc87" + +[[package]] +name = "httpdate" +version = "1.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "df3b46402a9d5adb4c86a0cf463f42e19994e3ee891101b1841f30a545cb49a9" + +[[package]] +name = "hyper" +version = "1.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2ab2d4f250c3d7b1c9fcdff1cece94ea4e2dfbec68614f7b87cb205f24ca9d11" +dependencies = [ + "atomic-waker", + "bytes", + "futures-channel", + "futures-core", + "h2", + "http", + "http-body", + "httparse", + "httpdate", + "itoa", + "pin-project-lite", + "pin-utils", + "smallvec", + "tokio", + "want", +] + +[[package]] +name = "hyper-http-proxy" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7ad4b0a1e37510028bc4ba81d0e38d239c39671b0f0ce9e02dfa93a8133f7c08" +dependencies = [ + "bytes", + "futures-util", + "headers", + "http", + "hyper", + "hyper-rustls", + "hyper-util", + "pin-project-lite", + "rustls-native-certs 0.7.3", + "tokio", + "tokio-rustls", + "tower-service", +] + +[[package]] +name = "hyper-rustls" +version = "0.27.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e3c93eb611681b207e1fe55d5a71ecf91572ec8a6705cdb6857f7d8d5242cf58" +dependencies = [ + "http", + "hyper", + "hyper-util", + "rustls", + "rustls-native-certs 0.8.3", + "rustls-pki-types", + "tokio", + "tokio-rustls", + "tower-service", + "webpki-roots", +] + +[[package]] +name = "hyper-timeout" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2b90d566bffbce6a75bd8b09a05aa8c2cb1fabb6cb348f8840c9e4c90a0d83b0" +dependencies = [ + "hyper", + "hyper-util", + "pin-project-lite", + "tokio", + "tower-service", +] + +[[package]] +name = "hyper-util" +version = "0.1.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "96547c2556ec9d12fb1578c4eaf448b04993e7fb79cbaad930a656880a6bdfa0" +dependencies = [ + "base64", + "bytes", + "futures-channel", + "futures-util", + "http", + "http-body", + "hyper", + "ipnet", + "libc", + "percent-encoding", + "pin-project-lite", + "socket2", + "tokio", + "tower-service", + "tracing", +] + +[[package]] +name = "iana-time-zone" +version = "0.1.65" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e31bc9ad994ba00e440a8aa5c9ef0ec67d5cb5e5cb0cc7f8b744a35b389cc470" +dependencies = [ + "android_system_properties", + "core-foundation-sys", + "iana-time-zone-haiku", + "js-sys", + "log", + "wasm-bindgen", + "windows-core", +] + +[[package]] +name = "iana-time-zone-haiku" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f31827a206f56af32e590ba56d5d2d085f558508192593743f16b2306495269f" +dependencies = [ + "cc", +] + +[[package]] +name = "icu_collections" +version = "2.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4c6b649701667bbe825c3b7e6388cb521c23d88644678e83c0c4d0a621a34b43" +dependencies = [ + "displaydoc", + "potential_utf", + "yoke", + "zerofrom", + "zerovec", +] + +[[package]] +name = "icu_locale_core" +version = "2.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "edba7861004dd3714265b4db54a3c390e880ab658fec5f7db895fae2046b5bb6" +dependencies = [ + "displaydoc", + "litemap", + "tinystr", + "writeable", + "zerovec", +] + +[[package]] +name = "icu_normalizer" +version = "2.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5f6c8828b67bf8908d82127b2054ea1b4427ff0230ee9141c54251934ab1b599" +dependencies = [ + "icu_collections", + "icu_normalizer_data", + "icu_properties", + "icu_provider", + "smallvec", + "zerovec", +] + +[[package]] +name = "icu_normalizer_data" +version = "2.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7aedcccd01fc5fe81e6b489c15b247b8b0690feb23304303a9e560f37efc560a" + +[[package]] +name = "icu_properties" +version = "2.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "020bfc02fe870ec3a66d93e677ccca0562506e5872c650f893269e08615d74ec" +dependencies = [ + "icu_collections", + "icu_locale_core", + "icu_properties_data", + "icu_provider", + "zerotrie", + "zerovec", +] + +[[package]] +name = "icu_properties_data" +version = "2.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "616c294cf8d725c6afcd8f55abc17c56464ef6211f9ed59cccffe534129c77af" + +[[package]] +name = "icu_provider" +version = "2.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85962cf0ce02e1e0a629cc34e7ca3e373ce20dda4c4d7294bbd0bf1fdb59e614" +dependencies = [ + "displaydoc", + "icu_locale_core", + "writeable", + "yoke", + "zerofrom", + "zerotrie", + "zerovec", +] + +[[package]] +name = "id-arena" +version = "2.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3d3067d79b975e8844ca9eb072e16b31c3c1c36928edf9c6789548c524d0d954" + +[[package]] +name = "idna" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3b0875f23caa03898994f6ddc501886a45c7d3d62d04d2d90788d47be1b1e4de" +dependencies = [ + "idna_adapter", + "smallvec", + "utf8_iter", +] + +[[package]] +name = "idna_adapter" +version = "1.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3acae9609540aa318d1bc588455225fb2085b9ed0c4f6bd0d9d5bcd86f1a0344" +dependencies = [ + "icu_normalizer", + "icu_properties", +] + +[[package]] +name = "indexmap" +version = "2.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7714e70437a7dc3ac8eb7e6f8df75fd8eb422675fc7678aff7364301092b1017" +dependencies = [ + "equivalent", + "hashbrown 0.16.1", + "serde", + "serde_core", +] + +[[package]] +name = "ipnet" +version = "2.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d98f6fed1fde3f8c21bc40a1abb88dd75e67924f9cffc3ef95607bad8017f8e2" + +[[package]] +name = "iri-string" +version = "0.7.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c91338f0783edbd6195decb37bae672fd3b165faffb89bf7b9e6942f8b1a731a" +dependencies = [ + "memchr", + "serde", +] + +[[package]] +name = "is-terminal" +version = "0.4.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3640c1c38b8e4e43584d8df18be5fc6b0aa314ce6ebf51b53313d4306cca8e46" +dependencies = [ + "hermit-abi", + "libc", + "windows-sys 0.61.2", +] + +[[package]] +name = "is_terminal_polyfill" +version = "1.70.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a6cb138bb79a146c1bd460005623e142ef0181e3d0219cb493e02f7d08a35695" + +[[package]] +name = "itertools" +version = "0.10.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b0fd2260e829bddf4cb6ea802289de2f86d6a7a690192fbe91b3f46e0f2c8473" +dependencies = [ + "either", +] + +[[package]] +name = "itertools" +version = "0.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2b192c782037fadd9cfa75548310488aabdbf3d2da73885b31bd0abd03351285" +dependencies = [ + "either", +] + +[[package]] +name = "itoa" +version = "1.0.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "92ecc6618181def0457392ccd0ee51198e065e016d1d527a7ac1b6dc7c1f09d2" + +[[package]] +name = "jobserver" +version = "0.1.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9afb3de4395d6b3e67a780b6de64b51c978ecf11cb9a462c66be7d4ca9039d33" +dependencies = [ + "getrandom 0.3.4", + "libc", +] + +[[package]] +name = "js-sys" +version = "0.3.91" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b49715b7073f385ba4bc528e5747d02e66cb39c6146efb66b781f131f0fb399c" +dependencies = [ + "once_cell", + "wasm-bindgen", +] + +[[package]] +name = "lazy_static" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bbd2bcb4c963f2ddae06a2efc7e9f3591312473c50c6685e1f298068316e66fe" + +[[package]] +name = "leb128fmt" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09edd9e8b54e49e587e4f6295a7d29c3ea94d469cb40ab8ca70b288248a81db2" + +[[package]] +name = "lexical-core" +version = "1.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7d8d125a277f807e55a77304455eb7b1cb52f2b18c143b60e766c120bd64a594" +dependencies = [ + "lexical-parse-float", + "lexical-parse-integer", + "lexical-util", + "lexical-write-float", + "lexical-write-integer", +] + +[[package]] +name = "lexical-parse-float" +version = "1.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "52a9f232fbd6f550bc0137dcb5f99ab674071ac2d690ac69704593cb4abbea56" +dependencies = [ + "lexical-parse-integer", + "lexical-util", +] + +[[package]] +name = "lexical-parse-integer" +version = "1.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9a7a039f8fb9c19c996cd7b2fcce303c1b2874fe1aca544edc85c4a5f8489b34" +dependencies = [ + "lexical-util", +] + +[[package]] +name = "lexical-util" +version = "1.0.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2604dd126bb14f13fb5d1bd6a66155079cb9fa655b37f875b3a742c705dbed17" + +[[package]] +name = "lexical-write-float" +version = "1.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "50c438c87c013188d415fbabbb1dceb44249ab81664efbd31b14ae55dabb6361" +dependencies = [ + "lexical-util", + "lexical-write-integer", +] + +[[package]] +name = "lexical-write-integer" +version = "1.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "409851a618475d2d5796377cad353802345cba92c867d9fbcde9cf4eac4e14df" +dependencies = [ + "lexical-util", +] + +[[package]] +name = "libc" +version = "0.2.186" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "68ab91017fe16c622486840e4c83c9a37afeff978bd239b5293d61ece587de66" + +[[package]] +name = "libm" +version = "0.2.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6d2cec3eae94f9f509c767b45932f1ada8350c4bdb85af2fcab4a3c14807981" + +[[package]] +name = "linux-raw-sys" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32a66949e030da00e8c7d4434b251670a91556f4144941d37452769c25d58a53" + +[[package]] +name = "litemap" +version = "0.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6373607a59f0be73a39b6fe456b8192fcc3585f602af20751600e974dd455e77" + +[[package]] +name = "log" +version = "0.4.29" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5e5032e24019045c762d3c0f28f5b6b8bbf38563a65908389bf7978758920897" + +[[package]] +name = "lru-slab" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "112b39cec0b298b6c1999fee3e31427f74f676e4cb9879ed1a121b43661a4154" + +[[package]] +name = "lz4_flex" +version = "0.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "db9a0d582c2874f68138a16ce1867e0ffde6c0bb0a0df85e1f36d04146db488a" +dependencies = [ + "twox-hash", +] + +[[package]] +name = "matchers" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d1525a2a28c7f4fa0fc98bb91ae755d1e2d1505079e05539e35bc876b5d65ae9" +dependencies = [ + "regex-automata", +] + +[[package]] +name = "matchit" +version = "0.8.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "47e1ffaa40ddd1f3ed91f717a33c8c0ee23fff369e3aa8772b9605cc1d22f4c3" + +[[package]] +name = "memchr" +version = "2.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8ca58f447f06ed17d5fc4043ce1b10dd205e060fb3ce5b979b8ed8e59ff3f79" + +[[package]] +name = "mime" +version = "0.3.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6877bb514081ee2a7ff5ef9de3281f14a4dd4bceac4c09388074a6b5df8a139a" + +[[package]] +name = "mio" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "50b7e5b27aa02a74bac8c3f23f448f8d87ff11f92d3aac1a6ed369ee08cc56c1" +dependencies = [ + "libc", + "wasi", + "windows-sys 0.61.2", +] + +[[package]] +name = "multimap" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1d87ecb2933e8aeadb3e3a02b828fed80a7528047e68b4f424523a0981a3a084" + +[[package]] +name = "nu-ansi-term" +version = "0.50.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7957b9740744892f114936ab4a57b3f487491bbeafaf8083688b16841a4240e5" +dependencies = [ + "windows-sys 0.61.2", +] + +[[package]] +name = "num-bigint" +version = "0.4.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a5e44f723f1133c9deac646763579fdb3ac745e418f2a7af9cd0c431da1f20b9" +dependencies = [ + "num-integer", + "num-traits", +] + +[[package]] +name = "num-complex" +version = "0.4.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "73f88a1307638156682bada9d7604135552957b7818057dcef22705b4d509495" +dependencies = [ + "num-traits", +] + +[[package]] +name = "num-integer" +version = "0.1.46" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7969661fd2958a5cb096e56c8e1ad0444ac2bbcd0061bd28660485a44879858f" +dependencies = [ + "num-traits", +] + +[[package]] +name = "num-traits" +version = "0.2.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "071dfc062690e90b734c0b2273ce72ad0ffa95f0c74596bc250dcfd960262841" +dependencies = [ + "autocfg", + "libm", +] + +[[package]] +name = "once_cell" +version = "1.21.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "42f5e15c9953c5e4ccceeb2e7382a716482c34515315f7b03532b8b4e8393d2d" + +[[package]] +name = "once_cell_polyfill" +version = "1.70.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "384b8ab6d37215f3c5301a95a4accb5d64aa607f1fcb26a11b5303878451b4fe" + +[[package]] +name = "oorandom" +version = "11.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d6790f58c7ff633d8771f42965289203411a5e5c68388703c06e14f24770b41e" + +[[package]] +name = "openssl-probe" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d05e27ee213611ffe7d6348b942e8f942b37114c00cc03cec254295a4a17852e" + +[[package]] +name = "openssl-probe" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7c87def4c32ab89d880effc9e097653c8da5d6ef28e6b539d313baaacfbafcbe" + +[[package]] +name = "ordered-float" +version = "2.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "68f19d67e5a2795c94e73e0bb1cc1a7edeb2e28efd39e2e1c9b7a40c1108b11c" +dependencies = [ + "num-traits", +] + +[[package]] +name = "percent-encoding" +version = "2.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9b4f627cb1b25917193a259e49bdad08f671f8d9708acfd5fe0a8c1455d87220" + +[[package]] +name = "petgraph" +version = "0.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8701b58ea97060d5e5b155d383a69952a60943f0e6dfe30b04c287beb0b27455" +dependencies = [ + "fixedbitset", + "hashbrown 0.15.5", + "indexmap", +] + +[[package]] +name = "pin-project" +version = "1.1.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f1749c7ed4bcaf4c3d0a3efc28538844fb29bcdd7d2b67b2be7e20ba861ff517" +dependencies = [ + "pin-project-internal", +] + +[[package]] +name = "pin-project-internal" +version = "1.1.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d9b20ed30f105399776b9c883e68e536ef602a16ae6f596d2c473591d6ad64c6" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "pin-project-lite" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a89322df9ebe1c1578d689c92318e070967d1042b512afbe49518723f4e6d5cd" + +[[package]] +name = "pin-utils" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b870d8c151b6f2fb93e84a13146138f05d02ed11c7e7c54f8826aaaf7c9f184" + +[[package]] +name = "pkg-config" +version = "0.3.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7edddbd0b52d732b21ad9a5fab5c704c14cd949e5e9a1ec5929a24fded1b904c" + +[[package]] +name = "plotters" +version = "0.3.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5aeb6f403d7a4911efb1e33402027fc44f29b5bf6def3effcc22d7bb75f2b747" +dependencies = [ + "num-traits", + "plotters-backend", + "plotters-svg", + "wasm-bindgen", + "web-sys", +] + +[[package]] +name = "plotters-backend" +version = "0.3.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "df42e13c12958a16b3f7f4386b9ab1f3e7933914ecea48da7139435263a4172a" + +[[package]] +name = "plotters-svg" +version = "0.3.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "51bae2ac328883f7acdfea3d66a7c35751187f870bc81f94563733a154d7a670" +dependencies = [ + "plotters-backend", +] + +[[package]] +name = "potential_utf" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b73949432f5e2a09657003c25bca5e19a0e9c84f8058ca374f49e0ebe605af77" +dependencies = [ + "zerovec", +] + +[[package]] +name = "ppv-lite86" +version = "0.2.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85eae3c4ed2f50dcfe72643da4befc30deadb458a9b590d720cde2f2b1e97da9" +dependencies = [ + "zerocopy", +] + +[[package]] +name = "prettyplease" +version = "0.2.37" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "479ca8adacdd7ce8f1fb39ce9ecccbfe93a3f1344b3d0d97f20bc0196208f62b" +dependencies = [ + "proc-macro2", + "syn", +] + +[[package]] +name = "proc-macro-crate" +version = "3.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e67ba7e9b2b56446f1d419b1d807906278ffa1a658a8a5d8a39dcb1f5a78614f" +dependencies = [ + "toml_edit 0.25.11+spec-1.1.0", +] + +[[package]] +name = "proc-macro2" +version = "1.0.106" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8fd00f0bb2e90d81d1044c2b32617f68fcb9fa3bb7640c23e9c748e53fb30934" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "prost" +version = "0.14.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d2ea70524a2f82d518bce41317d0fae74151505651af45faf1ffbd6fd33f0568" +dependencies = [ + "bytes", + "prost-derive", +] + +[[package]] +name = "prost-build" +version = "0.14.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "343d3bd7056eda839b03204e68deff7d1b13aba7af2b2fd16890697274262ee7" +dependencies = [ + "heck 0.5.0", + "itertools 0.14.0", + "log", + "multimap", + "petgraph", + "prettyplease", + "prost", + "prost-types", + "pulldown-cmark", + "pulldown-cmark-to-cmark", + "regex", + "syn", + "tempfile", +] + +[[package]] +name = "prost-derive" +version = "0.14.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "27c6023962132f4b30eb4c172c91ce92d933da334c59c23cddee82358ddafb0b" +dependencies = [ + "anyhow", + "itertools 0.14.0", + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "prost-reflect" +version = "0.16.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b89455ef41ed200cafc47c76c552ee7792370ac420497e551f16123a9135f76e" +dependencies = [ + "base64", + "prost", + "prost-types", + "serde", + "serde-value", +] + +[[package]] +name = "prost-types" +version = "0.14.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8991c4cbdb8bc5b11f0b074ffe286c30e523de90fee5ba8132f1399f23cb3dd7" +dependencies = [ + "prost", +] + +[[package]] +name = "protoc-bin-vendored" +version = "3.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d1c381df33c98266b5f08186583660090a4ffa0889e76c7e9a5e175f645a67fa" +dependencies = [ + "protoc-bin-vendored-linux-aarch_64", + "protoc-bin-vendored-linux-ppcle_64", + "protoc-bin-vendored-linux-s390_64", + "protoc-bin-vendored-linux-x86_32", + "protoc-bin-vendored-linux-x86_64", + "protoc-bin-vendored-macos-aarch_64", + "protoc-bin-vendored-macos-x86_64", + "protoc-bin-vendored-win32", +] + +[[package]] +name = "protoc-bin-vendored-linux-aarch_64" +version = "3.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c350df4d49b5b9e3ca79f7e646fde2377b199e13cfa87320308397e1f37e1a4c" + +[[package]] +name = "protoc-bin-vendored-linux-ppcle_64" +version = "3.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a55a63e6c7244f19b5c6393f025017eb5d793fd5467823a099740a7a4222440c" + +[[package]] +name = "protoc-bin-vendored-linux-s390_64" +version = "3.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1dba5565db4288e935d5330a07c264a4ee8e4a5b4a4e6f4e83fad824cc32f3b0" + +[[package]] +name = "protoc-bin-vendored-linux-x86_32" +version = "3.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8854774b24ee28b7868cd71dccaae8e02a2365e67a4a87a6cd11ee6cdbdf9cf5" + +[[package]] +name = "protoc-bin-vendored-linux-x86_64" +version = "3.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b38b07546580df720fa464ce124c4b03630a6fb83e05c336fea2a241df7e5d78" + +[[package]] +name = "protoc-bin-vendored-macos-aarch_64" +version = "3.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "89278a9926ce312e51f1d999fee8825d324d603213344a9a706daa009f1d8092" + +[[package]] +name = "protoc-bin-vendored-macos-x86_64" +version = "3.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "81745feda7ccfb9471d7a4de888f0652e806d5795b61480605d4943176299756" + +[[package]] +name = "protoc-bin-vendored-win32" +version = "3.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "95067976aca6421a523e491fce939a3e65249bac4b977adee0ee9771568e8aa3" + +[[package]] +name = "pulldown-cmark" +version = "0.13.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7c3a14896dfa883796f1cb410461aef38810ea05f2b2c33c5aded3649095fdad" +dependencies = [ + "bitflags", + "memchr", + "unicase", +] + +[[package]] +name = "pulldown-cmark-to-cmark" +version = "22.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "50793def1b900256624a709439404384204a5dc3a6ec580281bfaac35e882e90" +dependencies = [ + "pulldown-cmark", +] + +[[package]] +name = "quinn" +version = "0.11.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b9e20a958963c291dc322d98411f541009df2ced7b5a4f2bd52337638cfccf20" +dependencies = [ + "bytes", + "cfg_aliases", + "pin-project-lite", + "quinn-proto", + "quinn-udp", + "rustc-hash", + "rustls", + "socket2", + "thiserror 2.0.18", + "tokio", + "tracing", + "web-time", +] + +[[package]] +name = "quinn-proto" +version = "0.11.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f1906b49b0c3bc04b5fe5d86a77925ae6524a19b816ae38ce1e426255f1d8a31" +dependencies = [ + "bytes", + "getrandom 0.3.4", + "lru-slab", + "rand 0.9.2", + "ring", + "rustc-hash", + "rustls", + "rustls-pki-types", + "slab", + "thiserror 2.0.18", + "tinyvec", + "tracing", + "web-time", +] + +[[package]] +name = "quinn-udp" +version = "0.5.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "addec6a0dcad8a8d96a771f815f0eaf55f9d1805756410b39f5fa81332574cbd" +dependencies = [ + "cfg_aliases", + "libc", + "once_cell", + "socket2", + "tracing", + "windows-sys 0.60.2", +] + +[[package]] +name = "quote" +version = "1.0.45" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "41f2619966050689382d2b44f664f4bc593e129785a36d6ee376ddf37259b924" +dependencies = [ + "proc-macro2", +] + +[[package]] +name = "r-efi" +version = "5.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "69cdb34c158ceb288df11e18b4bd39de994f6657d83847bdffdbd7f346754b0f" + +[[package]] +name = "r-efi" +version = "6.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8dcc9c7d52a811697d2151c701e0d08956f92b0e24136cf4cf27b57a6a0d9bf" + +[[package]] +name = "rand" +version = "0.8.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "34af8d1a0e25924bc5b7c43c079c942339d8f0a8b57c39049bef581b46327404" +dependencies = [ + "libc", + "rand_chacha 0.3.1", + "rand_core 0.6.4", +] + +[[package]] +name = "rand" +version = "0.9.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6db2770f06117d490610c7488547d543617b21bfa07796d7a12f6f1bd53850d1" +dependencies = [ + "rand_chacha 0.9.0", + "rand_core 0.9.5", +] + +[[package]] +name = "rand_chacha" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6c10a63a0fa32252be49d21e7709d4d4baf8d231c2dbce1eaa8141b9b127d88" +dependencies = [ + "ppv-lite86", + "rand_core 0.6.4", +] + +[[package]] +name = "rand_chacha" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d3022b5f1df60f26e1ffddd6c66e8aa15de382ae63b3a0c1bfc0e4d3e3f325cb" +dependencies = [ + "ppv-lite86", + "rand_core 0.9.5", +] + +[[package]] +name = "rand_core" +version = "0.6.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ec0be4795e2f6a28069bec0b5ff3e2ac9bafc99e6a9a7dc3547996c5c816922c" +dependencies = [ + "getrandom 0.2.17", +] + +[[package]] +name = "rand_core" +version = "0.9.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "76afc826de14238e6e8c374ddcc1fa19e374fd8dd986b0d2af0d02377261d83c" +dependencies = [ + "getrandom 0.3.4", +] + +[[package]] +name = "regex" +version = "1.12.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e10754a14b9137dd7b1e3e5b0493cc9171fdd105e0ab477f51b72e7f3ac0e276" +dependencies = [ + "aho-corasick", + "memchr", + "regex-automata", + "regex-syntax", +] + +[[package]] +name = "regex-automata" +version = "0.4.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6e1dd4122fc1595e8162618945476892eefca7b88c52820e74af6262213cae8f" +dependencies = [ + "aho-corasick", + "memchr", + "regex-syntax", +] + +[[package]] +name = "regex-syntax" +version = "0.8.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dc897dd8d9e8bd1ed8cdad82b5966c3e0ecae09fb1907d58efaa013543185d0a" + +[[package]] +name = "relative-path" +version = "1.9.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ba39f3699c378cd8970968dcbff9c43159ea4cfbd88d43c00b22f2ef10a435d2" + +[[package]] +name = "reqwest" +version = "0.12.28" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eddd3ca559203180a307f12d114c268abf583f59b03cb906fd0b3ff8646c1147" +dependencies = [ + "base64", + "bytes", + "futures-core", + "http", + "http-body", + "http-body-util", + "hyper", + "hyper-rustls", + "hyper-util", + "js-sys", + "log", + "percent-encoding", + "pin-project-lite", + "quinn", + "rustls", + "rustls-native-certs 0.8.3", + "rustls-pki-types", + "serde", + "serde_json", + "serde_urlencoded", + "sync_wrapper", + "tokio", + "tokio-rustls", + "tower", + "tower-http", + "tower-service", + "url", + "wasm-bindgen", + "wasm-bindgen-futures", + "web-sys", + "webpki-roots", +] + +[[package]] +name = "ring" +version = "0.17.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a4689e6c2294d81e88dc6261c768b63bc4fcdb852be6d1352498b114f61383b7" +dependencies = [ + "cc", + "cfg-if", + "getrandom 0.2.17", + "libc", + "untrusted", + "windows-sys 0.52.0", +] + +[[package]] +name = "rstest" +version = "0.24.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "03e905296805ab93e13c1ec3a03f4b6c4f35e9498a3d5fa96dc626d22c03cd89" +dependencies = [ + "futures-timer", + "futures-util", + "rstest_macros", + "rustc_version", +] + +[[package]] +name = "rstest_macros" +version = "0.24.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ef0053bbffce09062bee4bcc499b0fbe7a57b879f1efe088d6d8d4c7adcdef9b" +dependencies = [ + "cfg-if", + "glob", + "proc-macro-crate", + "proc-macro2", + "quote", + "regex", + "relative-path", + "rustc_version", + "syn", + "unicode-ident", +] + +[[package]] +name = "rustc-hash" +version = "2.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "357703d41365b4b27c590e3ed91eabb1b663f07c4c084095e60cbed4362dff0d" + +[[package]] +name = "rustc_version" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cfcb3a22ef46e85b45de6ee7e79d063319ebb6594faafcf1c225ea92ab6e9b92" +dependencies = [ + "semver", +] + +[[package]] +name = "rustix" +version = "1.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6fe4565b9518b83ef4f91bb47ce29620ca828bd32cb7e408f0062e9930ba190" +dependencies = [ + "bitflags", + "errno", + "libc", + "linux-raw-sys", + "windows-sys 0.61.2", +] + +[[package]] +name = "rustls" +version = "0.23.37" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "758025cb5fccfd3bc2fd74708fd4682be41d99e5dff73c377c0646c6012c73a4" +dependencies = [ + "log", + "once_cell", + "ring", + "rustls-pki-types", + "rustls-webpki", + "subtle", + "zeroize", +] + +[[package]] +name = "rustls-native-certs" +version = "0.7.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e5bfb394eeed242e909609f56089eecfe5fda225042e8b171791b9c95f5931e5" +dependencies = [ + "openssl-probe 0.1.6", + "rustls-pemfile", + "rustls-pki-types", + "schannel", + "security-framework 2.11.1", +] + +[[package]] +name = "rustls-native-certs" +version = "0.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "612460d5f7bea540c490b2b6395d8e34a953e52b491accd6c86c8164c5932a63" +dependencies = [ + "openssl-probe 0.2.1", + "rustls-pki-types", + "schannel", + "security-framework 3.7.0", +] + +[[package]] +name = "rustls-pemfile" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dce314e5fee3f39953d46bb63bb8a46d40c2f8fb7cc5a3b6cab2bde9721d6e50" +dependencies = [ + "rustls-pki-types", +] + +[[package]] +name = "rustls-pki-types" +version = "1.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "be040f8b0a225e40375822a563fa9524378b9d63112f53e19ffff34df5d33fdd" +dependencies = [ + "web-time", + "zeroize", +] + +[[package]] +name = "rustls-webpki" +version = "0.103.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d7df23109aa6c1567d1c575b9952556388da57401e4ace1d15f79eedad0d8f53" +dependencies = [ + "ring", + "rustls-pki-types", + "untrusted", +] + +[[package]] +name = "rustversion" +version = "1.0.22" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b39cdef0fa800fc44525c84ccb54a029961a8215f9619753635a9c0d2538d46d" + +[[package]] +name = "ryu" +version = "1.0.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9774ba4a74de5f7b1c1451ed6cd5285a32eddb5cccb8cc655a4e50009e06477f" + +[[package]] +name = "same-file" +version = "1.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "93fc1dc3aaa9bfed95e02e6eadabb4baf7e3078b0bd1b4d7b6b0b68378900502" +dependencies = [ + "winapi-util", +] + +[[package]] +name = "schannel" +version = "0.1.28" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "891d81b926048e76efe18581bf793546b4c0eaf8448d72be8de2bbee5fd166e1" +dependencies = [ + "windows-sys 0.61.2", +] + +[[package]] +name = "security-framework" +version = "2.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "897b2245f0b511c87893af39b033e5ca9cce68824c4d7e7630b5a1d339658d02" +dependencies = [ + "bitflags", + "core-foundation 0.9.4", + "core-foundation-sys", + "libc", + "security-framework-sys", +] + +[[package]] +name = "security-framework" +version = "3.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b7f4bc775c73d9a02cde8bf7b2ec4c9d12743edf609006c7facc23998404cd1d" +dependencies = [ + "bitflags", + "core-foundation 0.10.1", + "core-foundation-sys", + "libc", + "security-framework-sys", +] + +[[package]] +name = "security-framework-sys" +version = "2.17.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6ce2691df843ecc5d231c0b14ece2acc3efb62c0a398c7e1d875f3983ce020e3" +dependencies = [ + "core-foundation-sys", + "libc", +] + +[[package]] +name = "self_cell" +version = "1.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b12e76d157a900eb52e81bc6e9f3069344290341720e9178cde2407113ac8d89" + +[[package]] +name = "semver" +version = "1.0.27" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d767eb0aabc880b29956c35734170f26ed551a859dbd361d140cdbeca61ab1e2" + +[[package]] +name = "serde" +version = "1.0.228" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9a8e94ea7f378bd32cbbd37198a4a91436180c5bb472411e48b5ec2e2124ae9e" +dependencies = [ + "serde_core", + "serde_derive", +] + +[[package]] +name = "serde-value" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f3a1a3341211875ef120e117ea7fd5228530ae7e7036a779fdc9117be6b3282c" +dependencies = [ + "ordered-float", + "serde", +] + +[[package]] +name = "serde_core" +version = "1.0.228" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "41d385c7d4ca58e59fc732af25c3983b67ac852c1a25000afe1175de458b67ad" +dependencies = [ + "serde_derive", +] + +[[package]] +name = "serde_derive" +version = "1.0.228" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d540f220d3187173da220f885ab66608367b6574e925011a9353e4badda91d79" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "serde_json" +version = "1.0.149" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "83fc039473c5595ace860d8c4fafa220ff474b3fc6bfdb4293327f1a37e94d86" +dependencies = [ + "itoa", + "memchr", + "serde", + "serde_core", + "zmij", +] + +[[package]] +name = "serde_spanned" +version = "0.6.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bf41e0cfaf7226dca15e8197172c295a782857fcb97fad1808a166870dee75a3" +dependencies = [ + "serde", +] + +[[package]] +name = "serde_urlencoded" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d3491c14715ca2294c4d6a88f15e84739788c1d030eed8c110436aafdaa2f3fd" +dependencies = [ + "form_urlencoded", + "itoa", + "ryu", + "serde", +] + +[[package]] +name = "sha1" +version = "0.10.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e3bf829a2d51ab4a5ddf1352d8470c140cadc8301b2ae1789db023f01cedd6ba" +dependencies = [ + "cfg-if", + "cpufeatures", + "digest", +] + +[[package]] +name = "sha2" +version = "0.10.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a7507d819769d01a365ab707794a4084392c824f54a7a6a7862f8c3d0892b283" +dependencies = [ + "cfg-if", + "cpufeatures", + "digest", +] + +[[package]] +name = "sharded-slab" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f40ca3c46823713e0d4209592e8d6e826aa57e928f09752619fc696c499637f6" +dependencies = [ + "lazy_static", +] + +[[package]] +name = "shlex" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0fda2ff0d084019ba4d7c6f371c95d8fd75ce3524c3cb8fb653a3023f6323e64" + +[[package]] +name = "slab" +version = "0.4.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0c790de23124f9ab44544d7ac05d60440adc586479ce501c1d6d7da3cd8c9cf5" + +[[package]] +name = "smallvec" +version = "1.15.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "67b1b7a3b5fe4f1376887184045fcf45c69e92af734b7aaddc05fb777b6fbd03" + +[[package]] +name = "socket2" +version = "0.6.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3a766e1110788c36f4fa1c2b71b387a7815aa65f88ce0229841826633d93723e" +dependencies = [ + "libc", + "windows-sys 0.61.2", +] + +[[package]] +name = "stable_deref_trait" +version = "1.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6ce2be8dc25455e1f91df71bfa12ad37d7af1092ae736f3a6cd0e37bc7810596" + +[[package]] +name = "strsim" +version = "0.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7da8b5736845d9f2fcb837ea5d9e2628564b3b043a70948a3f0b778838c5fb4f" + +[[package]] +name = "strum" +version = "0.27.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "af23d6f6c1a224baef9d3f61e287d2761385a5b88fdab4eb4c6f11aeb54c4bcf" +dependencies = [ + "strum_macros", +] + +[[package]] +name = "strum_macros" +version = "0.27.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7695ce3845ea4b33927c055a39dc438a45b059f7c1b3d91d38d10355fb8cbca7" +dependencies = [ + "heck 0.5.0", + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "subtle" +version = "2.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "13c2bddecc57b384dee18652358fb23172facb8a2c51ccc10d74c157bdea3292" + +[[package]] +name = "syn" +version = "2.0.117" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e665b8803e7b1d2a727f4023456bbbbe74da67099c585258af0ad9c5013b9b99" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "sync_wrapper" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0bf256ce5efdfa370213c1dabab5935a12e49f2c58d15e9eac2870d3b4f27263" +dependencies = [ + "futures-core", +] + +[[package]] +name = "synstructure" +version = "0.13.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "728a70f3dbaf5bab7f0c4b1ac8d7ae5ea60a4b5549c8a5914361c99147a709d2" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "tempfile" +version = "3.27.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32497e9a4c7b38532efcdebeef879707aa9f794296a4f0244f6f69e9bc8574bd" +dependencies = [ + "fastrand", + "getrandom 0.4.2", + "once_cell", + "rustix", + "windows-sys 0.61.2", +] + +[[package]] +name = "thiserror" +version = "1.0.69" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6aaf5339b578ea85b50e080feb250a3e8ae8cfcdff9a461c9ec2904bc923f52" +dependencies = [ + "thiserror-impl 1.0.69", +] + +[[package]] +name = "thiserror" +version = "2.0.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4288b5bcbc7920c07a1149a35cf9590a2aa808e0bc1eafaade0b80947865fbc4" +dependencies = [ + "thiserror-impl 2.0.18", +] + +[[package]] +name = "thiserror-impl" +version = "1.0.69" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4fee6c4efc90059e10f81e6d42c60a18f76588c3d74cb83a0b242a2b6c7504c1" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "thiserror-impl" +version = "2.0.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ebc4ee7f67670e9b64d05fa4253e753e016c6c95ff35b89b7941d6b856dec1d5" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "thread_local" +version = "1.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f60246a4944f24f6e018aa17cdeffb7818b76356965d03b07d6a9886e8962185" +dependencies = [ + "cfg-if", +] + +[[package]] +name = "tiny-keccak" +version = "2.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2c9d3793400a45f954c52e73d068316d76b6f4e36977e3fcebb13a2721e80237" +dependencies = [ + "crunchy", +] + +[[package]] +name = "tinystr" +version = "0.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "42d3e9c45c09de15d06dd8acf5f4e0e399e85927b7f00711024eb7ae10fa4869" +dependencies = [ + "displaydoc", + "zerovec", +] + +[[package]] +name = "tinytemplate" +version = "1.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "be4d6b5f19ff7664e8c98d03e2139cb510db9b0a60b55f8e8709b689d939b6bc" +dependencies = [ + "serde", + "serde_json", +] + +[[package]] +name = "tinyvec" +version = "1.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bfa5fdc3bce6191a1dbc8c02d5c8bffcf557bafa17c124c5264a458f1b0613fa" +dependencies = [ + "tinyvec_macros", +] + +[[package]] +name = "tinyvec_macros" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20" + +[[package]] +name = "tokio" +version = "1.52.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "110a78583f19d5cdb2c5ccf321d1290344e71313c6c37d43520d386027d18386" +dependencies = [ + "bytes", + "libc", + "mio", + "pin-project-lite", + "socket2", + "tokio-macros", + "windows-sys 0.61.2", +] + +[[package]] +name = "tokio-macros" +version = "2.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "385a6cb71ab9ab790c5fe8d67f1645e6c450a7ce006a33de03daa956cf70a496" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "tokio-retry" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7f57eb36ecbe0fc510036adff84824dd3c24bb781e21bfa67b69d556aa85214f" +dependencies = [ + "pin-project", + "rand 0.8.5", + "tokio", +] + +[[package]] +name = "tokio-rustls" +version = "0.26.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1729aa945f29d91ba541258c8df89027d5792d85a8841fb65e8bf0f4ede4ef61" +dependencies = [ + "rustls", + "tokio", +] + +[[package]] +name = "tokio-stream" +version = "0.1.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32da49809aab5c3bc678af03902d4ccddea2a87d028d86392a4b1560c6906c70" +dependencies = [ + "futures-core", + "pin-project-lite", + "tokio", +] + +[[package]] +name = "tokio-util" +version = "0.7.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9ae9cec805b01e8fc3fd2fe289f89149a9b66dd16786abd8b19cfa7b48cb0098" +dependencies = [ + "bytes", + "futures-core", + "futures-sink", + "futures-util", + "pin-project-lite", + "tokio", +] + +[[package]] +name = "toml" +version = "0.8.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dc1beb996b9d83529a9e75c17a1686767d148d70663143c7854d8b4a09ced362" +dependencies = [ + "serde", + "serde_spanned", + "toml_datetime 0.6.11", + "toml_edit 0.22.27", +] + +[[package]] +name = "toml_datetime" +version = "0.6.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "22cddaf88f4fbc13c51aebbf5f8eceb5c7c5a9da2ac40a13519eb5b0a0e8f11c" +dependencies = [ + "serde", +] + +[[package]] +name = "toml_datetime" +version = "1.1.1+spec-1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3165f65f62e28e0115a00b2ebdd37eb6f3b641855f9d636d3cd4103767159ad7" +dependencies = [ + "serde_core", +] + +[[package]] +name = "toml_edit" +version = "0.22.27" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "41fe8c660ae4257887cf66394862d21dbca4a6ddd26f04a3560410406a2f819a" +dependencies = [ + "indexmap", + "serde", + "serde_spanned", + "toml_datetime 0.6.11", + "toml_write", + "winnow 0.7.14", +] + +[[package]] +name = "toml_edit" +version = "0.25.11+spec-1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b59c4d22ed448339746c59b905d24568fcbb3ab65a500494f7b8c3e97739f2b" +dependencies = [ + "indexmap", + "toml_datetime 1.1.1+spec-1.1.0", + "toml_parser", + "winnow 1.0.3", +] + +[[package]] +name = "toml_parser" +version = "1.1.2+spec-1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a2abe9b86193656635d2411dc43050282ca48aa31c2451210f4202550afb7526" +dependencies = [ + "winnow 1.0.3", +] + +[[package]] +name = "toml_write" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5d99f8c9a7727884afe522e9bd5edbfc91a3312b36a77b5fb8926e4c31a41801" + +[[package]] +name = "tonic" +version = "0.14.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fec7c61a0695dc1887c1b53952990f3ad2e3a31453e1f49f10e75424943a93ec" +dependencies = [ + "async-trait", + "axum", + "base64", + "bytes", + "h2", + "http", + "http-body", + "http-body-util", + "hyper", + "hyper-timeout", + "hyper-util", + "percent-encoding", + "pin-project", + "rustls-native-certs 0.8.3", + "socket2", + "sync_wrapper", + "tokio", + "tokio-rustls", + "tokio-stream", + "tower", + "tower-layer", + "tower-service", + "tracing", +] + +[[package]] +name = "tonic-build" +version = "0.14.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1882ac3bf5ef12877d7ed57aad87e75154c11931c2ba7e6cde5e22d63522c734" +dependencies = [ + "prettyplease", + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "tonic-prost" +version = "0.14.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a55376a0bbaa4975a3f10d009ad763d8f4108f067c7c2e74f3001fb49778d309" +dependencies = [ + "bytes", + "prost", + "tonic", +] + +[[package]] +name = "tonic-prost-build" +version = "0.14.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f3144df636917574672e93d0f56d7edec49f90305749c668df5101751bb8f95a" +dependencies = [ + "prettyplease", + "proc-macro2", + "prost-build", + "prost-types", + "quote", + "syn", + "tempfile", + "tonic-build", +] + +[[package]] +name = "tower" +version = "0.5.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ebe5ef63511595f1344e2d5cfa636d973292adc0eec1f0ad45fae9f0851ab1d4" +dependencies = [ + "futures-core", + "futures-util", + "indexmap", + "pin-project-lite", + "slab", + "sync_wrapper", + "tokio", + "tokio-util", + "tower-layer", + "tower-service", + "tracing", +] + +[[package]] +name = "tower-http" +version = "0.6.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d4e6559d53cc268e5031cd8429d05415bc4cb4aefc4aa5d6cc35fbf5b924a1f8" +dependencies = [ + "bitflags", + "bytes", + "futures-util", + "http", + "http-body", + "iri-string", + "pin-project-lite", + "tower", + "tower-layer", + "tower-service", +] + +[[package]] +name = "tower-layer" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "121c2a6cda46980bb0fcd1647ffaf6cd3fc79a013de288782836f6df9c48780e" + +[[package]] +name = "tower-service" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8df9b6e13f2d32c91b9bd719c00d1958837bc7dec474d94952798cc8e69eeec3" + +[[package]] +name = "tracing" +version = "0.1.44" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "63e71662fa4b2a2c3a26f570f037eb95bb1f85397f3cd8076caed2f026a6d100" +dependencies = [ + "pin-project-lite", + "tracing-attributes", + "tracing-core", +] + +[[package]] +name = "tracing-attributes" +version = "0.1.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7490cfa5ec963746568740651ac6781f701c9c5ea257c58e057f3ba8cf69e8da" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "tracing-core" +version = "0.1.36" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "db97caf9d906fbde555dd62fa95ddba9eecfd14cb388e4f491a66d74cd5fb79a" +dependencies = [ + "once_cell", + "valuable", +] + +[[package]] +name = "tracing-log" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ee855f1f400bd0e5c02d150ae5de3840039a3f54b025156404e34c23c03f47c3" +dependencies = [ + "log", + "once_cell", + "tracing-core", +] + +[[package]] +name = "tracing-subscriber" +version = "0.3.22" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2f30143827ddab0d256fd843b7a66d164e9f271cfa0dde49142c5ca0ca291f1e" +dependencies = [ + "matchers", + "nu-ansi-term", + "once_cell", + "regex-automata", + "sharded-slab", + "smallvec", + "thread_local", + "tracing", + "tracing-core", + "tracing-log", +] + +[[package]] +name = "try-lock" +version = "0.2.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e421abadd41a4225275504ea4d6566923418b7f05506fbc9c0fe86ba7396114b" + +[[package]] +name = "twox-hash" +version = "2.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9ea3136b675547379c4bd395ca6b938e5ad3c3d20fad76e7fe85f9e0d011419c" + +[[package]] +name = "typenum" +version = "1.19.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "562d481066bde0658276a35467c4af00bdc6ee726305698a55b86e61d7ad82bb" + +[[package]] +name = "unicase" +version = "2.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dbc4bc3a9f746d862c45cb89d705aa10f187bb96c76001afab07a0d35ce60142" + +[[package]] +name = "unicode-ident" +version = "1.0.24" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6e4313cd5fcd3dad5cafa179702e2b244f760991f45397d14d4ebf38247da75" + +[[package]] +name = "unicode-xid" +version = "0.2.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ebc1c04c71510c7f702b52b7c350734c9ff1295c464a03335b00bb84fc54f853" + +[[package]] +name = "untrusted" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8ecb6da28b8a351d773b68d5825ac39017e680750f980f3a1a85cd8dd28a47c1" + +[[package]] +name = "url" +version = "2.5.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ff67a8a4397373c3ef660812acab3268222035010ab8680ec4215f38ba3d0eed" +dependencies = [ + "form_urlencoded", + "idna", + "percent-encoding", + "serde", +] + +[[package]] +name = "utf8_iter" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6c140620e7ffbb22c2dee59cafe6084a59b5ffc27a8859a5f0d494b5d52b6be" + +[[package]] +name = "utf8parse" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "06abde3611657adf66d383f00b093d7faecc7fa57071cce2578660c9f1010821" + +[[package]] +name = "valuable" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ba73ea9cf16a25df0c8caa16c51acb937d5712a8429db78a3ee29d5dcacd3a65" + +[[package]] +name = "version_check" +version = "0.9.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a" + +[[package]] +name = "walkdir" +version = "2.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "29790946404f91d9c5d06f9874efddea1dc06c5efe94541a7d6863108e3a5e4b" +dependencies = [ + "same-file", + "winapi-util", +] + +[[package]] +name = "want" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bfa7760aed19e106de2c7c0b581b509f2f25d3dacaf737cb82ac61bc6d760b0e" +dependencies = [ + "try-lock", +] + +[[package]] +name = "wasi" +version = "0.11.1+wasi-snapshot-preview1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ccf3ec651a847eb01de73ccad15eb7d99f80485de043efb2f370cd654f4ea44b" + +[[package]] +name = "wasip2" +version = "1.0.2+wasi-0.2.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9517f9239f02c069db75e65f174b3da828fe5f5b945c4dd26bd25d89c03ebcf5" +dependencies = [ + "wit-bindgen", +] + +[[package]] +name = "wasip3" +version = "0.4.0+wasi-0.3.0-rc-2026-01-06" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5428f8bf88ea5ddc08faddef2ac4a67e390b88186c703ce6dbd955e1c145aca5" +dependencies = [ + "wit-bindgen", +] + +[[package]] +name = "wasm-bindgen" +version = "0.2.114" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6532f9a5c1ece3798cb1c2cfdba640b9b3ba884f5db45973a6f442510a87d38e" +dependencies = [ + "cfg-if", + "once_cell", + "rustversion", + "wasm-bindgen-macro", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-futures" +version = "0.4.64" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e9c5522b3a28661442748e09d40924dfb9ca614b21c00d3fd135720e48b67db8" +dependencies = [ + "cfg-if", + "futures-util", + "js-sys", + "once_cell", + "wasm-bindgen", + "web-sys", +] + +[[package]] +name = "wasm-bindgen-macro" +version = "0.2.114" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "18a2d50fcf105fb33bb15f00e7a77b772945a2ee45dcf454961fd843e74c18e6" +dependencies = [ + "quote", + "wasm-bindgen-macro-support", +] + +[[package]] +name = "wasm-bindgen-macro-support" +version = "0.2.114" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "03ce4caeaac547cdf713d280eda22a730824dd11e6b8c3ca9e42247b25c631e3" +dependencies = [ + "bumpalo", + "proc-macro2", + "quote", + "syn", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-shared" +version = "0.2.114" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "75a326b8c223ee17883a4251907455a2431acc2791c98c26279376490c378c16" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "wasm-encoder" +version = "0.244.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "990065f2fe63003fe337b932cfb5e3b80e0b4d0f5ff650e6985b1048f62c8319" +dependencies = [ + "leb128fmt", + "wasmparser", +] + +[[package]] +name = "wasm-metadata" +version = "0.244.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bb0e353e6a2fbdc176932bbaab493762eb1255a7900fe0fea1a2f96c296cc909" +dependencies = [ + "anyhow", + "indexmap", + "wasm-encoder", + "wasmparser", +] + +[[package]] +name = "wasmparser" +version = "0.244.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "47b807c72e1bac69382b3a6fb3dbe8ea4c0ed87ff5629b8685ae6b9a611028fe" +dependencies = [ + "bitflags", + "hashbrown 0.15.5", + "indexmap", + "semver", +] + +[[package]] +name = "web-sys" +version = "0.3.91" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "854ba17bb104abfb26ba36da9729addc7ce7f06f5c0f90f3c391f8461cca21f9" +dependencies = [ + "js-sys", + "wasm-bindgen", +] + +[[package]] +name = "web-time" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5a6580f308b1fad9207618087a65c04e7a10bc77e02c8e84e9b00dd4b12fa0bb" +dependencies = [ + "js-sys", + "wasm-bindgen", +] + +[[package]] +name = "webpki-roots" +version = "1.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "22cfaf3c063993ff62e73cb4311efde4db1efb31ab78a3e5c457939ad5cc0bed" +dependencies = [ + "rustls-pki-types", +] + +[[package]] +name = "winapi-util" +version = "0.1.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c2a7b1c03c876122aa43f3020e6c3c3ee5c05081c9a00739faf7503aeba10d22" +dependencies = [ + "windows-sys 0.61.2", +] + +[[package]] +name = "windows-core" +version = "0.62.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b8e83a14d34d0623b51dce9581199302a221863196a1dde71a7663a4c2be9deb" +dependencies = [ + "windows-implement", + "windows-interface", + "windows-link", + "windows-result", + "windows-strings", +] + +[[package]] +name = "windows-implement" +version = "0.60.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "053e2e040ab57b9dc951b72c264860db7eb3b0200ba345b4e4c3b14f67855ddf" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "windows-interface" +version = "0.59.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f316c4a2570ba26bbec722032c4099d8c8bc095efccdc15688708623367e358" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "windows-link" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f0805222e57f7521d6a62e36fa9163bc891acd422f971defe97d64e70d0a4fe5" + +[[package]] +name = "windows-result" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7781fa89eaf60850ac3d2da7af8e5242a5ea78d1a11c49bf2910bb5a73853eb5" +dependencies = [ + "windows-link", +] + +[[package]] +name = "windows-strings" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7837d08f69c77cf6b07689544538e017c1bfcf57e34b4c0ff58e6c2cd3b37091" +dependencies = [ + "windows-link", +] + +[[package]] +name = "windows-sys" +version = "0.52.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "282be5f36a8ce781fad8c8ae18fa3f9beff57ec1b52cb3de0789201425d9a33d" +dependencies = [ + "windows-targets 0.52.6", +] + +[[package]] +name = "windows-sys" +version = "0.60.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f2f500e4d28234f72040990ec9d39e3a6b950f9f22d3dba18416c35882612bcb" +dependencies = [ + "windows-targets 0.53.5", +] + +[[package]] +name = "windows-sys" +version = "0.61.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ae137229bcbd6cdf0f7b80a31df61766145077ddf49416a728b02cb3921ff3fc" +dependencies = [ + "windows-link", +] + +[[package]] +name = "windows-targets" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9b724f72796e036ab90c1021d4780d4d3d648aca59e491e6b98e725b84e99973" +dependencies = [ + "windows_aarch64_gnullvm 0.52.6", + "windows_aarch64_msvc 0.52.6", + "windows_i686_gnu 0.52.6", + "windows_i686_gnullvm 0.52.6", + "windows_i686_msvc 0.52.6", + "windows_x86_64_gnu 0.52.6", + "windows_x86_64_gnullvm 0.52.6", + "windows_x86_64_msvc 0.52.6", +] + +[[package]] +name = "windows-targets" +version = "0.53.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4945f9f551b88e0d65f3db0bc25c33b8acea4d9e41163edf90dcd0b19f9069f3" +dependencies = [ + "windows-link", + "windows_aarch64_gnullvm 0.53.1", + "windows_aarch64_msvc 0.53.1", + "windows_i686_gnu 0.53.1", + "windows_i686_gnullvm 0.53.1", + "windows_i686_msvc 0.53.1", + "windows_x86_64_gnu 0.53.1", + "windows_x86_64_gnullvm 0.53.1", + "windows_x86_64_msvc 0.53.1", +] + +[[package]] +name = "windows_aarch64_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32a4622180e7a0ec044bb555404c800bc9fd9ec262ec147edd5989ccd0c02cd3" + +[[package]] +name = "windows_aarch64_gnullvm" +version = "0.53.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a9d8416fa8b42f5c947f8482c43e7d89e73a173cead56d044f6a56104a6d1b53" + +[[package]] +name = "windows_aarch64_msvc" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09ec2a7bb152e2252b53fa7803150007879548bc709c039df7627cabbd05d469" + +[[package]] +name = "windows_aarch64_msvc" +version = "0.53.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b9d782e804c2f632e395708e99a94275910eb9100b2114651e04744e9b125006" + +[[package]] +name = "windows_i686_gnu" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e9b5ad5ab802e97eb8e295ac6720e509ee4c243f69d781394014ebfe8bbfa0b" + +[[package]] +name = "windows_i686_gnu" +version = "0.53.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "960e6da069d81e09becb0ca57a65220ddff016ff2d6af6a223cf372a506593a3" + +[[package]] +name = "windows_i686_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0eee52d38c090b3caa76c563b86c3a4bd71ef1a819287c19d586d7334ae8ed66" + +[[package]] +name = "windows_i686_gnullvm" +version = "0.53.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fa7359d10048f68ab8b09fa71c3daccfb0e9b559aed648a8f95469c27057180c" + +[[package]] +name = "windows_i686_msvc" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "240948bc05c5e7c6dabba28bf89d89ffce3e303022809e73deaefe4f6ec56c66" + +[[package]] +name = "windows_i686_msvc" +version = "0.53.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e7ac75179f18232fe9c285163565a57ef8d3c89254a30685b57d83a38d326c2" + +[[package]] +name = "windows_x86_64_gnu" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "147a5c80aabfbf0c7d901cb5895d1de30ef2907eb21fbbab29ca94c5b08b1a78" + +[[package]] +name = "windows_x86_64_gnu" +version = "0.53.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9c3842cdd74a865a8066ab39c8a7a473c0778a3f29370b5fd6b4b9aa7df4a499" + +[[package]] +name = "windows_x86_64_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "24d5b23dc417412679681396f2b49f3de8c1473deb516bd34410872eff51ed0d" + +[[package]] +name = "windows_x86_64_gnullvm" +version = "0.53.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0ffa179e2d07eee8ad8f57493436566c7cc30ac536a3379fdf008f47f6bb7ae1" + +[[package]] +name = "windows_x86_64_msvc" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec" + +[[package]] +name = "windows_x86_64_msvc" +version = "0.53.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d6bbff5f0aada427a1e5a6da5f1f98158182f26556f345ac9e04d36d0ebed650" + +[[package]] +name = "winnow" +version = "0.7.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5a5364e9d77fcdeeaa6062ced926ee3381faa2ee02d3eb83a5c27a8825540829" +dependencies = [ + "memchr", +] + +[[package]] +name = "winnow" +version = "1.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0592e1c9d151f854e6fd382574c3a0855250e1d9b2f99d9281c6e6391af352f1" +dependencies = [ + "memchr", +] + +[[package]] +name = "wit-bindgen" +version = "0.51.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d7249219f66ced02969388cf2bb044a09756a083d0fab1e566056b04d9fbcaa5" +dependencies = [ + "wit-bindgen-rust-macro", +] + +[[package]] +name = "wit-bindgen-core" +version = "0.51.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ea61de684c3ea68cb082b7a88508a8b27fcc8b797d738bfc99a82facf1d752dc" +dependencies = [ + "anyhow", + "heck 0.5.0", + "wit-parser", +] + +[[package]] +name = "wit-bindgen-rust" +version = "0.51.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b7c566e0f4b284dd6561c786d9cb0142da491f46a9fbed79ea69cdad5db17f21" +dependencies = [ + "anyhow", + "heck 0.5.0", + "indexmap", + "prettyplease", + "syn", + "wasm-metadata", + "wit-bindgen-core", + "wit-component", +] + +[[package]] +name = "wit-bindgen-rust-macro" +version = "0.51.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0c0f9bfd77e6a48eccf51359e3ae77140a7f50b1e2ebfe62422d8afdaffab17a" +dependencies = [ + "anyhow", + "prettyplease", + "proc-macro2", + "quote", + "syn", + "wit-bindgen-core", + "wit-bindgen-rust", +] + +[[package]] +name = "wit-component" +version = "0.244.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9d66ea20e9553b30172b5e831994e35fbde2d165325bec84fc43dbf6f4eb9cb2" +dependencies = [ + "anyhow", + "bitflags", + "indexmap", + "log", + "serde", + "serde_derive", + "serde_json", + "wasm-encoder", + "wasm-metadata", + "wasmparser", + "wit-parser", +] + +[[package]] +name = "wit-parser" +version = "0.244.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ecc8ac4bc1dc3381b7f59c34f00b67e18f910c2c0f50015669dde7def656a736" +dependencies = [ + "anyhow", + "id-arena", + "indexmap", + "log", + "semver", + "serde", + "serde_derive", + "serde_json", + "unicode-xid", + "wasmparser", +] + +[[package]] +name = "writeable" +version = "0.6.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9edde0db4769d2dc68579893f2306b26c6ecfbe0ef499b013d731b7b9247e0b9" + +[[package]] +name = "yoke" +version = "0.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "72d6e5c6afb84d73944e5cedb052c4680d5657337201555f9f2a16b7406d4954" +dependencies = [ + "stable_deref_trait", + "yoke-derive", + "zerofrom", +] + +[[package]] +name = "yoke-derive" +version = "0.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b659052874eb698efe5b9e8cf382204678a0086ebf46982b79d6ca3182927e5d" +dependencies = [ + "proc-macro2", + "quote", + "syn", + "synstructure", +] + +[[package]] +name = "zerobus-ffi" +version = "1.3.0" +dependencies = [ + "arrow-ipc", + "async-trait", + "bytes", + "cbindgen", + "databricks-zerobus-ingest-sdk", + "libc", + "once_cell", + "prost", + "prost-reflect", + "prost-types", + "serde_json", + "tokio", + "tracing-subscriber", +] + +[[package]] +name = "zerocopy" +version = "0.8.40" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a789c6e490b576db9f7e6b6d661bcc9799f7c0ac8352f56ea20193b2681532e5" +dependencies = [ + "zerocopy-derive", +] + +[[package]] +name = "zerocopy-derive" +version = "0.8.40" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f65c489a7071a749c849713807783f70672b28094011623e200cb86dcb835953" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "zerofrom" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "50cc42e0333e05660c3587f3bf9d0478688e15d870fab3346451ce7f8c9fbea5" +dependencies = [ + "zerofrom-derive", +] + +[[package]] +name = "zerofrom-derive" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d71e5d6e06ab090c67b5e44993ec16b72dcbaabc526db883a360057678b48502" +dependencies = [ + "proc-macro2", + "quote", + "syn", + "synstructure", +] + +[[package]] +name = "zeroize" +version = "1.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b97154e67e32c85465826e8bcc1c59429aaaf107c1e4a9e53c8d8ccd5eff88d0" + +[[package]] +name = "zerotrie" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2a59c17a5562d507e4b54960e8569ebee33bee890c70aa3fe7b97e85a9fd7851" +dependencies = [ + "displaydoc", + "yoke", + "zerofrom", +] + +[[package]] +name = "zerovec" +version = "0.11.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6c28719294829477f525be0186d13efa9a3c602f7ec202ca9e353d310fb9a002" +dependencies = [ + "yoke", + "zerofrom", + "zerovec-derive", +] + +[[package]] +name = "zerovec-derive" +version = "0.11.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eadce39539ca5cb3985590102671f2567e659fca9666581ad3411d59207951f3" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "zmij" +version = "1.0.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b8848ee67ecc8aedbaf3e4122217aff892639231befc6a1b58d29fff4c2cabaa" + +[[package]] +name = "zstd" +version = "0.13.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e91ee311a569c327171651566e07972200e76fcfe2242a4fa446149a3881c08a" +dependencies = [ + "zstd-safe", +] + +[[package]] +name = "zstd-safe" +version = "7.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f49c4d5f0abb602a93fb8736af2a4f4dd9512e36f7f570d66e65ff867ed3b9d" +dependencies = [ + "zstd-sys", +] + +[[package]] +name = "zstd-sys" +version = "2.0.16+zstd.1.5.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "91e19ebc2adc8f83e43039e79776e3fda8ca919132d68a1fed6a5faca2683748" +dependencies = [ + "cc", + "pkg-config", +] diff --git a/lib/zerobus-ffi-1.3.0/rust/Cargo.toml b/lib/zerobus-ffi-1.3.0/rust/Cargo.toml new file mode 100644 index 00000000000..6902680efd8 --- /dev/null +++ b/lib/zerobus-ffi-1.3.0/rust/Cargo.toml @@ -0,0 +1,62 @@ +[workspace] +members = [ + "sdk", + "ffi", +] +resolver = "2" + +[workspace.dependencies] +# Protobuf / gRPC +prost = "0.14" +prost-types = "0.14" +prost-reflect = "0.16" +tonic = "0.14" +tonic-prost = "0.14" +tonic-prost-build = "0.14" +protoc-bin-vendored = "3.0.0" + +# Async runtime +async-trait = "0.1" +tokio = "1.52" +tokio-stream = "0.1.18" +tokio-retry = "0.3" +tokio-util = "0.7.18" + +# Serialization +serde = { version = "1.0", features = ["derive"] } +serde_json = "1.0" + +# HTTP / networking +reqwest = { version = "0.12", features = ["json", "rustls-tls-native-roots", "rustls-tls-webpki-roots"], default-features = false } +hyper-http-proxy = { version = "1.1", default-features = false, features = ["rustls-tls-native-roots"] } +hyper-util = { version = "0.1", features = ["client-legacy", "tokio"] } + +# Error handling +thiserror = "1.0" + +# Observability +tracing = "0.1.41" +tracing-subscriber = { version = "0.3", features = ["env-filter"] } + +# Arrow +arrow-flight = { version = "58.2", default-features = false } +arrow-array = { version = "58.2", default-features = false } +arrow-schema = { version = "58.2", default-features = false } +arrow-ipc = { version = "58.2", default-features = false } + +# Misc +futures = "0.3" +chrono = "0.4" +sha2 = "0.10" +once_cell = "1.21" +bytes = "1.11" +smallvec = "1.15.1" +libc = "0.2" +cbindgen = "0.27" + +# Tools +anyhow = "1" +clap = { version = "4.6", features = ["derive"] } +urlencoding = "2.1" +tempfile = "3.27" +jni = "0.21" diff --git a/lib/zerobus-ffi-1.3.0/rust/LICENSE b/lib/zerobus-ffi-1.3.0/rust/LICENSE new file mode 100644 index 00000000000..253d123149d --- /dev/null +++ b/lib/zerobus-ffi-1.3.0/rust/LICENSE @@ -0,0 +1,201 @@ + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright (2025) Databricks, Inc. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/lib/zerobus-ffi-1.3.0/rust/NOTICE b/lib/zerobus-ffi-1.3.0/rust/NOTICE new file mode 100644 index 00000000000..366e804e4da --- /dev/null +++ b/lib/zerobus-ffi-1.3.0/rust/NOTICE @@ -0,0 +1,43 @@ +Copyright (2025) Databricks, Inc. + +This Software includes software developed at Databricks (https://www.databricks.com/) and its use is subject to the included LICENSE file. + +--- + +This Software includes the following open source components: + +## MIT License + +- **tokio**: Copyright (c) 2019 Tokio Contributors (https://github.com/tokio-rs/tokio) +- **tokio-stream**: Copyright (c) 2019 Tokio Contributors (https://github.com/tokio-rs/tokio) +- **tokio-util**: Copyright (c) 2019 Tokio Contributors (https://github.com/tokio-rs/tokio) +- **tonic**: Copyright (c) 2019 Lucio Franco (https://github.com/hyperium/tonic) +- **tracing**: Copyright (c) 2019 Tokio Contributors (https://github.com/tokio-rs/tracing) +- **async-trait**: Copyright (c) 2019 David Tolnay (https://github.com/dtolnay/async-trait) +- **thiserror**: Copyright (c) 2019 David Tolnay (https://github.com/dtolnay/thiserror) +- **serde**: Copyright (c) 2014 David Tolnay (https://github.com/serde-rs/serde) +- **serde_json**: Copyright (c) 2014 David Tolnay (https://github.com/serde-rs/json) +- **bytes**: Copyright (c) 2018 Carl Lerche (https://github.com/tokio-rs/bytes) +- **smallvec**: Copyright (c) 2018 The Servo Project Developers (https://github.com/servo/rust-smallvec) +- **self_cell**: Copyright (c) 2020 Lukas Bergdoll (https://github.com/Voultapher/self_cell) + +## Apache License 2.0 + +- **prost**: Copyright (c) 2017 Dan Burkert (https://github.com/tokio-rs/prost) +- **prost-types**: Copyright (c) 2017 Dan Burkert (https://github.com/tokio-rs/prost) +- **tokio-retry**: Copyright (c) 2018 Sam Rijs (https://github.com/srijs/rust-tokio-retry) +- **reqwest**: Copyright (c) 2016 Sean McArthur (https://github.com/seanmonstar/reqwest) +- **hyper-http-proxy**: Copyright (c) 2019 Trent Mick (https://github.com/tafia/hyper-http-proxy) +- **hyper-util**: Copyright (c) 2023 Sean McArthur (https://github.com/hyperium/hyper-util) + +## Optional: Arrow Flight support (Apache License 2.0) + +These components are only included when the `arrow-flight` feature is enabled: + +- **arrow-flight**: Copyright (c) 2020 Apache Arrow contributors (https://github.com/apache/arrow-rs) +- **arrow-array**: Copyright (c) 2020 Apache Arrow contributors (https://github.com/apache/arrow-rs) +- **arrow-schema**: Copyright (c) 2020 Apache Arrow contributors (https://github.com/apache/arrow-rs) +- **arrow-ipc**: Copyright (c) 2020 Apache Arrow contributors (https://github.com/apache/arrow-rs) +- **futures**: Copyright (c) 2016 Alex Crichton (https://github.com/rust-lang/futures-rs) + +For a complete list of dependencies and their licenses, see `sdk/Cargo.toml`. diff --git a/lib/zerobus-ffi-1.3.0/rust/README.md b/lib/zerobus-ffi-1.3.0/rust/README.md new file mode 100644 index 00000000000..eb9788c021d --- /dev/null +++ b/lib/zerobus-ffi-1.3.0/rust/README.md @@ -0,0 +1,1094 @@ +# Zerobus Rust SDK + +A high-performance Rust client for streaming data ingestion into Databricks Delta tables using the Zerobus service. + +## Table of Contents + +- [Overview](#overview) +- [Features](#features) +- [Installation](#installation) +- [Quick Start](#quick-start) +- [Repository Structure](#repository-structure) +- [How It Works](#how-it-works) +- [Usage Guide](#usage-guide) + - [1. Generate Protocol Buffer Schema](#1-generate-protocol-buffer-schema) + - [2. Initialize the SDK](#2-initialize-the-sdk) + - [3. Configure Authentication](#3-configure-authentication) + - [4. Create a Stream](#4-create-a-stream) + - [5. Ingest Data](#5-ingest-data) + - [6. Handle Acknowledgments](#6-handle-acknowledgments) + - [7. Close the Stream](#7-close-the-stream) +- [Client-side warnings](#client-side-warnings) +- [Configuration Options](#configuration-options) +- [Error Handling](#error-handling) +- [Examples](#examples) +- [Best Practices](#best-practices) +- [API Reference](#api-reference) +- [Language Bindings](#language-bindings) +- [Building from Source](#building-from-source) +- [Community and Contributing](#community-and-contributing) +- [License](#license) + +## Overview + +The Zerobus Rust SDK provides a robust, async-first interface for ingesting large volumes of data into Databricks Delta tables. It abstracts the complexity of the Zerobus service and handles authentication, retries, stream recovery, and acknowledgment tracking automatically. + +**What is Zerobus?** See the [project overview](https://github.com/databricks/zerobus-sdk/blob/main/README.md#what-is-zerobus) for details on the Zerobus service. + +## Features + +- **Async/Await Support** - Built on Tokio for efficient concurrent I/O operations +- **Automatic OAuth 2.0 Authentication** - Seamless token management with Unity Catalog +- **Built-in Recovery** - Automatic retry and reconnection for transient failures +- **High Throughput** - Configurable inflight record limits for optimal performance +- **Batch Ingestion** - Ingest multiple records at once with all-or-nothing semantics for maximum throughput +- **Flexible Serialization** - Support for both JSON (simple) and Protocol Buffers (type-safe) data formats +- **Type Safety** - Protocol Buffers ensure schema validation at compile time +- **Schema Generation** - CLI tool to generate protobuf schemas from Unity Catalog tables +- **Flexible Configuration** - Fine-tune timeouts, retries, and recovery behavior +- **Graceful Stream Management** - Proper flushing and acknowledgment tracking +- **Acknowledgment Callbacks** - Receive notifications when records are acknowledged or encounter errors +- **Arrow Flight Ingestion** *(Beta, opt-in)* — Stream Apache Arrow `RecordBatch` data directly to Zerobus over the Arrow Flight protocol on the same gRPC connection. Enable with `features = ["arrow-flight"]`; see [`examples/arrow/`](https://github.com/databricks/zerobus-sdk/tree/main/rust/examples/arrow). +- **Zeroparser** *(opt-in)* — Zero-copy, single-pass protobuf parser for runtime-known schemas. Enable with `features = ["zeroparser"]`; see [`sdk/src/zeroparser/README.md`](https://github.com/databricks/zerobus-sdk/blob/main/rust/sdk/src/zeroparser/README.md). + +## Installation + +Add the SDK to your `Cargo.toml`: + +```bash +cargo add databricks-zerobus-ingest-sdk +cargo add prost prost-types +cargo add tokio --features macros,rt-multi-thread +``` +**Why these dependencies?** +- **`databricks-zerobus-ingest-sdk`** - The SDK itself +- **`prost`** and **`prost-types`** - Required for encoding your data to Protocol Buffers and loading schema descriptors +- **`tokio`** - Async runtime required for running async functions (the SDK is fully async) + +> **What's in the crates.io package?** The published crate contains only the core Zerobus ingestion SDK. Tools for schema generation (`tools/generate_files`) and working examples (`examples/`) are only available in the [GitHub repository](https://github.com/databricks/zerobus-sdk). You'll need to clone the repo to generate protobuf schemas from your Unity Catalog tables. + +### For Local Development + +Clone the repository and use a path dependency: + +```bash +git clone https://github.com/databricks/zerobus-sdk.git +cd your_project +``` + +Then in your `Cargo.toml`: + +```toml +[dependencies] +databricks-zerobus-ingest-sdk = { path = "../zerobus-sdk/rust/sdk" } +prost = "0.14" +prost-types = "0.14" +tokio = { version = "1.52", features = ["macros", "rt-multi-thread"] } +``` + +## Quick Start + +The SDK supports two serialization formats and two ingestion methods: + +**Serialization:** +- **JSON** (Recommended for getting started): Simpler approach using JSON strings, no schema generation required +- **Protocol Buffers** (Recommended for production): Type-safe approach with schema validation at compile time + +**Ingestion Methods:** +- **Single-record** (`ingest_record_offset`): Ingest records one at a time with per-record acknowledgment +- **Batch** (`ingest_records_offset`): Ingest multiple records at once with all-or-nothing semantics for higher throughput + +See [`examples/README.md`](https://github.com/databricks/zerobus-sdk/blob/main/rust/examples/README.md) for detailed setup instructions and examples for all combinations. + +## Repository Structure + +``` +zerobus_rust_sdk/ +├── sdk/ # Core SDK library +│ ├── src/ +│ │ ├── lib.rs # Main SDK and stream implementation +│ │ ├── builder/ # Builder pattern for SDK initialization +│ │ ├── default_token_factory.rs # OAuth 2.0 token handling +│ │ ├── errors.rs # Error types and retryable logic +│ │ ├── headers_provider.rs # Trait for custom authentication headers +│ │ ├── callbacks.rs # Ack/error callback traits +│ │ ├── record_types.rs # Record encoding types (JSON, proto, raw) +│ │ ├── schema.rs # Unity Catalog → proto/Arrow schema generation +│ │ ├── stream_configuration.rs # Stream options +│ │ ├── stream_options.rs # Shared stream configuration +│ │ ├── tls_config.rs # TLS configuration strategies +│ │ ├── landing_zone.rs # Inflight record buffer +│ │ ├── offset_generator.rs # Logical offset tracking +│ │ ├── proxy.rs # HTTP proxy support +│ │ ├── arrow_stream.rs # Arrow Flight stream (feature: arrow-flight) +│ │ ├── arrow_configuration.rs # Arrow Flight options (feature: arrow-flight) +│ │ ├── arrow_metadata.rs # Arrow Flight metadata (feature: arrow-flight) +│ │ └── zeroparser/ # Descriptor-driven protobuf parser (feature: zeroparser) +│ ├── zerobus_service.proto # gRPC protocol definition +│ ├── build.rs # Build script for protobuf compilation +│ └── Cargo.toml +│ +├── ffi/ # C FFI bindings for other languages +│ ├── src/ +│ │ ├── lib.rs # FFI implementation +│ │ └── tests.rs # FFI unit tests +│ ├── zerobus.h # Generated C header +│ ├── cbindgen.toml # Header generation config +│ ├── build.rs # Build script for header generation +│ └── Cargo.toml +│ +├── jni/ # JNI bindings for Java SDK +│ ├── src/ +│ │ └── lib.rs # JNI implementation +│ └── Cargo.toml +│ +├── tools/ +│ └── generate_files/ # Schema generation CLI tool (package `tools`) +│ ├── src/ +│ │ ├── main.rs # CLI entry point +│ │ └── generate.rs # Unity Catalog -> Proto conversion +│ ├── README.md # Tool documentation +│ └── Cargo.toml +│ +├── examples/ +│ ├── README.md # Examples documentation +│ ├── json/ # JSON examples (single Cargo package) +│ │ ├── README.md +│ │ ├── Cargo.toml +│ │ ├── single.rs # JSON single-record example +│ │ └── batch.rs # JSON batch ingestion example +│ ├── proto/ # Protocol Buffers examples (single Cargo package) +│ │ ├── README.md +│ │ ├── Cargo.toml +│ │ ├── single.rs # Protocol Buffers single-record example +│ │ ├── batch.rs # Protocol Buffers batch ingestion example +│ │ └── output/ # Generated schema files (shared) +│ └── arrow/ # Arrow Flight example (feature: arrow-flight, Beta) +│ ├── README.md +│ ├── Cargo.toml +│ └── src/main.rs # Arrow `RecordBatch` ingestion example +│ +├── tests/ # Integration tests crate +│ ├── src/ +│ │ ├── mock_grpc.rs # Mock Zerobus gRPC server +│ │ ├── mock_arrow_flight.rs # Mock Arrow Flight server +│ │ ├── rust_tests.rs # Core SDK test suite +│ │ ├── proxy_tests.rs # HTTP proxy tests +│ │ ├── arrow_tests.rs # Arrow Flight test suite +│ │ └── utils.rs # Shared test utilities +│ ├── build.rs +│ └── Cargo.toml +│ +├── Cargo.toml # Workspace configuration +└── README.md # This file +``` + +### Key Components + +- **`sdk/`** - The main library crate containing all SDK functionality +- **`ffi/`** - C FFI bindings for building language wrappers (Go, C#, C++, etc.) +- **`jni/`** - JNI bindings for the Java SDK +- **`tools/`** - CLI tool for generating Protocol Buffer schemas from Unity Catalog tables +- **`examples/`** - Complete working examples demonstrating SDK usage +- **Workspace** - Root `Cargo.toml` defines a Cargo workspace for unified builds + +## How It Works + +### Architecture Overview + +``` ++-----------------+ +| Your App | ++-----------------+ + | 1. stream_builder().build() + v ++-----------------+ +| ZerobusSdk | +| - Manages TLS | +| - Creates | +| channels | ++-----------------+ + | 2. Opens bidirectional gRPC stream + v ++--------------------------------------+ +| ZerobusStream | +| +----------------------------------+ | +| | Supervisor | | Manages lifecycle, recovery +| +----------------------------------+ | +| | | +| +-----------+-----------+ | +| v v | +| +----------+ +----------+ | +| | Sender | | Receiver | | Parallel tasks +| | Task | | Task | | +| +----------+ +----------+ | +| ^ | | +| | v | +| +----------------------------------+ | +| | Landing Zone | | Inflight buffer +| +----------------------------------+ | ++--------------------------------------+ + | 3. gRPC stream + v ++-----------------------+ +| Databricks | +| Zerobus Service | ++-----------------------+ +``` + +### Data Flow + +1. **Ingestion** - Your app calls `stream.ingest_record_offset(data)` or `stream.ingest_records_offset(batch)` +2. **Buffering** - Records are placed in the landing zone with logical offsets +3. **Sending** - Sender task sends records over gRPC with physical offsets +4. **Acknowledgment** - Receiver task gets server ack; callers wait via `stream.wait_for_offset(offset)` +5. **Recovery** - If connection fails, supervisor reconnects and resends unacked records + +### Authentication Flow + +The SDK uses OAuth 2.0 client credentials flow: + +1. SDK constructs authorization request with Unity Catalog privileges +2. Sends request to `{uc_endpoint}/oidc/v1/token` with client credentials +3. Token includes scoped permissions for the specific table +4. Token is attached to gRPC metadata as Bearer token +5. Tokens are cached per table and reused across connections until they near expiry (see Token Caching below) + +### Token Caching + +OAuth tokens minted via Unity Catalog have a lifetime chosen by Unity Catalog (currently one hour) that the SDK cannot configure, while a single stream lives at most ~15 minutes. By default the SDK caches the token for each table on the `ZerobusSdk` instance and reuses it across stream creations and recoveries, refreshing it only once it nears the expiry the server reported (so it adapts to whatever lifetime UC returns). This avoids minting a fresh token on every stream and reduces load on the Unity Catalog token endpoint. + +Caching applies only to the built-in OAuth path (`.oauth(...)`). Tokens are shared only across streams created from the same `ZerobusSdk`, so reuse a single SDK instance rather than constructing a new one per stream. Custom `HeadersProvider` implementations manage their own caching. + +Two builder options tune the behavior: + +```rust +use databricks_zerobus_ingest_sdk::ZerobusSdk; +use std::time::Duration; + +let sdk = ZerobusSdk::builder() + .endpoint("https://.zerobus..cloud.databricks.com") + .unity_catalog_url("https://.cloud.databricks.com") + // Refresh a cached token once it is within 10 minutes of expiry (default: 5 minutes). + .token_refresh_buffer(Duration::from_secs(600)) + // Or disable caching entirely to mint a fresh token per stream. + // .token_cache_enabled(false) + .build()?; +# Ok::<(), databricks_zerobus_ingest_sdk::ZerobusError>(()) +``` + +### Custom Authentication + +For advanced use cases, you can implement the `HeadersProvider` trait to supply your own authentication headers. This is useful for integrating with a different OAuth provider, using a centralized token caching service, or implementing alternative authentication mechanisms. + +> **Note:** The headers you provide must still conform to the authentication protocol expected by the Zerobus service. The default implementation, `OAuthHeadersProvider`, serves as the reference for the required headers (`authorization` and `x-databricks-zerobus-table-name`). This feature provides flexibility in *how* you source your credentials, not in changing the authentication protocol itself. + +**Example:** + +```rust +use databricks_zerobus_ingest_sdk::*; +use std::collections::HashMap; +use std::sync::Arc; +use async_trait::async_trait; + +struct MyCustomAuthProvider; + +#[async_trait] +impl HeadersProvider for MyCustomAuthProvider { + async fn get_headers(&self) -> ZerobusResult> { + let mut headers = HashMap::new(); + // Custom logic to fetch and cache a token would go here. + headers.insert("authorization", "Bearer ".to_string()); + headers.insert("x-databricks-zerobus-table-name", "".to_string()); + Ok(headers) + } +} + +async fn example(sdk: ZerobusSdk) -> ZerobusResult<()> { + let custom_provider = Arc::new(MyCustomAuthProvider {}); + let stream = sdk + .stream_builder().table("catalog.schema.table") + .headers_provider(custom_provider) + .json() + .build() + .await?; + Ok(()) +} +``` + +## Usage Guide + +The SDK supports two approaches for data serialization: + +1. **JSON** - Simpler approach that uses JSON strings. No schema generation required, making it ideal for quick prototyping. See [`examples/README.md`](https://github.com/databricks/zerobus-sdk/blob/main/rust/examples/README.md) for a complete example. +2. **Protocol Buffers** - Type-safe approach with schema validation at compile time. Recommended for production use cases. This guide focuses on the Protocol Buffers approach. + +For JSON-based ingestion, you can skip the schema generation step and directly pass JSON strings to `ingest_record_offset()`. + +### 1. Generate Protocol Buffer Schema (Protocol Buffers approach only) + +> **Important Note**: The schema generation tool and examples are **only available in the GitHub repository**. The crate published on [crates.io](https://crates.io/crates/databricks-zerobus-ingest-sdk) contains only the core Zerobus ingestion SDK logic. To generate protobuf schemas or see working examples, clone the repository: +> +> ```bash +> git clone https://github.com/databricks/zerobus-sdk.git +> cd zerobus-sdk/rust +> ``` + +Use the included tool to generate schema files from your Unity Catalog table: + +```bash +cd tools/generate_files + +# For AWS +cargo run -- \ + --uc-endpoint "https://.cloud.databricks.com" \ + --client-id "your-client-id" \ + --client-secret "your-client-secret" \ + --table "catalog.schema.table" \ + --output-dir "../../output" + +# For Azure +cargo run -- \ + --uc-endpoint "https://.azuredatabricks.net" \ + --client-id "your-client-id" \ + --client-secret "your-client-secret" \ + --table "catalog.schema.table" \ + --output-dir "../../output" +``` + +This generates three files: +- `{table}.proto` - Protocol Buffer schema definition +- `{table}.rs` - Rust structs with serialization code +- `{table}.descriptor` - Binary descriptor for runtime validation + +See [`tools/generate_files/README.md`](https://github.com/databricks/zerobus-sdk/blob/main/rust/tools/generate_files/README.md) for supported data types and limitations. + +See [`examples/README.md`](https://github.com/databricks/zerobus-sdk/blob/main/rust/examples/README.md) for more information on how to get OAuth credentials. + +### 2. Initialize the SDK + +Create an SDK instance using the builder pattern: + +```rust +// For AWS +let sdk = ZerobusSdk::builder() + .endpoint("https://.zerobus..cloud.databricks.com") + .unity_catalog_url("https://.cloud.databricks.com") + .build()?; + +// For Azure +let sdk = ZerobusSdk::builder() + .endpoint("https://.zerobus..azuredatabricks.net") + .unity_catalog_url("https://.azuredatabricks.net") + .build()?; +``` + +**Note:** The workspace ID is automatically extracted from the Zerobus endpoint. The `https://` scheme is optional — if omitted, the SDK automatically prepends `https://`. + +#### TLS Configuration + +By default, the SDK uses `SecureTlsConfig` which enables TLS with the operating system's trusted CA certificates. For testing against a local `http://` server, use `NoTlsConfig` (requires the `testing` feature): + +```rust +use databricks_zerobus_ingest_sdk::{ZerobusSdk, NoTlsConfig}; +use std::sync::Arc; + +let sdk = ZerobusSdk::builder() + .endpoint("http://localhost:50051") + .tls_config(Arc::new(NoTlsConfig)) + .build()?; +``` + +For custom certificate handling, implement the `TlsConfig` trait: + +```rust +use databricks_zerobus_ingest_sdk::{TlsConfig, ZerobusResult}; +use tonic::transport::Endpoint; + +struct MyTlsConfig { /* ... */ } + +impl TlsConfig for MyTlsConfig { + fn configure_endpoint(&self, endpoint: Endpoint) -> ZerobusResult { + // Custom TLS configuration logic + Ok(endpoint) + } +} +``` + +### 3. Configure Authentication + +The SDK handles authentication automatically. You just need to provide: +- **Client ID** - Your OAuth client ID +- **Client Secret** - Your OAuth client secret +- **Unity Catalog Endpoint** - Passed to SDK constructor +- **Table Name** - Included in table properties + +```rust +let client_id = "your-client-id".to_string(); +let client_secret = "your-client-secret".to_string(); +``` + +See [`examples/README.md`](https://github.com/databricks/zerobus-sdk/blob/main/rust/examples/README.md) for more information on how to get these credentials. + +### 4. Create a Stream + +Use the `stream_builder()` API to create a stream: + +#### JSON Stream + +```rust +let mut stream = sdk + .stream_builder().table("catalog.schema.orders") + .oauth(client_id, client_secret) + .json() + .max_inflight_requests(10_000) + .recovery_timeout_ms(15_000) + .recovery_backoff_ms(2_000) + .recovery_retries(4) + .build() + .await?; +``` + +#### Protocol Buffers Stream + +```rust +use std::fs; +use prost::Message; +use prost_types::{FileDescriptorSet, DescriptorProto}; + +// Load descriptor from generated files +fn load_descriptor(path: &str, file: &str, msg: &str) -> DescriptorProto { + let bytes = fs::read(path).expect("Failed to read descriptor"); + let file_set = FileDescriptorSet::decode(bytes.as_ref()).unwrap(); + + let file_desc = file_set.file.into_iter() + .find(|f| f.name.as_deref() == Some(file)) + .unwrap(); + + file_desc.message_type.into_iter() + .find(|m| m.name.as_deref() == Some(msg)) + .unwrap() +} + +let descriptor_proto = load_descriptor( + "output/orders.descriptor", + "orders.proto", + "table_Orders", +); + +let mut stream = sdk + .stream_builder().table("catalog.schema.orders") + .oauth(client_id, client_secret) + .compiled_proto(descriptor_proto) + .max_inflight_requests(10_000) + .recovery_timeout_ms(15_000) + .recovery_backoff_ms(2_000) + .recovery_retries(4) + .build() + .await?; +``` + +Setters can be called in any order. The builder validates at `build()` time that both authentication and format have been configured. + +### 5. Ingest Data + +The SDK provides flexible ways to ingest data with different levels of abstraction: + +| Wrapper | Format | Description | +|---------|--------|-------------| +| `ProtoMessage` | Proto | Auto-encoding: pass structs, SDK handles encoding | +| `ProtoBytes` | Proto | Pre-encoded: pass bytes with explicit wrapper | +| `Vec` | Proto | Backward-compatible: raw bytes without wrapper | +| `JsonValue` | JSON | Auto-serializing: pass structs, SDK handles JSON conversion | +| `JsonString` | JSON | Pre-serialized: pass JSON strings with explicit wrapper | +| `String` | JSON | Backward-compatible: raw strings without wrapper | + +#### Single Record Ingestion + +```rust +use databricks_zerobus_ingest_sdk::ProtoMessage; + +let record = YourMessage { id: Some(1), name: Some("Alice".to_string()) }; + +// Ingest and get offset (after queuing) +let offset = stream.ingest_record_offset(ProtoMessage(record)).await?; + +// Wait for server acknowledgment +stream.wait_for_offset(offset).await?; +``` + +#### Batch Ingestion + +Ingest multiple records at once for higher throughput with all-or-nothing semantics. + +```rust +use databricks_zerobus_ingest_sdk::ProtoMessage; + +let records: Vec> = vec![ + ProtoMessage(YourMessage { id: Some(1), /* ... */ }), + ProtoMessage(YourMessage { id: Some(2), /* ... */ }), + ProtoMessage(YourMessage { id: Some(3), /* ... */ }), +]; + +// Returns Some(offset) for non-empty batches, None for empty batches +if let Some(offset) = stream.ingest_records_offset(records).await? { + stream.wait_for_offset(offset).await?; +} +``` + +#### High Throughput Pattern + +Ingest many records without waiting for each acknowledgment, then flush periodically: + +```rust +for i in 0..100_000 { + let record = YourMessage { id: Some(i), /* ... */ }; + let _offset = stream.ingest_record_offset(ProtoMessage(record)).await?; + + // Periodically flush to avoid unbounded memory growth + if (i + 1) % 10_000 == 0 { + stream.flush().await?; + } +} +stream.flush().await?; +``` + +See [`examples/`](https://github.com/databricks/zerobus-sdk/tree/main/rust/examples) for complete working examples with all wrapper types, serialization formats, and ingestion patterns. + +### 6. Handle Acknowledgments + +The recommended `ingest_record_offset()` and `ingest_records_offset()` methods return offsets directly (after queuing): +- `ingest_record_offset()` returns `OffsetId` (the logical offset) +- `ingest_records_offset()` returns `Option` (None if the batch is empty) + +```rust +// Ingest and get offset, after queuing the record. +let offset_id = stream.ingest_record_offset(data).await?; +println!("Record sent with offset Id: {}", offset_id); + +// Wait for acknowledgment when needed. +stream.wait_for_offset(offset_id).await?; +println!("Record committed at offset: {}", offset_id); + +// For batches, the method returns Option. +// None if the batch is empty. +let batch = vec![data1, data2, data3]; +if let Some(offset_id) = stream.ingest_records_offset(batch).await? { + println!("Batch sent with last offset: {}", offset_id); + stream.wait_for_offset(offset_id).await?; + println!("Batch committed"); +} else { + println!("Empty batch, no records ingested"); +} + +// High-throughput: collect offsets and wait selectively. +let mut offsets = Vec::new(); +for i in 0..1000 { + let offset = stream.ingest_record_offset(record).await?; + offsets.push(offset); +} +// Wait for specific offsets as needed. +for offset in offsets { + stream.wait_for_offset(offset).await?; +} + +// Or use flush() to wait for all pending acknowledgments at once. +stream.flush().await?; +``` + +#### Using Acknowledgment Callbacks + +For scenarios where you need to track acknowledgments without explicitly waiting (e.g., for metrics or logging), you can use callbacks: + +```rust +use databricks_zerobus_ingest_sdk::{AckCallback, OffsetId}; +use std::sync::Arc; + +// Define a callback that implements the AckCallback trait +struct MyCallback; + +impl AckCallback for MyCallback { + fn on_ack(&self, offset_id: OffsetId) { + // Called when a record is acknowledged + println!("✓ Acknowledged offset: {}", offset_id); + } + + fn on_error(&self, offset_id: OffsetId, error_message: &str) { + // Called when a record encounters an error + eprintln!("✗ Error for offset {}: {}", offset_id, error_message); + } +} + +// Configure stream with callback +let mut stream = sdk + .stream_builder().table("catalog.schema.orders") + .oauth(client_id, client_secret) + .compiled_proto(descriptor_proto) + .max_inflight_requests(10_000) + .ack_callback(Arc::new(MyCallback)) + .build() + .await?; + +for i in 0..1000 { + let record = YourMessage { id: Some(i), /* ... */ }; + stream.ingest_record_offset(ProtoMessage(record)).await?; + // Callback fires when this record is acknowledged +} + +stream.flush().await?; +``` + +**Important:** Callbacks run synchronously in a dedicated callback handler task. Keep them lightweight (simple logging, metrics increment) to avoid callback backlog. For heavy work like database writes or network calls, send data to a channel for processing in a separate task: + +```rust +use tokio::sync::mpsc; + +struct ChannelCallback { + tx: mpsc::UnboundedSender, +} + +impl AckCallback for ChannelCallback { + fn on_ack(&self, offset_id: OffsetId) { + // Lightweight: just send to channel + let _ = self.tx.send(offset_id); + } + + fn on_error(&self, offset_id: OffsetId, error_message: &str) { + eprintln!("Error: {}", error_message); + } +} + +let (tx, mut rx) = mpsc::unbounded_channel(); +let callback = Arc::new(ChannelCallback { tx }); + +// Heavy processing in separate task +tokio::spawn(async move { + while let Some(offset) = rx.recv().await { + // Heavy work here (database writes, API calls, etc.) + write_to_database(offset).await; + } +}); +``` + +### 7. Close the Stream + +Always close streams to ensure data is flushed: + +```rust +// Close gracefully (flushes automatically) +stream.close().await?; +``` + +If the stream fails, retrieve unacknowledged records: + +```rust +match stream.close().await { + Err(_) => { + // Option 1: Get individual records (flattened) + let unacked = stream.get_unacked_records().await?; + let total_records = unacked.count(); + println!("Failed to ack {} records", total_records); + + // Option 2: Get records grouped by batch (preserves batch structure) + let unacked_batches = stream.get_unacked_batches().await?; + let total_records: usize = unacked_batches.iter().map(|batch| batch.get_record_count()).sum(); + println!("Failed to ack {} records in {} batches", total_records, unacked_batches.len()); + + // Retry with a new stream + } + Ok(_) => println!("Stream closed successfully"), +} +``` + +## Client-side warnings + +The SDK logs a `WARN`-level message via [`tracing`](https://docs.rs/tracing) when **100 or more** streams for the same table are opened within a 60-second sliding window. This usually indicates a "one stream per record" misuse pattern. The warning fires again if the rate drops below the threshold and later surges again. + +To suppress, set the environment variable before starting the process: + +```bash +ZEROBUS_SDK_WARNINGS_ENABLED=false +``` + +Also accepts `0` or `no`. + +## Configuration Options + +### StreamConfigurationOptions + +| Field | Type | Default | Description | +|-------|------|---------|-------------| +| `max_inflight_requests` | `usize` | 1,000,000 | Maximum unacknowledged requests in flight | +| `recovery` | `bool` | true | Enable automatic stream recovery on failure | +| `recovery_timeout_ms` | `u64` | 15,000 | Timeout for recovery operations (ms) | +| `recovery_backoff_ms` | `u64` | 2,000 | Delay between recovery retry attempts (ms) | +| `recovery_retries` | `u32` | 4 | Maximum number of recovery attempts | +| `server_lack_of_ack_timeout_ms` | `u64` | 60,000 | Timeout waiting for server acks (ms) | +| `flush_timeout_ms` | `u64` | 300,000 | Timeout for flush operations (ms) | +| `record_type` | `RecordType` | `RecordType::Proto` | Record serialization format (Proto or Json) | +| `stream_paused_max_wait_time_ms` | `Option` | `None` | Max time to wait during graceful close (`None` = full server duration, `Some(0)` = immediate, `Some(x)` = min(x, server_duration)) | +| `ack_callback` | `Option>` | `None` | Optional callback for acknowledgment notifications | +| `callback_max_wait_time_ms` | `Option` | `None` | Maximum time to wait for callback processing to complete after closing the stream (`None` = wait indefinitely, `Some(x)` = wait up to `x` ms) | + + +**Example:** + +```rust +let stream = sdk + .stream_builder() + .table("catalog.schema.orders") + .oauth(client_id, client_secret) + .json() + .max_inflight_requests(50_000) + .recovery(true) + .recovery_timeout_ms(20_000) + .recovery_retries(5) + .flush_timeout_ms(600_000) + .build() + .await?; +``` + +## Error Handling + +The SDK categorizes errors as **retryable** or **non-retryable**: + +### Retryable Errors +Auto-recovered if `recovery` is enabled: +- Network failures +- Connection timeouts +- Temporary server errors +- Stream closed by server + +### Non-Retryable Errors +Require manual intervention: +- `InvalidUCTokenError` - Invalid OAuth credentials +- `InvalidTableName` - Table doesn't exist or invalid format +- `InvalidArgument` - Invalid parameters or schema mismatch +- `Code::Unauthenticated` - Authentication failure +- `Code::PermissionDenied` - Insufficient table permissions +- `ChannelCreationError` - Failed to establish TLS connection + +**Check if an error is retryable:** + +```rust +match stream.ingest_record_offset(payload).await { + Ok(offset) => { + stream.wait_for_offset(offset).await?; + } + Err(e) if e.is_retryable() => { + eprintln!("Retryable error, SDK will auto-recover: {}", e); + } + Err(e) => { + eprintln!("Fatal error, manual intervention needed: {}", e); + return Err(e.into()); + } +} +``` + +## Examples + +### Complete Working Examples + +The `examples/` directory contains four working examples covering different serialization formats and ingestion patterns: + +| Example | Serialization | Ingestion | Run with | +|---------|--------------|-----------|----------| +| `json/single.rs` | JSON | Single-record | `cargo run -p rust-examples-json --example json_single` | +| `json/batch.rs` | JSON | Batch | `cargo run -p rust-examples-json --example json_batch` | +| `proto/single.rs` | Protocol Buffers | Single-record | `cargo run -p rust-examples-proto --example proto_single` | +| `proto/batch.rs` | Protocol Buffers | Batch | `cargo run -p rust-examples-proto --example proto_batch` | + + +Check [`examples/README.md`](https://github.com/databricks/zerobus-sdk/blob/main/rust/examples/README.md) for setup instructions and detailed comparisons. + +### Stream Recovery + +```rust +let sdk = ZerobusSdk::builder() + .endpoint(endpoint) + .unity_catalog_url(uc_endpoint) + .build()?; + +let mut stream = sdk + .stream_builder().table("catalog.schema.table") + .oauth(client_id, client_secret) + .json() + .build() + .await?; + +// Ingest data... +match stream.close().await { + Err(_) => { + // Stream failed, recreate with unacked records + stream = sdk.recreate_stream(&stream).await?; + } + Ok(_) => println!("Closed successfully"), +} +``` + +## Tests + +Integration tests live in the `tests/` crate and run against a lightweight mock Zerobus gRPC server. + +- Mock server: `tests/src/mock_grpc.rs` +- Test suite: `tests/src/rust_tests.rs` + +Run tests with logs: + +```bash +cargo test -p tests -- --nocapture +``` + +## Best Practices + +1. **Reuse SDK Instances** - Create one `ZerobusSdk` per application and reuse for multiple streams +2. **Always Close Streams** - Use `stream.close().await?` to ensure all data is flushed +3. **Choose the Right Ingestion Method**: + - Use `ingest_records_offset()` for high throughput batch ingestion + - Use `ingest_record_offset()` when processing records individually + - Both return offsets directly; use `wait_for_offset()` to explicitly wait for acknowledgments +4. **Tune Inflight Limits** - Adjust `max_inflight_requests` based on memory and throughput needs +5. **Enable Recovery** - Always set `recovery: true` in production environments +6. **Handle Ack Futures** - Use `tokio::spawn` for fire-and-forget or batch-wait for verification +7. **Monitor Errors** - Log and alert on non-retryable errors +8. **Validate Schemas** - Use the schema generation tool to ensure type safety (for Protocol Buffers) +9. **Secure Credentials** - Never hardcode secrets; use environment variables or secret managers +10. **Test Recovery** - Simulate failures to verify your error handling logic + +## API Reference + +### `ZerobusSdk` + +Main entry point for the SDK. + +**Builder:** +```rust +let sdk = ZerobusSdk::builder() + .endpoint("https://workspace.zerobus.databricks.com") // Required + .unity_catalog_url("https://workspace.cloud.databricks.com") // Optional with custom headers + .tls_config(Arc::new(SecureTlsConfig::new())) // Optional, defaults to SecureTlsConfig + .build()?; +``` + +**Methods:** + +```rust +pub fn stream_builder(&self) -> StreamBuilder<'_> +``` +Returns a fluent builder for creating ingestion streams. All setters can be called in any order. Call `validate()` to check the configuration without opening a stream, or `build()` / `build_arrow()` to open the stream. See [Create a Stream](#4-create-a-stream) for usage. + +```rust +pub async fn recreate_stream( + &self, + stream: &ZerobusStream +) -> ZerobusResult +``` +Recreates a failed stream, preserving and re-ingesting unacknowledged records. + +### `ZerobusStream` + +Represents an active ingestion stream. + +**Methods:** +```rust +pub async fn ingest_record_offset( + &self, + payload: impl Into +) -> ZerobusResult +``` +Ingests a single encoded record (Protocol Buffers or JSON). The await queues the record for sending and returns the logical offset ID directly. Use `wait_for_offset()` to explicitly wait for server acknowledgment of this offset. + +```rust +pub async fn ingest_records_offset( + &self, + payloads: Vec> +) -> ZerobusResult> +``` +Ingests multiple encoded records as a batch with all-or-nothing semantics. The entire batch either succeeds or fails as a unit. The await queues the batch for sending and returns the logical offset ID directly (or `None` for empty batches). Use `wait_for_offset()` to explicitly wait for server acknowledgment. + +```rust +pub async fn wait_for_offset(&self, offset_id: OffsetId) -> ZerobusResult<()> +``` +Waits for acknowledgment of a specific logical offset. Use this method with offsets returned from `ingest_record_offset()` or `ingest_records_offset()` to explicitly wait for server acknowledgment. + +```rust +pub async fn flush(&self) -> ZerobusResult<()> +``` +Flushes all pending records and waits for acknowledgment. + +```rust +pub async fn close(&mut self) -> ZerobusResult<()> +``` +Flushes and closes the stream gracefully. + +```rust +pub async fn get_unacked_records(&self) -> ZerobusResult> +``` +Returns an iterator over all unacknowledged records as individual `EncodedRecord` items. This flattens batches into individual records. Only call after stream failure. + +```rust +pub async fn get_unacked_batches(&self) -> ZerobusResult> +``` +Returns unacknowledged records grouped by batch, preserving the original batch structure. Records ingested together remain grouped: +- Each `ingest_record_offset()` call creates a batch containing one record +- Each `ingest_records_offset()` call creates a batch containing multiple records + +Only call after stream failure. + +### `StreamBuilder` + +Configure stream parameters via fluent setters; all configuration goes through the builder. See [Create a Stream](#4-create-a-stream) for usage and [Configuration Options](#configuration-options) for the full list of available setters and their defaults. + +### `AckCallback` + +Trait for receiving acknowledgment notifications. + +**Methods:** +```rust +pub trait AckCallback: Send + Sync { + fn on_ack(&self, offset_id: OffsetId); + fn on_error(&self, offset_id: OffsetId, error_message: &str); +} +``` + +- `on_ack()` - Called when a record/batch is successfully acknowledged +- `on_error()` - Called when a record/batch encounters an error + +### `HeadersProvider` + +Trait for custom authentication header providers. + +**Methods:** +```rust +#[async_trait] +pub trait HeadersProvider: Send + Sync { + async fn get_headers(&self) -> ZerobusResult>; +} +``` + +Implement this trait to provide custom authentication headers. The default implementation (`OAuthHeadersProvider`) handles OAuth 2.0 token management. Use this for: +- Custom token caching strategies +- Alternative authentication mechanisms +- Integration with centralized credential services + +See [Custom Authentication](#custom-authentication) section for usage examples. + +### `TlsConfig` + +Trait for TLS configuration strategies. + +**Implementations:** +- `SecureTlsConfig` (default) - Production TLS with system CA certificates +- `NoTlsConfig` - No-op TLS for testing with `http://` endpoints (requires `testing` feature) + +```rust +pub trait TlsConfig: Send + Sync { + fn configure_endpoint(&self, endpoint: Endpoint) -> ZerobusResult; +} +``` + +### `ZerobusError` + +Error type for all SDK operations. + +**Methods:** +```rust +pub fn is_retryable(&self) -> bool +``` +Returns `true` if the error can be automatically recovered by the SDK. + +## Language Bindings + +This repository includes bindings for building SDKs in other languages. + +### C FFI (`ffi/`) + +C Foreign Function Interface for languages that can call C functions (Go, C#, C++, etc.). + +```bash +# Build static and dynamic libraries +cargo build -p zerobus-ffi --release + +# Output: +# target/release/libzerobus_ffi.a (static library for Go, C++) +# target/release/libzerobus_ffi.so (Linux dynamic library for C#) +# target/release/libzerobus_ffi.dylib (macOS dynamic library for C#) +# target/release/zerobus_ffi.dll (Windows dynamic library for C#) +# ffi/zerobus.h (C header file) +``` + +Pre-built binaries for all platforms are available in [GitHub Releases](https://github.com/databricks/zerobus-sdk/releases) with tags `ffi/vX.X.X`. + +### JNI (`jni/`) + +Java Native Interface bindings for the [Zerobus Java SDK](https://github.com/databricks/zerobus-sdk/tree/main/java). + +```bash +# Build JNI library +cargo build -p zerobus-jni --release + +# Output: +# target/release/libzerobus_jni.so (Linux) +# target/release/libzerobus_jni.dylib (macOS) +# target/release/zerobus_jni.dll (Windows) +``` + +Pre-built binaries are available in [GitHub Releases](https://github.com/databricks/zerobus-sdk/releases) with tags `jni/vX.X.X`. + +## Building from Source + +For contributors or those who want to build and test the SDK: + +```bash +git clone https://github.com/databricks/zerobus-sdk.git +cd zerobus-sdk/rust +cargo build --workspace +``` + +**Build specific components:** + +```bash +# Build only SDK +cargo build -p databricks-zerobus-ingest-sdk + +# Build only schema tool +cargo build -p tools + +# Build and run JSON single-record example +cargo run -p rust-examples-json --example json_single + +# Build and run JSON batch example +cargo run -p rust-examples-json --example json_batch + +# Build and run Protocol Buffers single-record example +cargo run -p rust-examples-proto --example proto_single + +# Build and run Protocol Buffers batch example +cargo run -p rust-examples-proto --example proto_batch +``` + +## Community and Contributing + +This is an open source project. We welcome contributions, feedback, and bug reports. + +- **[Contributing Guide](https://github.com/databricks/zerobus-sdk/blob/main/rust/CONTRIBUTING.md)**: Rust-specific development setup and workflow. +- **[General Contributing Guide](https://github.com/databricks/zerobus-sdk/blob/main/CONTRIBUTING.md)**: Pull request process, commit requirements, and policies. +- **[Changelog](https://github.com/databricks/zerobus-sdk/blob/main/rust/CHANGELOG.md)**: See the history of changes in the SDK. +- **[Security Policy](https://github.com/databricks/zerobus-sdk/blob/main/SECURITY.md)**: Read about our security process and how to report vulnerabilities. +- **[Developer Certificate of Origin (DCO)](https://github.com/databricks/zerobus-sdk/blob/main/DCO)**: Understand the agreement for contributions. +- **[Open Source Attributions](https://github.com/databricks/zerobus-sdk/blob/main/rust/NOTICE)**: See a list of the open source libraries we use. + +## License + +This SDK is licensed under the Apache License 2.0. See [LICENSE](LICENSE) for the full text. + +## Requirements + +- **Rust** 1.70 or higher (2021 edition) +- **TLS** - Uses native OS certificate store by default (configurable via `TlsConfig` trait) +- See [prerequisites](https://github.com/databricks/zerobus-sdk/blob/main/README.md#prerequisites) for Databricks workspace and credential requirements + + +--- + +For issues, questions, or contributions, please visit the [GitHub repository](https://github.com/databricks/zerobus-sdk). See the [monorepo README](https://github.com/databricks/zerobus-sdk/blob/main/README.md) for an overview of all available SDKs. diff --git a/lib/zerobus-ffi-1.3.0/rust/ffi/CHANGELOG.md b/lib/zerobus-ffi-1.3.0/rust/ffi/CHANGELOG.md new file mode 100644 index 00000000000..ee96bc1fbe4 --- /dev/null +++ b/lib/zerobus-ffi-1.3.0/rust/ffi/CHANGELOG.md @@ -0,0 +1,104 @@ +# Version changelog + +## Release v1.3.0 + +### Major Changes + +### New Features and Improvements + +- **C-builder API for SDK construction**: `zerobus_sdk_builder_new`, per-option setters (`_endpoint`, `_unity_catalog_url`, `_sdk_identifier`, `_application_name`, `_disable_tls`), and `_build` / `_free`. Mirrors the Rust `ZerobusSdkBuilder`; new options are added as setters without ABI breaks. Legacy `zerobus_sdk_new` is retained and delegates to the builder. +- **Dynamic protobuf from a Unity Catalog schema**: a pure-C consumer can now build a protobuf descriptor from UC table metadata and encode records without a companion Rust crate. New opaque type `CZerobusProtoSchema` and functions: + - `zerobus_proto_schema_from_uc_json` — build a schema handle from UC table-metadata JSON (the body of `GET /api/2.1/unity-catalog/tables/{name}`). + - `zerobus_proto_schema_descriptor_bytes` — borrow the serialized `DescriptorProto` to pass straight to `zerobus_sdk_create_stream` (byte-identical to the descriptor the encoder uses). + - `zerobus_proto_schema_encode_json` — encode one JSON record into protobuf bytes; unknown keys are ignored. `DATE`/`TIMESTAMP`/`TIMESTAMP_NTZ` columns are integers (days / micros since epoch), `BINARY` is a base64 string, `DECIMAL` is a string, and large 64-bit integers are accepted as JSON strings (the protobuf-JSON canonical form) to avoid precision loss in producers that emit numbers as IEEE-754 doubles. Top-level non-nullable scalar/struct columns are proto2 `required`; a record missing one is rejected (ARRAY/MAP map to `repeated`, which has no presence, so an omitted one encodes as empty). + - `zerobus_free_proto_bytes` / `zerobus_proto_schema_free` — free an encoded buffer / a schema handle. + +### Bug Fixes + +### Documentation + +### Internal Changes + +### Behavior Changes + +### Breaking Changes + +### Deprecations + +### API Changes + +## Release v1.2.1 + +### Major Changes + +### New Features and Improvements + +### Bug Fixes + +- **`zerobus_arrow_stream_ingest_batch_via_record_batch` now works correctly on compression-enabled streams.** Previously the function performed its own IPC deserialization and called `ingest_batch` directly, bypassing the compression re-encoding step. It now delegates to `ingest_ipc_batch`, which handles compression transparently. The function is now fully equivalent to `zerobus_arrow_stream_ingest_batch` regardless of stream configuration. + +### Documentation + +### Internal Changes + +### Breaking Changes + +### Deprecations + +### API Changes + +## Release v1.2.0 + +### Major Changes + +### New Features and Improvements + +- **Arrow stream options (C API)**: `CArrowStreamConfigurationOptions.stream_paused_max_wait_time_ms` (`int64_t`) configures graceful-close paused wait: `-1` = None (full server duration), `0` = immediate recovery, `>0` = capped wait (see `zerobus.h` comments). +- **Zero-copy Arrow IPC ingestion**: `zerobus_arrow_stream_ingest_batch` now forwards IPC bytes directly via `ingest_ipc_batch`, skipping the deserialization round-trip. Use `zerobus_arrow_stream_ingest_batch_via_record_batch` for compression-enabled streams. +- **Fire-and-forget ingestion**: Added nowait variants that spawn a background task and return immediately — `zerobus_stream_ingest_proto_record_nowait`, `zerobus_stream_ingest_json_record_nowait`, `zerobus_stream_ingest_proto_records_nowait`, `zerobus_stream_ingest_json_records_nowait`. + +### Bug Fixes + +- **Arrow IPC compression fix**: Added `zerobus_arrow_stream_ingest_batch_via_record_batch` for streams created with `LZ4_FRAME` or `ZSTD` compression. The existing `zerobus_arrow_stream_ingest_batch` uses the zero-copy path and does not apply compression; callers must use the new function when compression is configured. This fixes a regression where compression was silently ignored. + +### Documentation + +### Internal Changes + +### Breaking Changes + +### Deprecations + +### API Changes + +- Added `zerobus_arrow_stream_ingest_batch_via_record_batch(stream, ipc_bytes, ipc_len, result)` for compression-enabled Arrow streams. +- Added `zerobus_stream_ingest_proto_record_nowait`, `zerobus_stream_ingest_json_record_nowait`, `zerobus_stream_ingest_proto_records_nowait`, `zerobus_stream_ingest_json_records_nowait` for fire-and-forget ingestion. + +## Release v1.1.0 + +### Major Changes + +- **License: Migrated from the Databricks License to the Apache License 2.0** +- Removed macOS x86_64 and macOS aarch64 support. + +### New Features and Improvements + +- Added dynamic library (.so / .dylib / .dll) output alongside static library + +## Release v1.0.1 + +Initial tracked release of the FFI C bindings for the Zerobus SDK. + +### Platforms + +- Linux x86_64 +- Linux aarch64 +- macOS x86_64 +- macOS aarch64 +- Windows x86_64 + +### Libraries + +- Static library (.a / .lib) +- Dynamic library (.so / .dylib / .dll) +- C header file (zerobus.h) diff --git a/lib/zerobus-ffi-1.3.0/rust/ffi/Cargo.toml b/lib/zerobus-ffi-1.3.0/rust/ffi/Cargo.toml new file mode 100644 index 00000000000..4f621812336 --- /dev/null +++ b/lib/zerobus-ffi-1.3.0/rust/ffi/Cargo.toml @@ -0,0 +1,37 @@ +[package] +name = "zerobus-ffi" +version = "1.3.0" +edition = "2021" +description = "C FFI bindings for the Zerobus Rust SDK" +license = "Apache-2.0" +publish = false + +[lib] +crate-type = ["staticlib", "cdylib"] + +[dependencies] +databricks-zerobus-ingest-sdk = { path = "../sdk", version = "2.0.1", features = ["arrow-flight", "testing"] } + +# Arrow IPC for serializing/deserializing RecordBatches across the FFI boundary +arrow-ipc.workspace = true + +# FFI helpers +tokio = { workspace = true, features = ["rt", "rt-multi-thread"] } +once_cell.workspace = true +prost.workspace = true +prost-types.workspace = true +async-trait.workspace = true +bytes.workspace = true +libc.workspace = true + +# Dynamic protobuf: build a descriptor pool from a UC-derived DescriptorProto +# and encode JSON records into protobuf bytes (UC schema -> proto path for +# pure-C consumers). serde_json parses the UC table metadata + record JSON. +prost-reflect = { workspace = true, features = ["serde"] } +serde_json.workspace = true + +# Logging +tracing-subscriber = { workspace = true, features = ["fmt"] } + +[build-dependencies] +cbindgen.workspace = true diff --git a/lib/zerobus-ffi-1.3.0/rust/ffi/NEXT_CHANGELOG.md b/lib/zerobus-ffi-1.3.0/rust/ffi/NEXT_CHANGELOG.md new file mode 100644 index 00000000000..29918b3b94d --- /dev/null +++ b/lib/zerobus-ffi-1.3.0/rust/ffi/NEXT_CHANGELOG.md @@ -0,0 +1,21 @@ +# NEXT CHANGELOG + +## Release v1.4.0 + +### Major Changes + +### New Features and Improvements + +### Bug Fixes + +### Documentation + +### Internal Changes + +### Behavior Changes + +### Breaking Changes + +### Deprecations + +### API Changes diff --git a/lib/zerobus-ffi-1.3.0/rust/ffi/README.md b/lib/zerobus-ffi-1.3.0/rust/ffi/README.md new file mode 100644 index 00000000000..9c7dc8736d1 --- /dev/null +++ b/lib/zerobus-ffi-1.3.0/rust/ffi/README.md @@ -0,0 +1,140 @@ +# Zerobus C FFI + +C Foreign Function Interface bindings for the Zerobus Rust SDK. + +## Building + +```bash +# Build both static and dynamic libraries +cargo build -p zerobus-ffi --release + +# Output: +# target/release/libzerobus_ffi.a (static library) +# target/release/libzerobus_ffi.so (Linux dynamic library) +# target/release/libzerobus_ffi.dylib (macOS dynamic library) +# target/release/zerobus_ffi.dll (Windows dynamic library) +``` + +## Cross-compilation + +```bash +# Linux ARM64 +rustup target add aarch64-unknown-linux-gnu +cargo build -p zerobus-ffi --release --target aarch64-unknown-linux-gnu + +# macOS ARM64 (Apple Silicon) +rustup target add aarch64-apple-darwin +cargo build -p zerobus-ffi --release --target aarch64-apple-darwin + +# Windows +rustup target add x86_64-pc-windows-gnu +cargo build -p zerobus-ffi --release --target x86_64-pc-windows-gnu +``` + +## Usage + +### Go (CGO with static library) + +```go +/* +#cgo LDFLAGS: -L${SRCDIR}/lib -lzerobus_ffi -ldl -lpthread -lm +#include "zerobus.h" +*/ +import "C" +``` + +### C# (P/Invoke with dynamic library) + +```csharp +[DllImport("zerobus_ffi", CallingConvention = CallingConvention.Cdecl)] +private static extern IntPtr zerobus_sdk_new(string endpoint, string ucUrl, ref CResult result); +``` + +### C++ + +```cpp +#include "zerobus.h" + +// Link with -lzerobus_ffi +``` + +### Dynamic protobuf from a Unity Catalog schema (pure C) + +Build a protobuf descriptor and encode records straight from Unity Catalog +table metadata — no pre-generated `.proto` file and no second Rust crate: + +```c +CResult r = {0}; + +/* init: fetch GET /api/2.1/unity-catalog/tables/{name} and pass its JSON body */ +CZerobusProtoSchema *schema = zerobus_proto_schema_from_uc_json(uc_table_json, &r); +/* on error schema == NULL; read r.error_message then zerobus_free_error_message(r.error_message) */ + +uintptr_t dlen; +const uint8_t *desc = zerobus_proto_schema_descriptor_bytes(schema, &dlen); +CZerobusStream *stream = zerobus_sdk_create_stream(sdk, table_name, desc, dlen, + client_id, client_secret, &opts, &r); + +/* per record, at flush time */ +uint8_t *buf; uintptr_t len; +if (zerobus_proto_schema_encode_json(schema, record_json, &buf, &len, &r)) { + /* collect buf/len into a batch, ingest via zerobus_stream_ingest_proto_records(...) */ + zerobus_free_proto_bytes(buf, len); +} + +/* shutdown */ +zerobus_proto_schema_free(schema); +``` + +Encoding contract: record object keys are matched to column names; unknown keys +are ignored (upstream records often carry extra non-column metadata). Records are +encoded through protobuf's JSON mapping, so a few column types need their JSON +value shaped accordingly: + +| Unity Catalog type | Proto type | JSON value to supply | +|-------------------------------|-----------------|------------------------------------------------------------------| +| `STRING` | `string` | string | +| `INT`/`INTEGER`, `SHORT`/`SMALLINT` | `int32` | number | +| `LONG`/`BIGINT` | `int64` | number **or string** — use a **string** above 2^53 (see below) | +| `FLOAT`, `DOUBLE` | `float`/`double`| number | +| `BOOLEAN` | `bool` | boolean | +| `BINARY` | `bytes` | **base64-encoded string** (not a JSON array of byte values) | +| `DATE` | `int32` | integer — days since the Unix epoch (not an ISO-8601 string) | +| `TIMESTAMP` | `int64` | integer — microseconds since the Unix epoch (not an ISO-8601 string) | +| `VARIANT` | `string` | **JSON-encoded string** — a string whose contents are the variant's JSON (objects, arrays, or primitives) | +| `ARRAY` | `repeated T` | JSON array of `T` values | +| `MAP` | `map` | JSON object (`K` must be an integral, bool, or string type) | +| `STRUCT<...>` | nested `message`| JSON object | + +This table mirrors the supported set in the [Zerobus type-support +reference](https://docs.databricks.com/aws/en/ingestion/zerobus-limits#type-support). +Beyond that reference, the encoder additionally accepts `BYTE`/`TINYINT` (→ +`int32`), `TIMESTAMP_NTZ` (→ `int64` micros, same wire shape as `TIMESTAMP`), +and `DECIMAL` (→ `string`, e.g. `"123.45"`, to preserve precision/scale). +Complex columns (`ARRAY`/`MAP`/`STRUCT`) require the column's `type_json` from +the Unity Catalog REST response to be present in the input. + +Two precision pitfalls worth calling out: + +- **64-bit integers above 2^53** (large `BIGINT` values, and `TIMESTAMP` micros + past year 2255) lose precision when emitted as a JSON number by most encoders. + Pass them as JSON **strings** (the canonical protobuf-JSON form for 64-bit + ints), which round-trip exactly. +- **`DATE`/`TIMESTAMP*` unit mismatches** are silent: writing milliseconds where + microseconds are expected shifts every row by 10³. + +Top-level non-nullable scalar and struct columns become proto2 `required` +fields; a record missing one is rejected rather than encoded. Non-nullable +`ARRAY`/`MAP` columns map to `repeated`, which has no presence, so an omitted one +encodes as empty rather than being rejected; required fields nested inside a +`STRUCT` are likewise not presence-checked. + +A handle may be shared by concurrent readers: worker threads may call +`zerobus_proto_schema_encode_json` and `zerobus_proto_schema_descriptor_bytes` +on the same handle concurrently. `zerobus_proto_schema_free` must be called +exactly once, after all in-flight calls on the handle have returned — it must +not race any other use of the handle. + +## API Reference + +See `zerobus.h` for the complete C API documentation. diff --git a/lib/zerobus-ffi-1.3.0/rust/ffi/build.rs b/lib/zerobus-ffi-1.3.0/rust/ffi/build.rs new file mode 100644 index 00000000000..9a3a4bc3361 --- /dev/null +++ b/lib/zerobus-ffi-1.3.0/rust/ffi/build.rs @@ -0,0 +1,19 @@ +extern crate cbindgen; + +use std::env; +use std::path::PathBuf; + +fn main() { + let crate_dir = env::var("CARGO_MANIFEST_DIR").unwrap(); + let output_file = PathBuf::from(env::var("OUT_DIR").unwrap()).join("zerobus.h"); + + cbindgen::Builder::new() + .with_crate(crate_dir) + .with_config(cbindgen::Config::from_file("cbindgen.toml").unwrap()) + .generate() + .expect("Unable to generate bindings") + .write_to_file(&output_file); + + println!("cargo:rerun-if-changed=src/lib.rs"); + println!("cargo:rerun-if-changed=cbindgen.toml"); +} diff --git a/lib/zerobus-ffi-1.3.0/rust/ffi/cbindgen.toml b/lib/zerobus-ffi-1.3.0/rust/ffi/cbindgen.toml new file mode 100644 index 00000000000..dabb48dedbc --- /dev/null +++ b/lib/zerobus-ffi-1.3.0/rust/ffi/cbindgen.toml @@ -0,0 +1,13 @@ +language = "C" +include_guard = "ZEROBUS_H" +pragma_once = true +autogen_warning = "/* Warning: This file is autogenerated by cbindgen. Don't modify this manually. */" +header = "/* Zerobus C FFI Interface */" +include_version = true +namespace = "zerobus" +cpp_compat = true + +[export] +include = ["CZerobusSdk", "CZerobusStream", "CResult", "CStreamConfigurationOptions", "CRecord", "CRecordArray", "CHeader", "CHeaders", "CArrowStream", "CArrowStreamConfigurationOptions", "CArrowBatchArray", "CZerobusProtoSchema"] + +[export.rename] diff --git a/lib/zerobus-ffi-1.3.0/rust/ffi/src/lib.rs b/lib/zerobus-ffi-1.3.0/rust/ffi/src/lib.rs new file mode 100644 index 00000000000..632c7d023c4 --- /dev/null +++ b/lib/zerobus-ffi-1.3.0/rust/ffi/src/lib.rs @@ -0,0 +1,2250 @@ +// Allow clippy warnings for FFI code where unsafe operations are unavoidable +#![allow(clippy::not_unsafe_ptr_arg_deref)] +#![allow(clippy::type_complexity)] + +use once_cell::sync::Lazy; +use std::collections::{HashMap, HashSet}; +use std::ffi::{CStr, CString}; +use std::os::raw::c_char; +use std::ptr; +use std::sync::atomic::{AtomicBool, Ordering}; +use std::sync::Mutex; +use tokio::runtime::Runtime; +use tracing_subscriber::{fmt, EnvFilter}; +extern crate libc; + +use arrow_ipc::{reader::StreamReader, writer::StreamWriter, CompressionType}; +use async_trait::async_trait; +use bytes::Bytes; +use databricks_zerobus_ingest_sdk::databricks::zerobus::RecordType; +use databricks_zerobus_ingest_sdk::schema::{descriptor_from_uc_schema, UcTableSchema}; +use databricks_zerobus_ingest_sdk::{ + EncodedRecord, HeadersProvider, NoTlsConfig, ZerobusError, ZerobusResult, ZerobusSdk, + ZerobusSdkBuilder, ZerobusStream, +}; +use databricks_zerobus_ingest_sdk::{RecordBatch, StreamBuilder, ZerobusArrowStream}; +use prost::Message; +use prost_reflect::{ + Cardinality, DescriptorPool, DeserializeOptions, DynamicMessage, MessageDescriptor, +}; +use std::sync::Arc; + +// Test module +#[cfg(test)] +mod tests; + +// ============================================================================ +// Arrow Flight FFI +// ============================================================================ + +/// Opaque handle for an Arrow Flight stream. +#[repr(C)] +pub struct CArrowStream { + _private: [u8; 0], +} + +/// Configuration options for Arrow Flight streams. +/// +/// `ipc_compression`: -1 = None, 0 = LZ4_FRAME, 1 = ZSTD +#[repr(C)] +#[derive(Clone, Copy)] +pub struct CArrowStreamConfigurationOptions { + pub max_inflight_batches: usize, + pub recovery: bool, + pub recovery_timeout_ms: u64, + pub recovery_backoff_ms: u64, + pub recovery_retries: u32, + pub server_lack_of_ack_timeout_ms: u64, + pub flush_timeout_ms: u64, + pub connection_timeout_ms: u64, + /// -1 = None, 0 = LZ4_FRAME, 1 = ZSTD + pub ipc_compression: i32, + /// Maximum time in milliseconds to wait during graceful stream close. + /// -1 = None (wait full server duration), 0 = immediate recovery, >0 = wait up to min(this, server_duration). + pub stream_paused_max_wait_time_ms: i64, +} + +fn c_to_compression(value: i32) -> Option { + match value { + 0 => Some(CompressionType::LZ4_FRAME), + 1 => Some(CompressionType::ZSTD), + _ => None, + } +} + +fn c_to_stream_paused_ms(value: i64) -> Option { + if value < 0 { + None + } else { + Some(value as u64) + } +} + +/// An array of Arrow IPC-encoded batches, returned by `zerobus_arrow_stream_get_unacked_batches`. +/// Must be freed with `zerobus_arrow_free_batch_array`. +#[repr(C)] +pub struct CArrowBatchArray { + /// Array of pointers to IPC-encoded batch bytes. + pub batches: *mut *mut u8, + /// Array of byte lengths, one per batch. + pub lengths: *mut usize, + /// Number of batches. + pub count: usize, +} + +// ---- Arrow pointer validation helpers ---- + +fn validate_arrow_stream_ptr<'a>( + stream: *mut CArrowStream, +) -> Result<&'a ZerobusArrowStream, &'static str> { + if stream.is_null() { + return Err("Arrow stream pointer is null"); + } + unsafe { Ok(&*(stream as *const ZerobusArrowStream)) } +} + +fn validate_arrow_stream_ptr_mut<'a>( + stream: *mut CArrowStream, +) -> Result<&'a mut ZerobusArrowStream, &'static str> { + if stream.is_null() { + return Err("Arrow stream pointer is null"); + } + unsafe { Ok(&mut *(stream as *mut ZerobusArrowStream)) } +} + +// ---- Arrow IPC helpers ---- + +/// Deserializes an `Arc` from Arrow IPC stream bytes (schema-only stream). +#[allow(clippy::result_large_err)] +fn ipc_bytes_to_schema( + bytes: &[u8], +) -> ZerobusResult> { + use std::io::Cursor; + let cursor = Cursor::new(bytes); + let reader = StreamReader::try_new(cursor, None).map_err(|e| { + ZerobusError::InvalidArgument(format!("Failed to parse Arrow IPC schema: {e}")) + })?; + Ok(reader.schema().clone()) +} + +/// Serializes a `RecordBatch` to Arrow IPC stream bytes (schema + one batch). +#[allow(clippy::result_large_err)] +fn record_batch_to_ipc_bytes(batch: &RecordBatch) -> ZerobusResult> { + let mut buf = Vec::new(); + let mut writer = StreamWriter::try_new(&mut buf, batch.schema().as_ref()).map_err(|e| { + ZerobusError::InvalidArgument(format!("Failed to create Arrow IPC writer: {e}")) + })?; + writer.write(batch).map_err(|e| { + ZerobusError::InvalidArgument(format!("Failed to write Arrow IPC batch: {e}")) + })?; + writer.finish().map_err(|e| { + ZerobusError::InvalidArgument(format!("Failed to finish Arrow IPC stream: {e}")) + })?; + Ok(buf) +} + +// Builder option application helpers + +fn apply_c_stream_options<'a>( + builder: StreamBuilder<'a>, + c: &CStreamConfigurationOptions, +) -> StreamBuilder<'a> { + builder + .max_inflight_requests(c.max_inflight_requests) + .recovery(c.recovery) + .recovery_timeout_ms(c.recovery_timeout_ms) + .recovery_backoff_ms(c.recovery_backoff_ms) + .recovery_retries(c.recovery_retries) + .server_lack_of_ack_timeout_ms(c.server_lack_of_ack_timeout_ms) + .flush_timeout_ms(c.flush_timeout_ms) + .stream_paused_max_wait_time_ms(if c.has_stream_paused_max_wait_time_ms { + Some(c.stream_paused_max_wait_time_ms) + } else { + None + }) + .callback_max_wait_time_ms(if c.has_callback_max_wait_time_ms { + Some(c.callback_max_wait_time_ms) + } else { + None + }) +} + +fn c_record_type(value: i32) -> RecordType { + match value { + 1 => RecordType::Proto, + 2 => RecordType::Json, + _ => RecordType::Unspecified, + } +} + +fn apply_c_arrow_stream_options<'a>( + builder: StreamBuilder<'a>, + c: &CArrowStreamConfigurationOptions, +) -> StreamBuilder<'a> { + builder + .max_inflight_batches(c.max_inflight_batches) + .recovery(c.recovery) + .recovery_timeout_ms(c.recovery_timeout_ms) + .recovery_backoff_ms(c.recovery_backoff_ms) + .recovery_retries(c.recovery_retries) + .server_lack_of_ack_timeout_ms(c.server_lack_of_ack_timeout_ms) + .flush_timeout_ms(c.flush_timeout_ms) + .connection_timeout_ms(c.connection_timeout_ms) + .ipc_compression(c_to_compression(c.ipc_compression)) + .stream_paused_max_wait_time_ms(c_to_stream_paused_ms(c.stream_paused_max_wait_time_ms)) +} + +// ---- Arrow FFI functions ---- + +/// Creates an Arrow Flight stream authenticated with OAuth client credentials. +/// +/// `schema_ipc_bytes` must point to Arrow IPC stream bytes encoding only the schema +/// (write an empty IPC stream with just the schema message). +#[no_mangle] +pub extern "C" fn zerobus_sdk_create_arrow_stream( + sdk: *mut CZerobusSdk, + table_name: *const c_char, + schema_ipc_bytes: *const u8, + schema_ipc_len: usize, + client_id: *const c_char, + client_secret: *const c_char, + options: *const CArrowStreamConfigurationOptions, + result: *mut CResult, +) -> *mut CArrowStream { + let sdk_ref = match validate_sdk_ptr(sdk) { + Ok(s) => s, + Err(msg) => { + write_error_result(result, msg, false); + return ptr::null_mut(); + } + }; + + let res = RUNTIME.block_on(async { + let table_name_str = unsafe { c_str_to_string(table_name).map_err(|e| e.to_string())? }; + let client_id_str = unsafe { c_str_to_string(client_id).map_err(|e| e.to_string())? }; + let client_secret_str = + unsafe { c_str_to_string(client_secret).map_err(|e| e.to_string())? }; + + if schema_ipc_bytes.is_null() || schema_ipc_len == 0 { + return Err("Schema IPC bytes are required for Arrow stream".to_string()); + } + let schema_bytes = unsafe { std::slice::from_raw_parts(schema_ipc_bytes, schema_ipc_len) }; + let schema = ipc_bytes_to_schema(schema_bytes).map_err(|e| e.to_string())?; + + let mut builder = sdk_ref + .stream_builder() + .table(table_name_str) + .oauth(client_id_str, client_secret_str) + .arrow(schema); + if !options.is_null() { + builder = apply_c_arrow_stream_options(builder, unsafe { &*options }); + } + + let stream = builder.build_arrow().await.map_err(|e| e.to_string())?; + + let boxed = Box::new(stream); + Ok::<*mut CArrowStream, String>(Box::into_raw(boxed) as *mut CArrowStream) + }); + + match res { + Ok(ptr) => { + write_success_result(result); + ptr + } + Err(err) => { + write_error_result(result, &err, false); + ptr::null_mut() + } + } +} + +/// Creates an Arrow Flight stream with a custom headers provider callback. +/// +/// `schema_ipc_bytes` must point to Arrow IPC stream bytes encoding only the schema. +#[no_mangle] +pub extern "C" fn zerobus_sdk_create_arrow_stream_with_headers_provider( + sdk: *mut CZerobusSdk, + table_name: *const c_char, + schema_ipc_bytes: *const u8, + schema_ipc_len: usize, + headers_callback: HeadersProviderCallback, + user_data: *mut std::ffi::c_void, + options: *const CArrowStreamConfigurationOptions, + result: *mut CResult, +) -> *mut CArrowStream { + let sdk_ref = match validate_sdk_ptr(sdk) { + Ok(s) => s, + Err(msg) => { + write_error_result(result, msg, false); + return ptr::null_mut(); + } + }; + + let res = RUNTIME.block_on(async { + let table_name_str = unsafe { c_str_to_string(table_name).map_err(|e| e.to_string())? }; + + if schema_ipc_bytes.is_null() || schema_ipc_len == 0 { + return Err("Schema IPC bytes are required for Arrow stream".to_string()); + } + let schema_bytes = unsafe { std::slice::from_raw_parts(schema_ipc_bytes, schema_ipc_len) }; + let schema = ipc_bytes_to_schema(schema_bytes).map_err(|e| e.to_string())?; + + let headers_provider: Arc = + Arc::new(CallbackHeadersProvider::new(headers_callback, user_data)); + + let mut builder = sdk_ref + .stream_builder() + .table(table_name_str) + .headers_provider(headers_provider) + .arrow(schema); + if !options.is_null() { + builder = apply_c_arrow_stream_options(builder, unsafe { &*options }); + } + + let stream = builder.build_arrow().await.map_err(|e| e.to_string())?; + + let boxed = Box::new(stream); + Ok::<*mut CArrowStream, String>(Box::into_raw(boxed) as *mut CArrowStream) + }); + + match res { + Ok(ptr) => { + write_success_result(result); + ptr + } + Err(err) => { + write_error_result(result, &err, false); + ptr::null_mut() + } + } +} + +/// Frees an Arrow Flight stream instance. +#[no_mangle] +pub extern "C" fn zerobus_arrow_stream_free(stream: *mut CArrowStream) { + if !stream.is_null() { + unsafe { + let _ = Box::from_raw(stream as *mut ZerobusArrowStream); + } + } +} + +/// Ingests one Arrow RecordBatch supplied as Arrow IPC stream bytes. +/// +/// `ipc_bytes` must be a valid Arrow IPC stream (schema + one record batch). +/// The bytes are deserialised to a RecordBatch internally. Works with all +/// compression settings. Returns the logical offset assigned to this batch, or -1 on error. +#[no_mangle] +pub extern "C" fn zerobus_arrow_stream_ingest_batch( + stream: *mut CArrowStream, + ipc_bytes: *const u8, + ipc_len: usize, + result: *mut CResult, +) -> i64 { + if ipc_bytes.is_null() || ipc_len == 0 { + write_error_result(result, "IPC bytes are required", false); + return -1; + } + + let stream_ref = match validate_arrow_stream_ptr(stream) { + Ok(s) => s, + Err(msg) => { + write_error_result(result, msg, false); + return -1; + } + }; + + let bytes = unsafe { std::slice::from_raw_parts(ipc_bytes, ipc_len) }; + + let offset_res = RUNTIME.block_on(async { + stream_ref + .ingest_ipc_batch(Bytes::copy_from_slice(bytes)) + .await + }); + + match offset_res { + Ok(offset) => { + write_success_result(result); + offset + } + Err(err) => { + if !result.is_null() { + unsafe { + *result = CResult::error(err); + } + } + -1 + } + } +} + +/// Ingests one Arrow RecordBatch supplied as Arrow IPC stream bytes. +/// +/// Equivalent to `zerobus_arrow_stream_ingest_batch`. Both functions deserialise the IPC +/// bytes to a `RecordBatch` and re-encode with the stream's compression settings, so +/// either works regardless of whether the stream was created with compression. +/// Returns the logical offset assigned to this batch, or -1 on error. +#[no_mangle] +pub extern "C" fn zerobus_arrow_stream_ingest_batch_via_record_batch( + stream: *mut CArrowStream, + ipc_bytes: *const u8, + ipc_len: usize, + result: *mut CResult, +) -> i64 { + if ipc_bytes.is_null() || ipc_len == 0 { + write_error_result(result, "IPC bytes are required", false); + return -1; + } + + let stream_ref = match validate_arrow_stream_ptr(stream) { + Ok(s) => s, + Err(msg) => { + write_error_result(result, msg, false); + return -1; + } + }; + + let bytes = unsafe { std::slice::from_raw_parts(ipc_bytes, ipc_len) }; + + let offset_res = RUNTIME.block_on(async { + stream_ref + .ingest_ipc_batch(bytes::Bytes::copy_from_slice(bytes)) + .await + }); + + match offset_res { + Ok(offset) => { + write_success_result(result); + offset + } + Err(err) => { + if !result.is_null() { + unsafe { + *result = CResult::error(err); + } + } + -1 + } + } +} + +/// Waits until the server acknowledges the batch at the given logical offset. +#[no_mangle] +pub extern "C" fn zerobus_arrow_stream_wait_for_offset( + stream: *mut CArrowStream, + offset: i64, + result: *mut CResult, +) -> bool { + let stream_ref = match validate_arrow_stream_ptr(stream) { + Ok(s) => s, + Err(msg) => { + write_error_result(result, msg, false); + return false; + } + }; + + let res = RUNTIME.block_on(async { stream_ref.wait_for_offset(offset).await }); + + match res { + Ok(()) => { + write_success_result(result); + true + } + Err(err) => { + if !result.is_null() { + unsafe { + *result = CResult::error(err); + } + } + false + } + } +} + +/// Flushes all pending batches and waits for their acknowledgment. +#[no_mangle] +pub extern "C" fn zerobus_arrow_stream_flush( + stream: *mut CArrowStream, + result: *mut CResult, +) -> bool { + let stream_ref = match validate_arrow_stream_ptr(stream) { + Ok(s) => s, + Err(msg) => { + write_error_result(result, msg, false); + return false; + } + }; + + let res = RUNTIME.block_on(async { stream_ref.flush().await }); + + match res { + Ok(()) => { + write_success_result(result); + true + } + Err(err) => { + if !result.is_null() { + unsafe { + *result = CResult::error(err); + } + } + false + } + } +} + +/// Gracefully closes the stream, flushing all pending batches first. +#[no_mangle] +pub extern "C" fn zerobus_arrow_stream_close( + stream: *mut CArrowStream, + result: *mut CResult, +) -> bool { + let stream_ref = match validate_arrow_stream_ptr_mut(stream) { + Ok(s) => s, + Err(msg) => { + write_error_result(result, msg, false); + return false; + } + }; + + let res = RUNTIME.block_on(async { stream_ref.close().await }); + + match res { + Ok(()) => { + write_success_result(result); + true + } + Err(err) => { + if !result.is_null() { + unsafe { + *result = CResult::error(err); + } + } + false + } + } +} + +/// Returns all unacknowledged batches from a closed or failed stream as Arrow IPC bytes. +/// +/// Each batch is serialized as a self-contained Arrow IPC stream (schema + one batch). +/// The returned array must be freed with `zerobus_arrow_free_batch_array`. +#[no_mangle] +pub extern "C" fn zerobus_arrow_stream_get_unacked_batches( + stream: *mut CArrowStream, + result: *mut CResult, +) -> CArrowBatchArray { + let empty = CArrowBatchArray { + batches: ptr::null_mut(), + lengths: ptr::null_mut(), + count: 0, + }; + + let stream_ref = match validate_arrow_stream_ptr(stream) { + Ok(s) => s, + Err(msg) => { + write_error_result(result, msg, false); + return empty; + } + }; + + let batches_res = RUNTIME.block_on(async { stream_ref.get_unacked_batches().await }); + + match batches_res { + Ok(batches) => { + if batches.is_empty() { + write_success_result(result); + return empty; + } + + let count = batches.len(); + let mut batch_ptrs: Vec<*mut u8> = Vec::with_capacity(count); + let mut batch_lens: Vec = Vec::with_capacity(count); + + for batch in &batches { + match record_batch_to_ipc_bytes(batch) { + Ok(bytes) => { + let len = bytes.len(); + let ptr = Box::into_raw(bytes.into_boxed_slice()) as *mut u8; + batch_ptrs.push(ptr); + batch_lens.push(len); + } + Err(e) => { + // Free already-allocated batches before returning error. + for (&ptr, &len) in batch_ptrs.iter().zip(batch_lens.iter()) { + if !ptr.is_null() && len > 0 { + // Safe: ptr came from Box::into_raw(bytes.into_boxed_slice()), + // so capacity == len. + unsafe { + let _ = + Box::from_raw(std::ptr::slice_from_raw_parts_mut(ptr, len)); + } + } + } + write_error_result(result, &e.to_string(), false); + return empty; + } + } + } + + // into_boxed_slice() shrinks to fit, guaranteeing capacity == len + // so the corresponding Box::from_raw in free_batch_array is sound. + let ptrs_box = batch_ptrs.into_boxed_slice(); + let lens_box = batch_lens.into_boxed_slice(); + let ptrs_ptr = Box::into_raw(ptrs_box) as *mut *mut u8; + let lens_ptr = Box::into_raw(lens_box) as *mut usize; + + write_success_result(result); + CArrowBatchArray { + batches: ptrs_ptr, + lengths: lens_ptr, + count, + } + } + Err(err) => { + if !result.is_null() { + unsafe { + *result = CResult::error(err); + } + } + empty + } + } +} + +/// Frees a `CArrowBatchArray` returned by `zerobus_arrow_stream_get_unacked_batches`. +#[no_mangle] +pub extern "C" fn zerobus_arrow_free_batch_array(array: CArrowBatchArray) { + if array.count == 0 { + return; + } + unsafe { + if !array.batches.is_null() && !array.lengths.is_null() { + // Reconstruct as Box<[T]> using the original length. This is safe because + // the pointers were produced by Box::into_raw(vec.into_boxed_slice()), + // which guarantees capacity == len. + let ptrs = Box::from_raw(std::ptr::slice_from_raw_parts_mut( + array.batches, + array.count, + )); + let lens = Box::from_raw(std::ptr::slice_from_raw_parts_mut( + array.lengths, + array.count, + )); + for (&ptr, &len) in ptrs.iter().zip(lens.iter()) { + if !ptr.is_null() && len > 0 { + // Each batch slice was produced by Box::into_raw(bytes.into_boxed_slice()), + // so capacity == len and this reconstruction is sound. + let _ = Box::from_raw(std::ptr::slice_from_raw_parts_mut(ptr, len)); + } + } + } + } +} + +/// Returns whether the Arrow stream has been closed. +#[no_mangle] +pub extern "C" fn zerobus_arrow_stream_is_closed(stream: *mut CArrowStream) -> bool { + match validate_arrow_stream_ptr(stream) { + Ok(s) => s.is_closed(), + Err(_) => true, + } +} + +/// Returns the default Arrow stream configuration options. +#[no_mangle] +pub extern "C" fn zerobus_arrow_get_default_config() -> CArrowStreamConfigurationOptions { + use databricks_zerobus_ingest_sdk::stream_options::defaults; + CArrowStreamConfigurationOptions { + max_inflight_batches: 1_000, + recovery: defaults::RECOVERY, + recovery_timeout_ms: defaults::RECOVERY_TIMEOUT_MS, + recovery_backoff_ms: defaults::RECOVERY_BACKOFF_MS, + recovery_retries: defaults::RECOVERY_RETRIES, + server_lack_of_ack_timeout_ms: defaults::SERVER_LACK_OF_ACK_TIMEOUT_MS, + flush_timeout_ms: defaults::FLUSH_TIMEOUT_MS, + connection_timeout_ms: defaults::CONNECTION_TIMEOUT_MS, + ipc_compression: -1, + stream_paused_max_wait_time_ms: -1, + } +} + +// Global Tokio runtime for handling async Rust calls +static RUNTIME: Lazy = + Lazy::new(|| Runtime::new().expect("Failed to create Tokio runtime")); + +// Flag to track if logging has been initialized +static LOGGING_INITIALIZED: AtomicBool = AtomicBool::new(false); + +/// Initialize tracing subscriber for Rust logs +/// Can be controlled via RUST_LOG environment variable +/// Examples: +/// RUST_LOG=info - Show info and above +/// RUST_LOG=debug - Show debug and above +/// RUST_LOG=trace - Show all logs +/// RUST_LOG=databricks_zerobus_ingest_sdk=debug - Show only SDK logs at debug level +fn init_logging() { + if LOGGING_INITIALIZED.swap(true, Ordering::SeqCst) { + return; + } + + let _ = fmt() + .with_env_filter(EnvFilter::from_default_env()) + .with_writer(std::io::stderr) + .try_init(); +} + +// Global cache for header keys to prevent memory leaks +// Header keys are typically a small set of constant strings (e.g., "Authorization", "Content-Type") +// We intern them once to avoid leaking memory on every callback +static HEADER_KEY_CACHE: Lazy>> = + Lazy::new(|| Mutex::new(HashSet::new())); + +/// Intern a header key string to prevent memory leaks +/// Only leaks memory for unique keys, not on every call +pub(crate) fn intern_header_key(key: String) -> &'static str { + let mut cache = HEADER_KEY_CACHE.lock().unwrap(); + + // Check if we already have this key + if let Some(&existing) = cache.iter().find(|&&k| k == key.as_str()) { + return existing; + } + + // Only leak if it's a new key (typically happens once per unique header name) + let static_key: &'static str = Box::leak(key.into_boxed_str()); + cache.insert(static_key); + static_key +} + +// Opaque types for Go +#[repr(C)] +pub struct CZerobusSdk { + _private: [u8; 0], +} + +#[repr(C)] +pub struct CZerobusStream { + _private: [u8; 0], +} + +// Result type for FFI calls +#[repr(C)] +pub struct CResult { + pub success: bool, + pub error_message: *mut c_char, + pub is_retryable: bool, +} + +/// Represents a single record (either Proto or JSON) +#[repr(C)] +pub struct CRecord { + pub is_json: bool, + pub data: *mut u8, + pub data_len: usize, +} + +/// Represents an array of records +#[repr(C)] +pub struct CRecordArray { + pub records: *mut CRecord, + pub len: usize, +} + +impl CResult { + fn success() -> Self { + CResult { + success: true, + error_message: ptr::null_mut(), + is_retryable: false, + } + } + + fn error(err: ZerobusError) -> Self { + let is_retryable = err.is_retryable(); + let message = CString::new(err.to_string()) + .unwrap_or_else(|_| CString::new("Unknown error").unwrap()); + + CResult { + success: false, + error_message: message.into_raw(), + is_retryable, + } + } +} + +// Configuration options +#[repr(C)] +#[derive(Clone, Copy)] +pub struct CStreamConfigurationOptions { + pub max_inflight_requests: usize, + pub recovery: bool, + pub recovery_timeout_ms: u64, + pub recovery_backoff_ms: u64, + pub recovery_retries: u32, + pub server_lack_of_ack_timeout_ms: u64, + pub flush_timeout_ms: u64, + pub record_type: i32, + pub stream_paused_max_wait_time_ms: u64, + pub has_stream_paused_max_wait_time_ms: bool, + pub callback_max_wait_time_ms: u64, + pub has_callback_max_wait_time_ms: bool, +} + +// Helper to convert C string to Rust String +unsafe fn c_str_to_string(c_str: *const c_char) -> Result { + if c_str.is_null() { + return Err("Null pointer passed"); + } + CStr::from_ptr(c_str) + .to_str() + .map(|s| s.to_string()) + .map_err(|_| "Invalid UTF-8 string") +} + +/// A single header key-value pair for C FFI +#[repr(C)] +pub struct CHeader { + pub key: *mut c_char, + pub value: *mut c_char, +} + +/// A collection of headers returned from Go callback +#[repr(C)] +pub struct CHeaders { + pub headers: *mut CHeader, + pub count: usize, + pub error_message: *mut c_char, +} + +/// Function pointer type for the headers provider callback +/// The callback should return a CHeaders struct +/// The caller is responsible for freeing the returned CHeaders using zerobus_free_headers +pub type HeadersProviderCallback = extern "C" fn(user_data: *mut std::ffi::c_void) -> CHeaders; + +/// Free headers returned from callback +#[no_mangle] +pub extern "C" fn zerobus_free_headers(headers: CHeaders) { + if !headers.headers.is_null() { + unsafe { + let headers_slice = std::slice::from_raw_parts_mut(headers.headers, headers.count); + for header in headers_slice { + if !header.key.is_null() { + let _ = CString::from_raw(header.key); + } + if !header.value.is_null() { + let _ = CString::from_raw(header.value); + } + } + libc::free(headers.headers as *mut std::ffi::c_void); + } + } + if !headers.error_message.is_null() { + unsafe { + let _ = CString::from_raw(headers.error_message); + } + } +} + +/// Rust struct that wraps a Go callback and implements HeadersProvider +pub(crate) struct CallbackHeadersProvider { + callback: HeadersProviderCallback, + user_data: *mut std::ffi::c_void, + in_use: AtomicBool, // Track concurrent access to detect thread-safety issues +} + +impl CallbackHeadersProvider { + pub(crate) fn new(callback: HeadersProviderCallback, user_data: *mut std::ffi::c_void) -> Self { + Self { + callback, + user_data, + in_use: AtomicBool::new(false), + } + } +} + +// Safety: We assume the Go callback is thread-safe, but we validate at runtime +unsafe impl Send for CallbackHeadersProvider {} +unsafe impl Sync for CallbackHeadersProvider {} + +#[async_trait] +impl HeadersProvider for CallbackHeadersProvider { + async fn get_headers(&self) -> ZerobusResult> { + // Check for concurrent access (indicates thread-safety issue) + if self.in_use.swap(true, Ordering::SeqCst) { + return Err(ZerobusError::InvalidArgument( + "Concurrent headers provider callback detected - Go callback must be thread-safe" + .to_string(), + )); + } + + // Call the Go callback (synchronous) + let c_headers = (self.callback)(self.user_data); + + // Release the lock before processing + self.in_use.store(false, Ordering::SeqCst); + + // Check for error + if !c_headers.error_message.is_null() { + let error_str = unsafe { + CStr::from_ptr(c_headers.error_message) + .to_string_lossy() + .into_owned() + }; + zerobus_free_headers(c_headers); + return Err(ZerobusError::InvalidArgument(format!( + "Headers provider error: {}", + error_str + ))); + } + + // Convert C headers to Rust HashMap + let mut headers = HashMap::new(); + if !c_headers.headers.is_null() && c_headers.count > 0 { + unsafe { + let headers_slice = std::slice::from_raw_parts(c_headers.headers, c_headers.count); + for header in headers_slice { + if !header.key.is_null() && !header.value.is_null() { + let key = CStr::from_ptr(header.key).to_string_lossy().into_owned(); + let value = CStr::from_ptr(header.value).to_string_lossy().into_owned(); + + // Use interned keys to minimize memory leaks + // Only unique header names are leaked (typically < 10 strings for lifetime of process) + let static_key = intern_header_key(key); + headers.insert(static_key, value); + } + } + } + } + + zerobus_free_headers(c_headers); + Ok(headers) + } +} + +// ============================================================================ +// SDK Functions +// ============================================================================ + +/// Safe wrapper to validate SDK pointer +pub(crate) fn validate_sdk_ptr<'a>(sdk: *mut CZerobusSdk) -> Result<&'a ZerobusSdk, &'static str> { + if sdk.is_null() { + return Err("SDK pointer is null"); + } + // Still unsafe, but centralized and validated + unsafe { Ok(&*(sdk as *const ZerobusSdk)) } +} + +/// Safe wrapper to validate stream pointer +pub(crate) fn validate_stream_ptr<'a>( + stream: *mut CZerobusStream, +) -> Result<&'a ZerobusStream, &'static str> { + if stream.is_null() { + return Err("Stream pointer is null"); + } + unsafe { Ok(&*(stream as *const ZerobusStream)) } +} + +/// Safe wrapper to validate mutable stream pointer +pub(crate) fn validate_stream_ptr_mut<'a>( + stream: *mut CZerobusStream, +) -> Result<&'a mut ZerobusStream, &'static str> { + if stream.is_null() { + return Err("Stream pointer is null"); + } + unsafe { Ok(&mut *(stream as *mut ZerobusStream)) } +} + +/// Helper to write error result +pub(crate) fn write_error_result(result: *mut CResult, message: &str, is_retryable: bool) { + if !result.is_null() { + unsafe { + *result = CResult { + success: false, + error_message: CString::new(message) + .unwrap_or_else(|_| CString::new("Error message contains null byte").unwrap()) + .into_raw(), + is_retryable, + }; + } + } +} + +/// Helper to write success result +pub(crate) fn write_success_result(result: *mut CResult) { + if !result.is_null() { + unsafe { + *result = CResult::success(); + } + } +} + +// ============================================================================ +// ZerobusSdkBuilder FFI +// ============================================================================ +// +// C-builder mirroring the Rust `ZerobusSdkBuilder`. New options are added as +// additive setter functions — no ABI breaks. +// +// Lifecycle: `_new` → zero or more `_` calls → `_build` (consumes) or +// `_free` (abandon). Single-owner; not safe to share across threads. + +/// Opaque handle for an SDK builder. Allocated by `_new`, consumed by +/// `_build`, or dropped by `_free`. Must not be used after either finalizer. +#[repr(C)] +pub struct CZerobusSdkBuilder { + _private: [u8; 0], +} + +/// Concrete type behind `*mut CZerobusSdkBuilder`. All cast sites in this +/// module must agree on this alias. +type SdkBuilderAlloc = ZerobusSdkBuilder; + +/// SAFETY: `b` must be a valid pointer from `_new` that hasn't been consumed +/// or freed. `mem::take` keeps the slot valid even if `f` panics. +unsafe fn with_builder(b: *mut CZerobusSdkBuilder, f: F) +where + F: FnOnce(ZerobusSdkBuilder) -> ZerobusSdkBuilder, +{ + if b.is_null() { + return; + } + let slot = &mut *(b as *mut SdkBuilderAlloc); + let taken = std::mem::take(slot); + *slot = f(taken); +} + +/// Allocates a new SDK builder. Must be terminated by exactly one of +/// `_build` or `_free`. +#[no_mangle] +pub extern "C" fn zerobus_sdk_builder_new() -> *mut CZerobusSdkBuilder { + init_logging(); + let boxed: Box = Box::new(ZerobusSdk::builder()); + Box::into_raw(boxed) as *mut CZerobusSdkBuilder +} + +/// Sets the Zerobus gRPC endpoint URL (required). No-op on null. +#[no_mangle] +pub extern "C" fn zerobus_sdk_builder_endpoint( + builder: *mut CZerobusSdkBuilder, + value: *const c_char, +) { + if value.is_null() { + return; + } + let s = match unsafe { c_str_to_string(value) } { + Ok(s) => s, + Err(_) => return, + }; + unsafe { with_builder(builder, |b| b.endpoint(s)) } +} + +/// Sets the Unity Catalog URL. Optional with a custom headers provider. +/// No-op on null. +#[no_mangle] +pub extern "C" fn zerobus_sdk_builder_unity_catalog_url( + builder: *mut CZerobusSdkBuilder, + value: *const c_char, +) { + if value.is_null() { + return; + } + let s = match unsafe { c_str_to_string(value) } { + Ok(s) => s, + Err(_) => return, + }; + unsafe { with_builder(builder, |b| b.unity_catalog_url(s)) } +} + +/// Overrides the SDK prefix of the `user-agent` header (default +/// `zerobus-sdk-rs/`). Wrappers pass their own identifier here. +/// Null and empty values are no-ops. +#[no_mangle] +pub extern "C" fn zerobus_sdk_builder_sdk_identifier( + builder: *mut CZerobusSdkBuilder, + value: *const c_char, +) { + if value.is_null() { + return; + } + let s = match unsafe { c_str_to_string(value) } { + Ok(s) if !s.is_empty() => s, + _ => return, + }; + unsafe { with_builder(builder, |b| b.sdk_identifier(s)) } +} + +/// Appends an application identifier to the `user-agent` header. Wire value +/// becomes ` `. Null and empty values are +/// no-ops. +#[no_mangle] +pub extern "C" fn zerobus_sdk_builder_application_name( + builder: *mut CZerobusSdkBuilder, + value: *const c_char, +) { + if value.is_null() { + return; + } + let s = match unsafe { c_str_to_string(value) } { + Ok(s) if !s.is_empty() => s, + _ => return, + }; + unsafe { with_builder(builder, |b| b.application_name(s)) } +} + +/// Selects a no-TLS gRPC channel. TLS is on by default. +#[no_mangle] +pub extern "C" fn zerobus_sdk_builder_disable_tls(builder: *mut CZerobusSdkBuilder) { + unsafe { with_builder(builder, |b| b.tls_config(Arc::new(NoTlsConfig))) } +} + +/// Consumes the builder and returns a `CZerobusSdk*`, or NULL on error. +/// Frees the builder on both paths — any further use of the pointer is +/// undefined behavior. Null `builder` writes an error to `result`. +#[no_mangle] +pub extern "C" fn zerobus_sdk_builder_build( + builder: *mut CZerobusSdkBuilder, + result: *mut CResult, +) -> *mut CZerobusSdk { + if builder.is_null() { + write_error_result(result, "Builder pointer is null", false); + return ptr::null_mut(); + } + // Reclaim ownership of the builder Box so it is dropped on every path, + // mirroring the Rust builder's consume-on-build semantics. + let inner = *unsafe { Box::from_raw(builder as *mut SdkBuilderAlloc) }; + match inner.build() { + Ok(sdk) => { + write_success_result(result); + Box::into_raw(Box::new(sdk)) as *mut CZerobusSdk + } + Err(err) => { + if !result.is_null() { + unsafe { + *result = CResult::error(err); + } + } + ptr::null_mut() + } + } +} + +/// Drops an unconsumed builder. No-op on null. +#[no_mangle] +pub extern "C" fn zerobus_sdk_builder_free(builder: *mut CZerobusSdkBuilder) { + if !builder.is_null() { + unsafe { + let _ = Box::from_raw(builder as *mut SdkBuilderAlloc); + } + } +} + +/// Creates a new ZerobusSdk with default user-agent and TLS settings. +/// +/// Retained for ABI back-compat with v1.2.x; new code should use the +/// `zerobus_sdk_builder_*` API. Does not infer TLS state from the endpoint +/// scheme — callers needing a plain-HTTP channel must use the builder API. +/// +/// Returns NULL on error; see `result` for details. +#[no_mangle] +pub extern "C" fn zerobus_sdk_new( + zerobus_endpoint: *const c_char, + unity_catalog_url: *const c_char, + result: *mut CResult, +) -> *mut CZerobusSdk { + let builder = zerobus_sdk_builder_new(); + zerobus_sdk_builder_endpoint(builder, zerobus_endpoint); + zerobus_sdk_builder_unity_catalog_url(builder, unity_catalog_url); + zerobus_sdk_builder_build(builder, result) +} + +/// Free the SDK instance +#[no_mangle] +pub extern "C" fn zerobus_sdk_free(sdk: *mut CZerobusSdk) { + if !sdk.is_null() { + unsafe { + let _ = Box::from_raw(sdk as *mut ZerobusSdk); + } + } +} + +/// Set whether to use TLS for connections. +/// +/// Deprecated: This function is a no-op. TLS is now controlled via the `TlsConfig` +/// trait passed to the SDK builder. This function is retained for ABI compatibility. +#[no_mangle] +pub extern "C" fn zerobus_sdk_set_use_tls(_sdk: *mut CZerobusSdk, _use_tls: bool) {} + +/// Create a stream with OAuth authentication +/// descriptor_proto_bytes: protobuf-encoded DescriptorProto (can be NULL for JSON streams) +#[no_mangle] +pub extern "C" fn zerobus_sdk_create_stream( + sdk: *mut CZerobusSdk, + table_name: *const c_char, + descriptor_proto_bytes: *const u8, + descriptor_proto_len: usize, + client_id: *const c_char, + client_secret: *const c_char, + options: *const CStreamConfigurationOptions, + result: *mut CResult, +) -> *mut CZerobusStream { + let sdk_ref = match validate_sdk_ptr(sdk) { + Ok(s) => s, + Err(msg) => { + write_error_result(result, msg, false); + return ptr::null_mut(); + } + }; + + let res = RUNTIME.block_on(async { + let table_name_str = unsafe { c_str_to_string(table_name).map_err(|e| e.to_string())? }; + let client_id_str = unsafe { c_str_to_string(client_id).map_err(|e| e.to_string())? }; + let client_secret_str = + unsafe { c_str_to_string(client_secret).map_err(|e| e.to_string())? }; + + let descriptor_proto = if !descriptor_proto_bytes.is_null() && descriptor_proto_len > 0 { + let bytes = + unsafe { std::slice::from_raw_parts(descriptor_proto_bytes, descriptor_proto_len) }; + Some(prost_types::DescriptorProto::decode(bytes).map_err(|e| e.to_string())?) + } else { + None + }; + + let c_opts = if !options.is_null() { + Some(unsafe { &*options }) + } else { + None + }; + let record_type = c_opts + .map(|c| c_record_type(c.record_type)) + .unwrap_or(RecordType::Proto); + + let base = sdk_ref + .stream_builder() + .table(table_name_str) + .oauth(client_id_str, client_secret_str); + let mut builder = match record_type { + RecordType::Proto => { + let desc = descriptor_proto.ok_or_else(|| { + "Proto descriptor is required for Proto record type".to_string() + })?; + base.compiled_proto(desc) + } + RecordType::Json => base.json(), + RecordType::Unspecified => return Err("Record type is not specified".to_string()), + }; + if let Some(c) = c_opts { + builder = apply_c_stream_options(builder, c); + } + + let stream = builder.build().await.map_err(|e| e.to_string())?; + + let arc = Arc::new(stream); + Ok::<*mut CZerobusStream, String>(Arc::into_raw(arc) as *mut CZerobusStream) + }); + + match res { + Ok(stream_ptr) => { + write_success_result(result); + stream_ptr + } + Err(err) => { + write_error_result(result, &err, false); + ptr::null_mut() + } + } +} + +/// Create a stream with a custom headers provider callback +/// This allows you to provide custom authentication headers via a Go callback function +#[no_mangle] +pub extern "C" fn zerobus_sdk_create_stream_with_headers_provider( + sdk: *mut CZerobusSdk, + table_name: *const c_char, + descriptor_proto_bytes: *const u8, + descriptor_proto_len: usize, + headers_callback: HeadersProviderCallback, + user_data: *mut std::ffi::c_void, + options: *const CStreamConfigurationOptions, + result: *mut CResult, +) -> *mut CZerobusStream { + let sdk_ref = match validate_sdk_ptr(sdk) { + Ok(s) => s, + Err(msg) => { + write_error_result(result, msg, false); + return ptr::null_mut(); + } + }; + + let res = RUNTIME.block_on(async { + let table_name_str = unsafe { c_str_to_string(table_name).map_err(|e| e.to_string())? }; + + let descriptor_proto = if !descriptor_proto_bytes.is_null() && descriptor_proto_len > 0 { + let bytes = + unsafe { std::slice::from_raw_parts(descriptor_proto_bytes, descriptor_proto_len) }; + Some(prost_types::DescriptorProto::decode(bytes).map_err(|e| e.to_string())?) + } else { + None + }; + + let c_opts = if !options.is_null() { + Some(unsafe { &*options }) + } else { + None + }; + let record_type = c_opts + .map(|c| c_record_type(c.record_type)) + .unwrap_or(RecordType::Proto); + + let headers_provider: Arc = + Arc::new(CallbackHeadersProvider::new(headers_callback, user_data)); + + let base = sdk_ref + .stream_builder() + .table(table_name_str) + .headers_provider(headers_provider); + let mut builder = match record_type { + RecordType::Proto => { + let desc = descriptor_proto.ok_or_else(|| { + "Proto descriptor is required for Proto record type".to_string() + })?; + base.compiled_proto(desc) + } + RecordType::Json => base.json(), + RecordType::Unspecified => return Err("Record type is not specified".to_string()), + }; + if let Some(c) = c_opts { + builder = apply_c_stream_options(builder, c); + } + + let stream = builder.build().await.map_err(|e| e.to_string())?; + + let arc = Arc::new(stream); + Ok::<*mut CZerobusStream, String>(Arc::into_raw(arc) as *mut CZerobusStream) + }); + + match res { + Ok(stream_ptr) => { + write_success_result(result); + stream_ptr + } + Err(err) => { + write_error_result(result, &err, false); + ptr::null_mut() + } + } +} + +/// Free a stream instance +#[no_mangle] +pub extern "C" fn zerobus_stream_free(stream: *mut CZerobusStream) { + if !stream.is_null() { + unsafe { + // Reconstruct the Arc and drop it. If nowait tasks still hold clones, + // the stream is not freed until the last Arc is dropped. + let _ = Arc::from_raw(stream as *const ZerobusStream); + } + } +} + +/// Ingest a record (protobuf encoded) +/// Returns the offset directly +/// Returns -1 on error +#[no_mangle] +pub extern "C" fn zerobus_stream_ingest_proto_record( + stream: *mut CZerobusStream, + data: *const u8, + data_len: usize, + result: *mut CResult, +) -> i64 { + if data.is_null() { + write_error_result(result, "Invalid data pointer", false); + return -1; + } + + let stream_ref = match validate_stream_ptr(stream) { + Ok(s) => s, + Err(msg) => { + write_error_result(result, msg, false); + return -1; + } + }; + + let data_slice = unsafe { std::slice::from_raw_parts(data, data_len) }; + let data_vec = data_slice.to_vec(); + + // Queue the record and get the offset directly + let offset_res = RUNTIME.block_on(async { + let payload = EncodedRecord::Proto(data_vec); + stream_ref.ingest_record_offset(payload).await + }); + + match offset_res { + Ok(offset) => { + write_success_result(result); + offset + } + Err(err) => { + if !result.is_null() { + unsafe { + *result = CResult::error(err); + } + } + -1 + } + } +} + +/// Ingest a JSON record +/// Returns the offset directly +/// Returns -1 on error +#[no_mangle] +pub extern "C" fn zerobus_stream_ingest_json_record( + stream: *mut CZerobusStream, + json_data: *const c_char, + result: *mut CResult, +) -> i64 { + let stream_ref = match validate_stream_ptr(stream) { + Ok(s) => s, + Err(msg) => { + write_error_result(result, msg, false); + return -1; + } + }; + + let json_str = match unsafe { c_str_to_string(json_data) } { + Ok(s) => s, + Err(e) => { + write_error_result(result, e, false); + return -1; + } + }; + + // Queue the record and get the offset directly + let offset_res = RUNTIME.block_on(async { + let payload = EncodedRecord::Json(json_str); + stream_ref.ingest_record_offset(payload).await + }); + + match offset_res { + Ok(offset) => { + write_success_result(result); + offset + } + Err(err) => { + if !result.is_null() { + unsafe { + *result = CResult::error(err); + } + } + -1 + } + } +} + +/// Ingest a batch of protobuf records +/// Returns the offset of the last record in the batch, or -1 on error +/// Returns -2 if batch is empty +#[no_mangle] +pub extern "C" fn zerobus_stream_ingest_proto_records( + stream: *mut CZerobusStream, + records: *const *const u8, + record_lens: *const usize, + num_records: usize, + result: *mut CResult, +) -> i64 { + if records.is_null() || record_lens.is_null() { + write_error_result(result, "Invalid records pointer", false); + return -1; + } + + if num_records == 0 { + write_success_result(result); + return -2; // Empty batch + } + + let stream_ref = match validate_stream_ptr(stream) { + Ok(s) => s, + Err(msg) => { + write_error_result(result, msg, false); + return -1; + } + }; + + // Convert array of C pointers to Vec> + let records_vec: Vec> = unsafe { + let records_slice = std::slice::from_raw_parts(records, num_records); + let lens_slice = std::slice::from_raw_parts(record_lens, num_records); + + records_slice + .iter() + .zip(lens_slice.iter()) + .map(|(ptr, len)| { + let data_slice = std::slice::from_raw_parts(*ptr, *len); + data_slice.to_vec() + }) + .collect() + }; + + // Queue the records and get the offset + let offset_res = RUNTIME.block_on(async { + let payloads: Vec = + records_vec.into_iter().map(EncodedRecord::Proto).collect(); + stream_ref.ingest_records_offset(payloads).await + }); + + match offset_res { + Ok(Some(offset)) => { + write_success_result(result); + offset + } + Ok(None) => { + write_success_result(result); + -2 // Empty batch + } + Err(err) => { + if !result.is_null() { + unsafe { + *result = CResult::error(err); + } + } + -1 + } + } +} + +/// Ingest a batch of JSON records +/// Returns the offset of the last record in the batch, or -1 on error +/// Returns -2 if batch is empty +#[no_mangle] +pub extern "C" fn zerobus_stream_ingest_json_records( + stream: *mut CZerobusStream, + json_records: *const *const c_char, + num_records: usize, + result: *mut CResult, +) -> i64 { + if json_records.is_null() { + write_error_result(result, "Invalid records pointer", false); + return -1; + } + + if num_records == 0 { + write_success_result(result); + return -2; // Empty batch + } + + let stream_ref = match validate_stream_ptr(stream) { + Ok(s) => s, + Err(msg) => { + write_error_result(result, msg, false); + return -1; + } + }; + + // Convert array of C strings to Vec + let json_vec: Result, _> = unsafe { + let json_slice = std::slice::from_raw_parts(json_records, num_records); + json_slice.iter().map(|ptr| c_str_to_string(*ptr)).collect() + }; + + let json_vec = match json_vec { + Ok(v) => v, + Err(e) => { + write_error_result(result, e, false); + return -1; + } + }; + + // Queue the records and get the offset + let offset_res = RUNTIME.block_on(async { + let payloads: Vec = json_vec.into_iter().map(EncodedRecord::Json).collect(); + stream_ref.ingest_records_offset(payloads).await + }); + + match offset_res { + Ok(Some(offset)) => { + write_success_result(result); + offset + } + Ok(None) => { + write_success_result(result); + -2 // Empty batch + } + Err(err) => { + if !result.is_null() { + unsafe { + *result = CResult::error(err); + } + } + -1 + } + } +} + +/// Clones the `Arc` from a raw `CZerobusStream` pointer without +/// consuming the pointer. The caller retains ownership of the original pointer; +/// the returned `Arc` will keep the stream alive until it is dropped. +/// +/// # Safety +/// `stream` must be a non-null pointer produced by `zerobus_sdk_create_stream` or +/// `zerobus_sdk_create_stream_with_headers_provider` and must not have been freed. +unsafe fn clone_stream_arc(stream: *mut CZerobusStream) -> Arc { + Arc::increment_strong_count(stream as *const ZerobusStream); + Arc::from_raw(stream as *const ZerobusStream) +} + +/// Ingest a protobuf record without waiting for the record to be queued (fire-and-forget). +/// +/// Spawns a background task to queue the record and returns immediately. +/// The result only reflects argument validation errors; ingestion errors are silently ignored. +/// +/// # Safety +/// The stream must remain valid until all background tasks spawned by this function complete. +#[no_mangle] +pub extern "C" fn zerobus_stream_ingest_proto_record_nowait( + stream: *mut CZerobusStream, + data: *const u8, + data_len: usize, + result: *mut CResult, +) { + if data.is_null() { + write_error_result(result, "Invalid data pointer", false); + return; + } + + if let Err(msg) = validate_stream_ptr(stream) { + write_error_result(result, msg, false); + return; + } + + let data_slice = unsafe { std::slice::from_raw_parts(data, data_len) }; + let data_vec = data_slice.to_vec(); + let stream_arc = unsafe { clone_stream_arc(stream) }; + + RUNTIME.spawn(async move { + let payload = EncodedRecord::Proto(data_vec); + let _ = stream_arc.ingest_record_offset(payload).await; + }); + + write_success_result(result); +} + +/// Ingest a JSON record without waiting for the record to be queued (fire-and-forget). +/// +/// Spawns a background task to queue the record and returns immediately. +/// The result only reflects argument validation errors; ingestion errors are silently ignored. +/// +/// # Safety +/// The stream must remain valid until all background tasks spawned by this function complete. +#[no_mangle] +pub extern "C" fn zerobus_stream_ingest_json_record_nowait( + stream: *mut CZerobusStream, + json_data: *const c_char, + result: *mut CResult, +) { + if let Err(msg) = validate_stream_ptr(stream) { + write_error_result(result, msg, false); + return; + } + + let json_str = match unsafe { c_str_to_string(json_data) } { + Ok(s) => s, + Err(e) => { + write_error_result(result, e, false); + return; + } + }; + + let stream_arc = unsafe { clone_stream_arc(stream) }; + + RUNTIME.spawn(async move { + let payload = EncodedRecord::Json(json_str); + let _ = stream_arc.ingest_record_offset(payload).await; + }); + + write_success_result(result); +} + +/// Ingest a batch of protobuf records without waiting (fire-and-forget). +/// +/// Copies all record data before spawning the background task, so the caller's +/// memory is safe to release immediately after this function returns. +/// +/// # Safety +/// The stream must remain valid until all background tasks spawned by this function complete. +#[no_mangle] +pub extern "C" fn zerobus_stream_ingest_proto_records_nowait( + stream: *mut CZerobusStream, + records: *const *const u8, + record_lens: *const usize, + num_records: usize, + result: *mut CResult, +) { + if records.is_null() || record_lens.is_null() { + write_error_result(result, "Invalid records pointer", false); + return; + } + + if let Err(msg) = validate_stream_ptr(stream) { + write_error_result(result, msg, false); + return; + } + + if num_records == 0 { + write_success_result(result); + return; + } + + let records_vec: Vec> = unsafe { + let records_slice = std::slice::from_raw_parts(records, num_records); + let lens_slice = std::slice::from_raw_parts(record_lens, num_records); + records_slice + .iter() + .zip(lens_slice.iter()) + .map(|(ptr, len)| std::slice::from_raw_parts(*ptr, *len).to_vec()) + .collect() + }; + + let stream_arc = unsafe { clone_stream_arc(stream) }; + + RUNTIME.spawn(async move { + let payloads: Vec = + records_vec.into_iter().map(EncodedRecord::Proto).collect(); + let _ = stream_arc.ingest_records_offset(payloads).await; + }); + + write_success_result(result); +} + +/// Ingest a batch of JSON records without waiting (fire-and-forget). +/// +/// Copies all strings before spawning the background task, so the caller's +/// memory is safe to release immediately after this function returns. +/// +/// # Safety +/// The stream must remain valid until all background tasks spawned by this function complete. +#[no_mangle] +pub extern "C" fn zerobus_stream_ingest_json_records_nowait( + stream: *mut CZerobusStream, + json_records: *const *const c_char, + num_records: usize, + result: *mut CResult, +) { + if json_records.is_null() { + write_error_result(result, "Invalid records pointer", false); + return; + } + + if let Err(msg) = validate_stream_ptr(stream) { + write_error_result(result, msg, false); + return; + } + + if num_records == 0 { + write_success_result(result); + return; + } + + let json_vec: Result, _> = unsafe { + let json_slice = std::slice::from_raw_parts(json_records, num_records); + json_slice.iter().map(|ptr| c_str_to_string(*ptr)).collect() + }; + + let json_vec = match json_vec { + Ok(v) => v, + Err(e) => { + write_error_result(result, e, false); + return; + } + }; + + let stream_arc = unsafe { clone_stream_arc(stream) }; + + RUNTIME.spawn(async move { + let payloads: Vec = json_vec.into_iter().map(EncodedRecord::Json).collect(); + let _ = stream_arc.ingest_records_offset(payloads).await; + }); + + write_success_result(result); +} + +/// Wait for a specific offset to be acknowledged by the server +#[no_mangle] +pub extern "C" fn zerobus_stream_wait_for_offset( + stream: *mut CZerobusStream, + offset: i64, + result: *mut CResult, +) -> bool { + let stream_ref = match validate_stream_ptr(stream) { + Ok(s) => s, + Err(msg) => { + write_error_result(result, msg, false); + return false; + } + }; + + let res = RUNTIME.block_on(async { stream_ref.wait_for_offset(offset).await }); + + match res { + Ok(()) => { + write_success_result(result); + true + } + Err(err) => { + if !result.is_null() { + unsafe { + *result = CResult::error(err); + } + } + false + } + } +} + +/// Flush all pending records +#[no_mangle] +pub extern "C" fn zerobus_stream_flush(stream: *mut CZerobusStream, result: *mut CResult) -> bool { + let stream_ref = match validate_stream_ptr(stream) { + Ok(s) => s, + Err(msg) => { + write_error_result(result, msg, false); + return false; + } + }; + + let res = RUNTIME.block_on(async { stream_ref.flush().await }); + + match res { + Ok(_) => { + write_success_result(result); + true + } + Err(err) => { + if !result.is_null() { + unsafe { + *result = CResult::error(err); + } + } + false + } + } +} + +/// Get unacknowledged records from a closed stream +/// Returns a CRecordArray that must be freed with zerobus_free_record_array +#[no_mangle] +pub extern "C" fn zerobus_stream_get_unacked_records( + stream: *mut CZerobusStream, + result: *mut CResult, +) -> CRecordArray { + let stream_ref = match validate_stream_ptr(stream) { + Ok(s) => s, + Err(msg) => { + write_error_result(result, msg, false); + return CRecordArray { + records: ptr::null_mut(), + len: 0, + }; + } + }; + + let records_res = RUNTIME.block_on(async { stream_ref.get_unacked_records().await }); + + match records_res { + Ok(records_iter) => { + // Collect into Vec + let records_vec: Vec = records_iter.collect(); + let len = records_vec.len(); + + // Convert to CRecords + let mut c_records: Vec = records_vec + .into_iter() + .map(|record| match record { + EncodedRecord::Proto(data) => { + let data_len = data.len(); + let data_ptr = Box::into_raw(data.into_boxed_slice()) as *mut u8; + CRecord { + is_json: false, + data: data_ptr, + data_len, + } + } + EncodedRecord::Json(json_str) => { + let bytes = json_str.into_bytes(); + let data_len = bytes.len(); + let data_ptr = Box::into_raw(bytes.into_boxed_slice()) as *mut u8; + CRecord { + is_json: true, + data: data_ptr, + data_len, + } + } + }) + .collect(); + + let records_ptr = c_records.as_mut_ptr(); + std::mem::forget(c_records); // Don't drop, Go will call free + + write_success_result(result); + CRecordArray { + records: records_ptr, + len, + } + } + Err(err) => { + if !result.is_null() { + unsafe { + *result = CResult::error(err); + } + } + CRecordArray { + records: ptr::null_mut(), + len: 0, + } + } + } +} + +/// Free a CRecordArray returned by zerobus_stream_get_unacked_records +#[no_mangle] +pub extern "C" fn zerobus_free_record_array(array: CRecordArray) { + if array.records.is_null() || array.len == 0 { + return; + } + + unsafe { + let records_vec = Vec::from_raw_parts(array.records, array.len, array.len); + for record in records_vec { + if !record.data.is_null() && record.data_len > 0 { + let _ = Vec::from_raw_parts(record.data, record.data_len, record.data_len); + } + } + } +} + +/// Close the stream gracefully +#[no_mangle] +pub extern "C" fn zerobus_stream_close(stream: *mut CZerobusStream, result: *mut CResult) -> bool { + let stream_ref = match validate_stream_ptr_mut(stream) { + Ok(s) => s, + Err(msg) => { + write_error_result(result, msg, false); + return false; + } + }; + + let res = RUNTIME.block_on(async { stream_ref.close().await }); + + match res { + Ok(_) => { + write_success_result(result); + true + } + Err(err) => { + if !result.is_null() { + unsafe { + *result = CResult::error(err); + } + } + false + } + } +} + +/// Free error message string +#[no_mangle] +pub extern "C" fn zerobus_free_error_message(message: *mut c_char) { + if !message.is_null() { + unsafe { + let _ = CString::from_raw(message); + } + } +} + +/// Get default stream configuration options +#[no_mangle] +pub extern "C" fn zerobus_get_default_config() -> CStreamConfigurationOptions { + use databricks_zerobus_ingest_sdk::stream_options::defaults; + CStreamConfigurationOptions { + max_inflight_requests: 1_000_000, + recovery: defaults::RECOVERY, + recovery_timeout_ms: defaults::RECOVERY_TIMEOUT_MS, + recovery_backoff_ms: defaults::RECOVERY_BACKOFF_MS, + recovery_retries: defaults::RECOVERY_RETRIES, + server_lack_of_ack_timeout_ms: defaults::SERVER_LACK_OF_ACK_TIMEOUT_MS, + flush_timeout_ms: defaults::FLUSH_TIMEOUT_MS, + record_type: 1, // RecordType::Proto + stream_paused_max_wait_time_ms: 0, + has_stream_paused_max_wait_time_ms: false, + callback_max_wait_time_ms: defaults::CALLBACK_MAX_WAIT_TIME_MS, + has_callback_max_wait_time_ms: true, + } +} + +// ============================================================================ +// Dynamic Protobuf FFI +// ============================================================================ +// +// Pure-C consumers can build a protobuf descriptor from Unity Catalog metadata +// and encode JSON records to protobuf bytes without a companion Rust crate. +// Lifecycle: `_from_uc_json` → `_descriptor_bytes` / `_encode_json` → `_free`. + +/// Opaque handle to a table's protobuf schema: its serialized descriptor plus a +/// prepared encoder. C code only ever holds a pointer to it; the backing +/// allocation is owned by the SDK and released by zerobus_proto_schema_free. +#[repr(C)] +pub struct CZerobusProtoSchema { + _private: [u8; 0], +} + +/// Concrete type behind `*mut CZerobusProtoSchema`. +struct ProtoSchema { + /// Serialized `DescriptorProto` bytes (passed to `zerobus_sdk_create_stream`). + descriptor_bytes: Vec, + /// Message descriptor for encoding JSON records to protobuf. + message: MessageDescriptor, +} + +/// Null-check a schema handle and borrow the [`ProtoSchema`] behind it. +/// +/// # Safety +/// +/// `schema` must be null or a live handle from +/// [`zerobus_proto_schema_from_uc_json`]. The caller must not free the handle +/// (via [`zerobus_proto_schema_free`]) for the lifetime of the returned borrow. +unsafe fn proto_schema_ref<'a>( + schema: *const CZerobusProtoSchema, +) -> Result<&'a ProtoSchema, &'static str> { + if schema.is_null() { + return Err("Proto schema pointer is null"); + } + Ok(&*(schema as *const ProtoSchema)) +} + +/// Builds a [`ProtoSchema`] from Unity Catalog table-metadata JSON. +fn build_proto_schema(uc_table_json: &str) -> Result { + let schema: UcTableSchema = serde_json::from_str(uc_table_json) + .map_err(|e| format!("failed to parse Unity Catalog table JSON: {e}"))?; + let descriptor = descriptor_from_uc_schema(&schema).map_err(|e| e.to_string())?; + let descriptor_bytes = descriptor.encode_to_vec(); + let message_name = descriptor.name().to_string(); + + let file = prost_types::FileDescriptorProto { + name: Some("zerobus_dynamic.proto".to_string()), + message_type: vec![descriptor], + ..Default::default() + }; + let mut pool = DescriptorPool::new(); + pool.add_file_descriptor_proto(file) + .map_err(|e| format!("failed to build descriptor pool: {e}"))?; + // No package on the synthetic file, so the fully-qualified name is the + // bare message name. + let message = pool + .get_message_by_name(&message_name) + .ok_or_else(|| format!("message '{message_name}' not found in descriptor pool"))?; + + Ok(ProtoSchema { + descriptor_bytes, + message, + }) +} + +/// Build a protobuf schema from Unity Catalog table metadata JSON. +/// Returns NULL on error; free with `zerobus_proto_schema_free`. +#[no_mangle] +pub extern "C" fn zerobus_proto_schema_from_uc_json( + uc_table_json: *const c_char, + result: *mut CResult, +) -> *mut CZerobusProtoSchema { + let json = match unsafe { c_str_to_string(uc_table_json) } { + Ok(s) => s, + Err(e) => { + write_error_result(result, e, false); + return ptr::null_mut(); + } + }; + + match build_proto_schema(&json) { + Ok(schema) => { + write_success_result(result); + // Hand ownership of the allocation to C as a raw pointer; it is + // reclaimed by zerobus_proto_schema_free. + Box::into_raw(Box::new(schema)) as *mut CZerobusProtoSchema + } + Err(err) => { + write_error_result(result, &err, false); + ptr::null_mut() + } + } +} + +/// Borrow the serialized descriptor bytes. Valid until `zerobus_proto_schema_free`. +/// Pass directly to `zerobus_sdk_create_stream`. +/// +/// `out_len` is required: the bytes are not null-terminated, so the caller needs +/// the length to read them. Returns NULL without touching `out_len` if it is +/// NULL, and NULL with `*out_len` set to 0 on a null handle. +#[no_mangle] +pub extern "C" fn zerobus_proto_schema_descriptor_bytes( + schema: *const CZerobusProtoSchema, + out_len: *mut usize, +) -> *const u8 { + // The bytes are not null-terminated, so a pointer without a length is + // unusable. Refuse rather than hand back something the caller can't size. + if out_len.is_null() { + return ptr::null(); + } + // SAFETY: caller upholds the handle contract (valid, unfreed handle). + let schema_ref = match unsafe { proto_schema_ref(schema) } { + Ok(s) => s, + Err(_) => { + unsafe { + *out_len = 0; + } + return ptr::null(); + } + }; + unsafe { + *out_len = schema_ref.descriptor_bytes.len(); + } + // Valid until the caller's `_free`, which owns the backing allocation. + schema_ref.descriptor_bytes.as_ptr() +} + +/// Encode JSON record to protobuf bytes. Unknown keys are ignored. +/// +/// Values follow protobuf's JSON mapping; a few column types need shaping: +/// - DATE/TIMESTAMP/TIMESTAMP_NTZ: integers (days / micros since epoch), not strings. +/// - BINARY: base64-encoded string, not a JSON array of bytes. +/// - DECIMAL: string (e.g. "123.45"), to preserve precision/scale. +/// - VARIANT: a JSON-encoded string (a string whose contents are the variant's JSON). +/// - ARRAY/MAP/STRUCT: JSON array / object / object respectively. +/// - LONG/BIGINT above 2^53: pass as a JSON string, else the value loses +/// precision as a JSON number. +/// +/// Presence is enforced only for top-level non-nullable scalar and struct +/// columns (proto2 `required`); a record omitting one fails. Non-nullable +/// ARRAY/MAP columns map to `repeated`, which has no presence, so an omitted one +/// encodes as empty rather than failing; required fields nested inside a STRUCT +/// are likewise not presence-checked. +/// Returns true on success; caller must free buffer with `zerobus_free_proto_bytes`. +/// On failure `*out_data` is set to NULL and `*out_len` to 0. +#[no_mangle] +pub extern "C" fn zerobus_proto_schema_encode_json( + schema: *const CZerobusProtoSchema, + record_json: *const c_char, + out_data: *mut *mut u8, + out_len: *mut usize, + result: *mut CResult, +) -> bool { + if out_data.is_null() || out_len.is_null() { + write_error_result(result, "Output pointers are null", false); + return false; + } + // Initialize outputs up front so every failure path leaves them null/0 — a + // caller that frees on failure then hits a no-op rather than a stale or + // uninitialized pointer. + unsafe { + *out_data = ptr::null_mut(); + *out_len = 0; + } + + // SAFETY: caller upholds the handle contract — a valid handle not freed for + // the duration of this call. + let schema_ref = match unsafe { proto_schema_ref(schema) } { + Ok(s) => s, + Err(msg) => { + write_error_result(result, msg, false); + return false; + } + }; + let json = match unsafe { c_str_to_string(record_json) } { + Ok(s) => s, + Err(e) => { + write_error_result(result, e, false); + return false; + } + }; + + let mut deserializer = serde_json::Deserializer::from_str(&json); + // Records carry extra non-column fields; ignore them rather than erroring. + let options = DeserializeOptions::new().deny_unknown_fields(false); + let message = match DynamicMessage::deserialize_with_options( + schema_ref.message.clone(), + &mut deserializer, + &options, + ) { + Ok(m) => m, + Err(e) => { + write_error_result(result, &format!("failed to encode record: {e}"), false); + return false; + } + }; + if let Err(e) = deserializer.end() { + write_error_result( + result, + &format!("unexpected trailing content in record JSON: {e}"), + false, + ); + return false; + } + + // Top-level non-nullable scalar/struct columns are proto2 `required`, but + // prost-reflect doesn't enforce presence on encode — reject a missing one + // here rather than emit wire bytes the server rejects. (ARRAY/MAP are + // `repeated`, which has no presence, and nested struct fields aren't walked.) + let missing: Vec = schema_ref + .message + .fields() + .filter(|f| matches!(f.cardinality(), Cardinality::Required) && !message.has_field(f)) + .map(|f| f.name().to_string()) + .collect(); + if !missing.is_empty() { + write_error_result( + result, + &format!("record missing required field(s): {}", missing.join(", ")), + false, + ); + return false; + } + + let bytes = message.encode_to_vec(); + let len = bytes.len(); + // into_boxed_slice() shrinks capacity to len so the matching + // zerobus_free_proto_bytes reconstruction is sound. + let data_ptr = Box::into_raw(bytes.into_boxed_slice()) as *mut u8; + unsafe { + *out_data = data_ptr; + *out_len = len; + } + write_success_result(result); + true +} + +/// Free a buffer returned by `zerobus_proto_schema_encode_json`. +#[no_mangle] +pub extern "C" fn zerobus_free_proto_bytes(data: *mut u8, len: usize) { + // An all-default record encodes to zero bytes: a non-null, zero-length + // boxed slice. Reconstruct on `!data.is_null()` alone; gating on `len > 0` + // would leak it. + if !data.is_null() { + unsafe { + // data came from Box::into_raw(bytes.into_boxed_slice()), so + // capacity == len and this reconstruction is sound (len 0 included). + let _ = Box::from_raw(std::ptr::slice_from_raw_parts_mut(data, len)); + } + } +} + +/// Free a handle from `zerobus_proto_schema_from_uc_json`. Call exactly once, +/// after every other call using this handle has returned. The handle may be +/// shared by concurrent readers (`descriptor_bytes`, `encode_json`), but `free` +/// must not race any of them. +#[no_mangle] +pub extern "C" fn zerobus_proto_schema_free(schema: *mut CZerobusProtoSchema) { + if !schema.is_null() { + unsafe { + // Reclaim the Box handed to C by from_uc_json and drop it. + let _ = Box::from_raw(schema as *mut ProtoSchema); + } + } +} diff --git a/lib/zerobus-ffi-1.3.0/rust/ffi/src/tests.rs b/lib/zerobus-ffi-1.3.0/rust/ffi/src/tests.rs new file mode 100644 index 00000000000..03cda573742 --- /dev/null +++ b/lib/zerobus-ffi-1.3.0/rust/ffi/src/tests.rs @@ -0,0 +1,1068 @@ +#[cfg(test)] +mod tests { + use crate::{ + c_record_type, intern_header_key, validate_sdk_ptr, validate_stream_ptr, + write_error_result, write_success_result, zerobus_free_error_message, + zerobus_get_default_config, zerobus_sdk_builder_application_name, + zerobus_sdk_builder_build, zerobus_sdk_builder_disable_tls, zerobus_sdk_builder_endpoint, + zerobus_sdk_builder_free, zerobus_sdk_builder_new, zerobus_sdk_builder_sdk_identifier, + zerobus_sdk_builder_unity_catalog_url, zerobus_sdk_free, CHeaders, CResult, + CallbackHeadersProvider, RecordType, ZerobusError, + }; + use databricks_zerobus_ingest_sdk::HeadersProvider; + use std::ffi::{CStr, CString}; + use std::ptr; + + // Helper for c_str_to_string since it's private + unsafe fn test_c_str_to_string( + c_str: *const std::os::raw::c_char, + ) -> Result { + if c_str.is_null() { + return Err("Null pointer passed"); + } + CStr::from_ptr(c_str) + .to_str() + .map(|s| s.to_string()) + .map_err(|_| "Invalid UTF-8 string") + } + + // ======================================================================== + // Safety Wrapper Tests + // ======================================================================== + + #[test] + fn test_validate_sdk_ptr_null() { + let result = validate_sdk_ptr(ptr::null_mut()); + assert!(result.is_err()); + assert_eq!(result.err().unwrap(), "SDK pointer is null"); + } + + #[test] + fn test_validate_stream_ptr_null() { + let result = validate_stream_ptr(ptr::null_mut()); + assert!(result.is_err()); + assert_eq!(result.err().unwrap(), "Stream pointer is null"); + } + + #[test] + fn test_write_error_result() { + let mut result = CResult { + success: true, + error_message: ptr::null_mut(), + is_retryable: false, + }; + + write_error_result(&mut result as *mut CResult, "Test error", true); + + assert!(!result.success); + assert!(!result.error_message.is_null()); + assert!(result.is_retryable); + + // Clean up + unsafe { + if !result.error_message.is_null() { + let _ = CString::from_raw(result.error_message); + } + } + } + + #[test] + fn test_write_success_result() { + let mut result = CResult { + success: false, + error_message: CString::new("error").unwrap().into_raw(), + is_retryable: true, + }; + + write_success_result(&mut result as *mut CResult); + + assert!(result.success); + assert!(result.error_message.is_null()); + assert!(!result.is_retryable); + } + + #[test] + fn test_write_error_result_with_null_pointer() { + // Should not panic when result pointer is null + write_error_result(ptr::null_mut(), "Test error", false); + // If we get here, test passed + } + + #[test] + fn test_write_success_result_with_null_pointer() { + // Should not panic when result pointer is null + write_success_result(ptr::null_mut()); + // If we get here, test passed + } + + // ======================================================================== + // Header Key Cache Tests + // ======================================================================== + + #[test] + fn test_intern_header_key_caches_keys() { + // First call - should create new entry + let key1 = intern_header_key("Authorization".to_string()); + + // Second call with same string - should return cached entry + let key2 = intern_header_key("Authorization".to_string()); + + // Should be the same pointer (same address in memory) + assert_eq!(key1.as_ptr(), key2.as_ptr()); + } + + #[test] + fn test_intern_header_key_different_keys() { + let key1 = intern_header_key("Authorization".to_string()); + let key2 = intern_header_key("Content-Type".to_string()); + + // Different keys should have different pointers + assert_ne!(key1.as_ptr(), key2.as_ptr()); + assert_eq!(key1, "Authorization"); + assert_eq!(key2, "Content-Type"); + } + + #[test] + fn test_intern_header_key_prevents_duplicate_leaks() { + // Clear the cache first (can't actually do this safely in test, but we can verify behavior) + let initial_key = intern_header_key("X-Test-Header".to_string()); + + // Call many times + for _ in 0..100 { + let key = intern_header_key("X-Test-Header".to_string()); + // All should point to the same memory location + assert_eq!(initial_key.as_ptr(), key.as_ptr()); + } + } + + // ======================================================================== + // CResult Tests + // ======================================================================== + + #[test] + fn test_cresult_success() { + let result = CResult::success(); + assert!(result.success); + assert!(result.error_message.is_null()); + assert!(!result.is_retryable); + } + + #[test] + fn test_cresult_error() { + let error = ZerobusError::InvalidArgument("Test error".to_string()); + let result = CResult::error(error); + + assert!(!result.success); + assert!(!result.error_message.is_null()); + + // Verify error message + let msg = unsafe { CStr::from_ptr(result.error_message).to_string_lossy() }; + assert!(msg.contains("Test error")); + + // Clean up + unsafe { + let _ = CString::from_raw(result.error_message); + } + } + + // ======================================================================== + // Configuration Tests + // ======================================================================== + + #[test] + fn test_c_record_type_mapping() { + assert_eq!(c_record_type(1), RecordType::Proto); + assert_eq!(c_record_type(2), RecordType::Json); + assert_eq!(c_record_type(999), RecordType::Unspecified); + assert_eq!(c_record_type(0), RecordType::Unspecified); + } + + // ======================================================================== + // zerobus_sdk_builder Tests + // ======================================================================== + + /// Builds an SDK via the C builder API. Caller frees the SDK and any + /// error message. + fn build_via_c_builder( + endpoint: &str, + unity_catalog_url: &str, + sdk_identifier: Option<&str>, + application_name: Option<&str>, + ) -> (*mut crate::CZerobusSdk, CResult) { + let endpoint_c = CString::new(endpoint).unwrap(); + let uc_c = CString::new(unity_catalog_url).unwrap(); + + let builder = zerobus_sdk_builder_new(); + assert!(!builder.is_null()); + + zerobus_sdk_builder_endpoint(builder, endpoint_c.as_ptr()); + zerobus_sdk_builder_unity_catalog_url(builder, uc_c.as_ptr()); + + if let Some(id) = sdk_identifier { + let id_c = CString::new(id).unwrap(); + zerobus_sdk_builder_sdk_identifier(builder, id_c.as_ptr()); + } + if let Some(app) = application_name { + let app_c = CString::new(app).unwrap(); + zerobus_sdk_builder_application_name(builder, app_c.as_ptr()); + } + + let mut result = CResult { + success: false, + error_message: ptr::null_mut(), + is_retryable: false, + }; + let sdk = zerobus_sdk_builder_build(builder, &mut result); + (sdk, result) + } + + #[test] + fn test_builder_minimal() { + let (sdk, result) = + build_via_c_builder("https://workspace.zerobus.databricks.com", "", None, None); + assert!(result.success, "expected success, got error"); + assert!(!sdk.is_null()); + zerobus_sdk_free(sdk); + } + + #[test] + fn test_builder_with_sdk_identifier() { + let (sdk, result) = build_via_c_builder( + "https://workspace.zerobus.databricks.com", + "", + Some("zerobus-sdk-go/1.3.0"), + None, + ); + assert!(result.success); + assert!(!sdk.is_null()); + zerobus_sdk_free(sdk); + } + + #[test] + fn test_builder_with_application_name() { + let (sdk, result) = build_via_c_builder( + "https://workspace.zerobus.databricks.com", + "", + None, + Some("my-app/1.0"), + ); + assert!(result.success); + assert!(!sdk.is_null()); + zerobus_sdk_free(sdk); + } + + #[test] + fn test_builder_both_user_agent_options() { + let (sdk, result) = build_via_c_builder( + "https://workspace.zerobus.databricks.com", + "", + Some("zerobus-sdk-go/1.3.0"), + Some("my-app/1.0"), + ); + assert!(result.success); + assert!(!sdk.is_null()); + zerobus_sdk_free(sdk); + } + + #[test] + fn test_builder_empty_strings_are_noops() { + // Empty identifier/application_name must not produce a trailing space. + let (sdk, result) = build_via_c_builder( + "https://workspace.zerobus.databricks.com", + "", + Some(""), + Some(""), + ); + assert!(result.success); + assert!(!sdk.is_null()); + zerobus_sdk_free(sdk); + } + + #[test] + fn test_builder_build_consumes_on_error() { + // Missing endpoint fails build. Builder must still be consumed — + // don't _free the pointer afterward (use-after-free). + let builder = zerobus_sdk_builder_new(); + let mut result = CResult { + success: false, + error_message: ptr::null_mut(), + is_retryable: false, + }; + let sdk = zerobus_sdk_builder_build(builder, &mut result); + assert!(sdk.is_null()); + assert!(!result.success); + zerobus_free_error_message(result.error_message); + } + + #[test] + fn test_builder_free_without_build() { + let builder = zerobus_sdk_builder_new(); + zerobus_sdk_builder_free(builder); + } + + #[test] + fn test_builder_free_on_null_is_safe() { + zerobus_sdk_builder_free(ptr::null_mut()); + } + + #[test] + fn test_builder_setters_on_null_are_safe() { + let s = CString::new("x").unwrap(); + zerobus_sdk_builder_endpoint(ptr::null_mut(), s.as_ptr()); + zerobus_sdk_builder_unity_catalog_url(ptr::null_mut(), s.as_ptr()); + zerobus_sdk_builder_sdk_identifier(ptr::null_mut(), s.as_ptr()); + zerobus_sdk_builder_application_name(ptr::null_mut(), s.as_ptr()); + zerobus_sdk_builder_disable_tls(ptr::null_mut()); + } + + #[test] + fn test_builder_disable_tls_for_plain_http() { + let endpoint_c = CString::new("http://localhost:50051").unwrap(); + let builder = zerobus_sdk_builder_new(); + zerobus_sdk_builder_endpoint(builder, endpoint_c.as_ptr()); + zerobus_sdk_builder_disable_tls(builder); + let mut result = CResult { + success: false, + error_message: ptr::null_mut(), + is_retryable: false, + }; + let sdk = zerobus_sdk_builder_build(builder, &mut result); + assert!(result.success); + assert!(!sdk.is_null()); + zerobus_sdk_free(sdk); + } + + #[test] + fn test_get_default_config() { + let config = zerobus_get_default_config(); + + // Verify it returns reasonable defaults + assert!(config.max_inflight_requests > 0); + assert_eq!(config.record_type, 1); // Proto + } + + // ======================================================================== + // C String Conversion Tests + // ======================================================================== + + #[test] + fn test_c_str_to_string_valid() { + let test_str = CString::new("Hello, World!").unwrap(); + let result = unsafe { test_c_str_to_string(test_str.as_ptr()) }; + + assert!(result.is_ok()); + assert_eq!(result.unwrap(), "Hello, World!"); + } + + #[test] + fn test_c_str_to_string_null() { + let result = unsafe { test_c_str_to_string(ptr::null()) }; + assert!(result.is_err()); + assert_eq!(result.unwrap_err(), "Null pointer passed"); + } + + #[test] + fn test_c_str_to_string_empty() { + let test_str = CString::new("").unwrap(); + let result = unsafe { test_c_str_to_string(test_str.as_ptr()) }; + + assert!(result.is_ok()); + assert_eq!(result.unwrap(), ""); + } + + // ======================================================================== + // Memory Management Tests + // ======================================================================== + + #[test] + fn test_zerobus_free_error_message_null() { + // Should not panic with null pointer + zerobus_free_error_message(ptr::null_mut()); + } + + #[test] + fn test_zerobus_free_error_message_valid() { + let msg = CString::new("Test error").unwrap().into_raw(); + zerobus_free_error_message(msg); + // If we get here without crashing, test passed + } + + // ======================================================================== + // Thread Safety Tests + // ======================================================================== + + #[test] + fn test_callback_headers_provider_sequential() { + extern "C" fn test_callback(_user_data: *mut std::ffi::c_void) -> CHeaders { + CHeaders { + headers: ptr::null_mut(), + count: 0, + error_message: ptr::null_mut(), + } + } + + let provider = CallbackHeadersProvider::new(test_callback, ptr::null_mut()); + + // Sequential calls should work fine + let rt = tokio::runtime::Runtime::new().unwrap(); + let result1 = rt.block_on(provider.get_headers()); + assert!(result1.is_ok()); + + let result2 = rt.block_on(provider.get_headers()); + assert!(result2.is_ok()); + } + + #[test] + fn test_callback_headers_provider_returns_headers() { + extern "C" fn test_callback(_user_data: *mut std::ffi::c_void) -> CHeaders { + // Create simple test headers + let auth_key = CString::new("Authorization").unwrap().into_raw(); + let auth_val = CString::new("Bearer test-token").unwrap().into_raw(); + + let header = Box::new(crate::CHeader { + key: auth_key, + value: auth_val, + }); + + CHeaders { + headers: Box::into_raw(header), + count: 1, + error_message: ptr::null_mut(), + } + } + + let provider = CallbackHeadersProvider::new(test_callback, ptr::null_mut()); + + let rt = tokio::runtime::Runtime::new().unwrap(); + let result = rt.block_on(provider.get_headers()); + + assert!(result.is_ok()); + let headers = result.unwrap(); + assert_eq!(headers.len(), 1); + assert!(headers.contains_key("Authorization")); + } + + // ======================================================================== + // Dynamic protobuf schema tests + // ======================================================================== + + use crate::{ + zerobus_free_proto_bytes, zerobus_proto_schema_descriptor_bytes, + zerobus_proto_schema_encode_json, zerobus_proto_schema_free, + zerobus_proto_schema_from_uc_json, + }; + use prost::Message; + use prost_reflect::{DescriptorPool, DynamicMessage, MessageDescriptor}; + + // Minimal Unity Catalog table-metadata JSON, shaped like the body of + // GET /api/2.1/unity-catalog/tables/{name}. + fn sample_uc_table_json() -> CString { + let json = r#"{ + "name": "events", + "catalog_name": "main", + "schema_name": "analytics", + "columns": [ + {"name": "id", "type_name": "BIGINT", "type_text": "bigint", "nullable": false, "position": 0}, + {"name": "payload", "type_name": "STRING", "type_text": "string", "nullable": true, "position": 1}, + {"name": "ts", "type_name": "TIMESTAMP", "type_text": "timestamp", "nullable": true, "position": 2} + ] + }"#; + CString::new(json).unwrap() + } + + // A CResult to be written into by the function under test. Starts as a + // failure so a success-path test proves the call flipped it to success. + fn unwritten_result() -> CResult { + CResult { + success: false, + error_message: ptr::null_mut(), + is_retryable: false, + } + } + + // As above but starts successful, so an error-path test proves the call + // flipped it to failure. + fn presumed_success_result() -> CResult { + CResult { + success: true, + error_message: ptr::null_mut(), + is_retryable: false, + } + } + + // Rebuild a MessageDescriptor from the bare DescriptorProto bytes the handle + // exposes, so a test can decode encoded records back and assert field values + // — proving the descriptor given to the server and the encoder agree. + fn message_descriptor_from_bytes(descriptor_bytes: &[u8]) -> MessageDescriptor { + let descriptor = prost_types::DescriptorProto::decode(descriptor_bytes).unwrap(); + let name = descriptor.name().to_string(); + let file = prost_types::FileDescriptorProto { + name: Some("test.proto".to_string()), + message_type: vec![descriptor], + ..Default::default() + }; + let mut pool = DescriptorPool::new(); + pool.add_file_descriptor_proto(file).unwrap(); + pool.get_message_by_name(&name).unwrap() + } + + #[test] + fn test_proto_schema_from_uc_json_roundtrip() { + let json = sample_uc_table_json(); + let mut result = unwritten_result(); + let schema = zerobus_proto_schema_from_uc_json(json.as_ptr(), &mut result as *mut CResult); + assert!(!schema.is_null(), "schema build failed"); + assert!(result.success); + + // Descriptor bytes must decode to the bare DescriptorProto that the + // server is given via zerobus_sdk_create_stream. + let mut dlen: usize = 0; + let dptr = zerobus_proto_schema_descriptor_bytes(schema, &mut dlen as *mut usize); + assert!(!dptr.is_null()); + assert!(dlen > 0); + let desc_bytes = unsafe { std::slice::from_raw_parts(dptr, dlen) }; + let descriptor = prost_types::DescriptorProto::decode(desc_bytes).unwrap(); + // schema_name + table_name, sanitized to PascalCase. + assert_eq!(descriptor.name(), "AnalyticsEvents"); + assert_eq!(descriptor.field.len(), 3); + + // Encode a record; unknown keys are ignored, timestamps are integers. + let record = + CString::new(r#"{"id": 7, "payload": "hello", "ts": 1700000000000000, "extra": "x"}"#) + .unwrap(); + let mut out_data: *mut u8 = ptr::null_mut(); + let mut out_len: usize = 0; + let mut enc_result = unwritten_result(); + let ok = zerobus_proto_schema_encode_json( + schema, + record.as_ptr(), + &mut out_data as *mut *mut u8, + &mut out_len as *mut usize, + &mut enc_result as *mut CResult, + ); + assert!(ok, "encode failed"); + assert!(enc_result.success); + assert!(!out_data.is_null()); + assert!(out_len > 0); + + // Decode the encoded bytes against the same descriptor and assert the + // values round-trip: the encoding is correct, not merely non-empty. + let encoded = unsafe { std::slice::from_raw_parts(out_data, out_len) }; + let msg_desc = message_descriptor_from_bytes(desc_bytes); + let decoded = DynamicMessage::decode(msg_desc, encoded).unwrap(); + assert_eq!(decoded.get_field_by_name("id").unwrap().as_i64(), Some(7)); + assert_eq!( + decoded.get_field_by_name("payload").unwrap().as_str(), + Some("hello") + ); + assert_eq!( + decoded.get_field_by_name("ts").unwrap().as_i64(), + Some(1700000000000000) + ); + + zerobus_free_proto_bytes(out_data, out_len); + zerobus_proto_schema_free(schema); + } + + #[test] + fn test_proto_schema_from_uc_json_invalid_json_errors() { + let bad = CString::new("not json").unwrap(); + let mut result = presumed_success_result(); + let schema = zerobus_proto_schema_from_uc_json(bad.as_ptr(), &mut result as *mut CResult); + assert!(schema.is_null()); + assert!(!result.success); + // A parse failure is a caller error, not a transient one. Assert on the + // error code rather than the message text, which is free to change. + assert!(!result.is_retryable); + assert!(!result.error_message.is_null()); + zerobus_free_error_message(result.error_message); + } + + #[test] + fn test_proto_schema_from_uc_json_unsupported_type_errors() { + // Parses cleanly into UcTableSchema but carries a column type the + // descriptor builder rejects — exercises the schema-conversion error + // path, distinct from a JSON parse failure. + let json = CString::new( + r#"{ + "name": "t", "catalog_name": "c", "schema_name": "s", + "columns": [ + {"name": "x", "type_name": "GEOGRAPHY", "type_text": "geography", "nullable": true, "position": 0} + ] + }"#, + ) + .unwrap(); + let mut result = presumed_success_result(); + let schema = zerobus_proto_schema_from_uc_json(json.as_ptr(), &mut result as *mut CResult); + assert!(schema.is_null()); + assert!(!result.success); + assert!(!result.error_message.is_null()); + zerobus_free_error_message(result.error_message); + } + + #[test] + fn test_proto_schema_from_uc_json_null_input_errors() { + let mut result = presumed_success_result(); + let schema = zerobus_proto_schema_from_uc_json(ptr::null(), &mut result as *mut CResult); + assert!(schema.is_null()); + assert!(!result.success); + zerobus_free_error_message(result.error_message); + } + + #[test] + fn test_proto_schema_descriptor_bytes_null_handle() { + // A null handle must yield a null pointer and zero the out-length so the + // caller never reads a stale length. + let mut len: usize = 123; + let dptr = zerobus_proto_schema_descriptor_bytes(ptr::null(), &mut len as *mut usize); + assert!(dptr.is_null()); + assert_eq!(len, 0); + } + + #[test] + fn test_proto_schema_descriptor_bytes_null_out_len() { + // The descriptor bytes are not null-terminated, so a null out_len leaves + // the caller no way to size them. A valid handle must still yield a null + // pointer rather than a length-less buffer. + let json = sample_uc_table_json(); + let mut build = unwritten_result(); + let schema = zerobus_proto_schema_from_uc_json(json.as_ptr(), &mut build as *mut CResult); + assert!(!schema.is_null(), "schema build failed"); + + let dptr = zerobus_proto_schema_descriptor_bytes(schema, ptr::null_mut()); + assert!(dptr.is_null()); + + zerobus_proto_schema_free(schema); + } + + #[test] + fn test_proto_schema_encode_null_schema_errors() { + let record = CString::new(r#"{"id": 1}"#).unwrap(); + // Seed the outputs with non-null/non-zero sentinels: a failed call must + // clear them so a caller that frees on error hits a no-op. The schema + // check fails before any encoding, exercising the earliest failure path. + let mut sentinel: u8 = 0; + let mut out_data: *mut u8 = &mut sentinel as *mut u8; + let mut out_len: usize = 999; + let mut result = presumed_success_result(); + let ok = zerobus_proto_schema_encode_json( + ptr::null(), + record.as_ptr(), + &mut out_data as *mut *mut u8, + &mut out_len as *mut usize, + &mut result as *mut CResult, + ); + assert!(!ok); + assert!(!result.success); + assert!(out_data.is_null(), "outputs must be cleared on failure"); + assert_eq!(out_len, 0, "outputs must be cleared on failure"); + zerobus_free_error_message(result.error_message); + } + + #[test] + fn test_proto_schema_encode_malformed_record_errors() { + let json = sample_uc_table_json(); + let mut build = unwritten_result(); + let schema = zerobus_proto_schema_from_uc_json(json.as_ptr(), &mut build as *mut CResult); + assert!(!schema.is_null()); + + let bad_record = CString::new("{ not valid json").unwrap(); + let mut out_data: *mut u8 = ptr::null_mut(); + let mut out_len: usize = 0; + let mut enc_result = presumed_success_result(); + let ok = zerobus_proto_schema_encode_json( + schema, + bad_record.as_ptr(), + &mut out_data as *mut *mut u8, + &mut out_len as *mut usize, + &mut enc_result as *mut CResult, + ); + assert!(!ok); + assert!(!enc_result.success); + assert!(!enc_result.error_message.is_null()); + assert!(out_data.is_null(), "no buffer should be allocated on error"); + assert_eq!(out_len, 0, "length must be cleared on error"); + zerobus_free_error_message(enc_result.error_message); + zerobus_proto_schema_free(schema); + } + + #[test] + fn test_proto_schema_encode_null_out_pointers_errors() { + let json = sample_uc_table_json(); + let mut build = unwritten_result(); + let schema = zerobus_proto_schema_from_uc_json(json.as_ptr(), &mut build as *mut CResult); + assert!(!schema.is_null()); + + let record = CString::new(r#"{"id": 1}"#).unwrap(); + let mut enc_result = presumed_success_result(); + let ok = zerobus_proto_schema_encode_json( + schema, + record.as_ptr(), + ptr::null_mut(), + ptr::null_mut(), + &mut enc_result as *mut CResult, + ); + assert!(!ok); + assert!(!enc_result.success); + zerobus_free_error_message(enc_result.error_message); + zerobus_proto_schema_free(schema); + } + + #[test] + fn test_proto_schema_encode_missing_required_field_errors() { + // `id` is non-nullable (proto2 `required`); a record omitting it must be + // rejected rather than encoded. + let json = sample_uc_table_json(); + let mut build = unwritten_result(); + let schema = zerobus_proto_schema_from_uc_json(json.as_ptr(), &mut build as *mut CResult); + assert!(!schema.is_null()); + + let record = CString::new(r#"{"payload": "hello"}"#).unwrap(); + let mut out_data: *mut u8 = ptr::null_mut(); + let mut out_len: usize = 0; + let mut enc_result = presumed_success_result(); + let ok = zerobus_proto_schema_encode_json( + schema, + record.as_ptr(), + &mut out_data as *mut *mut u8, + &mut out_len as *mut usize, + &mut enc_result as *mut CResult, + ); + assert!(!ok); + assert!(!enc_result.success); + // A missing required field is a caller error, not a transient one. Assert + // on the error code rather than the message text, which is free to change. + assert!(!enc_result.is_retryable); + assert!(!enc_result.error_message.is_null()); + assert!(out_data.is_null(), "no buffer should be allocated on error"); + assert_eq!(out_len, 0, "length must be cleared on error"); + zerobus_free_error_message(enc_result.error_message); + zerobus_proto_schema_free(schema); + } + + // UC table JSON for the type-contract tests: a required key plus one column + // of the type under test. + fn uc_table_json_with_column(col_name: &str, type_name: &str) -> CString { + let json = format!( + r#"{{ + "name": "t", "catalog_name": "c", "schema_name": "s", + "columns": [ + {{"name": "k", "type_name": "BIGINT", "type_text": "bigint", "nullable": false, "position": 0}}, + {{"name": "{col_name}", "type_name": "{type_name}", "type_text": "{type_name}", "nullable": true, "position": 1}} + ] + }}"# + ); + CString::new(json).unwrap() + } + + // Build a schema + encode one record, returning the decoded message so a test + // can assert how a given JSON value lands on the wire. + fn encode_and_decode(table_json: &CString, record_json: &str) -> DynamicMessage { + let mut build = unwritten_result(); + let schema = + zerobus_proto_schema_from_uc_json(table_json.as_ptr(), &mut build as *mut CResult); + assert!(!schema.is_null(), "schema build failed"); + + let mut dlen: usize = 0; + let dptr = zerobus_proto_schema_descriptor_bytes(schema, &mut dlen as *mut usize); + let desc_bytes = unsafe { std::slice::from_raw_parts(dptr, dlen) }; + let msg_desc = message_descriptor_from_bytes(desc_bytes); + + let record = CString::new(record_json).unwrap(); + let mut out_data: *mut u8 = ptr::null_mut(); + let mut out_len: usize = 0; + let mut enc_result = unwritten_result(); + let ok = zerobus_proto_schema_encode_json( + schema, + record.as_ptr(), + &mut out_data as *mut *mut u8, + &mut out_len as *mut usize, + &mut enc_result as *mut CResult, + ); + assert!(ok, "encode failed"); + let encoded = unsafe { std::slice::from_raw_parts(out_data, out_len) }; + let decoded = DynamicMessage::decode(msg_desc, encoded).unwrap(); + + zerobus_free_proto_bytes(out_data, out_len); + zerobus_proto_schema_free(schema); + decoded + } + + #[test] + fn test_proto_schema_encode_binary_is_base64_string() { + // BINARY maps to proto `bytes`; prost-reflect's serde layer accepts it + // only as a base64-encoded string (not a JSON array of byte values). + let table = uc_table_json_with_column("blob", "BINARY"); + // "aGVsbG8=" is base64 for "hello". + let decoded = encode_and_decode(&table, r#"{"k": 1, "blob": "aGVsbG8="}"#); + assert_eq!( + decoded + .get_field_by_name("blob") + .unwrap() + .as_bytes() + .map(|b| b.as_ref()), + Some(b"hello".as_slice()) + ); + } + + #[test] + fn test_proto_schema_encode_decimal_is_string() { + // DECIMAL maps to proto `string`; the value must be passed as a JSON + // string to preserve precision and scale. + let table = uc_table_json_with_column("price", "DECIMAL"); + let decoded = encode_and_decode(&table, r#"{"k": 1, "price": "123.45"}"#); + assert_eq!( + decoded.get_field_by_name("price").unwrap().as_str(), + Some("123.45") + ); + } + + #[test] + fn test_proto_schema_encode_large_int64_as_string_preserves_precision() { + // int64 above 2^53 loses precision as a JSON number; passing it as a + // string round-trips exactly. + let table = uc_table_json_with_column("big", "BIGINT"); + let decoded = encode_and_decode(&table, r#"{"k": 1, "big": "9223372036854775807"}"#); + assert_eq!( + decoded.get_field_by_name("big").unwrap().as_i64(), + Some(9223372036854775807) + ); + } + + #[test] + fn test_proto_schema_encode_variant_is_json_encoded_string() { + // VARIANT maps to proto `string`; the value is a JSON-encoded string + // (a string whose contents are the variant's JSON). + let table = uc_table_json_with_column("v", "VARIANT"); + let decoded = encode_and_decode(&table, r#"{"k": 1, "v": "{\"a\":1,\"b\":[2,3]}"}"#); + assert_eq!( + decoded.get_field_by_name("v").unwrap().as_str(), + Some(r#"{"a":1,"b":[2,3]}"#) + ); + } + + #[test] + fn test_proto_schema_encode_array_is_json_array() { + // ARRAY maps to `repeated T`; the value is a JSON array. Complex + // columns carry their shape in `type_json`, so build the table directly. + let table = CString::new( + r#"{ + "name": "t", "catalog_name": "c", "schema_name": "s", + "columns": [ + {"name": "k", "type_name": "BIGINT", "type_text": "bigint", "nullable": false, "position": 0}, + {"name": "tags", "type_name": "ARRAY", "type_text": "array", "nullable": true, "position": 1, + "type_json": "{\"type\":\"array\",\"elementType\":\"integer\",\"containsNull\":false}"} + ] + }"#, + ) + .unwrap(); + let decoded = encode_and_decode(&table, r#"{"k": 1, "tags": [10, 20, 30]}"#); + let list = decoded.get_field_by_name("tags").unwrap(); + let values: Vec = list + .as_list() + .unwrap() + .iter() + .map(|v| v.as_i32().unwrap() as i64) + .collect(); + assert_eq!(values, vec![10, 20, 30]); + } + + #[test] + fn test_proto_schema_encode_map_roundtrip() { + // MAP maps to a synthetic map-entry message + `repeated`; the value + // is a JSON object. Protobuf-JSON map keys are always strings on the wire. + let table = CString::new( + r#"{ + "name": "t", "catalog_name": "c", "schema_name": "s", + "columns": [ + {"name": "k", "type_name": "BIGINT", "type_text": "bigint", "nullable": false, "position": 0}, + {"name": "attrs", "type_name": "MAP", "type_text": "map", "nullable": true, "position": 1, + "type_json": "{\"type\":\"map\",\"keyType\":\"string\",\"valueType\":\"integer\",\"valueContainsNull\":false}"} + ] + }"#, + ) + .unwrap(); + let decoded = encode_and_decode(&table, r#"{"k": 1, "attrs": {"a": 1, "b": 2}}"#); + let field = decoded.get_field_by_name("attrs").unwrap(); + let map = field.as_map().unwrap(); + let mut pairs: Vec<(String, i32)> = map + .iter() + .map(|(k, v)| (k.as_str().unwrap().to_string(), v.as_i32().unwrap())) + .collect(); + pairs.sort(); + assert_eq!(pairs, vec![("a".to_string(), 1), ("b".to_string(), 2)]); + } + + #[test] + fn test_proto_schema_encode_struct_roundtrip() { + // STRUCT<...> maps to a nested message; the value is a JSON object. + let table = CString::new( + r#"{ + "name": "t", "catalog_name": "c", "schema_name": "s", + "columns": [ + {"name": "k", "type_name": "BIGINT", "type_text": "bigint", "nullable": false, "position": 0}, + {"name": "addr", "type_name": "STRUCT", "type_text": "struct", "nullable": true, "position": 1, + "type_json": "{\"type\":\"struct\",\"fields\":[{\"name\":\"city\",\"type\":\"string\",\"nullable\":true,\"metadata\":{}},{\"name\":\"zip\",\"type\":\"integer\",\"nullable\":true,\"metadata\":{}}]}"} + ] + }"#, + ) + .unwrap(); + let decoded = + encode_and_decode(&table, r#"{"k": 1, "addr": {"city": "NYC", "zip": 10001}}"#); + let field = decoded.get_field_by_name("addr").unwrap(); + let addr = field.as_message().unwrap(); + assert_eq!( + addr.get_field_by_name("city").unwrap().as_str(), + Some("NYC") + ); + assert_eq!(addr.get_field_by_name("zip").unwrap().as_i32(), Some(10001)); + } + + #[test] + fn test_proto_schema_encode_array_of_struct_roundtrip() { + // ARRAY> maps to `repeated `; the value is a + // JSON array of objects. + let table = CString::new( + r#"{ + "name": "t", "catalog_name": "c", "schema_name": "s", + "columns": [ + {"name": "k", "type_name": "BIGINT", "type_text": "bigint", "nullable": false, "position": 0}, + {"name": "items", "type_name": "ARRAY", "type_text": "array>", "nullable": true, "position": 1, + "type_json": "{\"type\":\"array\",\"elementType\":{\"type\":\"struct\",\"fields\":[{\"name\":\"id\",\"type\":\"integer\",\"nullable\":true,\"metadata\":{}}]},\"containsNull\":false}"} + ] + }"#, + ) + .unwrap(); + let decoded = encode_and_decode(&table, r#"{"k": 1, "items": [{"id": 1}, {"id": 2}]}"#); + let field = decoded.get_field_by_name("items").unwrap(); + let ids: Vec = field + .as_list() + .unwrap() + .iter() + .map(|v| { + v.as_message() + .unwrap() + .get_field_by_name("id") + .unwrap() + .as_i32() + .unwrap() + }) + .collect(); + assert_eq!(ids, vec![1, 2]); + } + + #[test] + fn test_proto_schema_encode_date_is_days_since_epoch() { + // DATE maps to proto `int32`; the value is days since the Unix epoch, an + // integer (not an ISO-8601 string). 19000 days ≈ 2022-01-08. + let table = uc_table_json_with_column("d", "DATE"); + let decoded = encode_and_decode(&table, r#"{"k": 1, "d": 19000}"#); + assert_eq!( + decoded.get_field_by_name("d").unwrap().as_i32(), + Some(19000) + ); + } + + #[test] + fn test_proto_schema_encode_timestamp_ntz_is_micros() { + // TIMESTAMP_NTZ maps to proto `int64` (same wire shape as TIMESTAMP); the + // value is microseconds since the epoch, an integer. + let table = uc_table_json_with_column("tsn", "TIMESTAMP_NTZ"); + let decoded = encode_and_decode(&table, r#"{"k": 1, "tsn": 1700000000000000}"#); + assert_eq!( + decoded.get_field_by_name("tsn").unwrap().as_i64(), + Some(1700000000000000) + ); + } + + #[test] + fn test_free_proto_bytes_handles_empty_encoding() { + // A record with no fields set encodes to zero bytes: out_len == 0 but + // out_data is a non-null, zero-length boxed slice. Freeing it must + // reclaim that allocation, not leak it. + let table = CString::new( + r#"{ + "name": "t", "catalog_name": "c", "schema_name": "s", + "columns": [ + {"name": "opt", "type_name": "INT", "type_text": "int", "nullable": true, "position": 0} + ] + }"#, + ) + .unwrap(); + let mut build = unwritten_result(); + let schema = zerobus_proto_schema_from_uc_json(table.as_ptr(), &mut build as *mut CResult); + assert!(!schema.is_null(), "schema build failed"); + + let record = CString::new(r#"{}"#).unwrap(); + let mut out_data: *mut u8 = ptr::null_mut(); + let mut out_len: usize = 0; + let mut enc_result = unwritten_result(); + let ok = zerobus_proto_schema_encode_json( + schema, + record.as_ptr(), + &mut out_data as *mut *mut u8, + &mut out_len as *mut usize, + &mut enc_result as *mut CResult, + ); + assert!(ok, "encode failed"); + assert_eq!( + out_len, 0, + "record with no fields set should encode to zero bytes" + ); + assert!( + !out_data.is_null(), + "buffer pointer should be non-null even when empty" + ); + + // The assertion is the absence of a leak/crash on free. + zerobus_free_proto_bytes(out_data, out_len); + zerobus_proto_schema_free(schema); + } + + #[test] + fn test_proto_schema_shared_across_threads() { + // The handle may be shared by concurrent readers: many threads encode + // through one handle at once. `free` is ordered after every worker has + // joined, so it never races an in-flight encode. + use std::thread; + + let json = sample_uc_table_json(); + let mut build = unwritten_result(); + let schema = zerobus_proto_schema_from_uc_json(json.as_ptr(), &mut build as *mut CResult); + assert!(!schema.is_null(), "schema build failed"); + + // Raw pointers aren't Send; pass the address as a usize and rebuild it + // per thread. Safe: threads only read, and the handle outlives them. + let handle_addr = schema as usize; + let mut workers = Vec::new(); + for t in 0..8 { + workers.push(thread::spawn(move || { + let handle = handle_addr as *const crate::CZerobusProtoSchema; + for i in 0..200 { + let record = CString::new(format!( + r#"{{"id": {}, "payload": "p{}"}}"#, + t * 1000 + i, + i + )) + .unwrap(); + let mut out_data: *mut u8 = ptr::null_mut(); + let mut out_len: usize = 0; + let mut enc = unwritten_result(); + let ok = zerobus_proto_schema_encode_json( + handle, + record.as_ptr(), + &mut out_data as *mut *mut u8, + &mut out_len as *mut usize, + &mut enc as *mut CResult, + ); + assert!(ok, "concurrent encode failed"); + zerobus_free_proto_bytes(out_data, out_len); + } + })); + } + for w in workers { + w.join().unwrap(); + } + zerobus_proto_schema_free(schema); + } +} diff --git a/lib/zerobus-ffi-1.3.0/rust/ffi/zerobus.h b/lib/zerobus-ffi-1.3.0/rust/ffi/zerobus.h new file mode 100644 index 00000000000..091729ab86c --- /dev/null +++ b/lib/zerobus-ffi-1.3.0/rust/ffi/zerobus.h @@ -0,0 +1,574 @@ +/* Zerobus C FFI Interface */ + +#ifndef ZEROBUS_H +#define ZEROBUS_H + +#pragma once + +/* Generated with cbindgen:0.27.0 */ + +/* Warning: This file is autogenerated by cbindgen. Don't modify this manually. */ + +#include +#include +#include +#include + +#ifdef __cplusplus +namespace zerobus { +#endif // __cplusplus + +/** + * Opaque handle for an Arrow Flight stream. + */ +typedef struct CArrowStream { + uint8_t _private[0]; +} CArrowStream; + +typedef struct CZerobusSdk { + uint8_t _private[0]; +} CZerobusSdk; + +/** + * Configuration options for Arrow Flight streams. + * + * `ipc_compression`: -1 = None, 0 = LZ4_FRAME, 1 = ZSTD + */ +typedef struct CArrowStreamConfigurationOptions { + uintptr_t max_inflight_batches; + bool recovery; + uint64_t recovery_timeout_ms; + uint64_t recovery_backoff_ms; + uint32_t recovery_retries; + uint64_t server_lack_of_ack_timeout_ms; + uint64_t flush_timeout_ms; + uint64_t connection_timeout_ms; + /** + * -1 = None, 0 = LZ4_FRAME, 1 = ZSTD + */ + int32_t ipc_compression; + /** + * Maximum time in milliseconds to wait during graceful stream close. + * -1 = None (wait full server duration), 0 = immediate recovery, >0 = wait up to min(this, server_duration). + */ + int64_t stream_paused_max_wait_time_ms; +} CArrowStreamConfigurationOptions; + +typedef struct CResult { + bool success; + char *error_message; + bool is_retryable; +} CResult; + +/** + * A single header key-value pair for C FFI + */ +typedef struct CHeader { + char *key; + char *value; +} CHeader; + +/** + * A collection of headers returned from Go callback + */ +typedef struct CHeaders { + struct CHeader *headers; + uintptr_t count; + char *error_message; +} CHeaders; + +/** + * Function pointer type for the headers provider callback + * The callback should return a CHeaders struct + * The caller is responsible for freeing the returned CHeaders using zerobus_free_headers + */ +typedef struct CHeaders (*HeadersProviderCallback)(void *user_data); + +/** + * An array of Arrow IPC-encoded batches, returned by `zerobus_arrow_stream_get_unacked_batches`. + * Must be freed with `zerobus_arrow_free_batch_array`. + */ +typedef struct CArrowBatchArray { + /** + * Array of pointers to IPC-encoded batch bytes. + */ + uint8_t **batches; + /** + * Array of byte lengths, one per batch. + */ + uintptr_t *lengths; + /** + * Number of batches. + */ + uintptr_t count; +} CArrowBatchArray; + +/** + * Opaque handle for an SDK builder. Allocated by `_new`, consumed by + * `_build`, or dropped by `_free`. Must not be used after either finalizer. + */ +typedef struct CZerobusSdkBuilder { + uint8_t _private[0]; +} CZerobusSdkBuilder; + +typedef struct CZerobusStream { + uint8_t _private[0]; +} CZerobusStream; + +typedef struct CStreamConfigurationOptions { + uintptr_t max_inflight_requests; + bool recovery; + uint64_t recovery_timeout_ms; + uint64_t recovery_backoff_ms; + uint32_t recovery_retries; + uint64_t server_lack_of_ack_timeout_ms; + uint64_t flush_timeout_ms; + int32_t record_type; + uint64_t stream_paused_max_wait_time_ms; + bool has_stream_paused_max_wait_time_ms; + uint64_t callback_max_wait_time_ms; + bool has_callback_max_wait_time_ms; +} CStreamConfigurationOptions; + +/** + * Represents a single record (either Proto or JSON) + */ +typedef struct CRecord { + bool is_json; + uint8_t *data; + uintptr_t data_len; +} CRecord; + +/** + * Represents an array of records + */ +typedef struct CRecordArray { + struct CRecord *records; + uintptr_t len; +} CRecordArray; + +/** + * Opaque handle to a table's protobuf schema: its serialized descriptor plus a + * prepared encoder. C code only ever holds a pointer to it; the backing + * allocation is owned by the SDK and released by zerobus_proto_schema_free. + */ +typedef struct CZerobusProtoSchema { + uint8_t _private[0]; +} CZerobusProtoSchema; + +#ifdef __cplusplus +extern "C" { +#endif // __cplusplus + +/** + * Creates an Arrow Flight stream authenticated with OAuth client credentials. + * + * `schema_ipc_bytes` must point to Arrow IPC stream bytes encoding only the schema + * (write an empty IPC stream with just the schema message). + */ +struct CArrowStream *zerobus_sdk_create_arrow_stream(struct CZerobusSdk *sdk, + const char *table_name, + const uint8_t *schema_ipc_bytes, + uintptr_t schema_ipc_len, + const char *client_id, + const char *client_secret, + const struct CArrowStreamConfigurationOptions *options, + struct CResult *result); + +/** + * Creates an Arrow Flight stream with a custom headers provider callback. + * + * `schema_ipc_bytes` must point to Arrow IPC stream bytes encoding only the schema. + */ +struct CArrowStream *zerobus_sdk_create_arrow_stream_with_headers_provider(struct CZerobusSdk *sdk, + const char *table_name, + const uint8_t *schema_ipc_bytes, + uintptr_t schema_ipc_len, + HeadersProviderCallback headers_callback, + void *user_data, + const struct CArrowStreamConfigurationOptions *options, + struct CResult *result); + +/** + * Frees an Arrow Flight stream instance. + */ +void zerobus_arrow_stream_free(struct CArrowStream *stream); + +/** + * Ingests one Arrow RecordBatch supplied as Arrow IPC stream bytes. + * + * `ipc_bytes` must be a valid Arrow IPC stream (schema + one record batch). + * The bytes are deserialised to a RecordBatch internally. Works with all + * compression settings. Returns the logical offset assigned to this batch, or -1 on error. + */ +int64_t zerobus_arrow_stream_ingest_batch(struct CArrowStream *stream, + const uint8_t *ipc_bytes, + uintptr_t ipc_len, + struct CResult *result); + +/** + * Ingests one Arrow RecordBatch supplied as Arrow IPC stream bytes. + * + * Equivalent to `zerobus_arrow_stream_ingest_batch`. Both functions deserialise the IPC + * bytes to a `RecordBatch` and re-encode with the stream's compression settings, so + * either works regardless of whether the stream was created with compression. + * Returns the logical offset assigned to this batch, or -1 on error. + */ +int64_t zerobus_arrow_stream_ingest_batch_via_record_batch(struct CArrowStream *stream, + const uint8_t *ipc_bytes, + uintptr_t ipc_len, + struct CResult *result); + +/** + * Waits until the server acknowledges the batch at the given logical offset. + */ +bool zerobus_arrow_stream_wait_for_offset(struct CArrowStream *stream, + int64_t offset, + struct CResult *result); + +/** + * Flushes all pending batches and waits for their acknowledgment. + */ +bool zerobus_arrow_stream_flush(struct CArrowStream *stream, struct CResult *result); + +/** + * Gracefully closes the stream, flushing all pending batches first. + */ +bool zerobus_arrow_stream_close(struct CArrowStream *stream, struct CResult *result); + +/** + * Returns all unacknowledged batches from a closed or failed stream as Arrow IPC bytes. + * + * Each batch is serialized as a self-contained Arrow IPC stream (schema + one batch). + * The returned array must be freed with `zerobus_arrow_free_batch_array`. + */ +struct CArrowBatchArray zerobus_arrow_stream_get_unacked_batches(struct CArrowStream *stream, + struct CResult *result); + +/** + * Frees a `CArrowBatchArray` returned by `zerobus_arrow_stream_get_unacked_batches`. + */ +void zerobus_arrow_free_batch_array(struct CArrowBatchArray array); + +/** + * Returns whether the Arrow stream has been closed. + */ +bool zerobus_arrow_stream_is_closed(struct CArrowStream *stream); + +/** + * Returns the default Arrow stream configuration options. + */ +struct CArrowStreamConfigurationOptions zerobus_arrow_get_default_config(void); + +/** + * Free headers returned from callback + */ +void zerobus_free_headers(struct CHeaders headers); + +/** + * Allocates a new SDK builder. Must be terminated by exactly one of + * `_build` or `_free`. + */ +struct CZerobusSdkBuilder *zerobus_sdk_builder_new(void); + +/** + * Sets the Zerobus gRPC endpoint URL (required). No-op on null. + */ +void zerobus_sdk_builder_endpoint(struct CZerobusSdkBuilder *builder, const char *value); + +/** + * Sets the Unity Catalog URL. Optional with a custom headers provider. + * No-op on null. + */ +void zerobus_sdk_builder_unity_catalog_url(struct CZerobusSdkBuilder *builder, const char *value); + +/** + * Overrides the SDK prefix of the `user-agent` header (default + * `zerobus-sdk-rs/`). Wrappers pass their own identifier here. + * Null and empty values are no-ops. + */ +void zerobus_sdk_builder_sdk_identifier(struct CZerobusSdkBuilder *builder, const char *value); + +/** + * Appends an application identifier to the `user-agent` header. Wire value + * becomes ` `. Null and empty values are + * no-ops. + */ +void zerobus_sdk_builder_application_name(struct CZerobusSdkBuilder *builder, const char *value); + +/** + * Selects a no-TLS gRPC channel. TLS is on by default. + */ +void zerobus_sdk_builder_disable_tls(struct CZerobusSdkBuilder *builder); + +/** + * Consumes the builder and returns a `CZerobusSdk*`, or NULL on error. + * Frees the builder on both paths — any further use of the pointer is + * undefined behavior. Null `builder` writes an error to `result`. + */ +struct CZerobusSdk *zerobus_sdk_builder_build(struct CZerobusSdkBuilder *builder, + struct CResult *result); + +/** + * Drops an unconsumed builder. No-op on null. + */ +void zerobus_sdk_builder_free(struct CZerobusSdkBuilder *builder); + +/** + * Creates a new ZerobusSdk with default user-agent and TLS settings. + * + * Retained for ABI back-compat with v1.2.x; new code should use the + * `zerobus_sdk_builder_*` API. Does not infer TLS state from the endpoint + * scheme — callers needing a plain-HTTP channel must use the builder API. + * + * Returns NULL on error; see `result` for details. + */ +struct CZerobusSdk *zerobus_sdk_new(const char *zerobus_endpoint, + const char *unity_catalog_url, + struct CResult *result); + +/** + * Free the SDK instance + */ +void zerobus_sdk_free(struct CZerobusSdk *sdk); + +/** + * Set whether to use TLS for connections. + * + * Deprecated: This function is a no-op. TLS is now controlled via the `TlsConfig` + * trait passed to the SDK builder. This function is retained for ABI compatibility. + */ +void zerobus_sdk_set_use_tls(struct CZerobusSdk *_sdk, bool _use_tls); + +/** + * Create a stream with OAuth authentication + * descriptor_proto_bytes: protobuf-encoded DescriptorProto (can be NULL for JSON streams) + */ +struct CZerobusStream *zerobus_sdk_create_stream(struct CZerobusSdk *sdk, + const char *table_name, + const uint8_t *descriptor_proto_bytes, + uintptr_t descriptor_proto_len, + const char *client_id, + const char *client_secret, + const struct CStreamConfigurationOptions *options, + struct CResult *result); + +/** + * Create a stream with a custom headers provider callback + * This allows you to provide custom authentication headers via a Go callback function + */ +struct CZerobusStream *zerobus_sdk_create_stream_with_headers_provider(struct CZerobusSdk *sdk, + const char *table_name, + const uint8_t *descriptor_proto_bytes, + uintptr_t descriptor_proto_len, + HeadersProviderCallback headers_callback, + void *user_data, + const struct CStreamConfigurationOptions *options, + struct CResult *result); + +/** + * Free a stream instance + */ +void zerobus_stream_free(struct CZerobusStream *stream); + +/** + * Ingest a record (protobuf encoded) + * Returns the offset directly + * Returns -1 on error + */ +int64_t zerobus_stream_ingest_proto_record(struct CZerobusStream *stream, + const uint8_t *data, + uintptr_t data_len, + struct CResult *result); + +/** + * Ingest a JSON record + * Returns the offset directly + * Returns -1 on error + */ +int64_t zerobus_stream_ingest_json_record(struct CZerobusStream *stream, + const char *json_data, + struct CResult *result); + +/** + * Ingest a batch of protobuf records + * Returns the offset of the last record in the batch, or -1 on error + * Returns -2 if batch is empty + */ +int64_t zerobus_stream_ingest_proto_records(struct CZerobusStream *stream, + const uint8_t *const *records, + const uintptr_t *record_lens, + uintptr_t num_records, + struct CResult *result); + +/** + * Ingest a batch of JSON records + * Returns the offset of the last record in the batch, or -1 on error + * Returns -2 if batch is empty + */ +int64_t zerobus_stream_ingest_json_records(struct CZerobusStream *stream, + const char *const *json_records, + uintptr_t num_records, + struct CResult *result); + +/** + * Ingest a protobuf record without waiting for the record to be queued (fire-and-forget). + * + * Spawns a background task to queue the record and returns immediately. + * The result only reflects argument validation errors; ingestion errors are silently ignored. + * + * # Safety + * The stream must remain valid until all background tasks spawned by this function complete. + */ +void zerobus_stream_ingest_proto_record_nowait(struct CZerobusStream *stream, + const uint8_t *data, + uintptr_t data_len, + struct CResult *result); + +/** + * Ingest a JSON record without waiting for the record to be queued (fire-and-forget). + * + * Spawns a background task to queue the record and returns immediately. + * The result only reflects argument validation errors; ingestion errors are silently ignored. + * + * # Safety + * The stream must remain valid until all background tasks spawned by this function complete. + */ +void zerobus_stream_ingest_json_record_nowait(struct CZerobusStream *stream, + const char *json_data, + struct CResult *result); + +/** + * Ingest a batch of protobuf records without waiting (fire-and-forget). + * + * Copies all record data before spawning the background task, so the caller's + * memory is safe to release immediately after this function returns. + * + * # Safety + * The stream must remain valid until all background tasks spawned by this function complete. + */ +void zerobus_stream_ingest_proto_records_nowait(struct CZerobusStream *stream, + const uint8_t *const *records, + const uintptr_t *record_lens, + uintptr_t num_records, + struct CResult *result); + +/** + * Ingest a batch of JSON records without waiting (fire-and-forget). + * + * Copies all strings before spawning the background task, so the caller's + * memory is safe to release immediately after this function returns. + * + * # Safety + * The stream must remain valid until all background tasks spawned by this function complete. + */ +void zerobus_stream_ingest_json_records_nowait(struct CZerobusStream *stream, + const char *const *json_records, + uintptr_t num_records, + struct CResult *result); + +/** + * Wait for a specific offset to be acknowledged by the server + */ +bool zerobus_stream_wait_for_offset(struct CZerobusStream *stream, + int64_t offset, + struct CResult *result); + +/** + * Flush all pending records + */ +bool zerobus_stream_flush(struct CZerobusStream *stream, struct CResult *result); + +/** + * Get unacknowledged records from a closed stream + * Returns a CRecordArray that must be freed with zerobus_free_record_array + */ +struct CRecordArray zerobus_stream_get_unacked_records(struct CZerobusStream *stream, + struct CResult *result); + +/** + * Free a CRecordArray returned by zerobus_stream_get_unacked_records + */ +void zerobus_free_record_array(struct CRecordArray array); + +/** + * Close the stream gracefully + */ +bool zerobus_stream_close(struct CZerobusStream *stream, struct CResult *result); + +/** + * Free error message string + */ +void zerobus_free_error_message(char *message); + +/** + * Get default stream configuration options + */ +struct CStreamConfigurationOptions zerobus_get_default_config(void); + +/** + * Build a protobuf schema from Unity Catalog table metadata JSON. + * Returns NULL on error; free with `zerobus_proto_schema_free`. + */ +struct CZerobusProtoSchema *zerobus_proto_schema_from_uc_json(const char *uc_table_json, + struct CResult *result); + +/** + * Borrow the serialized descriptor bytes. Valid until `zerobus_proto_schema_free`. + * Pass directly to `zerobus_sdk_create_stream`. + * + * `out_len` is required: the bytes are not null-terminated, so the caller needs + * the length to read them. Returns NULL without touching `out_len` if it is + * NULL, and NULL with `*out_len` set to 0 on a null handle. + */ +const uint8_t *zerobus_proto_schema_descriptor_bytes(const struct CZerobusProtoSchema *schema, + uintptr_t *out_len); + +/** + * Encode JSON record to protobuf bytes. Unknown keys are ignored. + * + * Values follow protobuf's JSON mapping; a few column types need shaping: + * - DATE/TIMESTAMP/TIMESTAMP_NTZ: integers (days / micros since epoch), not strings. + * - BINARY: base64-encoded string, not a JSON array of bytes. + * - DECIMAL: string (e.g. "123.45"), to preserve precision/scale. + * - VARIANT: a JSON-encoded string (a string whose contents are the variant's JSON). + * - ARRAY/MAP/STRUCT: JSON array / object / object respectively. + * - LONG/BIGINT above 2^53: pass as a JSON string, else the value loses + * precision as a JSON number. + * + * Presence is enforced only for top-level non-nullable scalar and struct + * columns (proto2 `required`); a record omitting one fails. Non-nullable + * ARRAY/MAP columns map to `repeated`, which has no presence, so an omitted one + * encodes as empty rather than failing; required fields nested inside a STRUCT + * are likewise not presence-checked. + * Returns true on success; caller must free buffer with `zerobus_free_proto_bytes`. + * On failure `*out_data` is set to NULL and `*out_len` to 0. + */ +bool zerobus_proto_schema_encode_json(const struct CZerobusProtoSchema *schema, + const char *record_json, + uint8_t **out_data, + uintptr_t *out_len, + struct CResult *result); + +/** + * Free a buffer returned by `zerobus_proto_schema_encode_json`. + */ +void zerobus_free_proto_bytes(uint8_t *data, uintptr_t len); + +/** + * Free a handle from `zerobus_proto_schema_from_uc_json`. Call exactly once, + * after every other call using this handle has returned. The handle may be + * shared by concurrent readers (`descriptor_bytes`, `encode_json`), but `free` + * must not race any of them. + */ +void zerobus_proto_schema_free(struct CZerobusProtoSchema *schema); + +#ifdef __cplusplus +} // extern "C" +#endif // __cplusplus + +#ifdef __cplusplus +} // namespace zerobus +#endif // __cplusplus + +#endif /* ZEROBUS_H */ diff --git a/lib/zerobus-ffi-1.3.0/rust/sdk/Cargo.toml b/lib/zerobus-ffi-1.3.0/rust/sdk/Cargo.toml new file mode 100644 index 00000000000..d9bf3af9631 --- /dev/null +++ b/lib/zerobus-ffi-1.3.0/rust/sdk/Cargo.toml @@ -0,0 +1,86 @@ +[package] +name = "databricks-zerobus-ingest-sdk" +version = "2.2.2" +authors = ["Databricks"] +edition = "2021" +rust-version = "1.70" +license = "Apache-2.0" +description = "A high-performance Rust client for streaming data ingestion into Databricks Delta tables using the Zerobus service" +readme = "../README.md" +repository = "https://github.com/databricks/zerobus-sdk" +keywords = ["databricks", "delta", "streaming", "ingestion", "zerobus"] +categories = ["database", "api-bindings"] +documentation = "https://docs.rs/databricks-zerobus-ingest-sdk" + +# Build docs on docs.rs with all optional features enabled so feature-gated +# modules (arrow-flight, zeroparser) are visible on the rendered page. +[package.metadata.docs.rs] +all-features = true + +[dependencies] +async-trait.workspace = true +prost.workspace = true +prost-types.workspace = true +reqwest.workspace = true +serde.workspace = true +serde_json.workspace = true +sha2.workspace = true +thiserror.workspace = true +tokio = { workspace = true, features = ["macros", "rt-multi-thread", "fs", "sync"] } +tokio-retry.workspace = true +tokio-stream.workspace = true +tokio-util = { workspace = true, features = ["rt"] } +tonic = { workspace = true, features = ["tls-native-roots", "transport"] } +tonic-prost.workspace = true +tracing.workspace = true +hyper-http-proxy.workspace = true +hyper-util.workspace = true +smallvec.workspace = true +bytes.workspace = true +self_cell = { version = "1.2", optional = true } + +# Arrow Flight dependencies +arrow-flight = { workspace = true, optional = true } +arrow-array = { workspace = true, optional = true } +arrow-schema = { workspace = true, optional = true } +arrow-ipc = { workspace = true, optional = true, features = ["lz4", "zstd"] } +futures = { workspace = true, optional = true } + +[dev-dependencies] +tracing-subscriber.workspace = true +# zeroparser e2e + bench dev-deps +criterion = { version = "0.5", default-features = false, features = ["html_reports"] } +plotters = { version = "0.3", default-features = false, features = ["svg_backend"] } +prost-reflect = { workspace = true, features = ["serde"] } +rstest = "0.24" +strum = { version = "0.27", features = ["derive"] } + +[build-dependencies] +tonic-prost-build.workspace = true +protoc-bin-vendored.workspace = true +prost-build = { version = "0.14", optional = true } + +[features] +default = [] +# Arrow Flight is in Beta +arrow-flight = ["dep:arrow-flight", "dep:arrow-array", "dep:arrow-schema", "dep:arrow-ipc", "dep:futures"] +# Zero-copy protobuf parser. +zeroparser = ["dep:self_cell", "dep:prost-build"] +testing = [] + +[[test]] +name = "zeroparser_e2e" +path = "src/zeroparser/tests/e2e.rs" +required-features = ["zeroparser"] + +[[bench]] +name = "zeroparser_parser_bench" +path = "src/zeroparser/benches/parser_bench.rs" +harness = false +required-features = ["zeroparser"] + +[[bench]] +name = "zeroparser_bench_plot" +path = "src/zeroparser/benches/bench_plot.rs" +harness = false +required-features = ["zeroparser"] diff --git a/lib/zerobus-ffi-1.3.0/rust/sdk/LICENSE b/lib/zerobus-ffi-1.3.0/rust/sdk/LICENSE new file mode 120000 index 00000000000..30cff7403da --- /dev/null +++ b/lib/zerobus-ffi-1.3.0/rust/sdk/LICENSE @@ -0,0 +1 @@ +../../LICENSE \ No newline at end of file diff --git a/lib/zerobus-ffi-1.3.0/rust/sdk/build.rs b/lib/zerobus-ffi-1.3.0/rust/sdk/build.rs new file mode 100644 index 00000000000..3e080ecc4ae --- /dev/null +++ b/lib/zerobus-ffi-1.3.0/rust/sdk/build.rs @@ -0,0 +1,14 @@ +use std::env; + +#[cfg(feature = "zeroparser")] +#[path = "src/zeroparser/proto_build.rs"] +mod zeroparser_proto_build; + +fn main() { + env::set_var("PROTOC", protoc_bin_vendored::protoc_bin_path().unwrap()); + tonic_prost_build::compile_protos("zerobus_service.proto") + .unwrap_or_else(|e| panic!("Failed to compile protos {:?}", e)); + + #[cfg(feature = "zeroparser")] + zeroparser_proto_build::compile(); +} diff --git a/lib/zerobus-ffi-1.3.0/rust/sdk/src/arrow_configuration.rs b/lib/zerobus-ffi-1.3.0/rust/sdk/src/arrow_configuration.rs new file mode 100644 index 00000000000..0d02840d4e5 --- /dev/null +++ b/lib/zerobus-ffi-1.3.0/rust/sdk/src/arrow_configuration.rs @@ -0,0 +1,133 @@ +//! Configuration options for Arrow Flight streams. +//! +//! **Beta**: Arrow Flight ingestion is in Beta. The API is stabilising but may +//! still change before reaching GA. + +use crate::stream_options::defaults; +use arrow_ipc::CompressionType; + +/// Configuration options for Arrow Flight stream creation and operation. +/// +/// These options control the behavior of Arrow Flight ingestion streams, including +/// backpressure limits, timeout settings, and recovery policies. +/// +/// **Do not construct this directly.** Configure Arrow streams via the builder API: +/// +/// ```rust,ignore +/// let stream = sdk +/// .stream_builder() +/// .table("catalog.schema.table") +/// .oauth("client-id", "client-secret") +/// .arrow(schema) +/// .max_inflight_batches(100) +/// .server_lack_of_ack_timeout_ms(30_000) +/// .recovery(true) +/// .build_arrow() +/// .await?; +/// ``` +#[derive(Clone, Debug)] +#[non_exhaustive] +pub struct ArrowStreamConfigurationOptions { + /// Maximum number of batches that can be in-flight (sent but not acknowledged). + /// + /// This limit controls memory usage and backpressure. When this limit is reached, + /// `ingest_batch()` calls will block until acknowledgments free up space. + /// + /// Default: 1,000 + pub max_inflight_batches: usize, + + /// Whether to enable automatic stream recovery on failure. + /// + /// When enabled, the SDK will automatically attempt to reconnect and recover + /// the stream when encountering retryable errors. + /// + /// Default: `true` + pub recovery: bool, + + /// Timeout in milliseconds for each stream recovery attempt. + /// + /// If a recovery attempt takes longer than this, it will be retried. + /// + /// Default: 15,000 (15 seconds) + pub recovery_timeout_ms: u64, + + /// Backoff time in milliseconds between stream recovery retry attempts. + /// + /// The SDK will wait this duration before attempting another recovery after a failure. + /// + /// Default: 2,000 (2 seconds) + pub recovery_backoff_ms: u64, + + /// Maximum number of recovery retry attempts before giving up. + /// + /// After this many failed attempts, the stream will close and return an error. + /// + /// Default: 4 + pub recovery_retries: u32, + + /// Timeout in milliseconds for waiting for server acknowledgements. + /// + /// If no acknowledgement is received within this time (and there are pending batches), + /// the stream will be considered failed and recovery will be triggered (if enabled). + /// + /// Default: 60,000 (60 seconds) + pub server_lack_of_ack_timeout_ms: u64, + + /// Timeout in milliseconds for flush operations. + /// + /// If a `flush()` call cannot complete within this time, it will return a timeout error. + /// + /// Default: 300,000 (5 minutes) + pub flush_timeout_ms: u64, + + /// Timeout in milliseconds for stream connection establishment. + /// + /// If the Arrow Flight stream cannot be established within this time, + /// stream creation will fail. + /// + /// Default: 30,000 (30 seconds) + pub connection_timeout_ms: u64, + + /// Optional Arrow IPC compression for Flight payloads. + /// + /// Supported compression types from `arrow_ipc::CompressionType`: + /// - `CompressionType::LZ4_FRAME` - LZ4 frame compression + /// - `CompressionType::ZSTD` - Zstandard compression + /// + /// Default: `None` + pub ipc_compression: Option, + + /// Maximum time in milliseconds to wait during graceful stream close. + /// + /// When the server sends a close stream signal indicating it will close the stream, + /// the SDK enters a "paused" state where it: + /// - Continues accepting and buffering new `ingest_batch()` calls + /// - Stops sending buffered batches to the server + /// - Continues processing acknowledgments for in-flight batches + /// - Waits for either all in-flight batches to be acknowledged or the timeout to expire + /// + /// Configuration values: + /// - `None`: Wait for the full server-specified duration (most graceful) + /// - `Some(0)`: Immediate recovery, close stream right away + /// - `Some(x)`: Wait up to min(x, server_duration) milliseconds + /// + /// Default: `None` (wait for full server duration) + pub stream_paused_max_wait_time_ms: Option, +} + +impl Default for ArrowStreamConfigurationOptions { + fn default() -> Self { + Self { + max_inflight_batches: 1_000, + recovery: defaults::RECOVERY, + recovery_timeout_ms: defaults::RECOVERY_TIMEOUT_MS, + recovery_backoff_ms: defaults::RECOVERY_BACKOFF_MS, + recovery_retries: defaults::RECOVERY_RETRIES, + server_lack_of_ack_timeout_ms: defaults::SERVER_LACK_OF_ACK_TIMEOUT_MS, + flush_timeout_ms: defaults::FLUSH_TIMEOUT_MS, + connection_timeout_ms: defaults::CONNECTION_TIMEOUT_MS, + ipc_compression: None, + stream_paused_max_wait_time_ms: None, + } + } +} diff --git a/lib/zerobus-ffi-1.3.0/rust/sdk/src/arrow_metadata.rs b/lib/zerobus-ffi-1.3.0/rust/sdk/src/arrow_metadata.rs new file mode 100644 index 00000000000..020c4e12c27 --- /dev/null +++ b/lib/zerobus-ffi-1.3.0/rust/sdk/src/arrow_metadata.rs @@ -0,0 +1,189 @@ +//! Metadata types for Arrow Flight offset tracking and acknowledgements. +//! +//! Arrow Flight uses the `app_metadata` field in `FlightData` and `PutResult` +//! messages to carry application-specific metadata. We use this for offset +//! tracking (similar to the existing gRPC/HTTP APIs). + +use serde::{Deserialize, Serialize}; + +use crate::errors::ZerobusError; +use crate::offset_generator::OffsetId; +use crate::ZerobusResult; + +/// Sentinel offset value indicating stream setup is complete but no batches have been acked yet. +/// The server sends this as the first `PutResult` after successful setup. +pub const STREAM_READY_OFFSET: OffsetId = -1; + +/// Metadata sent with each FlightData batch from the client. +/// +/// This is serialized to JSON and sent in `FlightData.app_metadata`. +/// The offset_id tracks the position in the stream for acknowledgements. +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct FlightBatchMetadata { + /// Client-provided offset ID for this batch. + /// Must be strictly sequential starting from 0 (0, 1, 2, ...). + pub offset_id: OffsetId, +} + +impl FlightBatchMetadata { + /// Create new batch metadata with the given offset ID. + pub fn new(offset_id: OffsetId) -> Self { + Self { offset_id } + } + + /// Serialize to JSON bytes for FlightData.app_metadata. + #[allow(clippy::result_large_err)] + pub fn to_bytes(&self) -> ZerobusResult> { + serde_json::to_vec(self).map_err(|e| { + ZerobusError::InvalidArgument(format!("Failed to serialize FlightBatchMetadata: {}", e)) + }) + } + + /// Deserialize from FlightData.app_metadata bytes. + #[cfg(test)] + #[allow(clippy::result_large_err)] + pub fn from_bytes(bytes: &[u8]) -> ZerobusResult { + if bytes.is_empty() { + return Err(ZerobusError::InvalidArgument( + "Empty app_metadata in FlightData - client must provide offset_id".to_string(), + )); + } + serde_json::from_slice(bytes).map_err(|e| { + ZerobusError::InvalidArgument(format!("Failed to parse FlightBatchMetadata: {}", e)) + }) + } +} + +/// Acknowledgement metadata sent back to the client. +/// +/// This is serialized to JSON and sent in `PutResult.app_metadata`. +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct FlightAckMetadata { + /// Offset ID up to which records are durably stored. + /// All records with offset_id <= this value are guaranteed durable. + /// + /// Special value [`STREAM_READY_OFFSET`] indicates a "ready" signal: stream setup succeeded + /// (schema validation, table access) but no batches acked yet. + /// The SDK waits for this signal during stream creation. + pub ack_up_to_offset: OffsetId, + /// Cumulative count of records durably stored up to this acknowledgment. + pub ack_up_to_records: u64, + /// Optional close stream signal with grace period duration in milliseconds. + /// + /// When present, the server is signaling that it will close the stream after + /// this duration. The client should enter a "paused" state: stop sending new + /// batches but continue processing acknowledgments for in-flight batches. + /// After the grace period (or when all in-flight batches are acked), the client + /// triggers stream recovery. + #[serde(skip_serializing_if = "Option::is_none")] + #[serde(default)] + pub close_stream_duration_ms: Option, +} + +impl FlightAckMetadata { + /// Create new acknowledgement metadata. + #[allow(dead_code)] + pub fn new(ack_up_to_offset: OffsetId, ack_up_to_records: u64) -> Self { + Self { + ack_up_to_offset, + ack_up_to_records, + close_stream_duration_ms: None, + } + } + + /// Create a "stream ready" signal indicating successful stream setup. + /// Uses [`STREAM_READY_OFFSET`] as the offset value. + #[allow(dead_code)] + pub fn stream_ready() -> Self { + Self { + ack_up_to_offset: STREAM_READY_OFFSET, + ack_up_to_records: 0, + close_stream_duration_ms: None, + } + } + + /// Returns true if this message contains a close stream signal. + pub fn is_close_signal(&self) -> bool { + self.close_stream_duration_ms.is_some() + } + + /// Returns true if this is a stream ready signal (setup complete, no batches acked). + pub fn is_stream_ready(&self) -> bool { + self.ack_up_to_offset == STREAM_READY_OFFSET + } + + /// Deserialize from PutResult.app_metadata bytes. + #[allow(clippy::result_large_err)] + pub fn from_bytes(bytes: &[u8]) -> ZerobusResult { + if bytes.is_empty() { + return Err(ZerobusError::InvalidArgument( + "Empty app_metadata in PutResult - server should provide ack offset".to_string(), + )); + } + serde_json::from_slice(bytes).map_err(|e| { + ZerobusError::InvalidArgument(format!("Failed to parse FlightAckMetadata: {}", e)) + }) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_flight_batch_metadata_roundtrip() { + let metadata = FlightBatchMetadata::new(42); + let bytes = metadata.to_bytes().unwrap(); + let parsed = FlightBatchMetadata::from_bytes(&bytes).unwrap(); + assert_eq!(parsed.offset_id, 42); + } + + #[test] + fn test_flight_batch_metadata_parse() { + let json = r#"{"offset_id": 42}"#; + let parsed = FlightBatchMetadata::from_bytes(json.as_bytes()).unwrap(); + assert_eq!(parsed.offset_id, 42); + } + + #[test] + fn test_flight_batch_metadata_empty_error() { + let result = FlightBatchMetadata::from_bytes(&[]); + assert!(result.is_err()); + } + + #[test] + fn test_flight_ack_metadata_parse_with_records() { + let json = r#"{"ack_up_to_offset": 99, "ack_up_to_records": 5000}"#; + let parsed = FlightAckMetadata::from_bytes(json.as_bytes()).unwrap(); + assert_eq!(parsed.ack_up_to_offset, 99); + assert_eq!(parsed.ack_up_to_records, 5000); + assert!(parsed.close_stream_duration_ms.is_none()); + assert!(!parsed.is_close_signal()); + } + + #[test] + fn test_flight_ack_metadata_with_close_signal() { + let json = r#"{"ack_up_to_offset": 5, "ack_up_to_records": 100, "close_stream_duration_ms": 2000}"#; + let parsed = FlightAckMetadata::from_bytes(json.as_bytes()).unwrap(); + assert_eq!(parsed.ack_up_to_offset, 5); + assert_eq!(parsed.ack_up_to_records, 100); + assert_eq!(parsed.close_stream_duration_ms, Some(2000)); + assert!(parsed.is_close_signal()); + } + + #[test] + fn test_flight_ack_metadata_close_signal_only() { + let json = + r#"{"ack_up_to_offset": -1, "ack_up_to_records": 0, "close_stream_duration_ms": 5000}"#; + let parsed = FlightAckMetadata::from_bytes(json.as_bytes()).unwrap(); + assert!(parsed.is_close_signal()); + assert!(parsed.is_stream_ready()); + assert_eq!(parsed.close_stream_duration_ms, Some(5000)); + } + + #[test] + fn test_flight_ack_metadata_empty_error() { + let result = FlightAckMetadata::from_bytes(&[]); + assert!(result.is_err()); + } +} diff --git a/lib/zerobus-ffi-1.3.0/rust/sdk/src/arrow_stream.rs b/lib/zerobus-ffi-1.3.0/rust/sdk/src/arrow_stream.rs new file mode 100644 index 00000000000..70ee1d266d7 --- /dev/null +++ b/lib/zerobus-ffi-1.3.0/rust/sdk/src/arrow_stream.rs @@ -0,0 +1,1711 @@ +//! Arrow Flight stream implementation for high-performance Arrow data ingestion. +//! +//! **Beta**: This module is in Beta. The API is stabilising but may still change +//! before reaching GA. +//! +//! This module provides `ZerobusArrowStream`, a client for ingesting Arrow `RecordBatch` +//! data into Databricks Delta tables using the Arrow Flight protocol. +//! Native Rust callers use `ingest_batch` with `RecordBatch` values; FFI callers +//! (Go, Python, Java, TypeScript) can use `ingest_ipc_batch` with pre-serialised +//! Arrow IPC bytes. + +use std::pin::Pin; +use std::sync::atomic::{AtomicBool, AtomicU32, AtomicU64, Ordering}; +use std::sync::Arc; + +use arrow_flight::encode::FlightDataEncoderBuilder; +use arrow_flight::error::FlightError; +use arrow_flight::{FlightClient, PutResult}; +use arrow_ipc::writer::IpcWriteOptions; +use bytes::Bytes; +use futures::{Stream, StreamExt}; +use tokio::sync::{mpsc, watch, Mutex}; +use tokio::time::{sleep, Duration}; +use tokio_retry::strategy::FixedInterval; +use tokio_retry::RetryIf; +use tonic::transport::Channel; +use tracing::{debug, error, info, instrument, warn}; + +// Re-export arrow types for public API +pub use arrow_array::RecordBatch; +pub use arrow_schema::{DataType, Field, Schema as ArrowSchema, TimeUnit}; + +use crate::arrow_configuration::ArrowStreamConfigurationOptions; +use crate::arrow_metadata::{FlightAckMetadata, FlightBatchMetadata}; +use crate::errors::ZerobusError; +use crate::headers_provider::HeadersProvider; +use crate::offset_generator::{OffsetId, OffsetIdGenerator}; +use crate::tls_config::TlsConfig; +use crate::ZerobusResult; + +/// Type alias for the batch sender channel, wrapped for thread-safe sharing. +type BatchSender = Arc>>>>; + +/// Properties for an Arrow Flight ingestion table. +/// +/// **Do not construct this directly.** Configure Arrow streams via the builder API: +/// `sdk.stream_builder().table("catalog.schema.table").arrow(schema)`. +#[derive(Debug, Clone)] +pub(crate) struct ArrowTableProperties { + /// The fully qualified table name (e.g., "catalog.schema.table"). + pub(crate) table_name: String, + /// The Arrow schema for the data being ingested. + /// This is used to validate RecordBatches before sending and is sent + /// as the first message in the Flight stream. + pub(crate) schema: Arc, +} + +/// A pending batch waiting for acknowledgment. +#[derive(Clone)] +struct PendingBatch { + batch: RecordBatch, + /// Offset ID assigned by the client for this batch. + offset_id: OffsetId, + /// Cumulative record count before this batch. + start_record: u64, + /// Cumulative record count after this batch. + /// Batch is fully acked when `acked_records >= end_record`. + end_record: u64, +} + +/// Returns the portion of a batch that needs to be replayed after recovery. +/// +/// - If batch is fully acked: returns `None` +/// - If batch is partially acked: returns sliced batch with only un-acked records +/// - If batch is fully un-acked: returns the full batch +fn slice_batch_for_recovery( + pb: &PendingBatch, + acked_before_disconnect: u64, +) -> Option { + if pb.start_record >= acked_before_disconnect { + // Fully un-acked + return Some(pb.batch.clone()); + } + + let records_already_acked = + (acked_before_disconnect - pb.start_record).min(pb.batch.num_rows() as u64); + let remaining_rows = pb + .batch + .num_rows() + .saturating_sub(records_already_acked as usize); + + if remaining_rows == 0 { + // Fully acked + None + } else { + // Partially acked - slice to get un-acked portion + debug!( + offset_id = pb.offset_id, + total_rows = pb.batch.num_rows(), + records_already_acked = records_already_acked, + remaining_rows = remaining_rows, + "Slicing partially-acked batch for recovery" + ); + Some( + pb.batch + .slice(records_already_acked as usize, remaining_rows), + ) + } +} + +/// Deserialises Arrow IPC stream bytes into a [`RecordBatch`]. +#[allow(clippy::result_large_err)] +fn materialize_ipc(bytes: &Bytes) -> ZerobusResult { + use std::io::Cursor; + let mut reader = arrow_ipc::reader::StreamReader::try_new(Cursor::new(bytes.as_ref()), None) + .map_err(|e| { + ZerobusError::InvalidArgument(format!("IPC: invalid Arrow IPC stream: {e}")) + })?; + let batch = match reader.next() { + None => { + return Err(ZerobusError::InvalidArgument( + "IPC stream contains no RecordBatch".into(), + )); + } + Some(Err(e)) => { + return Err(ZerobusError::InvalidArgument(format!( + "IPC: record batch read failed: {e}" + ))); + } + Some(Ok(b)) => b, + }; + match reader.next() { + None => Ok(batch), + Some(Ok(_)) => Err(ZerobusError::InvalidArgument( + "IPC stream must contain exactly one RecordBatch (found extra batch)".into(), + )), + Some(Err(e)) => Err(ZerobusError::InvalidArgument(format!( + "IPC: trailing message read failed: {e}" + ))), + } +} + +/// Builds [`IpcWriteOptions`] for the given optional compression codec. +#[allow(clippy::result_large_err)] +fn make_ipc_write_options( + compression: Option, +) -> ZerobusResult { + match compression { + None => Ok(IpcWriteOptions::default()), + Some(c) => IpcWriteOptions::default() + .try_with_compression(Some(c)) + .map_err(|e| { + ZerobusError::InvalidArgument(format!( + "Failed to enable Arrow IPC compression: {e}" + )) + }), + } +} + +/// An Arrow Flight stream for ingesting Arrow RecordBatches into a Delta table. +/// +/// This stream provides a high-performance interface for streaming Arrow data +/// to Databricks Delta tables using the Arrow Flight protocol. +/// +/// # Lifecycle +/// +/// 1. Create a stream via `ZerobusSdk::create_arrow_stream()` +/// 2. Ingest RecordBatches with `ingest_batch()` and await acknowledgments +/// 3. Optionally call `flush()` to ensure all batches are persisted +/// 4. Close the stream with `close()` to release resources +/// +/// # Recovery +/// +/// When recovery is enabled (default), the stream will automatically attempt to +/// reconnect and replay unacknowledged batches on transient failures. If recovery +/// fails after the configured number of retries, use `get_unacked_batches()` to +/// retrieve the failed batches for manual handling. +/// +/// # Examples +/// +/// ```no_run +/// # use databricks_zerobus_ingest_sdk::*; +/// # use arrow_array::RecordBatch; +/// # async fn example(mut stream: ZerobusArrowStream, batch: RecordBatch) -> Result<(), ZerobusError> { +/// // Ingest a single RecordBatch +/// let offset = stream.ingest_batch(batch).await?; +/// println!("Batch queued at offset: {}", offset); +/// +/// // Wait for acknowledgment +/// stream.wait_for_offset(offset).await?; +/// println!("Batch acknowledged at offset: {}", offset); +/// +/// // Close the stream gracefully +/// stream.close().await?; +/// # Ok(()) +/// # } +/// ``` +#[non_exhaustive] +pub struct ZerobusArrowStream { + /// Table properties including name and schema. + pub(crate) table_properties: ArrowTableProperties, + /// Configuration options for this stream. + pub(crate) options: ArrowStreamConfigurationOptions, + /// Channel to send RecordBatches to the encoder task. + batch_tx: BatchSender, + /// Generator for offset IDs returned from `ingest_batch` / `ingest_ipc_batch`. + offset_generator: OffsetIdGenerator, + /// Watch channel for tracking the last acknowledged offset. + last_ack_tx: tokio::sync::watch::Sender>, + /// Receiver for the watch channel (kept alive to prevent sender errors). + _last_ack_rx: tokio::sync::watch::Receiver>, + /// Flag indicating if the stream has been closed. + is_closed: Arc, + /// Handle to the receiver task processing server responses. + receiver_task: Arc>>>>, + /// Batches that have been sent but not yet acknowledged (for recovery). + pending_batches: Arc>>, + /// Batches that failed and couldn't be recovered. + failed_batches: Arc>>, + /// Count of recovery attempts. + recovery_attempts: Arc, + /// Connection details for recovery. + endpoint: String, + /// TLS configuration for the connection. + tls_config: Arc, + headers_provider: Arc, + /// Synchronization mutex for serializing ingest operations. + ingest_mutex: Arc>, + /// Last error received from the server (watch channel for race-free access). + /// When process_acks receives a server error, it sends to this channel. + /// When ingest_batch has a send failure, it can immediately check the current value. + server_error_tx: watch::Sender>, + server_error_rx: watch::Receiver>, + /// Cumulative count of records sent (for record-based ack tracking). + cumulative_records_sent: Arc, + /// Last acknowledged cumulative record count (for recovery slicing). + last_acked_records: Arc, + /// Flag indicating the stream is paused due to a server close signal. + /// When true, new `ingest_batch()` calls are still accepted and buffered, + /// but the receiver continues draining in-flight acks before triggering recovery. + is_paused: Arc, + /// Final value sent as the HTTP `user-agent` header on every request. + /// Either `"zerobus-sdk-rs/"` or `"zerobus-sdk-rs/ "`. + /// Re-applied to each fresh Channel built during recovery. + sdk_identifier: Arc, +} + +impl ZerobusArrowStream { + /// Creates a new Arrow Flight stream. + /// + /// This is typically called internally by `ZerobusSdk::create_arrow_stream()`. + /// + /// If `recovery` is enabled in options, initial connection will be retried + /// up to `recovery_retries` times with `recovery_backoff_ms` delay between attempts. + #[instrument(level = "debug", skip_all, fields(table_name = %table_properties.table_name))] + pub(crate) async fn new( + endpoint: &str, + tls_config: Arc, + table_properties: ArrowTableProperties, + headers_provider: Arc, + options: ArrowStreamConfigurationOptions, + sdk_identifier: Arc, + ) -> ZerobusResult { + let (last_ack_tx, _last_ack_rx) = tokio::sync::watch::channel(None); + let is_closed = Arc::new(AtomicBool::new(false)); + let pending_batches = Arc::new(Mutex::new(Vec::new())); + let failed_batches = Arc::new(Mutex::new(Vec::new())); + let recovery_attempts = Arc::new(AtomicU32::new(0)); + let batch_tx = Arc::new(Mutex::new(None)); + let receiver_task = Arc::new(Mutex::new(None)); + let cumulative_records_sent = Arc::new(AtomicU64::new(0)); + let last_acked_records = Arc::new(AtomicU64::new(0)); + let is_paused = Arc::new(AtomicBool::new(false)); + + let (server_error_tx, server_error_rx) = watch::channel(None); + + let stream = Self { + table_properties, + options, + batch_tx, + offset_generator: OffsetIdGenerator::default(), + last_ack_tx, + _last_ack_rx, + is_closed, + receiver_task, + pending_batches, + failed_batches, + recovery_attempts, + endpoint: endpoint.to_string(), + tls_config, + headers_provider, + ingest_mutex: Arc::new(Mutex::new(())), + server_error_tx, + server_error_rx, + cumulative_records_sent, + last_acked_records, + is_paused, + sdk_identifier, + }; + + // Initialize the connection with retry logic. + let endpoint = stream.endpoint.clone(); + let tls_config = Arc::clone(&stream.tls_config); + let table_properties = stream.table_properties.clone(); + let options = stream.options.clone(); + let headers_provider = Arc::clone(&stream.headers_provider); + let strategy = FixedInterval::from_millis(options.recovery_backoff_ms) + .take(options.recovery_retries as usize); + + let create_attempt = || { + let endpoint = endpoint.clone(); + let tls_config = Arc::clone(&tls_config); + let table_properties = table_properties.clone(); + let options = options.clone(); + let headers_provider = Arc::clone(&headers_provider); + let sdk_identifier = Arc::clone(&stream.sdk_identifier); + + async move { + tokio::time::timeout( + Duration::from_millis(options.recovery_timeout_ms), + Self::try_connect( + &endpoint, + &tls_config, + &table_properties, + &options, + &headers_provider, + &sdk_identifier, + ), + ) + .await + .map_err(|_| { + ZerobusError::CreateStreamError(tonic::Status::deadline_exceeded( + "Stream creation timed out", + )) + })? + } + }; + let should_retry = |e: &ZerobusError| options.recovery && e.is_retryable(); + let creation = RetryIf::spawn(strategy, create_attempt, should_retry).await; + + let (response_stream, tx) = match creation { + Ok(result) => result, + Err(e) => { + error!("Arrow Flight stream creation failed after retries: {}", e); + return Err(e); + } + }; + + // Store the sender. + { + let mut batch_tx = stream.batch_tx.lock().await; + *batch_tx = Some(tx); + } + + // Spawn the supervisor task. + let task = Self::spawn_supervisor_task( + stream.endpoint.clone(), + Arc::clone(&stream.tls_config), + stream.table_properties.clone(), + stream.options.clone(), + Arc::clone(&stream.headers_provider), + Arc::clone(&stream.batch_tx), + Arc::clone(&stream.is_closed), + stream.last_ack_tx.clone(), + Arc::clone(&stream.pending_batches), + Arc::clone(&stream.failed_batches), + Arc::clone(&stream.recovery_attempts), + stream.server_error_tx.clone(), + Arc::clone(&stream.cumulative_records_sent), + Arc::clone(&stream.last_acked_records), + Arc::clone(&stream.is_paused), + Arc::clone(&stream.ingest_mutex), + response_stream, + Arc::clone(&stream.sdk_identifier), + ); + + { + let mut receiver_task = stream.receiver_task.lock().await; + *receiver_task = Some(task); + } + + info!( + table_name = %stream.table_properties.table_name, + "Arrow Flight stream created successfully" + ); + + Ok(stream) + } + + /// Attempts to establish a Flight connection. + /// Returns the response stream and batch sender on success. + async fn try_connect( + endpoint: &str, + tls_config: &Arc, + table_properties: &ArrowTableProperties, + options: &ArrowStreamConfigurationOptions, + headers_provider: &Arc, + sdk_identifier: &str, + ) -> ZerobusResult<( + Pin> + Send>>, + mpsc::Sender>, + )> { + let client = Self::create_flight_client( + endpoint, + tls_config, + table_properties, + options, + headers_provider, + sdk_identifier, + ) + .await?; + + let result = Self::start_stream_connection(client, table_properties, options).await; + + // Drop the rejected token so the next attempt re-mints. + if let Err(err) = &result { + if err.is_auth_rejection() { + headers_provider.invalidate().await; + } + } + result + } + + /// Creates a Flight client connected to the endpoint. + async fn create_flight_client( + endpoint: &str, + tls_config: &Arc, + table_properties: &ArrowTableProperties, + options: &ArrowStreamConfigurationOptions, + headers_provider: &Arc, + sdk_identifier: &str, + ) -> ZerobusResult { + let connection_timeout = Duration::from_millis(options.connection_timeout_ms); + + let base_endpoint = Channel::from_shared(endpoint.to_string()) + .map_err(|e| ZerobusError::ChannelCreationError(e.to_string()))? + .user_agent(sdk_identifier) + .map_err(|e| ZerobusError::ChannelCreationError(e.to_string()))? + .connect_timeout(connection_timeout) + .timeout(connection_timeout); + + let channel = tls_config.configure_endpoint(base_endpoint)?.connect_lazy(); + + let mut client = FlightClient::new(channel); + + // Add headers from the provider first, filtering out reserved headers. + // The table name header is authoritative and must not be overridden. + const TABLE_NAME_HEADER: &str = "x-databricks-zerobus-table-name"; + let headers = headers_provider.get_headers().await?; + for (key, value) in headers { + if key.eq_ignore_ascii_case(TABLE_NAME_HEADER) { + warn!( + "HeadersProvider attempted to set reserved header '{}', ignoring", + TABLE_NAME_HEADER + ); + continue; + } + client.add_header(key, &value).map_err(|e| { + ZerobusError::InvalidArgument(format!("Failed to add header '{}': {}", key, e)) + })?; + } + + // Add the required table name header (authoritative, added last to ensure it's set). + client + .add_header(TABLE_NAME_HEADER, &table_properties.table_name) + .map_err(|e| { + ZerobusError::InvalidArgument(format!("Failed to add table name header: {}", e)) + })?; + + Ok(client) + } + + /// Starts the Flight stream with the given client. + /// Returns the response stream and batch sender for use by the supervisor. + /// + /// This method waits for the server's "ready" signal (ack_up_to_offset = -1) + /// to confirm that stream setup succeeded (auth, schema validation, table access). + /// This allows setup errors to be detected during stream creation rather than + /// later during batch ingestion. + async fn start_stream_connection( + mut client: FlightClient, + table_properties: &ArrowTableProperties, + options: &ArrowStreamConfigurationOptions, + ) -> ZerobusResult<( + Pin> + Send>>, + mpsc::Sender>, + )> { + // Create channel for sending RecordBatches. + let (batch_tx, batch_rx) = + mpsc::channel::>(options.max_inflight_batches); + + let ipc_write_options = make_ipc_write_options(options.ipc_compression)?; + let schema = Arc::clone(&table_properties.schema); + let batch_stream = tokio_stream::wrappers::ReceiverStream::new(batch_rx); + + // Build the Flight data stream. FlightDataEncoderBuilder handles schema + // framing, dictionary encoding, and automatic batch chunking at 2 MiB. + // Each non-schema FlightData message gets a sequential wire offset in + // its app_metadata (index 0 is the schema message; data messages start at 1). + let offset_counter = Arc::new(std::sync::atomic::AtomicI64::new(0)); + let offset_counter_clone = Arc::clone(&offset_counter); + let flight_data_stream = FlightDataEncoderBuilder::new() + .with_schema(schema) + .with_options(ipc_write_options) + .build(batch_stream) + .enumerate() + .map(move |(idx, result)| { + result.map(|mut flight_data| { + // Skip schema message (idx 0); add metadata to data messages. + if idx > 0 { + let offset = + offset_counter_clone.fetch_add(1, std::sync::atomic::Ordering::Relaxed); + let metadata = FlightBatchMetadata::new(offset); + if let Ok(bytes) = metadata.to_bytes() { + flight_data.app_metadata = bytes.into(); + } + } + flight_data + }) + }); + + // Start the DoPut stream. + let mut response_stream = client + .do_put(flight_data_stream) + .await + // `.into()` preserves the inner gRPC code; `Status::from_error` would + // flatten it to `Unknown` and break auth/retry classification. + .map_err(|e| ZerobusError::CreateStreamError(e.into()))?; + + // Wait for server's "ready" signal to confirm setup succeeded. + // The server sends ack_up_to_offset = -1 after successful auth, schema validation, + // and stream setup. This allows us to detect setup errors early. + let setup_timeout = Duration::from_millis(options.connection_timeout_ms); + match tokio::time::timeout(setup_timeout, response_stream.next()).await { + Ok(Some(Ok(put_result))) => { + // Parse the ack metadata to verify it's the ready signal. + match FlightAckMetadata::from_bytes(&put_result.app_metadata) { + Ok(metadata) if metadata.is_stream_ready() => { + info!("Stream setup confirmed by server (ready signal received)"); + } + Ok(metadata) => { + // Unexpected: got a real ack before sending any batches - protocol error. + error!( + "Unexpected ack during setup (offset {}), expected ready signal", + metadata.ack_up_to_offset + ); + return Err(ZerobusError::UnexpectedStreamResponseError(format!( + "Expected ready signal, got ack for offset {}", + metadata.ack_up_to_offset + ))); + } + Err(e) => { + // Malformed metadata - protocol error. + error!("Failed to parse setup response metadata: {}", e); + return Err(ZerobusError::UnexpectedStreamResponseError(format!( + "Malformed setup response metadata: {}", + e + ))); + } + } + } + Ok(Some(Err(flight_error))) => { + // Server sent an error during setup (auth failed, schema mismatch, blocked table, etc.) + error!("Stream setup failed: {:?}", flight_error); + return Err(ZerobusError::CreateStreamError(flight_error.into())); + } + Ok(None) => { + // Server closed the stream without sending anything. + error!("Server closed stream during setup without response"); + return Err(ZerobusError::StreamClosedError(tonic::Status::internal( + "Server closed stream during setup", + ))); + } + Err(_timeout) => { + // Timeout waiting for server response. + error!( + "Timed out waiting for server setup confirmation ({}ms)", + options.connection_timeout_ms + ); + return Err(ZerobusError::ConnectionTimeout(format!( + "Timed out waiting for server setup confirmation ({}ms)", + options.connection_timeout_ms + ))); + } + } + + Ok((response_stream, batch_tx)) + } + + /// Spawns the supervisor task that manages the stream lifecycle and recovery. + /// + /// The supervisor runs a loop that: + /// 1. Processes acknowledgments from the server + /// 2. When the ack processor returns with a retriable error, attempts recovery + /// 3. Continues until stream is closed or max retries exceeded + #[allow(clippy::too_many_arguments)] + fn spawn_supervisor_task( + endpoint: String, + tls_config: Arc, + table_properties: ArrowTableProperties, + options: ArrowStreamConfigurationOptions, + headers_provider: Arc, + batch_tx: BatchSender, + is_closed: Arc, + last_ack_tx: tokio::sync::watch::Sender>, + pending_batches: Arc>>, + failed_batches: Arc>>, + recovery_attempts: Arc, + server_error_tx: watch::Sender>, + cumulative_records_sent: Arc, + last_acked_records: Arc, + is_paused: Arc, + ingest_mutex: Arc>, + initial_response_stream: Pin> + Send>>, + sdk_identifier: Arc, + ) -> tokio::task::JoinHandle> { + tokio::spawn(async move { + let ack_timeout = Duration::from_millis(options.server_lack_of_ack_timeout_ms); + let mut response_stream = initial_response_stream; + + loop { + if is_closed.load(Ordering::Relaxed) { + debug!("Supervisor: Stream closed, exiting"); + return Ok(()); + } + + // Run process_acks until it returns (error or stream closed). + let result = Self::process_acks( + response_stream, + Arc::clone(&is_closed), + last_ack_tx.clone(), + Arc::clone(&pending_batches), + ack_timeout, + server_error_tx.clone(), + Arc::clone(&last_acked_records), + Arc::clone(&is_paused), + &options, + ) + .await; + + // Check if stream was closed during processing. + if is_closed.load(Ordering::Relaxed) { + debug!("Supervisor: Stream closed after process_acks, exiting"); + return result; + } + + // Handle the result. + match result { + Ok(()) => { + // Stream ended gracefully. + debug!("Supervisor: process_acks completed successfully"); + return Ok(()); + } + Err(ref error) if error.is_retryable() && options.recovery => { + // Retriable error - attempt recovery. + let attempts = recovery_attempts.fetch_add(1, Ordering::Relaxed); + if attempts >= options.recovery_retries { + error!( + attempts = attempts, + max_retries = options.recovery_retries, + "Supervisor: Max recovery retries exceeded" + ); + is_closed.store(true, Ordering::Relaxed); + // Move pending batches to failed and fail the ack futures. + Self::move_pending_to_failed(&pending_batches, &failed_batches).await; + return result; + } + + info!( + attempt = attempts + 1, + max_retries = options.recovery_retries, + error = %error, + "Supervisor: Attempting recovery after retriable error" + ); + + // Pause ingest before reconnect; gate is lifted inside reconnect(). + is_paused.store(true, Ordering::Relaxed); + + // Backoff before retry. + sleep(Duration::from_millis(options.recovery_backoff_ms)).await; + + // Clear the server error. + let _ = server_error_tx.send(None); + + // Close old sender. + { + let mut tx_guard = batch_tx.lock().await; + *tx_guard = None; + } + + // Create new connection. + let reconnect_result = tokio::time::timeout( + Duration::from_millis(options.recovery_timeout_ms), + Self::reconnect( + &endpoint, + &tls_config, + &table_properties, + &options, + &headers_provider, + &batch_tx, + &pending_batches, + &cumulative_records_sent, + &last_acked_records, + &sdk_identifier, + &ingest_mutex, + &is_paused, + ), + ) + .await; + + match reconnect_result { + Ok(Ok(new_response_stream)) => { + info!("Supervisor: Recovery successful, resuming"); + recovery_attempts.store(0, Ordering::Relaxed); + // is_paused was already cleared inside reconnect(). + response_stream = new_response_stream; + // Loop continues with new stream. + } + Ok(Err(e)) => { + // Mirror the initial-connect path: drop the cached + // token on auth rejection so recovery re-mints. + if e.is_auth_rejection() { + headers_provider.invalidate().await; + } + warn!("Supervisor: Reconnection failed: {}", e); + // Loop continues, will retry if retries remain. + // Create a dummy stream that immediately errors. + response_stream = Box::pin(futures::stream::once(async move { + Err(FlightError::Tonic(Box::new(tonic::Status::unavailable( + "Reconnection failed", + )))) + })); + } + Err(_timeout) => { + warn!("Supervisor: Reconnection timed out"); + // Loop continues, will retry if retries remain. + response_stream = Box::pin(futures::stream::once(async move { + Err(FlightError::Tonic(Box::new( + tonic::Status::deadline_exceeded("Reconnection timed out"), + ))) + })); + } + } + } + Err(error) => { + // Non-retriable error or recovery disabled. + error!("Supervisor: Non-retriable error, closing stream: {}", error); + is_closed.store(true, Ordering::Relaxed); + // A mid-stream auth rejection means the cached token is no + // longer accepted; drop it so the next stream re-mints. + if error.is_auth_rejection() { + headers_provider.invalidate().await; + } + // Move pending batches to failed and fail the ack futures. + Self::move_pending_to_failed(&pending_batches, &failed_batches).await; + return Err(error); + } + } + } + }) + } + + /// Reconnects to the server and replays pending batches. + /// + /// Holds `ingest_mutex` for the entire replay and clears `is_paused` before + /// releasing the mutex. This guarantees that any `ingest_batch` caller that + /// acquires the mutex after this function returns sees `is_paused = false` and + /// sends normally — there is no window in which a batch can be buffered but + /// never sent. + #[allow(clippy::too_many_arguments)] + async fn reconnect( + endpoint: &str, + tls_config: &Arc, + table_properties: &ArrowTableProperties, + options: &ArrowStreamConfigurationOptions, + headers_provider: &Arc, + batch_tx: &BatchSender, + pending_batches: &Arc>>, + cumulative_records_sent: &Arc, + last_acked_records: &Arc, + sdk_identifier: &str, + ingest_mutex: &Arc>, + is_paused: &Arc, + ) -> ZerobusResult> + Send>>> { + // Create new client. + let client = Self::create_flight_client( + endpoint, + tls_config, + table_properties, + options, + headers_provider, + sdk_identifier, + ) + .await?; + + // Create new channel. + let (tx, batch_rx) = + mpsc::channel::>(options.max_inflight_batches); + + let ipc_write_options = make_ipc_write_options(options.ipc_compression)?; + let schema = Arc::clone(&table_properties.schema); + let batch_stream = tokio_stream::wrappers::ReceiverStream::new(batch_rx); + + let offset_counter = Arc::new(std::sync::atomic::AtomicI64::new(0)); + let offset_counter_clone = Arc::clone(&offset_counter); + let flight_data_stream = FlightDataEncoderBuilder::new() + .with_schema(schema) + .with_options(ipc_write_options) + .build(batch_stream) + .enumerate() + .map(move |(idx, result)| { + result.map(|mut flight_data| { + if idx > 0 { + let offset = + offset_counter_clone.fetch_add(1, std::sync::atomic::Ordering::Relaxed); + let metadata = FlightBatchMetadata::new(offset); + if let Ok(bytes) = metadata.to_bytes() { + flight_data.app_metadata = bytes.into(); + } + } + flight_data + }) + }); + + // Start the DoPut stream. + let mut flight_client = client; + let mut response_stream = flight_client + .do_put(flight_data_stream) + .await + // `.into()` preserves the inner gRPC code; `Status::from_error` would + // flatten it to `Unknown` and break auth/retry classification. + .map_err(|e| ZerobusError::CreateStreamError(e.into()))?; + + // Wait for server's "ready" signal to confirm reconnection succeeded. + let setup_timeout = Duration::from_millis(options.connection_timeout_ms); + match tokio::time::timeout(setup_timeout, response_stream.next()).await { + Ok(Some(Ok(put_result))) => { + // Verify it's the ready signal. + match FlightAckMetadata::from_bytes(&put_result.app_metadata) { + Ok(metadata) if metadata.is_stream_ready() => { + info!("Reconnection confirmed by server (ready signal received)"); + } + Ok(metadata) => { + error!( + "Unexpected ack during reconnect (offset {}), expected ready signal", + metadata.ack_up_to_offset + ); + return Err(ZerobusError::UnexpectedStreamResponseError(format!( + "Expected ready signal, got ack for offset {}", + metadata.ack_up_to_offset + ))); + } + Err(e) => { + error!("Failed to parse reconnect response metadata: {}", e); + return Err(ZerobusError::UnexpectedStreamResponseError(format!( + "Malformed reconnect response metadata: {}", + e + ))); + } + } + } + Ok(Some(Err(flight_error))) => { + error!("Reconnection setup failed: {:?}", flight_error); + return Err(ZerobusError::CreateStreamError(flight_error.into())); + } + Ok(None) => { + error!("Server closed stream during reconnect without response"); + return Err(ZerobusError::StreamClosedError(tonic::Status::internal( + "Server closed stream during reconnect", + ))); + } + Err(_timeout) => { + error!( + "Timed out waiting for server reconnect confirmation ({}ms)", + options.connection_timeout_ms + ); + return Err(ZerobusError::ConnectionTimeout(format!( + "Timed out waiting for server reconnect confirmation ({}ms)", + options.connection_timeout_ms + ))); + } + } + + // Store the new sender. + { + let mut tx_guard = batch_tx.lock().await; + *tx_guard = Some(tx.clone()); + } + + // Get the last acked record count before the disconnect. + // This tells us how many records were durably stored. + let acked_before_disconnect = last_acked_records.load(Ordering::Relaxed); + // Reset for the new connection to avoid reusing stale values. + last_acked_records.store(0, Ordering::Relaxed); + + // Reset cumulative_records_sent for the new connection. + // It will be recalculated as we replay batches. + cumulative_records_sent.store(0, Ordering::Relaxed); + + // Replay pending batches, slicing partially-acked ones if present. + // We rebuild the pending list to drop fully-acked batches. + // Lock order matches ingest_batch: ingest_mutex -> pending_batches. + let _ingest_guard = ingest_mutex.lock().await; + { + let mut pending = pending_batches.lock().await; + if !pending.is_empty() { + info!( + batch_count = pending.len(), + acked_records = acked_before_disconnect, + "Replaying pending batches after recovery" + ); + + let mut new_pending = Vec::with_capacity(pending.len()); + let mut new_cumulative: u64 = 0; + + for pb in pending.drain(..) { + let Some(batch) = slice_batch_for_recovery(&pb, acked_before_disconnect) else { + debug!(offset_id = pb.offset_id, "Skipping fully-acked batch"); + continue; + }; + + if tx.send(Ok(batch.clone())).await.is_err() { + return Err(ZerobusError::StreamClosedError(tonic::Status::internal( + "Failed to replay batch during recovery", + ))); + } + + let num_records = batch.num_rows() as u64; + let start_record = new_cumulative; + let end_record = new_cumulative + num_records; + new_cumulative = end_record; + + new_pending.push(PendingBatch { + batch, + offset_id: pb.offset_id, + start_record, + end_record, + }); + } + + *pending = new_pending; + cumulative_records_sent.store(new_cumulative, Ordering::Relaxed); + } + } + + #[cfg(debug_assertions)] + { + let pending = pending_batches.lock().await; + let mut expected_start: u64 = 0; + for pb in pending.iter() { + debug_assert_eq!( + pb.start_record, expected_start, + "pending_batches has non-contiguous record ranges after recovery \ + (expected start_record = {}, found {} for offset_id {}); \ + possible orphaned buffered batch from pause-gate handoff race", + expected_start, pb.start_record, pb.offset_id, + ); + expected_start = pb.end_record; + } + } + + // Clear the pause gate while still holding ingest_mutex. + is_paused.store(false, Ordering::Relaxed); + + Ok(response_stream) + } + + /// Moves all pending batches to the failed batches list. + async fn move_pending_to_failed( + pending_batches: &Arc>>, + failed_batches: &Arc>>, + ) { + let pending: Vec = { + let mut pending_guard = pending_batches.lock().await; + std::mem::take(&mut *pending_guard) + }; + let mut failed = failed_batches.lock().await; + for pb in pending { + failed.push(pb.batch); + } + } + + /// Processes acknowledgments from the server response stream. + /// + /// Uses record-based tracking: the server sends `ack_up_to_records` indicating + /// the cumulative number of records durably stored. We match this against + /// pending batches' record ranges to determine which batches are fully acked. + /// This correctly handles batches that were split into multiple Flight chunks + /// by `FlightDataEncoderBuilder`. + #[allow(clippy::too_many_arguments)] + async fn process_acks( + mut response_stream: Pin> + Send>>, + is_closed: Arc, + last_ack_tx: tokio::sync::watch::Sender>, + pending_batches: Arc>>, + ack_timeout: Duration, + server_error_tx: watch::Sender>, + last_acked_records: Arc, + is_paused: Arc, + options: &ArrowStreamConfigurationOptions, + ) -> ZerobusResult<()> { + let mut pause_deadline: Option = None; + + loop { + if is_closed.load(Ordering::Relaxed) { + debug!("Stream closed, stopping ack processor"); + return Ok(()); + } + + // Check pause state: exit when deadline reached or all batches acked. + // Returns a retriable error to trigger recovery in the supervisor. + if let Some(deadline) = pause_deadline { + let now = tokio::time::Instant::now(); + let all_acked = pending_batches.lock().await.is_empty(); + + if now >= deadline { + info!("Graceful close timeout reached. Triggering recovery."); + return Err(ZerobusError::StreamClosedError(tonic::Status::unavailable( + "Graceful close timeout reached", + ))); + } else if all_acked { + info!("All in-flight batches acknowledged during graceful close. Triggering recovery."); + return Err(ZerobusError::StreamClosedError(tonic::Status::unavailable( + "All in-flight batches acked during graceful close", + ))); + } + } + + let result = if let Some(deadline) = pause_deadline { + tokio::select! { + biased; + _ = tokio::time::sleep_until(deadline) => { + continue; + } + res = tokio::time::timeout(ack_timeout, response_stream.next()) => res, + } + } else { + tokio::time::timeout(ack_timeout, response_stream.next()).await + }; + + match result { + Ok(Some(Ok(put_result))) => { + match FlightAckMetadata::from_bytes(&put_result.app_metadata) { + Ok(ack) => { + // Handle close stream signal. + if ack.is_close_signal() { + if options.recovery { + let server_duration_ms = + ack.close_stream_duration_ms.unwrap_or(0); + + let wait_duration_ms = match options + .stream_paused_max_wait_time_ms + { + None => server_duration_ms, + Some(0) => { + info!( + "Server will close the stream in {}ms. Triggering stream recovery.", + server_duration_ms + ); + return Err(ZerobusError::StreamClosedError( + tonic::Status::unavailable( + "Immediate recovery on close signal", + ), + )); + } + Some(max_wait) => { + std::cmp::min(max_wait, server_duration_ms) + } + }; + + if wait_duration_ms == 0 { + info!("Server will close the stream. Triggering immediate recovery."); + return Err(ZerobusError::StreamClosedError( + tonic::Status::unavailable( + "Immediate recovery on close signal", + ), + )); + } + + is_paused.store(true, Ordering::Relaxed); + pause_deadline = Some( + tokio::time::Instant::now() + + Duration::from_millis(wait_duration_ms), + ); + info!( + "Server will close the stream in {}ms. Entering graceful close period (waiting up to {}ms for in-flight acks).", + server_duration_ms, wait_duration_ms + ); + } + // Process any ack data that came with the close signal. + // Fall through to ack processing below only if there's + // meaningful ack data (non-zero records count). + if ack.ack_up_to_records == 0 { + continue; + } + } + + let acked_records = ack.ack_up_to_records; + debug!( + ack_up_to_offset = ack.ack_up_to_offset, + ack_up_to_records = acked_records, + "Received acknowledgment" + ); + + // Update last_acked_records for recovery slicing. + last_acked_records.store(acked_records, Ordering::Relaxed); + + // Find and remove batches that are fully acknowledged. + // A batch is fully acked when ack_up_to_records >= batch.end_record. + let mut max_acked_offset: Option = None; + { + let mut pending = pending_batches.lock().await; + pending.retain(|pb| { + if acked_records >= pb.end_record { + // Batch is fully acknowledged + max_acked_offset = Some( + max_acked_offset + .map_or(pb.offset_id, |o| o.max(pb.offset_id)), + ); + false // Remove from pending + } else { + true // Keep in pending + } + }); + } + + // Notify waiters of the highest acknowledged offset. + if let Some(offset) = max_acked_offset { + let _ = last_ack_tx.send(Some(offset)); + } + } + Err(e) => { + warn!("Failed to parse ack metadata: {}", e); + } + } + } + Ok(Some(Err(e))) => { + // During graceful close, errors are expected (server closes after grace period). + // Return retriable error to trigger recovery. + if pause_deadline.is_some() { + info!( + "Stream error during graceful close period, triggering recovery: {}", + e + ); + return Err(ZerobusError::StreamClosedError(tonic::Status::unavailable( + "Stream error during graceful close", + ))); + } + error!("Flight stream error: {}", e); + let status: tonic::Status = e.into(); + let error = ZerobusError::StreamClosedError(status); + let _ = server_error_tx.send(Some(error.clone())); + return Err(error); + } + Ok(None) => { + // During graceful close, stream end is expected. + // Return retriable error to trigger recovery. + if pause_deadline.is_some() { + info!("Server closed stream during graceful close period, triggering recovery."); + return Err(ZerobusError::StreamClosedError(tonic::Status::unavailable( + "Server closed stream during graceful close", + ))); + } + debug!("Server closed the stream"); + let error = ZerobusError::StreamClosedError(tonic::Status::unknown( + "Server closed the stream", + )); + return Err(error); + } + Err(_timeout) => { + // During graceful close, ack timeout is not an error. + if pause_deadline.is_some() { + continue; + } + // Check if there are pending acks that should have been received. + let pending = pending_batches.lock().await; + if !pending.is_empty() { + error!( + pending_count = pending.len(), + "Server ack timeout with pending batches" + ); + let error = ZerobusError::StreamClosedError( + tonic::Status::deadline_exceeded("Server ack timeout"), + ); + return Err(error); + } + } + } + } + } + + /// Ingests a single Arrow RecordBatch into the stream. + /// + /// This method queues the batch for transmission and returns the assigned offset + /// immediately. Use `wait_for_offset()` to explicitly wait for server acknowledgment + /// of this batch when needed. + /// + /// # Arguments + /// + /// * `batch` - An Arrow RecordBatch to ingest + /// + /// # Returns + /// + /// The offset ID assigned to this batch. + /// + /// # Errors + /// + /// * `StreamClosedError` - If the stream has been closed + /// * `InvalidArgument` - If the batch schema doesn't match the stream schema + /// + /// # Examples + /// + /// ```no_run + /// # use databricks_zerobus_ingest_sdk::*; + /// # use arrow_array::RecordBatch; + /// # async fn example(stream: ZerobusArrowStream, batch: RecordBatch) -> Result<(), ZerobusError> { + /// // Ingest and get offset immediately + /// let offset = stream.ingest_batch(batch).await?; + /// + /// // Later, wait for acknowledgment + /// stream.wait_for_offset(offset).await?; + /// println!("Batch at offset {} has been acknowledged", offset); + /// # Ok(()) + /// # } + /// ``` + #[instrument(level = "debug", skip_all, fields(table_name = %self.table_properties.table_name))] + pub async fn ingest_batch(&self, batch: RecordBatch) -> ZerobusResult { + if self.is_closed.load(Ordering::Relaxed) { + return Err(ZerobusError::StreamClosedError(tonic::Status::internal( + "Stream is closed", + ))); + } + + // Validate schema matches. + if batch.schema() != self.table_properties.schema { + return Err(ZerobusError::InvalidArgument(format!( + "RecordBatch schema does not match stream schema. Expected: {:?}, Got: {:?}", + self.table_properties.schema, + batch.schema() + ))); + } + + // Serialize ingestion operations. + let _guard = self.ingest_mutex.lock().await; + + let offset_id = self.offset_generator.next(); + let record_count = batch.num_rows() as u64; + let start_record = self + .cumulative_records_sent + .fetch_add(record_count, Ordering::Relaxed); + let end_record = start_record + record_count; + + // Store in pending batches for recovery with record range for ack matching. + { + let mut pending = self.pending_batches.lock().await; + pending.push(PendingBatch { + batch: batch.clone(), + offset_id, + start_record, + end_record, + }); + } + + // When paused (graceful close or pre-reconnect), buffer the batch. + // It will be replayed by reconnect() after recovery. + if self.is_paused.load(Ordering::Relaxed) { + return Ok(offset_id); + } + + let sender = { + let guard = self.batch_tx.lock().await; + guard.clone() + }; + + let sender = match sender { + Some(s) => s, + None => { + if let Some(server_error) = self.server_error_rx.borrow().clone() { + return Err(server_error); + } + return Err(ZerobusError::StreamClosedError(tonic::Status::internal( + "Stream sender is closed", + ))); + } + }; + + if let Err(e) = sender.send(Ok(batch)).await { + warn!("Send failed: {}", e); + if self.options.recovery { + debug!( + offset_id = offset_id, + "Send failed but recovery enabled - supervisor will handle recovery" + ); + return Ok(offset_id); + } else { + { + let mut pending = self.pending_batches.lock().await; + pending.retain(|pb| pb.offset_id != offset_id); + } + let _ = tokio::time::timeout( + Duration::from_millis(100), + self.server_error_rx.clone().changed(), + ) + .await; + if let Some(server_error) = self.server_error_rx.borrow().clone() { + return Err(server_error); + } + return Err(ZerobusError::StreamClosedError(tonic::Status::internal( + "Failed to send batch", + ))); + } + } + + debug!(offset_id = offset_id, "Batch queued for ingestion"); + Ok(offset_id) + } + + /// Ingests a single Arrow RecordBatch supplied as raw Arrow IPC stream bytes. + /// + /// Convenience wrapper for callers that already hold IPC-serialised bytes. + /// Deserialises the bytes to a [`RecordBatch`] and delegates to `ingest_batch`. + /// Prefer `ingest_batch` directly when you already have a [`RecordBatch`]. + /// + /// The `ipc_bytes` must be a valid Arrow IPC *stream* containing exactly one + /// RecordBatch (i.e. the output of `pyarrow.RecordBatch.serialize()`, + /// `tableToIPC(table, 'stream')`, etc.). Dictionary messages between the schema and + /// the RecordBatch are supported. Trailing stream metadata (such as an end-of-stream + /// marker after `finish()`) is allowed after that batch. + #[instrument(level = "debug", skip_all, fields(table_name = %self.table_properties.table_name))] + pub async fn ingest_ipc_batch(&self, ipc_bytes: Bytes) -> ZerobusResult { + if self.is_closed.load(Ordering::Relaxed) { + return Err(ZerobusError::StreamClosedError(tonic::Status::internal( + "Stream is closed", + ))); + } + + // Deserialise IPC bytes into a RecordBatch. + let batch = materialize_ipc(&ipc_bytes) + .map_err(|e| ZerobusError::InvalidArgument(format!("Invalid Arrow IPC bytes: {e}")))?; + + // Validate schema matches the stream schema. + if batch.schema() != self.table_properties.schema { + return Err(ZerobusError::InvalidArgument(format!( + "IPC batch schema does not match stream schema. Expected: {:?}, Got: {:?}", + self.table_properties.schema, + batch.schema() + ))); + } + + self.ingest_batch(batch).await + } + + /// Internal method to wait for a specific offset to be acknowledged. + /// Used by both `flush()` and `wait_for_offset()`. + async fn wait_for_offset_internal( + &self, + offset_to_wait: OffsetId, + operation_name: &str, + ) -> ZerobusResult<()> { + let flush_timeout = Duration::from_millis(self.options.flush_timeout_ms); + let mut offset_rx = self.last_ack_tx.subscribe(); + let mut error_rx = self.server_error_rx.clone(); + + let wait_future = async { + loop { + if self.is_closed.load(Ordering::Relaxed) { + return Err(ZerobusError::StreamClosedError(tonic::Status::internal( + format!("Stream closed during {}", operation_name.to_lowercase()), + ))); + } + + let current_ack = *offset_rx.borrow_and_update(); + if let Some(ack_offset) = current_ack { + if ack_offset >= offset_to_wait { + debug!( + ack_offset = ack_offset, + target_offset = offset_to_wait, + "{} completed", + operation_name + ); + return Ok(()); + } + debug!( + current_ack = ack_offset, + target_offset = offset_to_wait, + "Waiting for more acks" + ); + } + + // Race between offset updates and server errors + tokio::select! { + result = offset_rx.changed() => { + if result.is_err() { + return Err(ZerobusError::StreamClosedError(tonic::Status::internal( + format!( + "Ack channel closed during {}", + operation_name.to_lowercase() + ), + ))); + } + // Loop continues to check new offset value + } + _ = error_rx.changed() => { + // Server error occurred - return it immediately if stream is closed + if let Some(server_error) = error_rx.borrow().clone() { + if self.is_closed.load(Ordering::Relaxed) { + return Err(server_error); + } + // Stream still active, recovery might succeed - keep waiting + } + // Error channel updated but no error (cleared by recovery) - continue waiting + } + } + } + }; + + tokio::time::timeout(flush_timeout, wait_future) + .await + .map_err(|_| { + error!("{} timed out", operation_name); + ZerobusError::StreamClosedError(tonic::Status::deadline_exceeded(format!( + "{} timed out", + operation_name + ))) + })? + } + + /// Flushes all currently pending batches and waits for their acknowledgments. + /// + /// This method captures the current highest offset and waits until all batches up to + /// that offset have been acknowledged by the server. Batches ingested during the flush + /// operation are not included in this flush. + /// + /// # Returns + /// + /// `Ok(())` when all pending batches at the time of the call have been acknowledged. + /// + /// # Errors + /// + /// * `StreamClosedError` - If the stream is closed or times out + /// + /// # Examples + /// + /// ```no_run + /// # use databricks_zerobus_ingest_sdk::*; + /// # use arrow_array::RecordBatch; + /// # async fn example(stream: ZerobusArrowStream, batches: Vec) -> Result<(), ZerobusError> { + /// // Ingest many batches without waiting for each one + /// for batch in batches { + /// let _offset = stream.ingest_batch(batch).await?; + /// } + /// + /// // Wait for all batches to be acknowledged + /// stream.flush().await?; + /// println!("All batches have been acknowledged"); + /// # Ok(()) + /// # } + /// ``` + #[instrument(level = "debug", skip_all, fields(table_name = %self.table_properties.table_name))] + pub async fn flush(&self) -> ZerobusResult<()> { + // Check if stream is closed first, before checking for batches. + if self.is_closed.load(Ordering::Relaxed) { + return Err(ZerobusError::StreamClosedError(tonic::Status::internal( + "Cannot flush: stream is closed", + ))); + } + + let target_offset = match self.offset_generator.last() { + Some(offset) => offset, + None => { + debug!("No batches to flush"); + return Ok(()); + } + }; + + self.wait_for_offset_internal(target_offset, "Flush").await + } + + /// Waits for server acknowledgment of a specific offset. + /// + /// This method blocks until the server has acknowledged the batch at the + /// specified offset. Use this with offsets returned from `ingest_batch()` to + /// explicitly control when to wait for acknowledgments. + /// + /// # Arguments + /// + /// * `offset` - The offset ID to wait for (returned from `ingest_batch()`) + /// + /// # Returns + /// + /// `Ok(())` when the batch at the specified offset has been acknowledged. + /// + /// # Errors + /// + /// * `StreamClosedError` - If the stream is closed or times out while waiting + /// + /// # Examples + /// + /// ```no_run + /// # use databricks_zerobus_ingest_sdk::*; + /// # use arrow_array::RecordBatch; + /// # async fn example(stream: ZerobusArrowStream, batches: Vec) -> Result<(), ZerobusError> { + /// // Ingest multiple batches and collect their offsets + /// let mut offsets = Vec::new(); + /// for batch in batches { + /// let offset = stream.ingest_batch(batch).await?; + /// offsets.push(offset); + /// } + /// + /// // Wait for specific offsets + /// for offset in offsets { + /// stream.wait_for_offset(offset).await?; + /// } + /// println!("All batches acknowledged"); + /// # Ok(()) + /// # } + /// ``` + pub async fn wait_for_offset(&self, offset: OffsetId) -> ZerobusResult<()> { + self.wait_for_offset_internal(offset, "Waiting for acknowledgement") + .await + } + + /// Closes the stream gracefully after flushing all pending batches. + /// + /// This method first calls `flush()` to ensure all pending batches are acknowledged, + /// then shuts down the stream and releases all resources. + /// + /// # Returns + /// + /// `Ok(())` if the stream was closed successfully after flushing all batches. + /// + /// # Errors + /// + /// Returns any errors from the flush operation. If flush fails, some batches + /// may not have been acknowledged. Use `get_unacked_batches()` to retrieve them. + /// + /// # Examples + /// + /// ```no_run + /// # use databricks_zerobus_ingest_sdk::*; + /// # async fn example(mut stream: ZerobusArrowStream) -> Result<(), ZerobusError> { + /// // After ingesting batches... + /// stream.close().await?; + /// # Ok(()) + /// # } + /// ``` + #[instrument(level = "debug", skip_all, fields(table_name = %self.table_properties.table_name))] + pub async fn close(&mut self) -> ZerobusResult<()> { + if self.is_closed.load(Ordering::Relaxed) { + return Ok(()); + } + + info!( + table_name = %self.table_properties.table_name, + "Closing Arrow Flight stream" + ); + + // Flush pending batches. + if let Err(e) = self.flush().await { + warn!( + "Flush failed during close: {}. Moving pending batches to failed.", + e + ); + // Move pending batches to failed (drain to avoid duplicates in get_unacked_batches). + Self::move_pending_to_failed(&self.pending_batches, &self.failed_batches).await; + } + + // Mark as closed. + self.is_closed.store(true, Ordering::Relaxed); + + // Drop the batch sender to signal end of stream. + { + let mut tx = self.batch_tx.lock().await; + *tx = None; + } + + // Abort the receiver task. + { + let mut task = self.receiver_task.lock().await; + if let Some(t) = task.take() { + t.abort(); + } + } + + Ok(()) + } + + /// Returns all batches that were ingested but not acknowledged by the server. + /// + /// This method should only be called after a stream has failed or been closed. + /// It's useful for implementing custom retry logic or persisting failed batches. + /// + /// # Returns + /// + /// A vector of `RecordBatch` items that were not acknowledged. + /// + /// # Errors + /// + /// * `InvalidStateError` - If the stream is still active + /// + /// # Examples + /// + /// ```no_run + /// # use databricks_zerobus_ingest_sdk::*; + /// # async fn example(sdk: ZerobusSdk, mut stream: ZerobusArrowStream) -> Result<(), ZerobusError> { + /// match stream.flush().await { + /// Err(_) => { + /// let failed_batches = stream.get_unacked_batches().await?; + /// println!("Failed to send {} batches", failed_batches.len()); + /// // You can recreate the stream and retry these batches + /// let new_stream = sdk.recreate_arrow_stream(&stream).await?; + /// for batch in failed_batches { + /// new_stream.ingest_batch(batch).await?; + /// } + /// } + /// Ok(_) => println!("All batches acknowledged"), + /// } + /// # Ok(()) + /// # } + /// ``` + pub async fn get_unacked_batches(&self) -> ZerobusResult> { + if !self.is_closed.load(Ordering::Relaxed) { + error!( + table_name = %self.table_properties.table_name, + "Cannot get unacked batches from an active stream. Stream must be closed first." + ); + return Err(ZerobusError::InvalidStateError( + "Cannot get unacked batches from an active stream. Stream must be closed first." + .to_string(), + )); + } + + let mut result = Vec::new(); + + { + let pending = self.pending_batches.lock().await; + for pb in pending.iter() { + result.push(pb.batch.clone()); + } + } + + { + let failed = self.failed_batches.lock().await; + result.extend(failed.iter().cloned()); + } + + Ok(result) + } + + /// Returns whether the stream has been closed. + pub fn is_closed(&self) -> bool { + self.is_closed.load(Ordering::Relaxed) + } + + /// Returns the table name for this stream. + pub fn table_name(&self) -> &str { + &self.table_properties.table_name + } + + /// Returns the Arrow schema for this stream. + pub fn schema(&self) -> &Arc { + &self.table_properties.schema + } + + /// Returns the configuration options for this stream. + pub fn options(&self) -> &ArrowStreamConfigurationOptions { + &self.options + } + + /// Returns the headers provider for this stream (for recreation). + pub(crate) fn headers_provider(&self) -> Arc { + Arc::clone(&self.headers_provider) + } +} + +impl Drop for ZerobusArrowStream { + fn drop(&mut self) { + self.is_closed.store(true, Ordering::Relaxed); + // Abort the background supervisor task to prevent zombie tasks. + // This is a hard abort, but outstanding oneshot receivers will get + // RecvError when their senders are dropped, and pending batches can + // still be retrieved via get_unacked_batches() before drop. + if let Ok(mut guard) = self.receiver_task.try_lock() { + if let Some(handle) = guard.take() { + handle.abort(); + } + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + use arrow_schema::{DataType, Field}; + + #[test] + fn test_arrow_table_properties() { + let schema = Arc::new(ArrowSchema::new(vec![ + Field::new("id", DataType::Int32, false), + Field::new("name", DataType::Utf8, true), + ])); + + let props = ArrowTableProperties { + table_name: "catalog.schema.table".to_string(), + schema, + }; + + assert_eq!(props.table_name, "catalog.schema.table"); + assert_eq!(props.schema.fields().len(), 2); + } +} diff --git a/lib/zerobus-ffi-1.3.0/rust/sdk/src/builder/mod.rs b/lib/zerobus-ffi-1.3.0/rust/sdk/src/builder/mod.rs new file mode 100644 index 00000000000..c995c8ae0a4 --- /dev/null +++ b/lib/zerobus-ffi-1.3.0/rust/sdk/src/builder/mod.rs @@ -0,0 +1,36 @@ +//! Builder API for creating Zerobus SDK instances and ingestion streams. +//! +//! This module provides fluent builder patterns for configuring and creating +//! SDK instances and streams. +//! +//! # Examples +//! +//! ## SDK Builder +//! +//! ```no_run +//! use databricks_zerobus_ingest_sdk::ZerobusSdkBuilder; +//! +//! let sdk = ZerobusSdkBuilder::new() +//! .endpoint("https://workspace.zerobus.databricks.com") +//! .unity_catalog_url("https://workspace.cloud.databricks.com") +//! .build()?; +//! # Ok::<(), databricks_zerobus_ingest_sdk::ZerobusError>(()) +//! ``` +//! +//! ## Stream Builder +//! +//! ```rust,ignore +//! let stream = sdk +//! .stream_builder() +//! .table("catalog.schema.table") +//! .oauth("client-id", "client-secret") +//! .json() +//! .build() +//! .await?; +//! ``` + +mod sdk_builder; +mod stream_builder; + +pub use sdk_builder::ZerobusSdkBuilder; +pub use stream_builder::StreamBuilder; diff --git a/lib/zerobus-ffi-1.3.0/rust/sdk/src/builder/sdk_builder.rs b/lib/zerobus-ffi-1.3.0/rust/sdk/src/builder/sdk_builder.rs new file mode 100644 index 00000000000..d4cea7fc925 --- /dev/null +++ b/lib/zerobus-ffi-1.3.0/rust/sdk/src/builder/sdk_builder.rs @@ -0,0 +1,408 @@ +//! Builder for creating [`ZerobusSdk`] instances. + +use std::sync::Arc; +use std::time::Duration; + +use crate::proxy::ConnectorFactory; +use crate::token_cache::DEFAULT_REFRESH_BUFFER; +use crate::{ + SecureTlsConfig, TlsConfig, ZerobusError, ZerobusResult, ZerobusSdk, DEFAULT_SDK_IDENTIFIER, +}; + +/// Builder for creating a [`ZerobusSdk`] instance with fluent configuration. +/// +/// # Examples +/// +/// ```no_run +/// use databricks_zerobus_ingest_sdk::ZerobusSdkBuilder; +/// +/// let sdk = ZerobusSdkBuilder::new() +/// .endpoint("https://workspace.zerobus.databricks.com") +/// .unity_catalog_url("https://workspace.cloud.databricks.com") +/// .build()?; +/// # Ok::<(), databricks_zerobus_ingest_sdk::ZerobusError>(()) +/// ``` +pub struct ZerobusSdkBuilder { + zerobus_endpoint: Option, + unity_catalog_url: Option, + tls_config: Option>, + connector_factory: Option, + application_name: Option, + sdk_identifier_override: Option, + token_cache_enabled: bool, + token_refresh_buffer: Duration, +} + +impl ZerobusSdkBuilder { + /// Creates a new SDK builder with default settings. + /// + /// TLS is enabled by default using `SecureTlsConfig`. + pub fn new() -> Self { + Self { + zerobus_endpoint: None, + unity_catalog_url: None, + tls_config: None, + connector_factory: None, + application_name: None, + sdk_identifier_override: None, + token_cache_enabled: true, + token_refresh_buffer: DEFAULT_REFRESH_BUFFER, + } + } + + /// Sets the Zerobus API endpoint URL. + /// + /// This is required. The workspace ID is automatically extracted from this URL. + /// + /// # Arguments + /// + /// * `endpoint` - The Zerobus endpoint URL (e.g., "https://workspace-id.zerobus.region.cloud.databricks.com") + pub fn endpoint(mut self, endpoint: impl Into) -> Self { + self.zerobus_endpoint = Some(endpoint.into()); + self + } + + /// Sets the Unity Catalog endpoint URL. + /// + /// This is only required when using OAuth authentication via `create_stream()`. + /// When using `create_stream_with_headers_provider()` with a custom headers + /// provider, this can be omitted. + /// + /// # Arguments + /// + /// * `url` - The Unity Catalog URL (e.g., "https://workspace.cloud.databricks.com") + pub fn unity_catalog_url(mut self, url: impl Into) -> Self { + self.unity_catalog_url = Some(url.into()); + self + } + + /// Sets a custom TLS configuration. + /// + /// Use this to provide custom certificate handling or other TLS settings. + /// If not set, the default `SecureTlsConfig` (system CA certificates) is used. + /// + /// # Arguments + /// + /// * `tls_config` - A TLS configuration implementing the `TlsConfig` trait + pub fn tls_config(mut self, tls_config: Arc) -> Self { + self.tls_config = Some(tls_config); + self + } + + /// Override gRPC channel connector construction; see + /// [`ConnectorFactory`] for semantics. + pub fn connector_factory(mut self, factory: ConnectorFactory) -> Self { + self.connector_factory = Some(factory); + self + } + + /// Sets a custom application identifier appended to the HTTP `user-agent` + /// header sent on every request. + /// + /// The default user-agent value is `zerobus-sdk-rs/`. When this is + /// set, the value sent becomes `zerobus-sdk-rs/ `, + /// preserving the SDK version prefix for server-side telemetry while + /// adding caller-supplied identification (e.g. `"my-app/1.0"`). + /// + /// The SDK owns the `user-agent` header at the tonic `Endpoint` level; + /// values returned by a [`HeadersProvider`](crate::HeadersProvider) cannot + /// override it. + /// + /// If [`sdk_identifier`](Self::sdk_identifier) is also set, the override + /// replaces the SDK prefix but this value is still appended — the wire + /// value becomes ` `. + /// + /// # Arguments + /// + /// * `name` - Application identifier, conventionally `/` + pub fn application_name(mut self, name: impl Into) -> Self { + self.application_name = Some(name.into()); + self + } + + /// Overrides the SDK prefix of the HTTP `user-agent` header, replacing the + /// default `zerobus-sdk-rs/`. + /// + /// Used by wrapper SDKs that need to replace the SDK identification itself. + /// + /// Empty values are ignored and the default identifier is used. If + /// [`application_name`](Self::application_name) is also set, it is still + /// appended after this override — the wire value becomes + /// ` `. + /// + /// # Arguments + /// + /// * `identifier` - Replacement SDK prefix (e.g. `"zerobus-sdk-py/2.0.0"`) + pub fn sdk_identifier(mut self, identifier: impl Into) -> Self { + self.sdk_identifier_override = Some(identifier.into()); + self + } + + /// Enables or disables caching of OAuth tokens for the default OAuth path. + /// + /// When enabled (the default), tokens obtained via `.oauth(...)` are cached + /// per table on the SDK instance and reused across stream creations and + /// recoveries until they near expiry, instead of minting a fresh token on + /// every stream. This reduces load on the Unity Catalog token endpoint for + /// clients that churn through many short-lived streams. + /// + /// Caching only applies to the built-in OAuth path. Custom + /// [`HeadersProvider`](crate::HeadersProvider) implementations are + /// responsible for their own caching. Tokens are shared only across streams + /// created from the same `ZerobusSdk` instance, so reuse the SDK rather than + /// constructing a new one per stream to benefit from the cache. + /// + /// # Arguments + /// + /// * `enabled` - Whether to cache OAuth tokens. + pub fn token_cache_enabled(mut self, enabled: bool) -> Self { + self.token_cache_enabled = enabled; + self + } + + /// Sets how long before a cached OAuth token's expiry it is refreshed. + /// + /// A cached token is re-minted on the next stream creation once it is within + /// this buffer of its expiry, providing headroom against clock skew and + /// token propagation delays. Defaults to 5 minutes. Has no effect when token + /// caching is disabled. + /// + /// # Arguments + /// + /// * `buffer` - Lead time before expiry at which to refresh. + pub fn token_refresh_buffer(mut self, buffer: Duration) -> Self { + self.token_refresh_buffer = buffer; + self + } + + /// Builds the [`ZerobusSdk`] instance. + /// + /// # Errors + /// + /// Returns an error if: + /// - The endpoint is not set + /// - The workspace ID cannot be extracted from the endpoint + #[allow(clippy::result_large_err)] + pub fn build(self) -> ZerobusResult { + let zerobus_endpoint = self + .zerobus_endpoint + .ok_or_else(|| ZerobusError::InvalidArgument("endpoint is required".to_string()))?; + + let zerobus_endpoint = if !zerobus_endpoint.starts_with("https://") + && !zerobus_endpoint.starts_with("http://") + { + format!("https://{}", zerobus_endpoint) + } else { + zerobus_endpoint + }; + + let unity_catalog_url = self.unity_catalog_url.unwrap_or_default(); + + let workspace_id = zerobus_endpoint + .strip_prefix("https://") + .or_else(|| zerobus_endpoint.strip_prefix("http://")) + .and_then(|s| s.split('.').next()) + .map(|s| s.to_string()) + .ok_or_else(|| { + ZerobusError::InvalidArgument( + "Failed to extract workspace ID from zerobus_endpoint".to_string(), + ) + })?; + + let tls_config = self + .tls_config + .unwrap_or_else(|| Arc::new(SecureTlsConfig::new())); + + let sdk_prefix: &str = match self.sdk_identifier_override.as_deref() { + Some(override_id) if !override_id.is_empty() => override_id, + _ => DEFAULT_SDK_IDENTIFIER, + }; + let sdk_identifier: Arc = match self.application_name.as_deref() { + Some(app) if !app.is_empty() => Arc::from(format!("{} {}", sdk_prefix, app)), + _ => Arc::from(sdk_prefix), + }; + + Ok(ZerobusSdk::new_with_config( + zerobus_endpoint, + unity_catalog_url, + workspace_id, + tls_config, + self.connector_factory, + sdk_identifier, + self.token_cache_enabled, + self.token_refresh_buffer, + )) + } +} + +impl Default for ZerobusSdkBuilder { + fn default() -> Self { + Self::new() + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_builder_with_all_fields() { + let sdk = ZerobusSdkBuilder::new() + .endpoint("https://my-workspace.zerobus.us-east-1.cloud.databricks.com") + .unity_catalog_url("https://my-workspace.cloud.databricks.com") + .build() + .expect("should build successfully"); + + assert_eq!( + sdk.zerobus_endpoint, + "https://my-workspace.zerobus.us-east-1.cloud.databricks.com" + ); + assert_eq!( + sdk.unity_catalog_url, + "https://my-workspace.cloud.databricks.com" + ); + } + + #[test] + fn test_builder_token_cache_options() { + // Both knobs are chainable and the SDK builds with non-default values. + let sdk = ZerobusSdkBuilder::new() + .endpoint("https://workspace.zerobus.databricks.com") + .token_cache_enabled(false) + .token_refresh_buffer(Duration::from_secs(120)) + .build() + .expect("should build with token cache options"); + + // The builder accepts both knobs and produces a usable SDK; assert the + // build succeeded via a field on the result. + assert_eq!(sdk.workspace_id, "workspace"); + } + + #[test] + fn test_builder_token_cache_enabled_by_default() { + let sdk = ZerobusSdkBuilder::new() + .endpoint("https://workspace.zerobus.databricks.com") + .build() + .expect("should build"); + // The cache is always present; enablement is internal state. + let _ = &sdk.token_cache; + } + + #[test] + fn test_builder_missing_endpoint() { + let result = ZerobusSdkBuilder::new() + .unity_catalog_url("https://workspace.cloud.databricks.com") + .build(); + + assert!(matches!( + result, + Err(ZerobusError::InvalidArgument(msg)) if msg.contains("endpoint is required") + )); + } + + #[test] + fn test_builder_schemeless_endpoint() { + // Endpoint without protocol prefix - https:// is prepended automatically + let sdk = ZerobusSdkBuilder::new() + .endpoint("my-workspace.zerobus.databricks.com") + .build() + .expect("should build successfully with schemeless endpoint"); + + assert_eq!(sdk.workspace_id, "my-workspace"); + assert_eq!( + sdk.zerobus_endpoint, + "https://my-workspace.zerobus.databricks.com" + ); + } + + #[test] + fn test_builder_without_unity_catalog_url() { + // Unity Catalog URL is optional for custom headers providers + let sdk = ZerobusSdkBuilder::new() + .endpoint("https://workspace.zerobus.databricks.com") + .build() + .expect("should build successfully without unity_catalog_url"); + + assert_eq!(sdk.unity_catalog_url, ""); + } + + #[test] + fn test_sdk_identifier_default() { + let sdk = ZerobusSdkBuilder::new() + .endpoint("https://workspace.zerobus.databricks.com") + .build() + .expect("should build"); + + assert_eq!(&*sdk.sdk_identifier, crate::DEFAULT_SDK_IDENTIFIER); + } + + #[test] + fn test_sdk_identifier_with_application_name() { + let sdk = ZerobusSdkBuilder::new() + .endpoint("https://workspace.zerobus.databricks.com") + .application_name("my-app/1.0") + .build() + .expect("should build"); + + let expected = format!("{} my-app/1.0", crate::DEFAULT_SDK_IDENTIFIER); + assert_eq!(&*sdk.sdk_identifier, expected); + } + + #[test] + fn test_sdk_identifier_empty_application_name_falls_back_to_default() { + let sdk = ZerobusSdkBuilder::new() + .endpoint("https://workspace.zerobus.databricks.com") + .application_name("") + .build() + .expect("should build"); + + assert_eq!(&*sdk.sdk_identifier, crate::DEFAULT_SDK_IDENTIFIER); + } + + #[test] + fn test_sdk_identifier_override_replaces_default() { + let sdk = ZerobusSdkBuilder::new() + .endpoint("https://workspace.zerobus.databricks.com") + .sdk_identifier("custom-agent/2.0") + .build() + .expect("should build"); + + assert_eq!(&*sdk.sdk_identifier, "custom-agent/2.0"); + } + + #[test] + fn test_sdk_identifier_override_with_application_name_combines() { + let sdk = ZerobusSdkBuilder::new() + .endpoint("https://workspace.zerobus.databricks.com") + .application_name("my-app/1.0") + .sdk_identifier("custom-agent/2.0") + .build() + .expect("should build"); + + assert_eq!(&*sdk.sdk_identifier, "custom-agent/2.0 my-app/1.0"); + } + + #[test] + fn test_sdk_identifier_empty_override_falls_back_to_default() { + let sdk = ZerobusSdkBuilder::new() + .endpoint("https://workspace.zerobus.databricks.com") + .sdk_identifier("") + .build() + .expect("should build"); + + assert_eq!(&*sdk.sdk_identifier, crate::DEFAULT_SDK_IDENTIFIER); + } + + #[test] + fn test_sdk_identifier_empty_override_with_application_name_uses_application_name() { + let sdk = ZerobusSdkBuilder::new() + .endpoint("https://workspace.zerobus.databricks.com") + .application_name("my-app/1.0") + .sdk_identifier("") + .build() + .expect("should build"); + + let expected = format!("{} my-app/1.0", crate::DEFAULT_SDK_IDENTIFIER); + assert_eq!(&*sdk.sdk_identifier, expected); + } +} diff --git a/lib/zerobus-ffi-1.3.0/rust/sdk/src/builder/stream_builder.rs b/lib/zerobus-ffi-1.3.0/rust/sdk/src/builder/stream_builder.rs new file mode 100644 index 00000000000..e90ab538886 --- /dev/null +++ b/lib/zerobus-ffi-1.3.0/rust/sdk/src/builder/stream_builder.rs @@ -0,0 +1,672 @@ +//! Fluent builder for creating Zerobus ingestion streams. +//! +//! All setters can be called in any order. The builder validates at +//! `build()` time that table name, authentication, and format have been +//! configured. You can also call `validate()` to check the builder state +//! without opening a stream. +//! +//! # Examples +//! +//! ```rust,ignore +//! let stream = sdk +//! .stream_builder() +//! .table("catalog.schema.table") +//! .oauth("client-id", "client-secret") +//! .json() +//! .max_inflight_requests(500_000) +//! .build() +//! .await?; +//! ``` + +use std::fmt; +use std::sync::Arc; + +use crate::callbacks::AckCallback; +use crate::databricks::zerobus::RecordType; +use crate::headers_provider::{HeadersProvider, OAuthHeadersProvider}; +use crate::stream_configuration::StreamConfigurationOptions; +use crate::{TableProperties, ZerobusError, ZerobusResult, ZerobusSdk, ZerobusStream}; + +#[cfg(feature = "arrow-flight")] +use crate::arrow_configuration::ArrowStreamConfigurationOptions; +#[cfg(feature = "arrow-flight")] +use crate::arrow_stream::{ArrowSchema, ArrowTableProperties, ZerobusArrowStream}; + +/// Internal representation of the authentication configuration. +enum AuthConfig { + OAuth { + client_id: String, + client_secret: String, + }, + HeadersProvider(Arc), +} + +/// Which record format was selected. +enum FormatConfig { + Json, + CompiledProto(Box), + #[cfg(feature = "arrow-flight")] + Arrow(Arc), +} + +/// A fluent builder for creating Zerobus ingestion streams. +/// +/// All setters can be called in any order. The builder validates at +/// `build()` time that table name, authentication, and format have been +/// configured. Use [`validate()`](Self::validate) to check the builder +/// state without opening a stream. +/// +/// # Examples +/// +/// ```rust,ignore +/// // JSON stream with OAuth +/// let stream = sdk +/// .stream_builder() +/// .table("catalog.schema.table") +/// .oauth("client-id", "client-secret") +/// .json() +/// .build() +/// .await?; +/// +/// // Proto stream with custom headers +/// let stream = sdk +/// .stream_builder() +/// .table("catalog.schema.table") +/// .headers_provider(my_provider) +/// .compiled_proto(descriptor) +/// .max_inflight_requests(500_000) +/// .build() +/// .await?; +/// +/// // Validate without opening a stream +/// let builder = sdk +/// .stream_builder() +/// .table("catalog.schema.table") +/// .oauth("client-id", "client-secret") +/// .json(); +/// builder.validate()?; +/// let stream = builder.build().await?; +/// ``` +#[must_use = "a StreamBuilder does nothing until `.build()` is called"] +pub struct StreamBuilder<'a> { + sdk: &'a ZerobusSdk, + table_name: String, + auth: Option, + format: Option, + grpc_config: StreamConfigurationOptions, + #[cfg(feature = "arrow-flight")] + arrow_config: ArrowStreamConfigurationOptions, +} + +impl fmt::Debug for StreamBuilder<'_> { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + let auth_kind = match &self.auth { + Some(AuthConfig::OAuth { .. }) => "OAuth", + Some(AuthConfig::HeadersProvider(_)) => "HeadersProvider", + None => "None", + }; + let format_kind = match &self.format { + Some(FormatConfig::Json) => "Json", + Some(FormatConfig::CompiledProto(_)) => "CompiledProto", + #[cfg(feature = "arrow-flight")] + Some(FormatConfig::Arrow(_)) => "Arrow", + None => "None", + }; + f.debug_struct("StreamBuilder") + .field("table_name", &self.table_name) + .field("auth", &auth_kind) + .field("format", &format_kind) + .finish_non_exhaustive() + } +} + +#[allow(clippy::result_large_err)] +impl<'a> StreamBuilder<'a> { + pub(crate) fn new(sdk: &'a ZerobusSdk) -> Self { + Self { + sdk, + table_name: String::new(), + auth: None, + format: None, + grpc_config: StreamConfigurationOptions::default(), + #[cfg(feature = "arrow-flight")] + arrow_config: ArrowStreamConfigurationOptions::default(), + } + } + + /// Set the fully-qualified Unity Catalog table name (e.g., `"catalog.schema.table"`). + pub fn table(mut self, table_name: impl Into) -> Self { + self.table_name = table_name.into(); + self + } + + /// Authenticate with OAuth client credentials. + pub fn oauth(mut self, client_id: impl Into, client_secret: impl Into) -> Self { + self.auth = Some(AuthConfig::OAuth { + client_id: client_id.into(), + client_secret: client_secret.into(), + }); + self + } + + /// Authenticate with a custom headers provider. + pub fn headers_provider(mut self, provider: Arc) -> Self { + self.auth = Some(AuthConfig::HeadersProvider(provider)); + self + } + + /// Select JSON record format. + pub fn json(mut self) -> Self { + self.format = Some(FormatConfig::Json); + self + } + + /// Select compiled protobuf record format. + pub fn compiled_proto(mut self, descriptor: prost_types::DescriptorProto) -> Self { + self.format = Some(FormatConfig::CompiledProto(Box::new(descriptor))); + self + } + + /// Select Arrow Flight record format. + #[cfg(feature = "arrow-flight")] + pub fn arrow(mut self, schema: Arc) -> Self { + self.format = Some(FormatConfig::Arrow(schema)); + self + } + + /// Enable or disable automatic stream recovery. + pub fn recovery(mut self, enabled: bool) -> Self { + self.grpc_config.recovery = enabled; + #[cfg(feature = "arrow-flight")] + { + self.arrow_config.recovery = enabled; + } + self + } + + /// Set the timeout in milliseconds for each recovery attempt. + pub fn recovery_timeout_ms(mut self, ms: u64) -> Self { + self.grpc_config.recovery_timeout_ms = ms; + #[cfg(feature = "arrow-flight")] + { + self.arrow_config.recovery_timeout_ms = ms; + } + self + } + + /// Set the backoff time in milliseconds between recovery retries. + pub fn recovery_backoff_ms(mut self, ms: u64) -> Self { + self.grpc_config.recovery_backoff_ms = ms; + #[cfg(feature = "arrow-flight")] + { + self.arrow_config.recovery_backoff_ms = ms; + } + self + } + + /// Set the maximum number of recovery retry attempts. + pub fn recovery_retries(mut self, n: u32) -> Self { + self.grpc_config.recovery_retries = n; + #[cfg(feature = "arrow-flight")] + { + self.arrow_config.recovery_retries = n; + } + self + } + + /// Set the timeout in milliseconds for server acknowledgement. + pub fn server_lack_of_ack_timeout_ms(mut self, ms: u64) -> Self { + self.grpc_config.server_lack_of_ack_timeout_ms = ms; + #[cfg(feature = "arrow-flight")] + { + self.arrow_config.server_lack_of_ack_timeout_ms = ms; + } + self + } + + /// Set the timeout in milliseconds for flush operations. + pub fn flush_timeout_ms(mut self, ms: u64) -> Self { + self.grpc_config.flush_timeout_ms = ms; + #[cfg(feature = "arrow-flight")] + { + self.arrow_config.flush_timeout_ms = ms; + } + self + } + + /// Set the maximum number of in-flight requests (gRPC streams only). + pub fn max_inflight_requests(mut self, n: usize) -> Self { + self.grpc_config.max_inflight_requests = n; + self + } + + /// Set the maximum wait time during graceful stream pause (JSON/proto and Arrow streams). + pub fn stream_paused_max_wait_time_ms(mut self, ms: Option) -> Self { + self.grpc_config.stream_paused_max_wait_time_ms = ms; + #[cfg(feature = "arrow-flight")] + { + self.arrow_config.stream_paused_max_wait_time_ms = ms; + } + self + } + + /// Set the acknowledgment callback (gRPC streams only). + pub fn ack_callback(mut self, callback: Arc) -> Self { + self.grpc_config.ack_callback = Some(callback); + self + } + + /// Set the maximum wait time for callbacks after stream close (gRPC streams only). + pub fn callback_max_wait_time_ms(mut self, ms: Option) -> Self { + self.grpc_config.callback_max_wait_time_ms = ms; + self + } + + /// Set the maximum number of in-flight Arrow batches (Arrow streams only). + #[cfg(feature = "arrow-flight")] + pub fn max_inflight_batches(mut self, n: usize) -> Self { + self.arrow_config.max_inflight_batches = n; + self + } + + /// Set the connection timeout in milliseconds for Arrow Flight (Arrow streams only). + #[cfg(feature = "arrow-flight")] + pub fn connection_timeout_ms(mut self, ms: u64) -> Self { + self.arrow_config.connection_timeout_ms = ms; + self + } + + /// Set the Arrow IPC compression type (Arrow streams only). + #[cfg(feature = "arrow-flight")] + pub fn ipc_compression(mut self, compression: Option) -> Self { + self.arrow_config.ipc_compression = compression; + self + } + + /// Validate that the builder has all required fields configured. + /// + /// Returns `Ok(())` if table name, authentication, and format are all set. + /// This performs the same checks as `build()` without actually opening + /// a stream — useful for fail-fast validation during startup or config + /// parsing. + /// + /// # Examples + /// + /// ```rust,ignore + /// let builder = sdk + /// .stream_builder() + /// .table("catalog.schema.table") + /// .oauth("client-id", "client-secret") + /// .json(); + /// + /// // Check configuration before opening the stream + /// builder.validate()?; + /// let stream = builder.build().await?; + /// ``` + pub fn validate(&self) -> ZerobusResult<()> { + if self.table_name.is_empty() { + return Err(ZerobusError::InvalidArgument( + "table name is required: call .table()".into(), + )); + } + if self.auth.is_none() { + return Err(ZerobusError::InvalidArgument( + "authentication is required: call .oauth() or .headers_provider()".into(), + )); + } + if self.format.is_none() { + return Err(ZerobusError::InvalidArgument( + "record format is required: call .json(), .compiled_proto(), or .arrow()".into(), + )); + } + Ok(()) + } + + /// Resolve the headers provider from the stored auth config. + fn resolve_headers_provider(&self) -> ZerobusResult> { + match self.auth.as_ref() { + Some(AuthConfig::OAuth { + client_id, + client_secret, + }) => Ok(Arc::new(OAuthHeadersProvider::with_cache( + client_id.clone(), + client_secret.clone(), + self.table_name.clone(), + self.sdk.workspace_id.clone(), + self.sdk.unity_catalog_url.clone(), + Arc::clone(&self.sdk.token_cache), + ))), + Some(AuthConfig::HeadersProvider(p)) => Ok(Arc::clone(p)), + None => Err(ZerobusError::InvalidArgument( + "authentication is required: call .oauth() or .headers_provider()".into(), + )), + } + } + + /// Build and open a gRPC ingestion stream (JSON or compiled protobuf). + /// + /// Returns an error if table name, authentication, or format has not been set, + /// or if an Arrow format was selected (use `build_arrow()` instead). + pub async fn build(mut self) -> ZerobusResult { + self.validate()?; + let headers_provider = self.resolve_headers_provider()?; + + let (record_type, descriptor_proto) = match self.format { + Some(FormatConfig::Json) => (RecordType::Json, None), + Some(FormatConfig::CompiledProto(desc)) => (RecordType::Proto, Some(*desc)), + #[cfg(feature = "arrow-flight")] + Some(FormatConfig::Arrow(_)) => { + return Err(ZerobusError::InvalidArgument( + "Arrow format requires .build_arrow() instead of .build()".into(), + )); + } + None => { + return Err(ZerobusError::InvalidArgument( + "record format is required: call .json() or .compiled_proto() before .build()" + .into(), + )); + } + }; + + self.grpc_config.record_type = record_type; + let table_properties = TableProperties { + table_name: self.table_name, + descriptor_proto, + }; + + let channel = self.sdk.get_or_create_channel_zerobus_client().await?; + let stream = ZerobusStream::new_stream( + channel, + table_properties, + headers_provider, + self.grpc_config, + ) + .await?; + crate::client_warnings::record_stream_creation(stream.table_properties.table_name.as_str()); + Ok(stream) + } + + /// Build and open an Arrow Flight ingestion stream. + /// + /// Returns an error if table name, authentication, or format has not been set, + /// or if a non-Arrow format was selected (use `build()` instead). + #[cfg(feature = "arrow-flight")] + pub async fn build_arrow(self) -> ZerobusResult { + self.validate()?; + let headers_provider = self.resolve_headers_provider()?; + + let schema = match self.format { + Some(FormatConfig::Arrow(schema)) => schema, + Some(_) => { + return Err(ZerobusError::InvalidArgument( + "non-Arrow format requires .build() instead of .build_arrow()".into(), + )); + } + None => { + return Err(ZerobusError::InvalidArgument( + "record format is required: call .arrow() before .build_arrow()".into(), + )); + } + }; + + let table_properties = ArrowTableProperties { + table_name: self.table_name, + schema, + }; + + let table_name = table_properties.table_name.clone(); + let stream = ZerobusArrowStream::new( + &self.sdk.zerobus_endpoint, + Arc::clone(&self.sdk.tls_config), + table_properties, + headers_provider, + self.arrow_config, + Arc::clone(&self.sdk.sdk_identifier), + ) + .await?; + crate::client_warnings::record_stream_creation(&table_name); + Ok(stream) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use std::collections::HashMap; + + fn test_sdk() -> ZerobusSdk { + ZerobusSdk::new_with_config( + "http://localhost:1234".to_string(), + "http://localhost:5678".to_string(), + "test-workspace".to_string(), + Arc::new(crate::tls_config::SecureTlsConfig::new()), + None, + Arc::from(crate::DEFAULT_SDK_IDENTIFIER), + true, + crate::token_cache::DEFAULT_REFRESH_BUFFER, + ) + } + + #[test] + fn json_oauth_builder() { + let sdk = test_sdk(); + let _builder = sdk + .stream_builder() + .table("catalog.schema.table") + .oauth("cid", "csec") + .json() + .max_inflight_requests(100); + } + + #[test] + fn compiled_proto_headers_provider() { + struct StubProvider; + #[async_trait::async_trait] + impl HeadersProvider for StubProvider { + async fn get_headers(&self) -> crate::ZerobusResult> { + Ok(HashMap::new()) + } + } + + let sdk = test_sdk(); + let provider: Arc = Arc::new(StubProvider); + let _builder = sdk + .stream_builder() + .table("catalog.schema.table") + .headers_provider(provider) + .compiled_proto(prost_types::DescriptorProto::default()); + } + + #[test] + fn any_order_format_before_auth() { + let sdk = test_sdk(); + let _builder = sdk + .stream_builder() + .table("catalog.schema.table") + .json() + .oauth("cid", "csec") + .max_inflight_requests(100); + } + + #[test] + fn any_order_config_before_format() { + let sdk = test_sdk(); + let _builder = sdk + .stream_builder() + .table("catalog.schema.table") + .max_inflight_requests(100) + .recovery(false) + .oauth("cid", "csec") + .json(); + } + + #[test] + fn config_setters_chain() { + let sdk = test_sdk(); + let _builder = sdk + .stream_builder() + .table("t") + .oauth("a", "b") + .json() + .recovery(false) + .recovery_timeout_ms(10_000) + .recovery_backoff_ms(1_000) + .recovery_retries(3) + .server_lack_of_ack_timeout_ms(30_000) + .flush_timeout_ms(60_000) + .max_inflight_requests(500) + .stream_paused_max_wait_time_ms(Some(5_000)) + .callback_max_wait_time_ms(None); + } + + #[test] + fn default_config_without_setters() { + let sdk = test_sdk(); + let builder = sdk.stream_builder().table("t").oauth("a", "b").json(); + assert_eq!(builder.grpc_config.max_inflight_requests, 1_000_000); + assert!(builder.grpc_config.recovery); + } + + #[tokio::test] + async fn build_without_auth_returns_error() { + let sdk = test_sdk(); + let result = sdk.stream_builder().table("t").json().build().await; + match result { + Err(ZerobusError::InvalidArgument(msg)) => { + assert!(msg.contains("authentication is required")); + } + _ => panic!("expected InvalidArgument error"), + } + } + + #[tokio::test] + async fn build_without_table_returns_error() { + let sdk = test_sdk(); + let result = sdk.stream_builder().oauth("a", "b").json().build().await; + match result { + Err(ZerobusError::InvalidArgument(msg)) => { + assert!(msg.contains("table name is required")); + } + _ => panic!("expected InvalidArgument error"), + } + } + + #[tokio::test] + async fn build_without_format_returns_error() { + let sdk = test_sdk(); + let result = sdk + .stream_builder() + .table("t") + .oauth("a", "b") + .build() + .await; + match result { + Err(ZerobusError::InvalidArgument(msg)) => { + assert!(msg.contains("record format is required")); + } + _ => panic!("expected InvalidArgument error"), + } + } + + #[test] + fn debug_impl_works() { + let sdk = test_sdk(); + let builder = sdk.stream_builder().table("t").oauth("a", "b").json(); + let debug_str = format!("{:?}", builder); + assert!(debug_str.contains("StreamBuilder")); + assert!(debug_str.contains("OAuth")); + assert!(debug_str.contains("Json")); + } + + #[tokio::test] + async fn resolve_headers_provider_with_custom_provider() { + struct TestProvider; + + #[async_trait::async_trait] + impl HeadersProvider for TestProvider { + async fn get_headers(&self) -> crate::ZerobusResult> { + let mut h = HashMap::new(); + h.insert("x-test", "value".to_string()); + Ok(h) + } + } + + let sdk = test_sdk(); + let builder = sdk + .stream_builder() + .table("catalog.schema.table") + .headers_provider(Arc::new(TestProvider)) + .json(); + + let provider = builder.resolve_headers_provider().unwrap(); + let headers = provider.get_headers().await.unwrap(); + assert_eq!(headers.get("x-test").unwrap(), "value"); + } + + #[tokio::test] + async fn resolve_headers_provider_with_oauth() { + let sdk = test_sdk(); + let builder = sdk + .stream_builder() + .table("catalog.schema.table") + .oauth("my-client-id", "my-secret") + .json(); + + let _provider = builder.resolve_headers_provider().unwrap(); + } + + #[cfg(feature = "arrow-flight")] + #[test] + fn arrow_builder() { + use arrow_schema::{DataType, Field, Schema as ArrowSchema}; + + let sdk = test_sdk(); + let schema = Arc::new(ArrowSchema::new(vec![Field::new( + "id", + DataType::Int32, + false, + )])); + let _builder = sdk + .stream_builder() + .table("t") + .oauth("a", "b") + .arrow(schema) + .max_inflight_batches(500) + .connection_timeout_ms(10_000); + } + + #[cfg(feature = "arrow-flight")] + #[test] + fn shared_setters_write_to_arrow_config() { + use arrow_schema::{DataType, Field, Schema as ArrowSchema}; + + let sdk = test_sdk(); + let schema = Arc::new(ArrowSchema::new(vec![Field::new( + "id", + DataType::Int32, + false, + )])); + let builder = sdk + .stream_builder() + .table("t") + .oauth("a", "b") + .arrow(schema) + .recovery(false) + .recovery_timeout_ms(5_000) + .recovery_backoff_ms(500) + .recovery_retries(2) + .server_lack_of_ack_timeout_ms(10_000) + .flush_timeout_ms(20_000) + .stream_paused_max_wait_time_ms(Some(5_000)); + assert!(!builder.arrow_config.recovery); + assert_eq!(builder.arrow_config.recovery_timeout_ms, 5_000); + assert_eq!(builder.arrow_config.recovery_backoff_ms, 500); + assert_eq!(builder.arrow_config.recovery_retries, 2); + assert_eq!(builder.arrow_config.server_lack_of_ack_timeout_ms, 10_000); + assert_eq!(builder.arrow_config.flush_timeout_ms, 20_000); + assert_eq!( + builder.arrow_config.stream_paused_max_wait_time_ms, + Some(5_000) + ); + } +} diff --git a/lib/zerobus-ffi-1.3.0/rust/sdk/src/callbacks.rs b/lib/zerobus-ffi-1.3.0/rust/sdk/src/callbacks.rs new file mode 100644 index 00000000000..acd4ba23d24 --- /dev/null +++ b/lib/zerobus-ffi-1.3.0/rust/sdk/src/callbacks.rs @@ -0,0 +1,117 @@ +//! Callback system for acknowledgment notifications. +//! +//! This module provides a callback interface for receiving notifications when +//! records or batches are acknowledged by the server. +//! +//! # Examples +//! +//! ``` +//! use databricks_zerobus_ingest_sdk::{AckCallback, OffsetId}; +//! +//! struct MyCallback; +//! +//! impl AckCallback for MyCallback { +//! fn on_ack(&self, offset_id: OffsetId) { +//! println!("Acknowledged offset: {}", offset_id); +//! } +//! +//! fn on_error(&self, offset_id: OffsetId, error_message: &str) { +//! eprintln!("Error for offset {}: {}", offset_id, error_message); +//! } +//! } +//! ``` + +use crate::offset_generator::OffsetId; + +/// Callback trait for receiving acknowledgment notifications. +/// +/// Implement this trait to receive callbacks when records/batches are acknowledged +/// by the server or when errors occur. +/// +/// # Thread Safety and Performance +/// +/// Implementations must be `Send + Sync` as callbacks are invoked from +/// a dedicated background callback handler task. +/// +/// **Important**: Callbacks are executed synchronously in a separate callback handler task. +/// Keep implementations lightweight (simple logging, metrics increment, etc.) to avoid +/// accumulating callback backlog. For heavy work like database writes, network calls, +/// or complex processing, consider using channels to send data to dedicated worker tasks. +/// +/// # Examples +/// +/// ``` +/// use databricks_zerobus_ingest_sdk::{AckCallback, OffsetId}; +/// use std::sync::atomic::{AtomicI64, Ordering}; +/// +/// struct CountingCallback { +/// ack_count: AtomicI64, +/// } +/// +/// impl AckCallback for CountingCallback { +/// fn on_ack(&self, offset_id: OffsetId) { +/// self.ack_count.fetch_add(1, Ordering::Relaxed); +/// } +/// +/// fn on_error(&self, offset_id: OffsetId, error_message: &str) { +/// eprintln!("Error: {}", error_message); +/// } +/// } +/// ``` +pub trait AckCallback: Send + Sync { + /// Called when a record/batch is successfully acknowledged by the server. + /// + /// **Note**: This runs synchronously in a dedicated callback handler task. + /// Keep it lightweight (e.g., logging, metrics) to avoid callback backlog. + /// + /// # Parameters + /// + /// * `offset_id` - The logical offset ID that was acknowledged + fn on_ack(&self, offset_id: OffsetId); + + /// Called when an error occurs for a specific record/batch. + /// + /// **Note**: This runs synchronously in a dedicated callback handler task. + /// Keep it reasonably lightweight (e.g., logging, metrics) to avoid callback backlog. + /// + /// # Parameters + /// + /// * `offset_id` - The logical offset ID that encountered an error + /// * `error_message` - Human-readable error description + fn on_error(&self, offset_id: OffsetId, error_message: &str); +} + +#[cfg(test)] +mod tests { + use super::*; + use std::sync::atomic::{AtomicBool, AtomicI64, Ordering}; + + struct TestCallback { + last_ack: AtomicI64, + error_called: AtomicBool, + } + + impl AckCallback for TestCallback { + fn on_ack(&self, offset_id: OffsetId) { + self.last_ack.store(offset_id, Ordering::Relaxed); + } + + fn on_error(&self, _offset_id: OffsetId, _error_message: &str) { + self.error_called.store(true, Ordering::Relaxed); + } + } + + #[test] + fn test_callback_trait() { + let callback = TestCallback { + last_ack: AtomicI64::new(0), + error_called: AtomicBool::new(false), + }; + + callback.on_ack(42); + assert_eq!(callback.last_ack.load(Ordering::Relaxed), 42); + + callback.on_error(43, "test error"); + assert!(callback.error_called.load(Ordering::Relaxed)); + } +} diff --git a/lib/zerobus-ffi-1.3.0/rust/sdk/src/client_warnings.rs b/lib/zerobus-ffi-1.3.0/rust/sdk/src/client_warnings.rs new file mode 100644 index 00000000000..c679af0431a --- /dev/null +++ b/lib/zerobus-ffi-1.3.0/rust/sdk/src/client_warnings.rs @@ -0,0 +1,292 @@ +//! Process-wide client-side warnings that surface common SDK misuse patterns. +//! +//! A warning is emitted via [`tracing::warn!`] when 100 or more streams for +//! the same table are opened within a 60-second sliding window, which usually +//! indicates a "one stream per record" misuse pattern. +//! +//! The warning is process-wide and keyed by table name. +//! +//! ## Opt-out +//! +//! Set the environment variable `ZEROBUS_SDK_WARNINGS_ENABLED=false` (or `0` +//! or `no`) before the process starts to suppress all warnings. + +use std::collections::{HashMap, VecDeque}; +use std::sync::{Mutex, OnceLock}; + +use tracing::warn; + +// ─── Opt-out ────────────────────────────────────────────────────────────────── + +fn warnings_enabled() -> bool { + static ENABLED: OnceLock = OnceLock::new(); + *ENABLED.get_or_init(|| { + std::env::var("ZEROBUS_SDK_WARNINGS_ENABLED") + .map(|v| !matches!(v.to_lowercase().as_str(), "false" | "0" | "no")) + .unwrap_or(true) + }) +} + +// ─── Stream churn monitor ───────────────────────────────────────────────────── + +/// Sliding window length in milliseconds. +const CHURN_WINDOW_MS: u64 = 60_000; +/// Logs a warning when this many streams are opened within [`CHURN_WINDOW_MS`]. +const CHURN_WARN_THRESHOLD: usize = 100; +/// Maximum number of distinct tables tracked; oldest is evicted when exceeded. +const CHURN_MAX_TABLES: usize = 1000; + +fn default_clock_ms() -> u64 { + use std::time::{SystemTime, UNIX_EPOCH}; + SystemTime::now() + .duration_since(UNIX_EPOCH) + .map(|d| d.as_millis() as u64) + .unwrap_or(0) +} + +struct StreamChurnState { + /// Replaceable clock; overridden in tests to control time. + clock: fn() -> u64, + /// Per-table queue of stream-open timestamps (ms since Unix epoch). + timestamps: HashMap>, + /// Tracks insertion order for eviction when `CHURN_MAX_TABLES` is reached. + insertion_order: VecDeque, +} + +impl StreamChurnState { + fn new() -> Self { + Self { + clock: default_clock_ms, + timestamps: HashMap::new(), + insertion_order: VecDeque::new(), + } + } +} + +static CHURN_MONITOR: OnceLock> = OnceLock::new(); + +fn churn_monitor() -> &'static Mutex { + CHURN_MONITOR.get_or_init(|| Mutex::new(StreamChurnState::new())) +} + +/// Records one user-initiated stream creation for `table_name`. +/// +/// Maintains a per-table sliding window of creation timestamps. Logs a +/// `WARN`-level message when the count within the window reaches +/// [`CHURN_WARN_THRESHOLD`] (100), and again each time the window drains +/// and a new surge reaches the threshold. +/// +/// Must only be called from user-facing stream creation paths, not from internal +/// reconnect / recovery paths. +pub(crate) fn record_stream_creation(table_name: &str) { + if !warnings_enabled() { + return; + } + + let count = { + let mut state = churn_monitor().lock().unwrap_or_else(|e| e.into_inner()); + let now = (state.clock)(); + + if !state.timestamps.contains_key(table_name) { + // Evict the oldest-tracked table when the cap is reached. + if state.insertion_order.len() >= CHURN_MAX_TABLES { + if let Some(oldest) = state.insertion_order.pop_front() { + state.timestamps.remove(&oldest); + } + } + state + .timestamps + .insert(table_name.to_string(), VecDeque::new()); + state.insertion_order.push_back(table_name.to_string()); + } + + let deque = state.timestamps.get_mut(table_name).unwrap(); + // Evict timestamps that have fallen outside the sliding window. + while deque + .front() + .map(|&t| now.saturating_sub(t) > CHURN_WINDOW_MS) + .unwrap_or(false) + { + deque.pop_front(); + } + deque.push_back(now); + deque.len() + }; + + // Fire exactly when the window count hits the threshold; re-fires each time + // the window drains below the threshold and a new surge reaches it. + if count == CHURN_WARN_THRESHOLD { + warn!( + "Zerobus SDK: {} ingest streams opened for table `{}` in the last {}s in this \ + process. If this is unexpected, check that streams are being reused across records.", + count, + table_name, + CHURN_WINDOW_MS / 1000 + ); + } +} + +// ─── Test utilities ─────────────────────────────────────────────────────────── + +#[cfg(test)] +pub(crate) fn reset_for_testing() { + if let Ok(mut state) = churn_monitor().lock() { + *state = StreamChurnState::new(); + } +} + +#[cfg(test)] +pub(crate) fn set_churn_clock_for_testing(f: fn() -> u64) { + if let Ok(mut state) = churn_monitor().lock() { + state.clock = f; + } +} + +#[cfg(test)] +pub(crate) fn open_count_in_window_for_testing(table_name: &str) -> usize { + let state = churn_monitor().lock().unwrap_or_else(|e| e.into_inner()); + let now = (state.clock)(); + state + .timestamps + .get(table_name) + .map(|deque| { + deque + .iter() + .filter(|&&t| now.saturating_sub(t) <= CHURN_WINDOW_MS) + .count() + }) + .unwrap_or(0) +} + +// ─── Unit tests ─────────────────────────────────────────────────────────────── + +#[cfg(test)] +mod tests { + use std::sync::atomic::{AtomicU64, Ordering}; + use std::sync::{Mutex, OnceLock}; + + use super::*; + + // All tests share a global fake clock and call reset_for_testing(). Each + // acquires the serial lock so they run one at a time without corrupting + // each other's clock or timestamp state. + static SERIAL: OnceLock> = OnceLock::new(); + fn serial() -> std::sync::MutexGuard<'static, ()> { + SERIAL + .get_or_init(|| Mutex::new(())) + .lock() + .unwrap_or_else(|e| e.into_inner()) + } + + static FAKE_CLOCK_MS: AtomicU64 = AtomicU64::new(0); + + fn fake_clock() -> u64 { + FAKE_CLOCK_MS.load(Ordering::Relaxed) + } + + #[test] + fn churn_open_count_tracks_opens_within_window() { + let _lock = serial(); + reset_for_testing(); + set_churn_clock_for_testing(fake_clock); + FAKE_CLOCK_MS.store(0, Ordering::Relaxed); + let table = "cat.sch.churn_tracks"; + + record_stream_creation(table); + record_stream_creation(table); + assert_eq!(open_count_in_window_for_testing(table), 2); + } + + #[test] + fn churn_entries_older_than_window_evicted_on_next_open() { + let _lock = serial(); + reset_for_testing(); + set_churn_clock_for_testing(fake_clock); + FAKE_CLOCK_MS.store(0, Ordering::Relaxed); + let table = "cat.sch.churn_evict"; + + for _ in 0..10 { + record_stream_creation(table); + } + assert_eq!(open_count_in_window_for_testing(table), 10); + + // Advance past the window; the next open evicts all 10 old entries. + FAKE_CLOCK_MS.store(61_000, Ordering::Relaxed); + record_stream_creation(table); + assert_eq!(open_count_in_window_for_testing(table), 1); + } + + #[test] + fn churn_warning_fires_at_exactly_threshold_not_before() { + let _lock = serial(); + reset_for_testing(); + set_churn_clock_for_testing(fake_clock); + FAKE_CLOCK_MS.store(0, Ordering::Relaxed); + let table = "cat.sch.churn_threshold"; + + for _ in 0..99 { + record_stream_creation(table); + } + assert_eq!(open_count_in_window_for_testing(table), 99); + + // 100th open crosses the threshold. + record_stream_creation(table); + assert_eq!(open_count_in_window_for_testing(table), 100); + + // 101st does not re-fire (count != CHURN_WARN_THRESHOLD). + record_stream_creation(table); + assert_eq!(open_count_in_window_for_testing(table), 101); + } + + #[test] + fn churn_warning_refires_after_window_rolls_below_threshold() { + let _lock = serial(); + reset_for_testing(); + set_churn_clock_for_testing(fake_clock); + FAKE_CLOCK_MS.store(0, Ordering::Relaxed); + let table = "cat.sch.churn_refire"; + + for _ in 0..100 { + record_stream_creation(table); + } + assert_eq!(open_count_in_window_for_testing(table), 100); + + // Advance past window so all previous opens are evicted. + FAKE_CLOCK_MS.store(61_000, Ordering::Relaxed); + for _ in 0..99 { + record_stream_creation(table); + } + assert_eq!(open_count_in_window_for_testing(table), 99); + + // 100th open in the new window crosses the threshold again. + record_stream_creation(table); + assert_eq!(open_count_in_window_for_testing(table), 100); + } + + #[test] + fn churn_two_tables_tracked_independently() { + let _lock = serial(); + reset_for_testing(); + set_churn_clock_for_testing(fake_clock); + FAKE_CLOCK_MS.store(0, Ordering::Relaxed); + let t1 = "cat.sch.churn_indep1"; + let t2 = "cat.sch.churn_indep2"; + + for _ in 0..5 { + record_stream_creation(t1); + } + for _ in 0..3 { + record_stream_creation(t2); + } + assert_eq!(open_count_in_window_for_testing(t1), 5); + assert_eq!(open_count_in_window_for_testing(t2), 3); + } + + #[test] + fn churn_unknown_table_returns_zero() { + assert_eq!( + open_count_in_window_for_testing("cat.sch.churn_unknown_xyz"), + 0 + ); + } +} diff --git a/lib/zerobus-ffi-1.3.0/rust/sdk/src/default_token_factory.rs b/lib/zerobus-ffi-1.3.0/rust/sdk/src/default_token_factory.rs new file mode 100644 index 00000000000..0b178cd7215 --- /dev/null +++ b/lib/zerobus-ffi-1.3.0/rust/sdk/src/default_token_factory.rs @@ -0,0 +1,423 @@ +use std::time::Duration; + +use tokio::time::Instant; +use tracing::{debug, info, warn}; + +use crate::{ZerobusError, ZerobusResult}; + +/// An access token together with its time-to-live as reported by the OAuth +/// server, if any. +pub(crate) struct FetchedToken { + /// The OAuth 2.0 access token. + pub(crate) token: String, + /// Lifetime of the token derived from the `expires_in` field of the OAuth + /// response. `None` if the server did not return a usable `expires_in`, in + /// which case the token must not be cached. + pub(crate) expires_in: Option, +} + +/// Why a token mint was triggered. Logged on the mint so operators can tell a +/// cold start from a proactive refresh from caching being off. +#[derive(Clone, Copy, Debug)] +pub(crate) enum MintReason { + /// No usable cached token (cold start or the cached token had expired). + ColdMiss, + /// A cached token entered the refresh window and was proactively renewed. + Refresh, + /// Token caching is disabled, so every stream creation mints. + CacheDisabled, + /// Minted outside the cache via the public `get_token`. + Direct, +} + +impl MintReason { + fn as_str(self) -> &'static str { + match self { + MintReason::ColdMiss => "cold_miss", + MintReason::Refresh => "refresh", + MintReason::CacheDisabled => "cache_disabled", + MintReason::Direct => "direct", + } + } +} + +/// Default OAuth 2.0 token factory for Unity Catalog authentication. +/// +/// This factory implements the OAuth 2.0 client credentials flow with Unity Catalog +/// authorization details to obtain access tokens for Zerobus API access. +pub struct DefaultTokenFactory {} + +impl DefaultTokenFactory { + /// Obtains an OAuth 2.0 access token for Zerobus API access. + /// + /// # Arguments + /// + /// * `uc_endpoint` - Unity Catalog endpoint URL + /// * `table_name` - Full table name in format "catalog.schema.table" + /// * `client_id` - OAuth client ID + /// * `client_secret` - OAuth client secret + /// * `workspace_id` - Databricks workspace ID + /// + /// # Returns + /// + /// Returns an access token string on success, or a `ZerobusError` on failure. + /// + /// # Errors + /// + /// * `InvalidUCTokenError` - If the token request fails or returns invalid data + pub async fn get_token( + uc_endpoint: &str, + table_name: &str, + client_id: &str, + client_secret: &str, + workspace_id: &str, + ) -> ZerobusResult { + Self::fetch_token( + uc_endpoint, + table_name, + client_id, + client_secret, + workspace_id, + MintReason::Direct, + ) + .await + .map(|fetched| fetched.token) + } + + /// Obtains an OAuth 2.0 access token along with its reported lifetime. + /// + /// This is the caching-aware variant of [`get_token`](Self::get_token): in + /// addition to the token it returns the `expires_in` value from the OAuth + /// response so callers can cache the token until it nears expiry. + pub(crate) async fn fetch_token( + uc_endpoint: &str, + table_name: &str, + client_id: &str, + client_secret: &str, + workspace_id: &str, + reason: MintReason, + ) -> ZerobusResult { + debug!(table = %table_name, "requesting UC OAuth token"); + let started = Instant::now(); + let result = Self::fetch_token_inner( + uc_endpoint, + table_name, + client_id, + client_secret, + workspace_id, + ) + .await; + let elapsed_ms = started.elapsed().as_millis() as u64; + match &result { + Ok(FetchedToken { + expires_in: Some(ttl), + .. + }) => info!( + table = %table_name, + reason = reason.as_str(), + expires_in_secs = ttl.as_secs(), + elapsed_ms, + "minted UC OAuth token" + ), + Ok(FetchedToken { + expires_in: None, .. + }) => warn!( + table = %table_name, + reason = reason.as_str(), + elapsed_ms, + "minted UC OAuth token but UC returned no expires_in; token will not be cached" + ), + Err(err) => warn!( + table = %table_name, + reason = reason.as_str(), + retryable = err.is_retryable(), + elapsed_ms, + "failed to mint UC OAuth token: {err}" + ), + } + result + } + + async fn fetch_token_inner( + uc_endpoint: &str, + table_name: &str, + client_id: &str, + client_secret: &str, + workspace_id: &str, + ) -> ZerobusResult { + let (catalog, schema, table) = Self::parse_table_name(table_name)?; + + let uc_endpoint = uc_endpoint.to_string(); + let databricks_client_id = client_id.to_string(); + let databricks_client_secret = client_secret.to_string(); + let workspace_id = workspace_id.to_string(); + + let authorization_details = serde_json::json!([ + { + "type": "unity_catalog_privileges", + "privileges": ["USE CATALOG"], + "object_type": "CATALOG", + "object_full_path": catalog + }, + { + "type": "unity_catalog_privileges", + "privileges": ["USE SCHEMA"], + "object_type": "SCHEMA", + "object_full_path": format!("{}.{}", catalog, schema) + }, + { + "type": "unity_catalog_privileges", + "privileges": ["SELECT", "MODIFY"], + "object_type": "TABLE", + "object_full_path": format!("{}.{}.{}", catalog, schema, table), + "operations": ["zerobuswrite"] + } + ]); + + let client = reqwest::Client::new(); + + let params = [ + ("grant_type", "client_credentials".to_string()), + ("scope", "all-apis".to_string()), + ( + "resource", + format!( + "api://databricks/workspaces/{}/zerobusDirectWriteApi", + workspace_id + ) + .to_string(), + ), + ("authorization_details", authorization_details.to_string()), + ]; + + let token_endpoint = format!("{}/oidc/v1/token", uc_endpoint); + let resp = client + .post(&token_endpoint) + .basic_auth(databricks_client_id, Some(databricks_client_secret)) + .form(¶ms) + .send() + .await + .map_err(Self::handle_http_error)?; + + if !resp.status().is_success() { + let status_code = resp.status().as_u16(); + let error_body = resp + .text() + .await + .unwrap_or_else(|_| "Failed to read error body".to_string()); + + return Err(Self::classify_status_code(status_code, error_body)); + } + + let body: serde_json::Value = resp.json().await.map_err(|e| { + ZerobusError::InvalidUCTokenError(format!("Parse failed with error: {}", e)) + })?; + + let token = body["access_token"] + .as_str() + .ok_or_else(|| ZerobusError::InvalidUCTokenError("access_token missing".to_string()))? + .to_string(); + + // Reject a token that can't be a header value before it is returned, so + // an unusable token never enters the cache and poisons it until expiry. + if !Self::is_usable_as_header(&token) { + return Err(ZerobusError::InvalidUCTokenError( + "access token is not a valid HTTP header value".to_string(), + )); + } + + let expires_in = Self::parse_expires_in(&body); + + Ok(FetchedToken { token, expires_in }) + } + + /// Reports whether `token` can be sent as the `authorization` header value + /// (`Bearer `). The gRPC and Arrow paths both encode it this way, so a + /// token that fails here is unusable and must not be cached. + fn is_usable_as_header(token: &str) -> bool { + tonic::metadata::AsciiMetadataValue::try_from(format!("Bearer {token}").as_str()).is_ok() + } + + /// Parses the OAuth `expires_in` field (token lifetime in seconds) into a + /// `Duration`. It is optional in the OAuth spec; if it is missing or not a + /// positive integer the token has no known TTL and must not be cached. + fn parse_expires_in(body: &serde_json::Value) -> Option { + body["expires_in"] + .as_u64() + .filter(|secs| *secs > 0) + .map(Duration::from_secs) + } + + /// Classifies HTTP status codes as retryable or non-retryable errors. + /// + /// # Arguments + /// + /// * `status_code` - HTTP status code (e.g., 404, 500) + /// * `message` - Error message or response body + /// + /// # Returns + /// + /// * `TokenFetchError` for 5xx server errors (retryable) + /// * `InvalidUCTokenError` for 4xx client errors (non-retryable) + fn classify_status_code(status_code: u16, message: String) -> ZerobusError { + if status_code >= 500 { + ZerobusError::TokenFetchError(format!( + "Unity catalog server error ({}): {}", + status_code, message + )) + } else { + ZerobusError::InvalidUCTokenError(format!( + "Client error ({}): {}", + status_code, message + )) + } + } + + /// Helper to classify HTTP errors as retryable (TokenFetchError) or non-retryable. + /// + /// Retryable: + /// - Network errors (timeout, connection failure) + /// - Server errors (5xx status codes) + /// + /// Non-retryable: + /// - Client errors (4xx status codes - bad credentials, invalid request, etc.) + fn handle_http_error(error: reqwest::Error) -> ZerobusError { + if error.is_timeout() || error.is_connect() { + return ZerobusError::TokenFetchError(format!("Network error: {}", error)); + } + if let Some(status) = error.status() { + return Self::classify_status_code(status.as_u16(), error.to_string()); + } + ZerobusError::InvalidUCTokenError(format!("Request failed: {}", error)) + } + + /// Parses a fully qualified table name into its components. + /// + /// # Arguments + /// + /// * `table_name` - Full table name in format "catalog.schema.table" + /// + /// # Returns + /// + /// Returns a tuple of (catalog, schema, table) on success. + /// + /// # Errors + /// + /// * `InvalidTableName` - If the table name doesn't have exactly 3 non-empty parts. + #[allow(clippy::result_large_err)] + fn parse_table_name(table_name: &str) -> Result<(String, String, String), ZerobusError> { + let parts: Vec<&str> = table_name.split('.').collect(); + + if parts.len() != 3 { + return Err(ZerobusError::InvalidTableName(format!( + "Table name must have exactly 3 parts (catalog.schema.table), found {} parts", + parts.len() + ))); + } + + let catalog = parts[0]; + let schema = parts[1]; + let table = parts[2]; + + if catalog.is_empty() { + return Err(ZerobusError::InvalidTableName( + "Catalog name cannot be empty".to_string(), + )); + } + if schema.is_empty() { + return Err(ZerobusError::InvalidTableName( + "Schema name cannot be empty".to_string(), + )); + } + if table.is_empty() { + return Err(ZerobusError::InvalidTableName( + "Table name cannot be empty".to_string(), + )); + } + + Ok((catalog.to_string(), schema.to_string(), table.to_string())) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_parse_table_name_valid() { + let result = DefaultTokenFactory::parse_table_name("catalog_1.schema_2.table_3"); + assert!(result.is_ok()); + let (catalog, schema, table) = result.unwrap(); + assert_eq!(catalog, "catalog_1"); + assert_eq!(schema, "schema_2"); + assert_eq!(table, "table_3"); + } + + #[test] + fn test_parse_expires_in() { + let with_ttl = serde_json::json!({ "expires_in": 3600 }); + assert_eq!( + DefaultTokenFactory::parse_expires_in(&with_ttl), + Some(Duration::from_secs(3600)) + ); + + let missing = serde_json::json!({ "access_token": "abc" }); + assert_eq!(DefaultTokenFactory::parse_expires_in(&missing), None); + + let zero = serde_json::json!({ "expires_in": 0 }); + assert_eq!(DefaultTokenFactory::parse_expires_in(&zero), None); + + // A string value (non-integer) is not usable and yields no TTL. + let non_numeric = serde_json::json!({ "expires_in": "3600" }); + assert_eq!(DefaultTokenFactory::parse_expires_in(&non_numeric), None); + } + + #[test] + fn test_is_usable_as_header() { + // A normal JWT-shaped token is a valid header value. + assert!(DefaultTokenFactory::is_usable_as_header( + "eyJhbGciOiJSUzI1NiJ9.eyJzdWIiOiIxIn0.sig-_value" + )); + // Control characters (e.g. an embedded newline) make it unusable. + assert!(!DefaultTokenFactory::is_usable_as_header("bad\ntoken")); + assert!(!DefaultTokenFactory::is_usable_as_header("bad\0token")); + } + + #[test] + fn test_parse_table_name_invalid() { + let invalid_cases = vec![ + ("catalog.schema.table.extra", "exactly 3 parts"), + ("catalog.schema.table.with.dots", "exactly 3 parts"), + ("catalog", "exactly 3 parts"), + ("catalog.schema", "exactly 3 parts"), + ("", "exactly 3 parts"), + (".schema.table", "Catalog name cannot be empty"), + ("catalog..table", "Schema name cannot be empty"), + ("catalog.schema.", "Table name cannot be empty"), + ("..", "Catalog name cannot be empty"), + ("..table", "Catalog name cannot be empty"), + ("catalog..", "Schema name cannot be empty"), + ]; + + for (input, expected_error) in invalid_cases { + let result = DefaultTokenFactory::parse_table_name(input); + assert!( + result.is_err(), + "Expected '{}' to be invalid, but it was parsed successfully", + input + ); + match result { + Err(ZerobusError::InvalidTableName(msg)) => { + assert!( + msg.contains(expected_error), + "For input '{}', expected error to contain '{}', but got: '{}'", + input, + expected_error, + msg + ); + } + _ => panic!("Expected InvalidTableName error for '{}'", input), + } + } + } +} diff --git a/lib/zerobus-ffi-1.3.0/rust/sdk/src/errors.rs b/lib/zerobus-ffi-1.3.0/rust/sdk/src/errors.rs new file mode 100644 index 00000000000..13d5bda6a2d --- /dev/null +++ b/lib/zerobus-ffi-1.3.0/rust/sdk/src/errors.rs @@ -0,0 +1,151 @@ +use thiserror::Error; + +/// Represents all possible errors that can occur when using Zerobus. +#[derive(Error, Debug, Clone)] +#[non_exhaustive] +pub enum ZerobusError { + /// Returned when the client failed to open a gRPC channel to the Zerobus endpoint. + #[error("Failed to open a channel: {0}.")] + ChannelCreationError(String), + /// Returned when the client failed to create a stream. + #[error("Failed to create stream: {0}.")] + CreateStreamError(tonic::Status), + /// Returned when TLS handshake failed during connection setup. + #[error("Failed to establish TLS connection.")] + FailedToEstablishTlsConnectionError, + /// Returned when the specified Zerobus endpoint is in invalid format. + #[error("The specified Zerobus endpoint is in invalid format: {0}.")] + InvalidZerobusEndpointError(String), + /// Returned when the specified Unity Catalog table name is invalid. + #[error("Specified UC table name is invalid: {0}.")] + InvalidTableName(String), + /// Returned when the specified Unity Catalog endpoint is in invalid format. + #[error("Specified UC endpoint is in invalid format: {0}.")] + InvalidUCEndpointError(String), + /// Returned when the specified Unity Catalog token is invalid. + #[error("Specified UC token is in invalid format: {0}.")] + InvalidUCTokenError(String), + /// Returned when the stream is closed. + #[error("Stream is closed: {0}")] + StreamClosedError(tonic::Status), + /// Returned when the client provided an invalid argument. + #[error("Invalid argument: {0}.")] + InvalidArgument(String), + /// Returned when the server returned an unexpected response. + #[error("Unexpected response from server. Response: {0}")] + UnexpectedStreamResponseError(String), + /// Returned when the stream is in an invalid state for a requested operation. + #[error("Stream is in invalid state: {0}")] + InvalidStateError(String), + /// Returned when a connection or setup operation times out. + #[error("Connection timeout: {0}")] + ConnectionTimeout(String), + /// Returned when OAuth token fetching fails due to network or server errors. + #[error("Token fetch failed: {0}")] + TokenFetchError(String), +} + +/// List of gRPC status codes that indicate unretriable errors. +const UNRETRIABLE_STATUS_CODES: &[tonic::Code] = &[ + tonic::Code::InvalidArgument, + tonic::Code::Unauthenticated, + tonic::Code::PermissionDenied, + tonic::Code::OutOfRange, + tonic::Code::Unimplemented, + tonic::Code::NotFound, +]; + +impl ZerobusError { + /// Determines whether this error can be automatically recovered through stream recovery. + /// + /// Retryable errors typically indicate transient issues like network failures or + /// temporary server problems. Non-retryable errors indicate permanent issues like + /// authentication failures or invalid configurations that require manual intervention. + /// + /// # Returns + /// + /// `true` if the SDK should attempt automatic recovery, `false` otherwise. + pub fn is_retryable(&self) -> bool { + match self { + ZerobusError::InvalidArgument(_) => false, + ZerobusError::StreamClosedError(status) => { + !UNRETRIABLE_STATUS_CODES.contains(&status.code()) + } + ZerobusError::CreateStreamError(status) => { + !UNRETRIABLE_STATUS_CODES.contains(&status.code()) + } + ZerobusError::ChannelCreationError(_) => true, + ZerobusError::FailedToEstablishTlsConnectionError => true, + ZerobusError::InvalidZerobusEndpointError(_) => false, + ZerobusError::InvalidTableName(_) => false, + ZerobusError::InvalidUCEndpointError(_) => false, + ZerobusError::InvalidUCTokenError(_) => false, + ZerobusError::UnexpectedStreamResponseError(_) => true, + ZerobusError::InvalidStateError(_) => false, + ZerobusError::ConnectionTimeout(_) => true, + ZerobusError::TokenFetchError(_) => true, + } + } + + /// Reports whether this is a server-side authentication/authorization + /// rejection (as opposed to a transient or unrelated failure). Used to + /// decide when to invalidate cached credentials so the next attempt + /// re-derives them. + pub(crate) fn is_auth_rejection(&self) -> bool { + matches!( + self, + ZerobusError::CreateStreamError(status) | ZerobusError::StreamClosedError(status) + if matches!( + status.code(), + tonic::Code::Unauthenticated | tonic::Code::PermissionDenied + ) + ) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn auth_rejection_classification() { + assert!( + ZerobusError::CreateStreamError(tonic::Status::unauthenticated("x")) + .is_auth_rejection() + ); + assert!( + ZerobusError::CreateStreamError(tonic::Status::permission_denied("x")) + .is_auth_rejection() + ); + assert!( + ZerobusError::StreamClosedError(tonic::Status::unauthenticated("x")) + .is_auth_rejection() + ); + // Non-auth gRPC codes are not rejections. + assert!(!ZerobusError::CreateStreamError(tonic::Status::internal("x")).is_auth_rejection()); + assert!( + !ZerobusError::CreateStreamError(tonic::Status::unavailable("x")).is_auth_rejection() + ); + // Other variants are never auth rejections. + assert!(!ZerobusError::TokenFetchError("x".to_string()).is_auth_rejection()); + } + + /// Pins the cross-crate invariant the Arrow path relies on: `FlightError -> + /// tonic::Status` via `From` preserves the inner gRPC code (unlike + /// `Status::from_error`, which flattens it to `Unknown`). A future + /// `arrow-flight` change to that `From` impl fails here instead of silently + /// disabling Arrow auth-rejection detection. + #[cfg(feature = "arrow-flight")] + #[test] + fn auth_rejection_survives_flight_error_conversion() { + use arrow_flight::error::FlightError; + + let auth: tonic::Status = + FlightError::Tonic(Box::new(tonic::Status::permission_denied("denied"))).into(); + assert!(ZerobusError::CreateStreamError(auth).is_auth_rejection()); + + let non_auth: tonic::Status = + FlightError::Tonic(Box::new(tonic::Status::unavailable("blip"))).into(); + assert!(!ZerobusError::CreateStreamError(non_auth).is_auth_rejection()); + } +} diff --git a/lib/zerobus-ffi-1.3.0/rust/sdk/src/headers_provider.rs b/lib/zerobus-ffi-1.3.0/rust/sdk/src/headers_provider.rs new file mode 100644 index 00000000000..49b78e18092 --- /dev/null +++ b/lib/zerobus-ffi-1.3.0/rust/sdk/src/headers_provider.rs @@ -0,0 +1,153 @@ +use crate::default_token_factory::DefaultTokenFactory; +use crate::token_cache::{TokenCache, DEFAULT_REFRESH_BUFFER}; +use crate::ZerobusResult; +use async_trait::async_trait; +use std::collections::HashMap; +use std::sync::Arc; + +/// A trait for providing custom headers for gRPC requests. +/// +/// This trait allows you to implement custom logic for generating authentication headers, +/// such as fetching tokens from different OAuth providers or using alternative +/// authentication mechanisms. +/// +/// The HTTP `user-agent` header is set by the SDK on the underlying tonic +/// `Endpoint` and cannot be overridden by values returned from `get_headers`. +/// Use [`ZerobusSdkBuilder::application_name`](crate::ZerobusSdkBuilder::application_name) +/// to customize it. +/// +/// # Examples +/// +/// ```no_run +/// # use databricks_zerobus_ingest_sdk::{HeadersProvider, ZerobusResult}; +/// # use std::collections::HashMap; +/// # use async_trait::async_trait; +/// +/// struct MyCustomAuthProvider; +/// +/// #[async_trait] +/// impl HeadersProvider for MyCustomAuthProvider { +/// async fn get_headers(&self) -> ZerobusResult> { +/// let mut headers = HashMap::new(); +/// headers.insert("some_key", "some_value".to_string()); +/// Ok(headers) +/// } +/// } +/// ``` +#[async_trait] +pub trait HeadersProvider: Send + Sync { + /// Asynchronously gets the headers for a request. + /// + /// # Returns + /// + /// A `ZerobusResult` containing a `HashMap` of header names and values. + /// + /// # Errors + /// + /// Returns a `ZerobusError` if header generation fails (e.g., token request fails). + async fn get_headers(&self) -> ZerobusResult>; + + /// Invalidates any cached authentication state so the next `get_headers` + /// call re-derives it from scratch. + /// + /// The SDK calls this when the server rejects the supplied credentials with + /// an authentication error during stream creation. The default is a no-op, + /// which is correct for providers that hold no cache; the built-in OAuth + /// provider overrides it to drop its cached token so the next call re-mints. + async fn invalidate(&self) {} +} + +/// The default headers provider that uses OAuth 2.0 with Unity Catalog. +/// +/// This provider implements the OAuth 2.0 client credentials flow to obtain +/// access tokens for authenticating with the Zerobus service. +pub struct OAuthHeadersProvider { + client_id: String, + client_secret: String, + table_name: String, + workspace_id: String, + unity_catalog_url: String, + token_cache: Arc, +} + +impl OAuthHeadersProvider { + /// Creates a new `OAuthHeadersProvider`. + /// + /// This standalone constructor caches tokens for the lifetime of the + /// returned provider only. When streams are created via + /// [`ZerobusSdk::stream_builder`](crate::ZerobusSdk::stream_builder) the SDK + /// supplies a shared cache so tokens are reused across streams; see + /// [`with_cache`](Self::with_cache). + pub fn new( + client_id: String, + client_secret: String, + table_name: String, + workspace_id: String, + unity_catalog_url: String, + ) -> Self { + Self::with_cache( + client_id, + client_secret, + table_name, + workspace_id, + unity_catalog_url, + Arc::new(TokenCache::new(true, DEFAULT_REFRESH_BUFFER)), + ) + } + + /// Creates a new `OAuthHeadersProvider` backed by a shared token cache. + /// + /// Used internally so all streams created from one `ZerobusSdk` reuse cached + /// tokens rather than minting a fresh one per stream. + pub(crate) fn with_cache( + client_id: String, + client_secret: String, + table_name: String, + workspace_id: String, + unity_catalog_url: String, + token_cache: Arc, + ) -> Self { + Self { + client_id, + client_secret, + table_name, + workspace_id, + unity_catalog_url, + token_cache, + } + } +} + +#[async_trait] +impl HeadersProvider for OAuthHeadersProvider { + async fn get_headers(&self) -> ZerobusResult> { + let token = self + .token_cache + .get_or_fetch( + &self.client_id, + &self.client_secret, + &self.table_name, + |reason| { + DefaultTokenFactory::fetch_token( + &self.unity_catalog_url, + &self.table_name, + &self.client_id, + &self.client_secret, + &self.workspace_id, + reason, + ) + }, + ) + .await?; + let mut headers = HashMap::new(); + headers.insert("authorization", format!("Bearer {}", token)); + headers.insert("x-databricks-zerobus-table-name", self.table_name.clone()); + Ok(headers) + } + + async fn invalidate(&self) { + self.token_cache + .invalidate(&self.client_id, &self.client_secret, &self.table_name) + .await; + } +} diff --git a/lib/zerobus-ffi-1.3.0/rust/sdk/src/landing_zone.rs b/lib/zerobus-ffi-1.3.0/rust/sdk/src/landing_zone.rs new file mode 100644 index 00000000000..86f25f2a680 --- /dev/null +++ b/lib/zerobus-ffi-1.3.0/rust/sdk/src/landing_zone.rs @@ -0,0 +1,420 @@ +use std::collections::VecDeque; +use std::sync::Arc; + +use thiserror::Error; +use tokio::sync::{Notify, OwnedSemaphorePermit, Semaphore}; + +#[derive(Debug, Error)] +pub enum LandingZoneError { + #[error("Attempted to remove non-observed element")] + RemovingNonObservedElement, +} + +/// Internal state for the landing zone. +/// +/// Maintains two queues: one for unobserved items and one for observed items +/// that are waiting for acknowledgement. +struct LandingZoneState { + /// Queue of items that haven't been observed yet. + queue: VecDeque, + /// Queue of items that have been observed but not yet removed. + observed_items: VecDeque, +} + +/// A thread-safe queue with observation semantics and backpressure control. +/// +/// The `LandingZone` provides a specialized queue where items must be "observed" +/// before they can be removed. This enables the sender and receiver tasks to +/// coordinate: the sender observes items from the queue and sends them over the +/// network, while the receiver removes observed items only after receiving +/// acknowledgements. +/// +/// Key features: +/// - **Observe before remove**: Items must be observed before removal +/// - **Reset capability**: Observed items can be moved back to the queue for retry +/// - **Backpressure**: Enforces a maximum number of inflight requests via semaphore +/// - **Thread-safe**: Safe for concurrent access from multiple tasks +pub struct LandingZone { + /// Synchronizes access to the landing zone. + state: Arc>>, + /// Notifies waiting `observe()` calls when new items are added. + new_item_notify: Arc, + /// Controls maximum number of inflight requests to enforce backpressure. + semaphore: Arc, + /// Tracks semaphore permits to release them when items are removed. + permits: std::sync::Mutex>, +} + +impl LandingZone { + /// Creates a new `LandingZone` with the specified capacity. + /// + /// # Arguments + /// + /// * `max_inflight_requests` - Maximum number of requests that can be in the landing zone + /// (both observed and unobserved) at any time. When this limit is reached, `add()` + /// calls will block until items are removed. + pub fn new(max_inflight_requests: usize) -> Self { + Self { + state: Arc::new(std::sync::Mutex::new(LandingZoneState { + queue: VecDeque::with_capacity(max_inflight_requests), + observed_items: VecDeque::with_capacity(max_inflight_requests), + })), + new_item_notify: Arc::new(Notify::new()), + semaphore: Arc::new(Semaphore::new(max_inflight_requests)), + permits: std::sync::Mutex::new(VecDeque::with_capacity(max_inflight_requests)), + } + } + + /// Removes all items from the landing zone, both observed and unobserved. + /// + /// This is typically used during stream failure to retrieve all pending records. + /// + /// # Returns + /// + /// A vector containing all items that were in the landing zone. + pub fn remove_all(&self) -> Vec { + let mut state = self.state.lock().expect("Lock poisoned"); + + let mut all_items = Vec::with_capacity(state.observed_items.len() + state.queue.len()); + all_items.extend(state.observed_items.drain(..)); + all_items.extend(state.queue.drain(..)); + + let mut permits = self.permits.lock().expect("Lock poisoned"); + permits.clear(); + + all_items + } + + /// Adds an item to the queue. + /// + /// This method will block if the maximum number of inflight requests has been reached, + /// providing automatic backpressure control. + /// + /// # Arguments + /// + /// * `request` - The item to add to the queue + pub async fn add(&self, request: T) { + let _permit = self + .semaphore + .clone() + .acquire_owned() + .await + .expect("Failed to acquire semaphore"); + let mut state = self.state.lock().expect("Lock poisoned"); + state.queue.push_back(request); + self.permits + .lock() + .expect("Lock poisoned") + .push_back(_permit); + // Unblock one of the waiting observe() calls. + self.new_item_notify.notify_one(); + } + + /// Removes and returns the next observed item. + /// + /// Items must be observed via `observe()` before they can be removed. This ensures + /// proper coordination between sender and receiver tasks. + /// + /// # Returns + /// + /// * `Ok(T)` - The removed item + /// * `Err(LandingZoneError::RemovingNonObservedElement)` - If no items have been observed + pub fn remove_observed(&self) -> Result { + let mut state = self.state.lock().expect("Lock poisoned"); + if let Some(item) = state.observed_items.pop_front() { + self.permits.lock().expect("Lock poisoned").pop_front(); + Ok(item) + } else { + Err(LandingZoneError::RemovingNonObservedElement) + } + } + + /// Observes the next item in the queue without removing it. + /// + /// This moves the item from the unobserved queue to the observed queue. The item + /// remains in the landing zone until `remove_observed()` is called. This allows the + /// sender to send items over the network while keeping them buffered for potential retry. + /// + /// This method will block if there are no items available to observe. + /// + /// # Returns + /// + /// The observed item (still retained in the landing zone). + pub async fn observe(&self) -> T { + loop { + let notified = self.new_item_notify.notified(); + { + let mut state = self.state.lock().expect("Lock poisoned"); + if let Some(elem) = state.queue.pop_front() { + state.observed_items.push_back(elem.clone()); + return elem; + } + } + notified.await; + } + } + + /// Resets observation by moving all observed items back to the queue. + /// + /// This is used during stream recovery to re-send items that were observed but + /// not yet acknowledged by the server. + pub fn reset_observe(&self) { + let mut state = self.state.lock().expect("Lock poisoned"); + while let Some(observed_item) = state.observed_items.pop_back() { + state.queue.push_front(observed_item); + } + } + + /// Checks if there are no observed items waiting for acknowledgement. + /// + /// # Returns + /// + /// `true` if the observed queue is empty, `false` otherwise. + pub fn is_observed_empty(&self) -> bool { + let state = self.state.lock().expect("Lock poisoned"); + state.observed_items.is_empty() + } + + /// Returns the number of in-flight requests in the landing zone. + pub fn len(&self) -> usize { + let state = self.state.lock().expect("Lock poisoned"); + state.queue.len() + state.observed_items.len() + } +} + +#[cfg(test)] +mod tests { + use std::sync::Arc; + + use tokio::time::{timeout, Duration}; + + use super::{LandingZone, LandingZoneError}; + + #[tokio::test] + async fn test_add_and_observe() { + let lz = Arc::new(LandingZone::new(10)); + + lz.add("test_item".to_string()).await; + + let observed = lz.observe().await; + assert_eq!(observed, "test_item"); + } + + #[tokio::test] + async fn test_observe_blocks_until_item_available() { + let lz = Arc::new(LandingZone::new(10)); + let lz_clone = lz.clone(); + + // Start observing in background. + let observe_task = tokio::spawn(async move { lz_clone.observe().await }); + + // Give it time to start waiting. + tokio::time::sleep(Duration::from_millis(10)).await; + + lz.add("delayed_item".to_string()).await; + + // Should unblock and return the item. + let result = timeout(Duration::from_millis(100), observe_task).await; + assert!(result.is_ok()); + assert_eq!(result.unwrap().unwrap(), "delayed_item"); + } + + #[tokio::test] + async fn test_remove_observed() { + let lz = Arc::new(LandingZone::new(10)); + + lz.add("item1".to_string()).await; + let _observed = lz.observe().await; + + let removed = lz.remove_observed().unwrap(); + assert_eq!(removed, "item1"); + } + + #[tokio::test] + async fn test_remove_non_observed_fails() { + let lz = Arc::new(LandingZone::::new(10)); + + let result = lz.remove_observed(); + assert!(matches!( + result, + Err(LandingZoneError::RemovingNonObservedElement) + )); + } + + #[tokio::test] + async fn test_remove_all() { + let lz = Arc::new(LandingZone::new(10)); + + lz.add("item1".to_string()).await; + lz.add("item2".to_string()).await; + + let _observed = lz.observe().await; + + let all_items = lz.remove_all(); + assert_eq!(all_items.len(), 2); + assert!(all_items.contains(&"item1".to_string())); + assert!(all_items.contains(&"item2".to_string())); + + assert!(lz.len() == 0); + } + + #[tokio::test] + async fn test_semaphore_limits_capacity() { + let lz = Arc::new(LandingZone::new(2)); + + lz.add("item1".to_string()).await; + lz.add("item2".to_string()).await; + + // Third add should block (test with timeout). + let mut add_task = tokio::spawn({ + let lz = lz.clone(); + async move { + lz.add("item3".to_string()).await; + } + }); + // Should timeout because semaphore is full. + tokio::select! { + _ = &mut add_task => { + panic!("add_task should not complete while semaphore is full"); + } + _ = tokio::time::sleep(Duration::from_millis(50)) => { + // This is expected, the task is still blocked. + } + }; + + // Remove one item to free up space. + let _observed = lz.observe().await; + let _removed = lz.remove_observed().unwrap(); + + // Now the add_task should complete. + add_task.await.unwrap(); + + let all_items = lz.remove_all(); + assert_eq!(all_items.len(), 2); + assert!(all_items.contains(&"item2".to_string())); + assert!(all_items.contains(&"item3".to_string())); + } + + #[tokio::test] + async fn test_reset_observe_with_concurrent_add() { + let lz = Arc::new(LandingZone::new(10)); + + lz.add("item1".to_string()).await; + lz.add("item2".to_string()).await; + lz.add("item3".to_string()).await; + + let observed = lz.observe().await; + assert_eq!(observed, "item1"); + + // In another thread, add 4th item. + let lz_clone = lz.clone(); + let add_task = tokio::spawn(async move { + lz_clone.add("item4".to_string()).await; + }); + + add_task.await.unwrap(); + + lz.reset_observe(); + + assert_eq!(lz.observe().await, "item1"); + assert_eq!(lz.observe().await, "item2"); + assert_eq!(lz.observe().await, "item3"); + assert_eq!(lz.observe().await, "item4"); + } + + #[tokio::test] + async fn test_semaphore_with_observe_reset() { + let lz = Arc::new(LandingZone::new(2)); + + lz.add("item1".to_string()).await; + lz.add("item2".to_string()).await; + + // Observe one (should not free semaphore permit yet). + let _observed = lz.observe().await; + + // Adding should still block because permit not released until remove_observed. + let add_task = tokio::spawn({ + let lz = lz.clone(); + async move { + lz.add("item3".to_string()).await; + } + }); + + let result = timeout(Duration::from_millis(50), add_task).await; + assert!(result.is_err()); // Should timeout. + + // Reset observe (item goes back to queue, still no permit freed). + lz.reset_observe(); + + // Remove observed should fail (nothing observed now). + assert!(lz.remove_observed().is_err()); + + // Only after actually removing an observed item should permit be freed and add_task should complete. + let _observed_again = lz.observe().await; + let _removed = lz.remove_observed().unwrap(); + + // Remove item_3. + let _observed_again_2 = lz.observe().await; + let _removed_2 = lz.remove_observed().unwrap(); + + // Now add should work. + lz.add("item4".to_string()).await; + } + + #[tokio::test] + async fn test_is_observed_empty() { + let lz = Arc::new(LandingZone::new(16)); + + // Initially, observed queue should be empty + assert!(lz.is_observed_empty()); + + lz.add("item1".to_string()).await; + // Still empty because we haven't observed yet + assert!(lz.is_observed_empty()); + + lz.observe().await; + // Now it should not be empty + assert!(!lz.is_observed_empty()); + + lz.remove_observed().unwrap(); + // After removal, should be empty again + assert!(lz.is_observed_empty()); + } + + #[tokio::test] + async fn test_concurrent_operations() { + let lz = Arc::new(LandingZone::new(100)); + + // Spawn multiple tasks that add items + let mut add_tasks = vec![]; + for i in 0..10 { + let lz_clone = lz.clone(); + add_tasks.push(tokio::spawn(async move { + lz_clone.add(format!("item{}", i)).await; + })); + } + + // Spawn multiple tasks that observe items + let mut observe_tasks = vec![]; + for _ in 0..10 { + let lz_clone = lz.clone(); + observe_tasks.push(tokio::spawn(async move { + lz_clone.observe().await; + })); + } + + // Wait for all add tasks to complete + for task in add_tasks { + task.await.unwrap(); + } + + // Wait for all observe tasks to complete + let mut observed_items = vec![]; + for task in observe_tasks { + observed_items.push(task.await); + } + + // All 10 items should have been observed + assert_eq!(observed_items.len(), 10); + } +} diff --git a/lib/zerobus-ffi-1.3.0/rust/sdk/src/lib.rs b/lib/zerobus-ffi-1.3.0/rust/sdk/src/lib.rs new file mode 100644 index 00000000000..530ea0bbc7a --- /dev/null +++ b/lib/zerobus-ffi-1.3.0/rust/sdk/src/lib.rs @@ -0,0 +1,1913 @@ +//! # Databricks Zerobus Ingest SDK +//! +//! A high-performance Rust client for streaming data ingestion into Databricks Delta tables. +//! +//! ## Quick Start +//! +//! ```rust,ignore +//! use databricks_zerobus_ingest_sdk::{ZerobusSdk, JsonValue}; +//! +//! let sdk = ZerobusSdk::builder() +//! .endpoint(zerobus_endpoint) +//! .unity_catalog_url(uc_endpoint) +//! .build()?; +//! +//! let stream = sdk +//! .stream_builder() +//! .table("catalog.schema.table") +//! .oauth(client_id, client_secret) +//! .json() +//! .build() +//! .await?; +//! +//! // Ingest a record and wait for acknowledgment +//! let offset = stream.ingest_record_offset(JsonValue(my_record)).await?; +//! stream.wait_for_offset(offset).await?; +//! +//! stream.close().await?; +//! ``` +//! +//! See the `examples/` directory for complete working examples. + +pub mod databricks { + pub mod zerobus { + include!(concat!(env!("OUT_DIR"), "/databricks.zerobus.rs")); + } +} + +#[cfg(feature = "arrow-flight")] +mod arrow_configuration; +#[cfg(feature = "arrow-flight")] +mod arrow_metadata; +#[cfg(feature = "arrow-flight")] +mod arrow_stream; +mod builder; +mod callbacks; +mod client_warnings; +mod default_token_factory; +mod errors; +mod headers_provider; +mod landing_zone; +mod offset_generator; +mod proxy; +mod record_types; +pub mod schema; +mod stream_configuration; +pub mod stream_options; +mod tls_config; +mod token_cache; + +use std::collections::HashMap; +use std::fmt::Debug; +use std::future::Future; +use std::sync::atomic::{AtomicBool, Ordering}; +use std::sync::Arc; + +use prost::Message; +use tokio::sync::RwLock; +use tokio::time::Duration; +use tokio_retry::strategy::FixedInterval; +use tokio_retry::RetryIf; +use tokio_stream::wrappers::ReceiverStream; +use tokio_util::sync::CancellationToken; +use tonic::metadata::MetadataValue; +use tonic::transport::{Channel, Endpoint}; +use tracing::{debug, error, info, instrument, span, trace, warn, Level}; + +use databricks::zerobus::ephemeral_stream_request::Payload as RequestPayload; +use databricks::zerobus::ephemeral_stream_response::Payload as ResponsePayload; +use databricks::zerobus::zerobus_client::ZerobusClient; +use databricks::zerobus::{ + CloseStreamSignal, CreateIngestStreamRequest, EphemeralStreamRequest, EphemeralStreamResponse, + IngestRecordResponse, RecordType, +}; +use landing_zone::LandingZone; + +/// **Beta**: Arrow Flight ingestion is in Beta. The API is stabilising but may +/// still change before reaching GA. +#[cfg(feature = "arrow-flight")] +pub use arrow_configuration::ArrowStreamConfigurationOptions; +#[cfg(feature = "arrow-flight")] +pub use arrow_stream::{ArrowSchema, DataType, Field, RecordBatch, TimeUnit, ZerobusArrowStream}; +pub use builder::{StreamBuilder, ZerobusSdkBuilder}; +pub use callbacks::AckCallback; +pub use default_token_factory::DefaultTokenFactory; +pub use errors::ZerobusError; +pub use headers_provider::{HeadersProvider, OAuthHeadersProvider}; +pub use offset_generator::{OffsetId, OffsetIdGenerator}; +pub use proxy::{ConnectorFactory, ProxyConnector}; +pub use record_types::{ + EncodedBatch, EncodedBatchIter, EncodedRecord, JsonEncodedRecord, JsonString, JsonValue, + ProtoBytes, ProtoEncodedRecord, ProtoMessage, +}; +pub use stream_configuration::StreamConfigurationOptions; +#[cfg(feature = "testing")] +pub use tls_config::NoTlsConfig; +pub use tls_config::{SecureTlsConfig, TlsConfig}; + +#[cfg(feature = "zeroparser")] +pub mod zeroparser; + +const SHUTDOWN_TIMEOUT_SECS: u64 = 1; + +/// Maximum time to wait for the receiver/sender tasks to finish during stream +/// teardown. +const STREAM_TEARDOWN_DRAIN_TIMEOUT_MS: u64 = 500; + +/// The type of the stream connection created with the server. +/// Currently we only support ephemeral streams on the server side, so we support only that in the SDK as well. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +#[non_exhaustive] +pub enum StreamType { + /// Ephemeral streams exist only for the duration of the connection. + /// They are not persisted and are not recoverable. + Ephemeral, + /// UNSUPPORTED: Persistent streams are durable and recoverable. + Persistent, +} + +/// The properties of the table to ingest to. +/// +/// Configure the table via the builder API: +/// `sdk.stream_builder().table("catalog.schema.table").compiled_proto(descriptor)`. +/// +/// # Common errors: +/// -`InvalidTableName`: table_name contains invalid characters or doesn't exist +/// -`PermissionDenied`: insufficient permissions to write to the specified table +/// -`InvalidArgument`: invalid or missing descriptor_proto or auth token +#[derive(Debug, Clone)] +pub(crate) struct TableProperties { + pub(crate) table_name: String, + pub(crate) descriptor_proto: Option, +} + +pub type ZerobusResult = Result; + +#[derive(Debug, Clone)] +struct IngestRequest { + payload: EncodedBatch, + offset_id: OffsetId, +} + +/// Map of logical offset to oneshot sender used to send acknowledgments back to the client. +type OneshotMap = HashMap>>; +/// Landing zone for ingest records. +type RecordLandingZone = Arc>>; + +/// Messages sent to the callback handler task. +#[derive(Debug, Clone)] +enum CallbackMessage { + /// Acknowledgment callback with logical offset ID. + Ack(OffsetId), + /// Error callback with logical offset ID and error message. + Error(OffsetId, String), +} + +/// Represents an active ingestion stream to a Databricks Delta table. +/// +/// A `ZerobusStream` manages a bidirectional gRPC stream for ingesting records into +/// a Unity Catalog table. It handles authentication, automatic recovery, acknowledgment +/// tracking, and graceful shutdown. +/// +/// # Lifecycle +/// +/// 1. Create a stream via `ZerobusSdk::stream_builder()` +/// 2. Ingest records with `ingest_record_offset()` and `wait_for_offset()` for acknowledgments +/// 3. Optionally call `flush()` to ensure all records are persisted +/// 4. Close the stream with `close()` to release resources +/// +/// # Examples +/// +/// ```no_run +/// # use databricks_zerobus_ingest_sdk::*; +/// # async fn example(mut stream: ZerobusStream, data: Vec) -> Result<(), ZerobusError> { +/// // Ingest a single record +/// let offset = stream.ingest_record_offset(data).await?; +/// println!("Record sent with offset: {}", offset); +/// +/// // Wait for acknowledgment +/// stream.wait_for_offset(offset).await?; +/// println!("Record acknowledged at offset: {}", offset); +/// +/// // Close the stream gracefully +/// stream.close().await?; +/// # Ok(()) +/// # } +/// ``` +#[non_exhaustive] +pub struct ZerobusStream { + /// This is a 128-bit UUID that is unique across all streams in the system, + /// not just within a single table. The server returns this ID in the CreateStreamResponse + /// after validating the table properties and establishing the gRPC connection. + stream_id: Option, + /// Type of gRPC stream that is used when sending records. + pub stream_type: StreamType, + /// Gets headers which are used in the first request to establish connection with the server. + pub headers_provider: Arc, + /// The stream configuration options related to recovery, fetching OAuth tokens, etc. + pub options: StreamConfigurationOptions, + /// The table properties - table name and descriptor of the table. + pub(crate) table_properties: TableProperties, + /// Logical landing zone that is used to store records that have been sent by user but not yet sent over the network. + landing_zone: RecordLandingZone, + /// Map of logical offset to oneshot sender. + oneshot_map: Arc>, + /// Supervisor task that manages the stream lifecycle such as stream creation, recovery, etc. + /// It orchestrates the receiver and sender tasks. + supervisor_task: tokio::task::JoinHandle>, + /// The generator of logical offset IDs. Used to generate monotonically increasing offset IDs, even if the stream recovers. + logical_offset_id_generator: OffsetIdGenerator, + /// Signal that the stream is caught up to the given offset. + logical_last_received_offset_id_tx: tokio::sync::watch::Sender>, + /// Persistent offset ID receiver to ensure at least one receiver exists, preventing SendError + _logical_last_received_offset_id_rx: tokio::sync::watch::Receiver>, + /// A vector of records that have failed to be acknowledged. + failed_records: Arc>>, + /// Flag indicating if the stream has been closed. + is_closed: Arc, + /// Sync mutex to ensure that offset generation and record ingestion happen atomically. + sync_mutex: Arc>, + /// Watch channel for last error received from the server. + server_error_rx: tokio::sync::watch::Receiver>, + /// Cancellation token to signal receiver and sender tasks to abort. It is sent either when stream is closed or dropped. + cancellation_token: CancellationToken, + /// Callback handler task that executes callbacks in a separate thread. + callback_handler_task: Option>, +} + +/// Default identifier the SDK sends as the HTTP `user-agent` header on every +/// request. Use [`ZerobusSdkBuilder::application_name`] to append an +/// application suffix. +pub const DEFAULT_SDK_IDENTIFIER: &str = concat!("zerobus-sdk-rs/", env!("CARGO_PKG_VERSION")); + +/// The main interface for interacting with the Zerobus API. +/// # Examples +/// ```rust,ignore +/// // Create SDK using the builder +/// let sdk = ZerobusSdk::builder() +/// .endpoint("https://your-workspace.zerobus.region.cloud.databricks.com") +/// .unity_catalog_url("https://your-workspace.cloud.databricks.com") +/// .build()?; +/// +/// // Create a stream via the stream builder +/// let stream = sdk +/// .stream_builder() +/// .table("catalog.schema.table") +/// .oauth("client-id", "client-secret") +/// .compiled_proto(descriptor_proto) +/// .max_inflight_requests(100) +/// .build() +/// .await?; +/// +/// // Ingest a single record +/// let offset_id = stream.ingest_record_offset(ProtoMessage(row)).await?; +/// +/// // Wait for acknowledgment +/// stream.wait_for_offset(offset_id).await?; +/// ``` +#[non_exhaustive] +pub struct ZerobusSdk { + pub zerobus_endpoint: String, + pub unity_catalog_url: String, + shared_channel: tokio::sync::Mutex>>, + pub(crate) workspace_id: String, + pub(crate) tls_config: Arc, + connector_factory: Option, + /// Final value sent as the HTTP `user-agent` header on every request. + /// Either `"zerobus-sdk-rs/"` or `"zerobus-sdk-rs/ "`. + pub(crate) sdk_identifier: Arc, + /// Shared cache of OAuth tokens, keyed per table, reused across all streams + /// created from this SDK instance via the default OAuth path. + pub(crate) token_cache: Arc, +} + +impl ZerobusSdk { + /// Creates a new SDK builder for fluent configuration. + /// + /// This is the recommended way to create a `ZerobusSdk` instance. + /// + /// # Examples + /// + /// ```no_run + /// use databricks_zerobus_ingest_sdk::ZerobusSdk; + /// + /// let sdk = ZerobusSdk::builder() + /// .endpoint("https://workspace.zerobus.databricks.com") + /// .unity_catalog_url("https://workspace.cloud.databricks.com") + /// .build()?; + /// # Ok::<(), databricks_zerobus_ingest_sdk::ZerobusError>(()) + /// ``` + pub fn builder() -> ZerobusSdkBuilder { + ZerobusSdkBuilder::new() + } + + /// Creates a new stream builder for configuring an ingestion stream. + /// + /// All setters can be called in any order. The builder validates at + /// `build()` time that table name, authentication, and format have + /// been configured. Use [`StreamBuilder::validate()`] to check the + /// configuration without opening a stream. + /// + /// # Examples + /// + /// ```rust,ignore + /// // JSON stream with OAuth + /// let stream = sdk + /// .stream_builder() + /// .table("catalog.schema.table") + /// .oauth("client-id", "client-secret") + /// .json() + /// .build() + /// .await?; + /// + /// // Proto stream with custom headers + /// let stream = sdk + /// .stream_builder() + /// .table("catalog.schema.table") + /// .headers_provider(my_provider) + /// .compiled_proto(descriptor) + /// .max_inflight_requests(500_000) + /// .build() + /// .await?; + /// ``` + pub fn stream_builder(&self) -> StreamBuilder<'_> { + StreamBuilder::new(self) + } + + /// Creates a new SDK instance with explicit configuration. + /// + /// This is used internally by the builder pattern. `sdk_identifier` is the + /// fully-resolved value sent as the HTTP `user-agent` header; the builder + /// is responsible for composing it from the default prefix and any + /// caller-supplied `application_name` or override. + #[allow(clippy::too_many_arguments)] + pub(crate) fn new_with_config( + zerobus_endpoint: String, + unity_catalog_url: String, + workspace_id: String, + tls_config: Arc, + connector_factory: Option, + sdk_identifier: Arc, + token_cache_enabled: bool, + token_refresh_buffer: Duration, + ) -> Self { + ZerobusSdk { + zerobus_endpoint, + unity_catalog_url, + workspace_id, + shared_channel: tokio::sync::Mutex::new(None), + tls_config, + connector_factory, + sdk_identifier, + token_cache: Arc::new(token_cache::TokenCache::new( + token_cache_enabled, + token_refresh_buffer, + )), + } + } + + /// Recreates a failed stream and re-ingests unacknowledged records. + /// + /// This is useful when a stream encounters an error and you want to preserve + /// unacknowledged records. The method creates a new stream with the same + /// configuration and automatically re-ingests all records that weren't acknowledged. + /// + /// # Arguments + /// + /// * `stream` - The failed stream to recreate + /// + /// # Returns + /// + /// A new `ZerobusStream` with unacknowledged records already submitted. + /// + /// # Errors + /// + /// Returns any errors from stream creation or re-ingestion. + /// + /// # Examples + /// + /// ```no_run + /// # use databricks_zerobus_ingest_sdk::*; + /// # async fn example(sdk: ZerobusSdk, mut stream: ZerobusStream) -> Result<(), ZerobusError> { + /// match stream.close().await { + /// Err(_) => { + /// // Stream failed, recreate it + /// let new_stream = sdk.recreate_stream(&stream).await?; + /// // Continue using new_stream + /// } + /// Ok(_) => println!("Stream closed successfully"), + /// } + /// # Ok(()) + /// # } + /// ``` + #[instrument(level = "debug", skip_all)] + pub async fn recreate_stream(&self, stream: &ZerobusStream) -> ZerobusResult { + let batches = stream.get_unacked_batches().await?; + let channel = self.get_or_create_channel_zerobus_client().await?; + let new_stream = ZerobusStream::new_stream( + channel, + stream.table_properties.clone(), + Arc::clone(&stream.headers_provider), + stream.options.clone(), + ) + .await; + + match new_stream { + Ok(new_stream) => { + if let Some(stream_id) = new_stream.stream_id.as_ref() { + info!(stream_id = %stream_id, "Successfully recreated ephemeral stream"); + } else { + error!("Successfully recreated a stream but stream_id is None"); + } + + for batch in batches { + let ack = new_stream.ingest_internal(batch).await?; + tokio::spawn(ack); + } + + Ok(new_stream) + } + Err(e) => { + error!("Stream recreation failed with error: {}", e); + Err(e) + } + } + } + + /// Recreates an Arrow Flight stream from a failed or closed stream, replaying any + /// unacknowledged batches. + /// + /// This method is useful when you want to manually recover from a stream failure + /// or continue ingestion after closing a stream with unacknowledged batches. + /// It creates a new stream with the same configuration and automatically ingests + /// any batches that were not acknowledged in the original stream. + /// + /// # Arguments + /// + /// * `stream` - A reference to the failed or closed Arrow Flight stream + /// + /// # Returns + /// + /// A new `ZerobusArrowStream` with the same configuration, with unacked batches + /// already queued for ingestion. + /// + /// # Errors + /// + /// * `InvalidStateError` - If the source stream is still active + /// * `CreateStreamError` - If stream creation fails + /// + /// # Examples + /// + /// ```no_run + /// # use databricks_zerobus_ingest_sdk::*; + /// # use arrow_array::RecordBatch; + /// # async fn example(sdk: ZerobusSdk, mut stream: ZerobusArrowStream) -> Result<(), ZerobusError> { + /// // Ingest some batches + /// // ... + /// + /// // Stream fails for some reason + /// match stream.flush().await { + /// Err(_) => { + /// // Close the failed stream + /// stream.close().await.ok(); + /// + /// // Recreate and retry + /// let new_stream = sdk.recreate_arrow_stream(&stream).await?; + /// new_stream.flush().await?; + /// } + /// Ok(_) => {} + /// } + /// # Ok(()) + /// # } + /// ``` + #[cfg(feature = "arrow-flight")] + #[instrument(level = "debug", skip_all)] + pub async fn recreate_arrow_stream( + &self, + stream: &ZerobusArrowStream, + ) -> ZerobusResult { + let batches = stream.get_unacked_batches().await?; + + let new_stream = ZerobusArrowStream::new( + &self.zerobus_endpoint, + Arc::clone(&self.tls_config), + stream.table_properties.clone(), + stream.headers_provider(), + stream.options().clone(), + Arc::clone(&self.sdk_identifier), + ) + .await; + + match new_stream { + Ok(new_stream) => { + info!( + table_name = %new_stream.table_name(), + "Successfully recreated Arrow Flight stream" + ); + + for batch in batches { + let _offset = new_stream.ingest_batch(batch).await?; + } + + Ok(new_stream) + } + Err(e) => { + error!("Arrow Flight stream recreation failed: {}", e); + Err(e) + } + } + } + + /// Gets or creates the shared Channel for all streams. + /// The first call creates the Channel, subsequent calls clone it. + /// All clones share the same underlying TCP connection via HTTP/2 multiplexing. + pub(crate) async fn get_or_create_channel_zerobus_client( + &self, + ) -> ZerobusResult> { + let mut guard = self.shared_channel.lock().await; + + if guard.is_none() { + // Create the channel for the first time. + let endpoint = Endpoint::from_shared(self.zerobus_endpoint.clone()) + .map_err(|err| ZerobusError::ChannelCreationError(err.to_string()))? + .user_agent(self.sdk_identifier.as_ref()) + .map_err(|err| ZerobusError::ChannelCreationError(err.to_string()))?; + + let endpoint = self.tls_config.configure_endpoint(endpoint)?; + + // A caller-supplied factory (from `ZerobusSdkBuilder::connector_factory`) + // fully replaces the default env-var proxy detection + // (`https_proxy`/`HTTPS_PROXY` and friends). + let host = endpoint.uri().host().unwrap_or_default().to_string(); + let proxy_connector = match &self.connector_factory { + Some(factory) => factory(&host).map(ProxyConnector::into_inner), + None if !proxy::is_no_proxy(&host) => proxy::create_proxy_connector(), + None => None, + }; + + let channel = match proxy_connector { + Some(pc) => endpoint.connect_with_connector_lazy(pc), + None => endpoint.connect_lazy(), + }; + + let client = ZerobusClient::new(channel) + .max_decoding_message_size(usize::MAX) + .max_encoding_message_size(usize::MAX); + + *guard = Some(client); + } + + Ok(guard + .as_ref() + .expect("Channel was just initialized") + .clone()) + } +} + +impl ZerobusStream { + /// Creates a new ephemeral stream for ingesting records. + #[instrument(level = "debug", skip_all)] + pub(crate) async fn new_stream( + channel: ZerobusClient, + table_properties: TableProperties, + headers_provider: Arc, + options: StreamConfigurationOptions, + ) -> ZerobusResult { + let (stream_init_result_tx, stream_init_result_rx) = + tokio::sync::oneshot::channel::>(); + + let (logical_last_received_offset_id_tx, _logical_last_received_offset_id_rx) = + tokio::sync::watch::channel(None); + let landing_zone = Arc::new(LandingZone::>::new( + options.max_inflight_requests, + )); + + let oneshot_map = Arc::new(tokio::sync::Mutex::new(HashMap::new())); + let is_closed = Arc::new(AtomicBool::new(false)); + let failed_records = Arc::new(RwLock::new(Vec::new())); + let logical_offset_id_generator = OffsetIdGenerator::default(); + + let (server_error_tx, server_error_rx) = tokio::sync::watch::channel(None); + let cancellation_token = CancellationToken::new(); + // Create callback channel and spawn callback handler task only if callback is defined + let (callback_tx, callback_handler_task) = if options.ack_callback.is_some() { + let (tx, rx) = tokio::sync::mpsc::unbounded_channel(); + let task = Self::spawn_callback_handler_task( + rx, + options.ack_callback.clone(), + cancellation_token.clone(), + ); + (Some(tx), Some(task)) + } else { + (None, None) + }; + + let supervisor_task = tokio::task::spawn(Self::supervisor_task( + channel, + table_properties.clone(), + Arc::clone(&headers_provider), + options.clone(), + Arc::clone(&landing_zone), + Arc::clone(&oneshot_map), + logical_last_received_offset_id_tx.clone(), + Arc::clone(&is_closed), + Arc::clone(&failed_records), + stream_init_result_tx, + server_error_tx, + cancellation_token.clone(), + callback_tx.clone(), + )); + let stream_id = Some(stream_init_result_rx.await.map_err(|_| { + ZerobusError::UnexpectedStreamResponseError( + "Supervisor task died before stream creation".to_string(), + ) + })??); + + let stream = Self { + stream_type: StreamType::Ephemeral, + headers_provider, + options: options.clone(), + table_properties, + stream_id, + landing_zone, + oneshot_map, + supervisor_task, + logical_offset_id_generator, + logical_last_received_offset_id_tx, + _logical_last_received_offset_id_rx, + failed_records, + is_closed, + sync_mutex: Arc::new(tokio::sync::Mutex::new(())), + server_error_rx, + cancellation_token, + callback_handler_task, + }; + + Ok(stream) + } + + /// Supervisor task is responsible for managing the stream lifecycle. + /// It handles stream creation, recovery, and error handling. + #[allow(clippy::too_many_arguments)] + #[instrument(level = "debug", skip_all, fields(table_name = %table_properties.table_name))] + async fn supervisor_task( + channel: ZerobusClient, + table_properties: TableProperties, + headers_provider: Arc, + options: StreamConfigurationOptions, + landing_zone: RecordLandingZone, + oneshot_map: Arc>, + logical_last_received_offset_id_tx: tokio::sync::watch::Sender>, + is_closed: Arc, + failed_records: Arc>>, + stream_init_result_tx: tokio::sync::oneshot::Sender>, + server_error_tx: tokio::sync::watch::Sender>, + cancellation_token: CancellationToken, + callback_tx: Option>, + ) -> ZerobusResult<()> { + let mut initial_stream_creation = true; + let mut stream_init_result_tx = Some(stream_init_result_tx); + + loop { + debug!("Supervisor task loop"); + + if cancellation_token.is_cancelled() { + debug!("Supervisor task cancelled, exiting"); + return Ok(()); + } + + let landing_zone_sender = Arc::clone(&landing_zone); + let landing_zone_receiver = Arc::clone(&landing_zone); + let landing_zone_recovery = Arc::clone(&landing_zone); + + // 1. Create a stream. + let strategy = FixedInterval::from_millis(options.recovery_backoff_ms) + .take(options.recovery_retries as usize); + + let create_attempt = || { + let channel = channel.clone(); + let table_properties = table_properties.clone(); + let headers_provider = Arc::clone(&headers_provider); + let record_type = options.record_type; + + async move { + tokio::time::timeout( + Duration::from_millis(options.recovery_timeout_ms), + Self::create_stream_connection( + channel, + &table_properties, + &headers_provider, + record_type, + ), + ) + .await + .map_err(|_| { + ZerobusError::CreateStreamError(tonic::Status::deadline_exceeded( + "Stream creation timed out", + )) + })? + } + }; + let should_retry = |e: &ZerobusError| options.recovery && e.is_retryable(); + let creation = RetryIf::spawn(strategy, create_attempt, should_retry).await; + + let (tx, response_grpc_stream, stream_id) = match creation { + Ok((tx, response_grpc_stream, stream_id)) => (tx, response_grpc_stream, stream_id), + Err(e) => { + if initial_stream_creation { + if let Some(tx) = stream_init_result_tx.take() { + let _ = tx.send(Err(e.clone())); + } + } else { + is_closed.store(true, Ordering::Relaxed); + Self::fail_all_pending_records( + landing_zone.clone(), + oneshot_map.clone(), + failed_records.clone(), + &e, + &callback_tx, + ) + .await; + } + return Err(e); + } + }; + if initial_stream_creation { + if let Some(stream_init_result_tx_inner) = stream_init_result_tx.take() { + let _ = stream_init_result_tx_inner.send(Ok(stream_id.clone())); + } + initial_stream_creation = false; + info!(stream_id = %stream_id, "Successfully created stream"); + } else { + info!(stream_id = %stream_id, "Successfully recovered stream"); + let _ = server_error_tx.send(None); + } + + // 2. Reset landing zone. + landing_zone_recovery.reset_observe(); + + // 3. Spawn receiver and sender task. + let is_paused = Arc::new(AtomicBool::new(false)); + + // Per-stream child token + let per_stream_token = cancellation_token.child_token(); + // Separate token for recv_task's close path + let recv_drain_token = CancellationToken::new(); + + let mut recv_task = Self::spawn_receiver_task( + response_grpc_stream, + logical_last_received_offset_id_tx.clone(), + landing_zone_receiver, + oneshot_map.clone(), + Arc::clone(&is_paused), + options.clone(), + server_error_tx.clone(), + recv_drain_token.clone(), + callback_tx.clone(), + ); + let mut send_task = Self::spawn_sender_task( + tx, + landing_zone_sender, + Arc::clone(&is_paused), + server_error_tx.clone(), + per_stream_token.clone(), + ); + + // 4. Wait for any of the two tasks to end. + let result = tokio::select! { + recv_result = &mut recv_task => { + per_stream_token.cancel(); + let _ = tokio::time::timeout( + Duration::from_millis(STREAM_TEARDOWN_DRAIN_TIMEOUT_MS), + &mut send_task, + ) + .await; + if !send_task.is_finished() { + send_task.abort(); + } + match recv_result { + Ok(Err(e)) => Err(e), + Err(e) => Err(ZerobusError::UnexpectedStreamResponseError( + format!("Receiver task panicked: {}", e) + )), + Ok(Ok(())) => { + info!("Receiver task completed successfully"); + Ok(()) + } + } + } + send_result = &mut send_task => { + // Draining the recv_task prevents RST_STREAM(CANCEL) from being sent alongside END_STREAM. + if matches!(send_result, Ok(Ok(()))) && cancellation_token.is_cancelled() { + recv_drain_token.cancel(); + let _ = tokio::time::timeout( + Duration::from_millis(STREAM_TEARDOWN_DRAIN_TIMEOUT_MS), + &mut recv_task, + ) + .await; + } + recv_task.abort(); + match send_result { + Ok(Err(e)) => Err(e), + Err(e) => Err(ZerobusError::UnexpectedStreamResponseError( + format!("Sender task panicked: {}", e) + )), + Ok(Ok(())) => Ok(()) // This only happens when the sender task receives a cancellation signal. + } + } + }; + + // 5. Handle errors. + if let Err(error) = result { + error!(stream_id = %stream_id, "Stream failure detected: {}", error); + let error = match &error { + // Mapping this to pass certain e2e tests. + // TODO: Remove this once we fix tests. + ZerobusError::StreamClosedError(status) + if status.code() == tonic::Code::InvalidArgument => + { + ZerobusError::InvalidArgument(status.message().to_string()) + } + _ => error, + }; + let _ = server_error_tx.send(Some(error.clone())); + if !error.is_retryable() || !options.recovery { + is_closed.store(true, Ordering::Relaxed); + // A mid-stream auth rejection means the cached token is no + // longer accepted; drop it so the next stream re-mints. + if error.is_auth_rejection() { + headers_provider.invalidate().await; + } + Self::fail_all_pending_records( + landing_zone.clone(), + oneshot_map.clone(), + failed_records.clone(), + &error, + &callback_tx, + ) + .await; + return Err(error); + } + } + } + } + + /// Creates a stream connection to the Zerobus API. + /// Returns a tuple containing the sender, response gRPC stream, and stream ID. + /// If the stream creation fails, it returns an error. + /// + /// On a server-side authentication rejection it asks the headers provider to + /// invalidate cached credentials so the next attempt re-derives them. This + /// covers IdP-revoked tokens, not a same-named table recreated within the + /// token's lifetime, which the server accepts. + async fn create_stream_connection( + channel: ZerobusClient, + table_properties: &TableProperties, + headers_provider: &Arc, + record_type: RecordType, + ) -> ZerobusResult<( + tokio::sync::mpsc::Sender, + tonic::Streaming, + String, + )> { + let result = Self::create_stream_connection_inner( + channel, + table_properties, + headers_provider, + record_type, + ) + .await; + if let Err(err) = &result { + if err.is_auth_rejection() { + headers_provider.invalidate().await; + } + } + result + } + + #[instrument(level = "debug", skip_all, fields(table_name = %table_properties.table_name))] + async fn create_stream_connection_inner( + mut channel: ZerobusClient, + table_properties: &TableProperties, + headers_provider: &Arc, + record_type: RecordType, + ) -> ZerobusResult<( + tokio::sync::mpsc::Sender, + tonic::Streaming, + String, + )> { + const CHANNEL_BUFFER_SIZE: usize = 2048; + let (tx, rx) = tokio::sync::mpsc::channel(CHANNEL_BUFFER_SIZE); + let mut request_stream = tonic::Request::new(ReceiverStream::new(rx)); + + let stream_metadata = request_stream.metadata_mut(); + let headers = headers_provider.get_headers().await?; + + for (key, value) in headers { + match key { + "x-databricks-zerobus-table-name" => { + let table_name = MetadataValue::try_from(value.as_str()) + .map_err(|e| ZerobusError::InvalidTableName(e.to_string()))?; + stream_metadata.insert("x-databricks-zerobus-table-name", table_name); + } + "authorization" => { + let mut auth_value = MetadataValue::try_from(value.as_str()).map_err(|_| { + error!(table_name = %table_properties.table_name, "authorization token is not a valid HTTP header value"); + ZerobusError::InvalidUCTokenError( + "authorization token is not a valid HTTP header value".to_string(), + ) + })?; + auth_value.set_sensitive(true); + stream_metadata.insert("authorization", auth_value); + } + other_key => { + let header_value = MetadataValue::try_from(value.as_str()) + .map_err(|_| ZerobusError::InvalidArgument(other_key.to_string()))?; + stream_metadata.insert(other_key, header_value); + } + } + } + + let mut response_grpc_stream = channel + .ephemeral_stream(request_stream) + .await + .map_err(ZerobusError::CreateStreamError)? + .into_inner(); + + let descriptor_proto = if record_type == RecordType::Proto { + Some( + table_properties + .descriptor_proto + .as_ref() + .ok_or_else(|| { + ZerobusError::InvalidArgument( + "Descriptor proto is required for Proto record type".to_string(), + ) + })? + .encode_to_vec(), + ) + } else { + None + }; + + let create_stream_request = RequestPayload::CreateStream(CreateIngestStreamRequest { + table_name: Some(table_properties.table_name.to_string()), + descriptor_proto, + record_type: Some(record_type.into()), + }); + + debug!("Sending CreateStream request."); + tx.send(EphemeralStreamRequest { + payload: Some(create_stream_request), + }) + .await + .map_err(|_| { + error!(table_name = %table_properties.table_name, "Failed to send CreateStream request"); + ZerobusError::StreamClosedError(tonic::Status::internal( + "Failed to send CreateStream request", + )) + })?; + debug!("Waiting for CreateStream response."); + let create_stream_response = response_grpc_stream.message().await; + + match create_stream_response { + Ok(Some(create_stream_response)) => match create_stream_response.payload { + Some(ResponsePayload::CreateStreamResponse(resp)) => { + if let Some(stream_id) = resp.stream_id { + info!(stream_id = %stream_id, "Successfully created stream"); + Ok((tx, response_grpc_stream, stream_id)) + } else { + error!("Successfully created a stream but stream_id is None"); + Err(ZerobusError::CreateStreamError(tonic::Status::internal( + "Successfully created a stream but stream_id is None", + ))) + } + } + unexpected_message => { + error!("Unexpected response from server {unexpected_message:?}"); + Err(ZerobusError::CreateStreamError(tonic::Status::internal( + "Unexpected response from server", + ))) + } + }, + Ok(None) => { + info!("Server closed the stream gracefully before sending CreateStream response"); + Err(ZerobusError::CreateStreamError(tonic::Status::ok( + "Stream closed gracefully by server", + ))) + } + Err(status) => { + error!("CreateStream RPC failed: {status:?}"); + Err(ZerobusError::CreateStreamError(status)) + } + } + } + + /// Ingests a single record and returns its logical offset directly. + /// + /// This is an alternative to `ingest_record()` that returns the logical offset directly + /// as an integer (after queuing) instead of wrapping it in a Future. Use `wait_for_offset()` + /// to explicitly wait for server acknowledgment of this offset when needed. + /// + /// # Arguments + /// + /// * `payload` - A record that can be converted to `EncodedRecord` (either JSON string or protobuf bytes) + /// + /// # Returns + /// + /// The logical offset ID assigned to this record. + /// + /// # Errors + /// + /// * `InvalidArgument` - If the record type doesn't match stream configuration + /// * `StreamClosedError` - If the stream has been closed + /// + /// # Examples + /// + /// ```no_run + /// # use databricks_zerobus_ingest_sdk::*; + /// # use prost::Message; + /// # async fn example(stream: ZerobusStream) -> Result<(), ZerobusError> { + /// # let my_record = vec![1, 2, 3]; // Example protobuf-encoded data + /// // Ingest and get offset immediately + /// let offset = stream.ingest_record_offset(my_record).await?; + /// + /// // Later, wait for acknowledgment + /// stream.wait_for_offset(offset).await?; + /// println!("Record at offset {} has been acknowledged", offset); + /// # Ok(()) + /// # } + /// ``` + pub async fn ingest_record_offset( + &self, + payload: impl Into, + ) -> ZerobusResult { + let encoded_batch = EncodedBatch::try_from_record(payload, self.options.record_type) + .ok_or_else(|| { + ZerobusError::InvalidArgument( + "Record type does not match stream configuration".to_string(), + ) + })?; + + self.ingest_internal_v2(encoded_batch).await + } + + /// Ingests a batch of records and returns the logical offset directly. + /// + /// This is an alternative to `ingest_records()` that returns the logical offset directly + /// (after queuing) instead of wrapping it in a Future. Use `wait_for_offset()` to explicitly + /// wait for server acknowledgment when needed. + /// + /// # Arguments + /// + /// * `payload` - An iterator of records (each item should be convertible to `EncodedRecord`) + /// + /// # Returns + /// + /// `Some(offset_id)` for non-empty batches, or `None` if the batch is empty. + /// + /// # Errors + /// + /// * `InvalidArgument` - If record types don't match stream configuration + /// * `StreamClosedError` - If the stream has been closed + /// + /// # Examples + /// + /// ```no_run + /// # use databricks_zerobus_ingest_sdk::*; + /// # use prost::Message; + /// # async fn example(stream: ZerobusStream) -> Result<(), ZerobusError> { + /// let records = vec![vec![1, 2, 3], vec![4, 5, 6]]; // Example protobuf-encoded data + /// + /// // Ingest batch and get offset immediately + /// if let Some(offset) = stream.ingest_records_offset(records).await? { + /// // Later, wait for batch acknowledgment + /// stream.wait_for_offset(offset).await?; + /// println!("Batch at offset {} has been acknowledged", offset); + /// } + /// # Ok(()) + /// # } + /// ``` + pub async fn ingest_records_offset(&self, payload: I) -> ZerobusResult> + where + I: IntoIterator, + T: Into, + { + let encoded_batch = EncodedBatch::try_from_batch(payload, self.options.record_type) + .ok_or_else(|| { + ZerobusError::InvalidArgument( + "Record type does not match stream configuration".to_string(), + ) + })?; + + if encoded_batch.is_empty() { + Ok(None) + } else { + self.ingest_internal_v2(encoded_batch) + .await + .map(Option::Some) + } + } + /// Internal unified method for ingesting records and batches + async fn ingest_internal( + &self, + encoded_batch: EncodedBatch, + ) -> ZerobusResult>> { + if self.is_closed.load(Ordering::Relaxed) { + error!(table_name = %self.table_properties.table_name, "Stream closed"); + return Err(ZerobusError::StreamClosedError(tonic::Status::internal( + "Stream closed", + ))); + } + + let _guard = self.sync_mutex.lock().await; + + let offset_id = self.logical_offset_id_generator.next(); + debug!( + offset_id = offset_id, + record_count = encoded_batch.get_record_count(), + "Ingesting record(s)" + ); + + if let Some(stream_id) = self.stream_id.as_ref() { + let (tx, rx) = tokio::sync::oneshot::channel(); + { + let mut map = self.oneshot_map.lock().await; + map.insert(offset_id, tx); + } + self.landing_zone + .add(Box::new(IngestRequest { + payload: encoded_batch, + offset_id, + })) + .await; + let stream_id = stream_id.to_string(); + Ok(async move { + rx.await.map_err(|err| { + error!(stream_id = %stream_id, "Failed to receive ack: {}", err); + ZerobusError::StreamClosedError(tonic::Status::internal( + "Failed to receive ack", + )) + })? + }) + } else { + error!("Stream ID is None"); + Err(ZerobusError::StreamClosedError(tonic::Status::internal( + "Stream ID is None", + ))) + } + } + + /// Internal unified method for ingesting records and batches + async fn ingest_internal_v2(&self, encoded_batch: EncodedBatch) -> ZerobusResult { + if self.is_closed.load(Ordering::Relaxed) { + error!(table_name = %self.table_properties.table_name, "Stream closed"); + return Err(ZerobusError::StreamClosedError(tonic::Status::internal( + "Stream closed", + ))); + } + + let _guard = self.sync_mutex.lock().await; + + let offset_id = self.logical_offset_id_generator.next(); + debug!( + offset_id = offset_id, + record_count = encoded_batch.get_record_count(), + "Ingesting record(s)" + ); + self.landing_zone + .add(Box::new(IngestRequest { + payload: encoded_batch, + offset_id, + })) + .await; + Ok(offset_id) + } + + /// Spawns a task that handles callback execution in a separate thread. + /// This task receives callback messages via a channel and executes them + /// without blocking the receiver task. + #[instrument(level = "debug", skip_all)] + fn spawn_callback_handler_task( + mut callback_rx: tokio::sync::mpsc::UnboundedReceiver, + ack_callback: Option>, + cancellation_token: CancellationToken, + ) -> tokio::task::JoinHandle<()> { + tokio::spawn(async move { + let span = span!(Level::DEBUG, "callback_handler"); + let _guard = span.enter(); + loop { + tokio::select! { + biased; + message = callback_rx.recv() => { + match message { + Some(message) => { + match message { + CallbackMessage::Ack(logical_offset) => { + if let Some(ref callback) = ack_callback { + callback.on_ack(logical_offset); + } + } + CallbackMessage::Error(logical_offset, error_message) => { + if let Some(ref callback) = ack_callback { + callback.on_error(logical_offset, &error_message); + } + } + } + } + None => { // This happens when all senders are dropped. + debug!("Callback handler task shutting down"); + return; + } + } + } + _ = cancellation_token.cancelled() => { + debug!("Callback handler task cancelled"); + return; + } + + } + } + }) + } + + /// Spawns a task that continuously reads from `response_grpc_stream` + /// and propagates the received durability acknowledgements to the + /// corresponding pending acks promises. + #[instrument(level = "debug", skip_all)] + #[allow(clippy::too_many_arguments)] + fn spawn_receiver_task( + mut response_grpc_stream: tonic::Streaming, + last_received_offset_id_tx: tokio::sync::watch::Sender>, + landing_zone: RecordLandingZone, + oneshot_map: Arc>, + is_paused: Arc, + options: StreamConfigurationOptions, + server_error_tx: tokio::sync::watch::Sender>, + recv_drain_token: CancellationToken, + callback_tx: Option>, + ) -> tokio::task::JoinHandle> { + tokio::spawn(async move { + let span = span!(Level::DEBUG, "inbound_stream_processor"); + let _guard = span.enter(); + let mut last_acked_offset = -1; + let mut pause_deadline: Option = None; + // Set when we exit because the supervisor signalled close (`recv_drain_token`). + // On that path we drain the response stream inline so the server sees END_STREAM + // instead of RST_STREAM. On all other exits (recovery / errors) the runtime is + // still up, so a detached drain is used to avoid blocking recovery. + let mut close_initiated = false; + + 'recv_loop: loop { + if let Some(deadline) = pause_deadline { + let now = tokio::time::Instant::now(); + let all_acked = landing_zone.is_observed_empty(); + + if now >= deadline { + info!("Graceful close timeout reached. Triggering recovery."); + break 'recv_loop; + } else if all_acked { + info!("All in-flight records acknowledged during graceful close. Triggering recovery."); + break 'recv_loop; + } + } + + let message_result = if let Some(deadline) = pause_deadline { + tokio::select! { + biased; + _ = recv_drain_token.cancelled() => { + close_initiated = true; + break 'recv_loop; + } + _ = tokio::time::sleep_until(deadline) => { + continue; + } + res = tokio::time::timeout( + Duration::from_millis(options.server_lack_of_ack_timeout_ms), + response_grpc_stream.message(), + ) => res, + } + } else { + tokio::select! { + biased; + _ = recv_drain_token.cancelled() => { + close_initiated = true; + break 'recv_loop; + } + res = tokio::time::timeout( + Duration::from_millis(options.server_lack_of_ack_timeout_ms), + response_grpc_stream.message(), + ) => res, + } + }; + + match message_result { + Ok(Ok(Some(ingest_record_response))) => match ingest_record_response.payload { + Some(ResponsePayload::IngestRecordResponse(IngestRecordResponse { + durability_ack_up_to_offset, + })) => { + let durability_ack_up_to_offset = match durability_ack_up_to_offset { + Some(offset) => offset, + None => { + error!("Missing ack offset in server response"); + let error = + ZerobusError::StreamClosedError(tonic::Status::internal( + "Missing ack offset in server response", + )); + let _ = server_error_tx.send(Some(error.clone())); + return Err(error); + } + }; + let mut last_logical_acked_offset = -2; + let mut map = oneshot_map.lock().await; + for _offset_to_ack in + (last_acked_offset + 1)..=durability_ack_up_to_offset + { + if let Ok(record) = landing_zone.remove_observed() { + let logical_offset = record.offset_id; + last_logical_acked_offset = logical_offset; + + if let Some(sender) = map.remove(&logical_offset) { + let _ = sender.send(Ok(logical_offset)); + } + + if let Some(ref tx) = callback_tx { + let _ = tx.send(CallbackMessage::Ack(logical_offset)); + } + } + } + drop(map); + last_acked_offset = durability_ack_up_to_offset; + if last_logical_acked_offset != -2 { + let _ignore_on_channel_break = last_received_offset_id_tx + .send(Some(last_logical_acked_offset)); + } + } + Some(ResponsePayload::CloseStreamSignal(CloseStreamSignal { + duration, + })) => { + if options.recovery { + let server_duration_ms = duration + .as_ref() + .map(|d| d.seconds as u64 * 1000 + d.nanos as u64 / 1_000_000) + .unwrap_or(0); + + let wait_duration_ms = match options.stream_paused_max_wait_time_ms + { + None => server_duration_ms, + Some(0) => { + // Immediate recovery + info!("Server will close the stream in {}ms. Triggering stream recovery.", server_duration_ms); + break 'recv_loop; + } + Some(max_wait) => std::cmp::min(max_wait, server_duration_ms), + }; + + if wait_duration_ms == 0 { + info!("Server will close the stream. Triggering immediate recovery."); + break 'recv_loop; + } + + is_paused.store(true, Ordering::Relaxed); + pause_deadline = Some( + tokio::time::Instant::now() + + Duration::from_millis(wait_duration_ms), + ); + info!( + "Server will close the stream in {}ms. Entering graceful close period (waiting up to {}ms for in-flight acks).", + server_duration_ms, wait_duration_ms + ); + } + } + unexpected_message => { + error!("Unexpected response from server {unexpected_message:?}"); + let error = ZerobusError::StreamClosedError(tonic::Status::internal( + "Unexpected response from server", + )); + let _ = server_error_tx.send(Some(error.clone())); + return Err(error); + } + }, + Ok(Ok(None)) => { + info!("Server closed the stream without errors."); + let error = ZerobusError::StreamClosedError(tonic::Status::ok( + "Stream closed by server without errors.", + )); + let _ = server_error_tx.send(Some(error.clone())); + return Err(error); + } + Ok(Err(status)) => { + error!("Unexpected response from server {status:?}"); + let error = ZerobusError::StreamClosedError(status); + let _ = server_error_tx.send(Some(error.clone())); + return Err(error); + } + Err(_timeout) => { + // No message received for server_lack_of_ack_timeout_ms. + if pause_deadline.is_none() && !landing_zone.is_observed_empty() { + error!( + "Server ack timeout: no response for {}ms", + options.server_lack_of_ack_timeout_ms + ); + let error = ZerobusError::StreamClosedError( + tonic::Status::deadline_exceeded("Server ack timeout"), + ); + let _ = server_error_tx.send(Some(error.clone())); + return Err(error); + } + } + } + } + + // Drain remaining server messages so the server sees END_STREAM instead of + // the client RST_STREAM-ing the response. Inline on close (runtime may exit + // right after); detached on recovery / errors so recovery isn't delayed. + if close_initiated { + let _ = tokio::time::timeout( + Duration::from_millis(STREAM_TEARDOWN_DRAIN_TIMEOUT_MS), + async { + while response_grpc_stream + .message() + .await + .ok() + .flatten() + .is_some() + {} + }, + ) + .await; + } else { + tokio::spawn(async move { + let _ = tokio::time::timeout( + Duration::from_millis(STREAM_TEARDOWN_DRAIN_TIMEOUT_MS), + async move { + while response_grpc_stream + .message() + .await + .ok() + .flatten() + .is_some() + {} + }, + ) + .await; + }); + } + Ok(()) + }) + } + + /// Spawns a task that continuously sends records to the Zerobus API by observing the landing zone + /// to get records and sending them through the outbound stream to the gRPC stream. + fn spawn_sender_task( + outbound_stream: tokio::sync::mpsc::Sender, + landing_zone: RecordLandingZone, + is_paused: Arc, + server_error_tx: tokio::sync::watch::Sender>, + cancellation_token: CancellationToken, + ) -> tokio::task::JoinHandle> { + tokio::spawn(async move { + let physical_offset_id_generator = OffsetIdGenerator::default(); + loop { + let item = tokio::select! { + biased; + _ = cancellation_token.cancelled() => return Ok(()), + item = async { + if is_paused.load(Ordering::Relaxed) { + std::future::pending().await // Wait until supervisor task aborts this task. + } else { + landing_zone.observe().await + } + } => item.clone(), + }; + let offset_id = physical_offset_id_generator.next(); + let request_payload = item.payload.into_request_payload(offset_id); + + let send_result = outbound_stream + .send(EphemeralStreamRequest { + payload: Some(request_payload), + }) + .await; + + if let Err(err) = send_result { + error!("Failed to send record: {}", err); + let error = ZerobusError::StreamClosedError(tonic::Status::internal( + "Failed to send record", + )); + let _ = server_error_tx.send(Some(error.clone())); + return Err(error); + } + } + }) + } + + /// Fails all pending records by removing them from the landing zone and sending error to all pending acks promises. + async fn fail_all_pending_records( + landing_zone: RecordLandingZone, + oneshot_map: Arc>, + failed_records: Arc>>, + error: &ZerobusError, + callback_tx: &Option>, + ) { + let mut failed_payloads = Vec::with_capacity(landing_zone.len()); + let records = landing_zone.remove_all(); + let mut map = oneshot_map.lock().await; + let error_message = error.to_string(); + for record in records { + failed_payloads.push(record.payload); + if let Some(sender) = map.remove(&record.offset_id) { + let _ = sender.send(Err(error.clone())); + } + if let Some(tx) = callback_tx { + let _ = tx.send(CallbackMessage::Error( + record.offset_id, + error_message.clone(), + )); + } + } + *failed_records.write().await = failed_payloads; + } + + /// Internal method to wait for a specific offset to be acknowledged. + /// Used by both `flush()` and `wait_for_offset()`. + async fn wait_for_offset_internal( + &self, + offset_to_wait: OffsetId, + operation_name: &str, + ) -> ZerobusResult<()> { + let wait_operation = async { + let mut offset_receiver = self.logical_last_received_offset_id_tx.subscribe(); + let mut error_rx = self.server_error_rx.clone(); + + loop { + let offset = *offset_receiver.borrow_and_update(); + + let stream_id = match self.stream_id.as_deref() { + Some(stream_id) => stream_id, + None => { + error!("Stream ID is None during {}", operation_name.to_lowercase()); + "None" + } + }; + if let Some(offset) = offset { + if offset >= offset_to_wait { + debug!(stream_id = %stream_id, "Stream is caught up to the given offset. {} completed.", operation_name); + return Ok(()); + } else { + trace!( + stream_id = %stream_id, + "Stream is caught up to offset {}. Waiting for offset {}.", + offset, offset_to_wait + ); + } + } else { + trace!( + stream_id = %stream_id, + "Stream is not caught up to any offset yet. Waiting for the first offset." + ); + } + if self.is_closed.load(Ordering::Relaxed) { + // Re-check offset before failing, it might have been updated. + let offset = *offset_receiver.borrow_and_update(); + if let Some(offset) = offset { + if offset >= offset_to_wait { + return Ok(()); + } + } + // The supervisor always sends the real error to server_error_tx + // before setting is_closed=true, so check error_rx first to + // return the actual error instead of a generic one. + if let Some(server_error) = error_rx.borrow().clone() { + return Err(server_error); + } + return Err(ZerobusError::StreamClosedError(tonic::Status::internal( + format!("Stream closed during {}", operation_name.to_lowercase()), + ))); + } + // Race between offset updates and server errors. + tokio::select! { + result = offset_receiver.changed() => { + // If offset_receiver channel is closed, break the loop. + if result.is_err() { + break; + } + // Loop continues to check new offset value. + } + _ = error_rx.changed() => { + // Server error occurred, return it immediately if stream is closed. + if let Some(server_error) = error_rx.borrow().clone() { + if self.is_closed.load(Ordering::Relaxed) { + // Re-check offset before failing, it might have been updated. + let offset = *offset_receiver.borrow_and_update(); + if let Some(offset) = offset { + if offset >= offset_to_wait { + return Ok(()); + } + } + return Err(server_error); + } + } + } + } + } + + if let Some(server_error) = error_rx.borrow().clone() { + if self.is_closed.load(Ordering::Relaxed) { + return Err(server_error); + } + } + + Err(ZerobusError::StreamClosedError(tonic::Status::internal( + format!("Stream closed during {}", operation_name.to_lowercase()), + ))) + }; + + match tokio::time::timeout( + Duration::from_millis(self.options.flush_timeout_ms), + wait_operation, + ) + .await + { + Ok(Ok(())) => Ok(()), + Ok(Err(e)) => Err(e), + Err(_) => { + if let Some(stream_id) = self.stream_id.as_deref() { + error!(stream_id = %stream_id, table_name = %self.table_properties.table_name, "{} timed out", operation_name); + } else { + error!(table_name = %self.table_properties.table_name, "{} timed out", operation_name); + } + Err(ZerobusError::StreamClosedError( + tonic::Status::deadline_exceeded(format!("{} timed out", operation_name)), + )) + } + } + } + + /// Flushes all currently pending records and waits for their acknowledgments. + /// + /// This method captures the current highest offset and waits until all records up to + /// that offset have been acknowledged by the server. Records ingested during the flush + /// operation are not included in this flush. + /// + /// # Returns + /// + /// `Ok(())` when all pending records at the time of the call have been acknowledged. + /// + /// # Errors + /// + /// * `StreamClosedError` - If the stream is closed or times out + /// + /// # Examples + /// + /// ```no_run + /// # use databricks_zerobus_ingest_sdk::*; + /// # async fn example(stream: ZerobusStream) -> Result<(), ZerobusError> { + /// // Ingest many records + /// for i in 0..1000 { + /// let _offset = stream.ingest_record_offset(vec![i as u8]).await?; + /// } + /// + /// // Wait for all to be acknowledged + /// stream.flush().await?; + /// println!("All 1000 records have been acknowledged"); + /// # Ok(()) + /// # } + /// ``` + #[instrument(level = "debug", skip_all, fields(table_name = %self.table_properties.table_name))] + pub async fn flush(&self) -> ZerobusResult<()> { + let offset_to_wait = match self.logical_offset_id_generator.last() { + Some(offset) => offset, + None => return Ok(()), // Nothing to flush. + }; + self.wait_for_offset_internal(offset_to_wait, "Flush").await + } + + /// Waits for server acknowledgment of a specific logical offset. + /// + /// This method blocks until the server has acknowledged the record or batch at the + /// specified offset. Use this with offsets returned from `ingest_record_offset()` or + /// `ingest_records_offset()` to explicitly control when to wait for acknowledgments. + /// + /// # Arguments + /// + /// * `offset` - The logical offset ID to wait for (returned from `ingest_record_offset()` or `ingest_records_offset()`) + /// + /// # Returns + /// + /// `Ok(())` when the record/batch at the specified offset has been acknowledged. + /// + /// # Errors + /// + /// * `StreamClosedError` - If the stream is closed or times out while waiting + /// + /// # Examples + /// + /// ```no_run + /// # use databricks_zerobus_ingest_sdk::*; + /// # async fn example(stream: ZerobusStream) -> Result<(), ZerobusError> { + /// # let my_record = vec![1, 2, 3]; + /// // Ingest multiple records and collect their offsets + /// let mut offsets = Vec::new(); + /// for i in 0..100 { + /// let offset = stream.ingest_record_offset(vec![i as u8]).await?; + /// offsets.push(offset); + /// } + /// + /// // Wait for specific offsets + /// for offset in offsets { + /// stream.wait_for_offset(offset).await?; + /// } + /// println!("All records acknowledged"); + /// # Ok(()) + /// # } + /// ``` + pub async fn wait_for_offset(&self, offset: OffsetId) -> ZerobusResult<()> { + self.wait_for_offset_internal(offset, "Waiting for acknowledgement") + .await + } + + /// Returns whether the stream has been closed. + pub fn is_closed(&self) -> bool { + self.is_closed.load(Ordering::Relaxed) + } + + /// Closes the stream gracefully after flushing all pending records. + /// + /// This method first calls `flush()` to ensure all pending records are acknowledged, + /// then shuts down the stream and releases all resources. Always call this method + /// when you're done with a stream to ensure data integrity. + /// + /// # Returns + /// + /// `Ok(())` if the stream was closed successfully after flushing all records. + /// + /// # Errors + /// + /// Returns any errors from the flush operation. If flush fails, some records + /// may not have been acknowledged. Use `get_unacked_records()` to retrieve them. + /// + /// # Examples + /// + /// ```no_run + /// # use databricks_zerobus_ingest_sdk::*; + /// # async fn example(mut stream: ZerobusStream) -> Result<(), ZerobusError> { + /// // After ingesting records... + /// stream.close().await?; + /// # Ok(()) + /// # } + /// ``` + pub async fn close(&mut self) -> ZerobusResult<()> { + if self.is_closed.load(Ordering::Relaxed) { + return Ok(()); + } + if let Some(stream_id) = self.stream_id.as_deref() { + info!(stream_id = %stream_id, "Closing stream"); + } else { + error!("Stream ID is None during closing"); + } + let flush_result = self.flush().await; + self.is_closed.store(true, Ordering::Relaxed); + self.shutdown_all_tasks_gracefully().await; + flush_result + } + + /// Gracefully shuts down the supervisor task. + /// + /// Signals cancellation and waits for the task to exit. If the timeout + /// is provided and expires, forcefully aborts the task. + async fn shutdown_all_tasks_gracefully(&mut self) { + self.cancellation_token.cancel(); + + // Shutdown supervisor task. + match tokio::time::timeout( + Duration::from_secs(SHUTDOWN_TIMEOUT_SECS), + &mut self.supervisor_task, + ) + .await + { + Ok(_) => { + debug!("Supervisor task exited gracefully"); + } + Err(_) => { + warn!("Supervisor task did not exit within timeout, aborting"); + self.supervisor_task.abort(); + } + } + // Shutdown callback handler task, if there are any callbacks. + if let Some(mut task) = self.callback_handler_task.take() { + if let Some(callback_max_wait_time_ms) = self.options.callback_max_wait_time_ms { + match tokio::time::timeout( + Duration::from_millis(callback_max_wait_time_ms), + &mut task, + ) + .await + { + Ok(_) => { + debug!("Callback handler task exited gracefully"); + } + Err(_) => { + debug!("Callback handler task did not exit within timeout, aborting"); + task.abort(); + } + } + } else { + debug!("Callback max wait time is not set, waiting indefinitely"); + let _ = (&mut task).await; + } + } + } + + /// Returns all records that were ingested but not acknowledged by the server. + /// + /// This method should only be called after a stream has failed or been closed. + /// It's useful for implementing custom retry logic or persisting failed records. + /// + /// **Note:** This method flattens all unacknowledged records into a single iterator, + /// losing the original batch grouping. + /// If you want to preserve the batch grouping, use `ZerobusStream::get_unacked_batches()` instead. + /// If you want to re-ingest unacknowledged records while preserving their batch + /// structure, use `ZerobusSdk::recreate_stream()` instead. + /// + /// + /// # Returns + /// + /// An iterator over individual `EncodedRecord` items. All unacknowledged records are + /// flattened into a single sequence, regardless of how they were originally ingested + /// (via `ingest_record()` or `ingest_records()`). + /// + /// # Errors + /// + /// * `InvalidStateError` - If called on an active (not closed) stream + /// + /// # Examples + /// + /// ```no_run + /// # use databricks_zerobus_ingest_sdk::*; + /// # async fn example(sdk: ZerobusSdk, mut stream: ZerobusStream) -> Result<(), ZerobusError> { + /// match stream.close().await { + /// Err(e) => { + /// // Stream failed, get unacked records + /// let unacked = stream.get_unacked_records().await?; + /// let total_records = unacked.into_iter().count(); + /// println!("Failed to acknowledge {} records", total_records); + /// + /// // For re-ingestion with preserved batch structure, use recreate_stream + /// let new_stream = sdk.recreate_stream(&stream).await?; + /// } + /// Ok(_) => println!("All records acknowledged"), + /// } + /// # Ok(()) + /// # } + /// ``` + pub async fn get_unacked_records(&self) -> ZerobusResult> { + Ok(self + .get_unacked_batches() + .await? + .into_iter() + .flat_map(|batch| batch.into_iter())) + } + + /// Returns all records that were ingested but not acknowledged by the server, grouped by batch. + /// + /// This method should only be called after a stream has failed or been closed. + /// It's useful for implementing custom retry logic or persisting failed records. + /// + /// **Note:** This method returns the unacknowledged records as a vector of `EncodedBatch` items, + /// where each batch corresponds to how records were ingested: + /// - Each `ingest_record()` call creates a single batch containing one record + /// - Each `ingest_records()` call creates a single batch containing multiple records + /// + /// For alternatives, see `ZerobusStream::get_unacked_records()` and `ZerobusSdk::recreate_stream()`. + /// + /// # Returns + /// + /// A vector of `EncodedBatch` items. Records are grouped by their original ingestion call. + pub async fn get_unacked_batches(&self) -> ZerobusResult> { + if self.is_closed.load(Ordering::Relaxed) { + let failed = self.failed_records.read().await.clone(); + return Ok(failed); + } + if let Some(stream_id) = self.stream_id.as_deref() { + error!(stream_id = %stream_id, "Cannot get unacked records from an active stream. Stream must be closed first."); + } else { + error!( + "Cannot get unacked records from an active stream. Stream must be closed first." + ); + } + Err(ZerobusError::InvalidStateError( + "Cannot get unacked records from an active stream. Stream must be closed first." + .to_string(), + )) + } +} + +impl Drop for ZerobusStream { + fn drop(&mut self) { + self.is_closed.store(true, Ordering::Relaxed); + self.cancellation_token.cancel(); + self.supervisor_task.abort(); + if let Some(callback_handler_task) = self.callback_handler_task.take() { + callback_handler_task.abort(); + } + } +} diff --git a/lib/zerobus-ffi-1.3.0/rust/sdk/src/offset_generator.rs b/lib/zerobus-ffi-1.3.0/rust/sdk/src/offset_generator.rs new file mode 100644 index 00000000000..97af91a5a68 --- /dev/null +++ b/lib/zerobus-ffi-1.3.0/rust/sdk/src/offset_generator.rs @@ -0,0 +1,172 @@ +use std::sync::atomic::{AtomicI64, Ordering}; + +/// Offset ID type representing a logical (or physical) position in the stream. +pub type OffsetId = i64; + +/// Generates monotonically increasing offset IDs for ingested records. +/// +/// This generator ensures that each record gets a unique, sequential offset ID +/// starting from 0. It's thread-safe and can be shared across multiple tasks. +/// +/// # Thread Safety +/// +/// This struct uses atomic operations and is safe to share across threads via `Arc`. +/// +/// # Examples +/// +/// ``` +/// use databricks_zerobus_ingest_sdk::OffsetIdGenerator; +/// +/// let generator = OffsetIdGenerator::default(); +/// assert_eq!(generator.next(), 0); +/// assert_eq!(generator.next(), 1); +/// assert_eq!(generator.next(), 2); +/// assert_eq!(generator.last(), Some(2)); +/// ``` +pub struct OffsetIdGenerator { + last_offset_id: AtomicI64, +} + +impl Default for OffsetIdGenerator { + fn default() -> Self { + Self { + last_offset_id: AtomicI64::new(-1), + } + } +} + +impl OffsetIdGenerator { + /// Generates and returns the next sequential offset ID. + /// + /// Each call increments the internal counter and returns the new value. + /// The first call returns 0. + /// + /// # Returns + /// + /// The next offset ID in the sequence. + pub fn next(&self) -> OffsetId { + self.last_offset_id.fetch_add(1, Ordering::SeqCst) + 1 + } + + /// Returns the last offset ID that was generated. + /// + /// # Returns + /// + /// * `Some(offset_id)` - If at least one offset has been generated + /// * `None` - If no offsets have been generated yet + pub fn last(&self) -> Option { + let last_offset = self.last_offset_id.load(Ordering::SeqCst); + if last_offset == -1 { + None + } else { + Some(last_offset) + } + } + + /// Repositions the generator so the next call to `next()` returns `next_value`. + /// + /// Used by the Arrow Flight stream recovery path: when the SDK reconnects and + /// replays N pending batches with wire offsets `0..N-1`, the in-memory generator + /// must be set so that subsequent fresh batches pick up at `N` rather than the + /// pre-recovery monotonic counter — otherwise the server rejects the next batch + /// with a non-sequential-offset error. + pub fn set_next(&self, next_value: OffsetId) { + self.last_offset_id.store(next_value - 1, Ordering::SeqCst); + } +} + +#[cfg(test)] +mod tests { + use std::sync::Arc; + use std::thread; + + use crate::OffsetIdGenerator; + + #[test] + fn test_initial_state() { + let generator = OffsetIdGenerator::default(); + assert_eq!(generator.last(), None); + } + + #[test] + fn test_first_next_is_zero() { + let generator = OffsetIdGenerator::default(); + assert_eq!(generator.next(), 0); + assert_eq!(generator.last(), Some(0)); + } + + #[test] + fn test_monotonic_sequence() { + let generator = OffsetIdGenerator::default(); + + assert_eq!(generator.next(), 0); + assert_eq!(generator.next(), 1); + assert_eq!(generator.next(), 2); + assert_eq!(generator.next(), 3); + //blblb + assert_eq!(generator.last(), Some(3)); + } + + #[test] + fn test_set_next_repositions_generator() { + let generator = OffsetIdGenerator::default(); + + for _ in 0..100 { + generator.next(); + } + assert_eq!(generator.last(), Some(99)); + + generator.set_next(5); + assert_eq!(generator.next(), 5); + assert_eq!(generator.next(), 6); + assert_eq!(generator.last(), Some(6)); + } + + #[test] + fn test_set_next_to_zero_after_empty_replay() { + let generator = OffsetIdGenerator::default(); + for _ in 0..42 { + generator.next(); + } + + generator.set_next(0); + assert_eq!(generator.next(), 0); + assert_eq!(generator.next(), 1); + } + + #[test] + fn test_thread_safety() { + let generator = Arc::new(OffsetIdGenerator::default()); + let mut handles = vec![]; + + // Spawn 10 threads, each generating 100 IDs. + for _ in 0..10 { + let gen = generator.clone(); + let handle = thread::spawn(move || { + let mut ids = vec![]; + for _ in 0..100 { + ids.push(gen.next()); + } + ids + }); + handles.push(handle); + } + + // Collect all generated IDs. + let mut all_ids = vec![]; + for handle in handles { + all_ids.extend(handle.join().unwrap()); + } + + // Should have 1000 unique IDs from 0 to 999. + all_ids.sort(); + assert_eq!(all_ids.len(), 1000); + assert_eq!(all_ids[0], 0); + assert_eq!(all_ids[999], 999); + + // Check no duplicates. + for i in 0..999 { + assert_eq!(all_ids[i] + 1, all_ids[i + 1]); + } + } +} diff --git a/lib/zerobus-ffi-1.3.0/rust/sdk/src/proxy.rs b/lib/zerobus-ffi-1.3.0/rust/sdk/src/proxy.rs new file mode 100644 index 00000000000..8b39983e997 --- /dev/null +++ b/lib/zerobus-ffi-1.3.0/rust/sdk/src/proxy.rs @@ -0,0 +1,196 @@ +use std::sync::Arc; + +use hyper_http_proxy::{Intercept, Proxy, ProxyConnector as HyperProxyConnector}; +use hyper_util::client::legacy::connect::HttpConnector; +use tracing::info; + +use crate::ZerobusError; + +pub(crate) type ProxiedConnector = HyperProxyConnector; + +/// A proxy connector for the gRPC channel. +/// +/// Construct with [`ProxyConnector::new`] and install via +/// [`crate::ZerobusSdkBuilder::connector_factory`] to override the SDK's +/// default env-var proxy detection. +/// +/// Supports both `http://` and `https://` proxy URIs — for HTTPS proxies, the +/// client→proxy hop does a TLS handshake using the system trust store, and +/// the CONNECT tunnel still carries raw TCP so tonic can layer its own TLS on +/// top of the target endpoint. +pub struct ProxyConnector(ProxiedConnector); + +impl ProxyConnector { + /// Build a proxy connector that routes all gRPC traffic through + /// `proxy_uri` (e.g. `"http://corp-proxy:3128"` or + /// `"https://corp-proxy:3128"`). + #[allow(clippy::result_large_err)] + pub fn new(proxy_uri: &str) -> Result { + build_connector(proxy_uri).map(Self) + } + + pub(crate) fn into_inner(self) -> ProxiedConnector { + self.0 + } +} + +#[allow(clippy::result_large_err)] +fn build_connector(proxy_uri: &str) -> Result { + let uri = proxy_uri.parse().map_err(|e| { + ZerobusError::InvalidArgument(format!("failed to parse proxy URL '{}': {}", proxy_uri, e)) + })?; + let mut proxy = Proxy::new(Intercept::All, uri); + // gRPC is HTTP/2 and cannot traverse a regular HTTP/1 forward proxy; + // force CONNECT tunneling for all targets (matches gRPC core behavior). + proxy.force_connect(); + let mut http_connector = HttpConnector::new(); + // Allow non-http target schemes (e.g. https:// CONNECT targets) through + // the underlying TCP connector; without this, HttpConnector rejects them. + http_connector.enforce_http(false); + // `from_proxy` (vs `from_proxy_unsecured`) attaches a TLS connector used + // only for the client→proxy hop when the proxy URL is https://. The + // CONNECT tunnel still carries raw TCP; tonic applies its own TLS to the + // target endpoint on top. + HyperProxyConnector::from_proxy(http_connector, proxy).map_err(|e| { + ZerobusError::ChannelCreationError(format!("failed to build proxy connector: {}", e)) + }) +} + +/// Signature for caller-supplied proxy selection. Given the target host, +/// return a configured connector or `None` for a direct connection. +/// +/// Set via [`crate::ZerobusSdkBuilder::connector_factory`]. When a factory is +/// installed it fully replaces the default env-var proxy detection — callers +/// own the complete proxy decision, including any no-proxy bypass rules. +pub type ConnectorFactory = Arc Option + Send + Sync>; + +/// Env var names checked for proxy URL, in gRPC core precedence order. +const PROXY_ENV_VARS: &[&str] = &[ + "grpc_proxy", + "GRPC_PROXY", + "https_proxy", + "HTTPS_PROXY", + "http_proxy", + "HTTP_PROXY", +]; + +/// Env var names checked for no-proxy list, in gRPC core precedence order. +const NO_PROXY_ENV_VARS: &[&str] = &["no_grpc_proxy", "NO_GRPC_PROXY", "no_proxy", "NO_PROXY"]; + +/// Reads the first non-empty value from the given env var names. +fn read_first_env(names: &[&str]) -> Option { + for name in names { + if let Ok(val) = std::env::var(name) { + if !val.is_empty() { + return Some(val); + } + } + } + None +} + +/// Reads proxy environment variables and returns a `ProxiedConnector` +/// if one is configured, or `None` for direct connections. +/// +/// Follows gRPC core precedence: `grpc_proxy` → `https_proxy` → `http_proxy`. +/// For each name the lowercase variant is checked first, then uppercase +/// (matching standard convention and gRPC core behavior). +/// +/// Uses `from_proxy` so `https://` proxy URLs work (TLS handshake on the +/// client→proxy hop using the system trust store). The CONNECT tunnel still +/// carries raw TCP; tonic applies TLS to the target on top. +pub(crate) fn create_proxy_connector() -> Option { + let proxy_url = read_first_env(PROXY_ENV_VARS)?; + info!("Using HTTP proxy: {}", proxy_url); + match build_connector(&proxy_url) { + Ok(pc) => Some(pc), + Err(e) => { + tracing::warn!("{}", e); + None + } + } +} + +/// Checks whether a given host should bypass the proxy. +/// +/// Follows gRPC core precedence: `no_grpc_proxy` → `no_proxy`. +/// For each name the lowercase variant is checked first, then uppercase. +/// A wildcard `*` matches all hosts. Otherwise entries are matched as +/// suffix of the target host (e.g. `example.com` matches `foo.example.com`). +pub(crate) fn is_no_proxy(host: &str) -> bool { + let no_proxy = read_first_env(NO_PROXY_ENV_VARS).unwrap_or_default(); + host_matches_no_proxy(host, &no_proxy) +} + +/// Pure logic for no-proxy matching, separated for testability. +fn host_matches_no_proxy(host: &str, no_proxy: &str) -> bool { + if no_proxy.is_empty() { + return false; + } + + if no_proxy.trim() == "*" { + return true; + } + + no_proxy.split(',').any(|entry| { + let entry = entry.trim().trim_start_matches('.'); + host == entry || host.ends_with(&format!(".{}", entry)) + }) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn no_proxy_empty_returns_false() { + assert!(!host_matches_no_proxy("example.com", "")); + } + + #[test] + fn no_proxy_wildcard_matches_everything() { + assert!(host_matches_no_proxy("anything.com", "*")); + assert!(host_matches_no_proxy("localhost", " * ")); + } + + #[test] + fn no_proxy_exact_match() { + assert!(host_matches_no_proxy("example.com", "example.com")); + assert!(!host_matches_no_proxy("other.com", "example.com")); + } + + #[test] + fn no_proxy_suffix_match() { + assert!(host_matches_no_proxy( + "workspace.cloud.databricks.com", + "databricks.com" + )); + assert!(host_matches_no_proxy("foo.example.com", "example.com")); + // Must be a subdomain, not just a string suffix + assert!(!host_matches_no_proxy("notexample.com", "example.com")); + } + + #[test] + fn no_proxy_leading_dot_stripped() { + assert!(host_matches_no_proxy("foo.example.com", ".example.com")); + assert!(host_matches_no_proxy("example.com", ".example.com")); + } + + #[test] + fn no_proxy_comma_separated() { + let no_proxy = "localhost, 127.0.0.1, .internal.corp"; + assert!(host_matches_no_proxy("localhost", no_proxy)); + assert!(host_matches_no_proxy("127.0.0.1", no_proxy)); + assert!(host_matches_no_proxy("service.internal.corp", no_proxy)); + assert!(!host_matches_no_proxy("external.com", no_proxy)); + } + + #[test] + fn no_proxy_whitespace_handling() { + assert!(host_matches_no_proxy("example.com", " example.com ")); + assert!(host_matches_no_proxy( + "example.com", + "other.com , example.com , more.com" + )); + } +} diff --git a/lib/zerobus-ffi-1.3.0/rust/sdk/src/record_types.rs b/lib/zerobus-ffi-1.3.0/rust/sdk/src/record_types.rs new file mode 100644 index 00000000000..6afd280e4b9 --- /dev/null +++ b/lib/zerobus-ffi-1.3.0/rust/sdk/src/record_types.rs @@ -0,0 +1,946 @@ +//! Record types and wrappers for the Zerobus SDK. +//! +//! This module contains all the types related to encoding records for ingestion: +//! - [`EncodedRecord`] - The core enum for encoded records (JSON or Proto) +//! - [`EncodedBatch`] - A batch of encoded records +//! - Wrapper types for ergonomic record creation: +//! - [`ProtoBytes`] - For pre-serialized protobuf bytes (you handle serialization) +//! - [`JsonString`] - For pre-serialized JSON strings (you handle serialization) +//! - [`ProtoMessage`] - For protobuf messages (SDK handles serialization automatically) +//! - [`JsonValue`] - For JSON-serializable objects (SDK handles serialization automatically) + +use prost::Message; +use smallvec::{smallvec, SmallVec}; + +use crate::databricks::zerobus::{ + ephemeral_stream_request::Payload as RequestPayload, + ingest_record_batch_request::Batch as IngestRequestBatch, + ingest_record_request::Record as IngestRequestRecord, IngestRecordBatchRequest, + IngestRecordRequest, JsonRecordBatch, ProtoEncodedRecordBatch, RecordType, +}; +use crate::OffsetId; + +/// A type alias for a protobuf-encoded record. +pub type ProtoEncodedRecord = Vec; + +/// A type alias for a JSON-encoded record. +pub type JsonEncodedRecord = String; + +#[derive(Clone, Debug, PartialEq, Eq, PartialOrd, Ord)] +pub enum EncodedRecord { + Json(JsonEncodedRecord), + Proto(ProtoEncodedRecord), +} + +impl From for EncodedRecord { + fn from(v: ProtoEncodedRecord) -> Self { + EncodedRecord::Proto(v) + } +} + +impl From for EncodedRecord { + fn from(s: JsonEncodedRecord) -> Self { + EncodedRecord::Json(s) + } +} + +/// Wrapper for pre-serialized protobuf bytes. +/// +/// Use this when you've already serialized the protobuf data yourself. +/// This is optional - you can also pass `Vec` directly. +/// +/// # Examples +/// +/// ```no_run +/// # use databricks_zerobus_ingest_sdk::{ZerobusStream, ProtoBytes}; +/// # async fn example(stream: &ZerobusStream) -> Result<(), Box> { +/// let proto_bytes = vec![1, 2, 3, 4]; +/// let offset = stream.ingest_record_offset(ProtoBytes(proto_bytes)).await?; +/// stream.wait_for_offset(offset).await?; +/// # Ok(()) +/// # } +/// ``` +pub struct ProtoBytes(pub Vec); + +impl From for EncodedRecord { + fn from(bytes: ProtoBytes) -> Self { + EncodedRecord::Proto(bytes.0) + } +} + +/// Wrapper for pre-serialized JSON strings. +/// +/// Use this when you've already serialized the JSON data yourself. +/// This is optional - you can also pass `String` directly. +/// +/// # Examples +/// +/// ```no_run +/// # use databricks_zerobus_ingest_sdk::{ZerobusStream, JsonString}; +/// # async fn example(stream: &ZerobusStream) -> Result<(), Box> { +/// let json_str = r#"{"name":"test","value":42}"#.to_string(); +/// let offset = stream.ingest_record_offset(JsonString(json_str)).await?; +/// stream.wait_for_offset(offset).await?; +/// # Ok(()) +/// # } +/// ``` +pub struct JsonString(pub String); + +impl From for EncodedRecord { + fn from(s: JsonString) -> Self { + EncodedRecord::Json(s.0) + } +} + +/// Wrapper for protobuf messages with automatic serialization. +/// +/// Use this when you want the SDK to handle serialization for you. +/// Pass any protobuf message that implements `prost::Message` and it will be +/// automatically serialized to bytes. +/// +/// # Examples +/// +/// ```no_run +/// # use databricks_zerobus_ingest_sdk::{ZerobusStream, ProtoMessage}; +/// # async fn example(stream: &ZerobusStream, my_proto_msg: impl prost::Message) -> Result<(), Box> { +/// // Ingest a protobuf message - it will be automatically serialized +/// let offset = stream.ingest_record_offset(ProtoMessage(my_proto_msg)).await?; +/// stream.wait_for_offset(offset).await?; +/// # Ok(()) +/// # } +/// ``` +pub struct ProtoMessage(pub T); + +impl From> for EncodedRecord { + fn from(msg: ProtoMessage) -> Self { + EncodedRecord::Proto(msg.0.encode_to_vec()) + } +} + +/// Wrapper for JSON-serializable objects with automatic serialization. +/// +/// Use this when you want the SDK to handle serialization for you. +/// Pass any Rust struct that implements `serde::Serialize` and it will be +/// automatically serialized to a JSON string. +/// +/// # Examples +/// +/// ```no_run +/// # use databricks_zerobus_ingest_sdk::{ZerobusStream, JsonValue}; +/// # use serde::Serialize; +/// # async fn example(stream: &ZerobusStream) -> Result<(), Box> { +/// #[derive(Serialize)] +/// struct MyData { +/// name: String, +/// value: i32, +/// } +/// +/// let my_data = MyData { name: "test".into(), value: 42 }; +/// // Ingest a JSON object - it will be automatically serialized +/// let offset = stream.ingest_record_offset(JsonValue(my_data)).await?; +/// stream.wait_for_offset(offset).await?; +/// # Ok(()) +/// # } +/// ``` +pub struct JsonValue(pub T); + +impl From> for EncodedRecord { + fn from(obj: JsonValue) -> Self { + let json_string = serde_json::to_string(&obj.0).expect( + "Failed to serialize to JSON - ensure your type implements serde::Serialize correctly", + ); + EncodedRecord::Json(json_string) + } +} + +#[derive(Clone, Debug, PartialEq, Eq, PartialOrd, Ord)] +pub enum EncodedBatch { + Proto(SmallVec<[ProtoEncodedRecord; 1]>), + Json(SmallVec<[JsonEncodedRecord; 1]>), +} + +impl EncodedBatch { + /// Try to convert a single record into an encoded batch of the provided type. + /// If the record type does not match the provided type, None is returned. + pub(crate) fn try_from_record>( + value: T, + record_type: RecordType, + ) -> Option { + match (value.into(), record_type) { + (EncodedRecord::Json(s), RecordType::Json) => Some(EncodedBatch::Json(smallvec![s])), + (EncodedRecord::Proto(v), RecordType::Proto) => Some(EncodedBatch::Proto(smallvec![v])), + _ => None, + } + } + + /// Try to convert records into an encoded batch of the provided type. + /// If the record type does not match the records' type, None is returned. + /// The returned batch will be empty if no records are provided. + pub(crate) fn try_from_batch(batch: B, record_type: RecordType) -> Option + where + B: IntoIterator, + R: Into, + { + let mut batch_iter = batch.into_iter(); + let (lower, upper) = batch_iter.size_hint(); + let size_hint = upper.unwrap_or(lower); + + match record_type { + RecordType::Json => batch_iter + .try_fold( + SmallVec::with_capacity(size_hint), + |mut vec, record| match record.into() { + EncodedRecord::Json(value) => { + vec.push(value); + Some(vec) + } + _ => None, + }, + ) + .map(EncodedBatch::Json), + RecordType::Proto => batch_iter + .try_fold( + SmallVec::with_capacity(size_hint), + |mut vec, record| match record.into() { + EncodedRecord::Proto(value) => { + vec.push(value); + Some(vec) + } + _ => None, + }, + ) + .map(EncodedBatch::Proto), + _ => None, + } + } + + pub(crate) fn into_request_payload(self, offset_id: OffsetId) -> RequestPayload { + match self { + EncodedBatch::Proto(records) if records.len() == 1 => { + RequestPayload::IngestRecord(IngestRecordRequest { + record: Some(IngestRequestRecord::ProtoEncodedRecord( + records.into_iter().next().unwrap(), + )), + offset_id: Some(offset_id), + }) + } + EncodedBatch::Proto(records) => { + RequestPayload::IngestRecordBatch(IngestRecordBatchRequest { + batch: Some(IngestRequestBatch::ProtoEncodedBatch( + ProtoEncodedRecordBatch { + records: records.into_vec(), + }, + )), + offset_id: Some(offset_id), + }) + } + EncodedBatch::Json(records) if records.len() == 1 => { + RequestPayload::IngestRecord(IngestRecordRequest { + record: Some(IngestRequestRecord::JsonRecord( + records.into_iter().next().unwrap(), + )), + offset_id: Some(offset_id), + }) + } + EncodedBatch::Json(records) => { + RequestPayload::IngestRecordBatch(IngestRecordBatchRequest { + batch: Some(IngestRequestBatch::JsonBatch(JsonRecordBatch { + records: records.into_vec(), + })), + offset_id: Some(offset_id), + }) + } + } + } + + /// Returns the number of records in this batch. + pub fn get_record_count(&self) -> usize { + match self { + EncodedBatch::Proto(records) => records.len(), + EncodedBatch::Json(records) => records.len(), + } + } + + pub fn is_empty(&self) -> bool { + self.get_record_count() == 0 + } +} + +impl IntoIterator for EncodedBatch { + type Item = EncodedRecord; + type IntoIter = EncodedBatchIter; + + fn into_iter(self) -> Self::IntoIter { + match self { + EncodedBatch::Proto(records) => EncodedBatchIter::Proto(records.into_iter()), + EncodedBatch::Json(records) => EncodedBatchIter::Json(records.into_iter()), + } + } +} + +pub enum EncodedBatchIter { + Proto(smallvec::IntoIter<[ProtoEncodedRecord; 1]>), + Json(smallvec::IntoIter<[JsonEncodedRecord; 1]>), +} + +impl Iterator for EncodedBatchIter { + type Item = EncodedRecord; + + fn next(&mut self) -> Option { + match self { + EncodedBatchIter::Proto(iter) => iter.next().map(EncodedRecord::Proto), + EncodedBatchIter::Json(iter) => iter.next().map(EncodedRecord::Json), + } + } + + fn size_hint(&self) -> (usize, Option) { + match self { + EncodedBatchIter::Proto(iter) => iter.size_hint(), + EncodedBatchIter::Json(iter) => iter.size_hint(), + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + use prost::Message as ProstMessage; + use serde::Serialize; + use smallvec::smallvec; + + #[derive(Clone, PartialEq, ProstMessage)] + struct TestMessage { + #[prost(string, tag = "1")] + name: String, + #[prost(int32, tag = "2")] + value: i32, + } + + #[derive(Serialize, PartialEq, Debug)] + struct TestData { + name: String, + value: i32, + } + + mod encoded_record_conversions { + use super::*; + + #[test] + fn test_vec_u8_to_encoded_record() { + let bytes = vec![1, 2, 3, 4, 5]; + let record: EncodedRecord = bytes.clone().into(); + + match record { + EncodedRecord::Proto(data) => assert_eq!(data, bytes), + _ => panic!("Expected Proto variant"), + } + } + + #[test] + fn test_proto_bytes_to_encoded_record() { + let bytes = vec![1, 2, 3, 4, 5]; + let proto_bytes = ProtoBytes(bytes.clone()); + let record: EncodedRecord = proto_bytes.into(); + + match record { + EncodedRecord::Proto(data) => assert_eq!(data, bytes), + _ => panic!("Expected Proto variant"), + } + } + + #[test] + fn test_proto_message_to_encoded_record() { + let message = TestMessage { + name: "test".to_string(), + value: 42, + }; + let expected_bytes = message.encode_to_vec(); + + let proto_message = ProtoMessage(message.clone()); + let record: EncodedRecord = proto_message.into(); + + match record { + EncodedRecord::Proto(data) => { + assert_eq!(data, expected_bytes); + let decoded = TestMessage::decode(&data[..]).unwrap(); + assert_eq!(decoded, message); + } + _ => panic!("Expected Proto variant"), + } + } + + #[test] + fn test_string_to_encoded_record() { + let json_str = r#"{"name":"test","value":42}"#.to_string(); + let record: EncodedRecord = json_str.clone().into(); + + match record { + EncodedRecord::Json(data) => assert_eq!(data, json_str), + _ => panic!("Expected Json variant"), + } + } + + #[test] + fn test_json_string_to_encoded_record() { + let json_str = r#"{"name":"test","value":42}"#.to_string(); + let json_string = JsonString(json_str.clone()); + let record: EncodedRecord = json_string.into(); + + match record { + EncodedRecord::Json(data) => assert_eq!(data, json_str), + _ => panic!("Expected Json variant"), + } + } + + #[test] + fn test_json_value_to_encoded_record() { + let test_data = TestData { + name: "test".to_string(), + value: 42, + }; + + let json_value = JsonValue(test_data); + let record: EncodedRecord = json_value.into(); + + match record { + EncodedRecord::Json(data) => { + let parsed: serde_json::Value = serde_json::from_str(&data).unwrap(); + assert_eq!(parsed["name"], "test"); + assert_eq!(parsed["value"], 42); + } + _ => panic!("Expected Json variant"), + } + } + + #[test] + fn test_wrapper_types_are_zero_cost() { + use std::mem::size_of; + + assert_eq!(size_of::(), size_of::>()); + assert_eq!(size_of::(), size_of::()); + } + } + + mod encoded_batch_try_from { + use super::*; + + #[test] + fn test_try_from_record_json_with_json_type() { + let json_str = r#"{"id": 1}"#.to_string(); + let batch = EncodedBatch::try_from_record(json_str.clone(), RecordType::Json); + + assert!(batch.is_some()); + let batch = batch.unwrap(); + assert_eq!(batch.get_record_count(), 1); + match batch { + EncodedBatch::Json(records) => assert_eq!(records[0], json_str), + _ => panic!("Expected Json batch"), + } + } + + #[test] + fn test_try_from_record_proto_with_proto_type() { + let bytes = vec![1, 2, 3]; + let batch = EncodedBatch::try_from_record(bytes.clone(), RecordType::Proto); + + assert!(batch.is_some()); + let batch = batch.unwrap(); + assert_eq!(batch.get_record_count(), 1); + match batch { + EncodedBatch::Proto(records) => assert_eq!(records[0], bytes), + _ => panic!("Expected Proto batch"), + } + } + + #[test] + fn test_try_from_record_json_with_proto_type_fails() { + let json_str = r#"{"id": 1}"#.to_string(); + let batch = EncodedBatch::try_from_record(json_str, RecordType::Proto); + + assert!(batch.is_none()); + } + + #[test] + fn test_try_from_record_proto_with_json_type_fails() { + let bytes = vec![1, 2, 3]; + let batch = EncodedBatch::try_from_record(bytes, RecordType::Json); + + assert!(batch.is_none()); + } + + #[test] + fn test_try_from_record_with_json_string_wrapper() { + let json_str = JsonString(r#"{"id": 1}"#.to_string()); + let batch = EncodedBatch::try_from_record(json_str, RecordType::Json); + + assert!(batch.is_some()); + assert_eq!(batch.unwrap().get_record_count(), 1); + } + + #[test] + fn test_try_from_record_with_proto_bytes_wrapper() { + let proto_bytes = ProtoBytes(vec![1, 2, 3]); + let batch = EncodedBatch::try_from_record(proto_bytes, RecordType::Proto); + + assert!(batch.is_some()); + assert_eq!(batch.unwrap().get_record_count(), 1); + } + + #[test] + fn test_try_from_record_with_json_value_wrapper() { + let test_data = TestData { + name: "test".to_string(), + value: 42, + }; + let batch = EncodedBatch::try_from_record(JsonValue(test_data), RecordType::Json); + + assert!(batch.is_some()); + let batch = batch.unwrap(); + match batch { + EncodedBatch::Json(records) => { + let parsed: serde_json::Value = serde_json::from_str(&records[0]).unwrap(); + assert_eq!(parsed["name"], "test"); + } + _ => panic!("Expected Json batch"), + } + } + + #[test] + fn test_try_from_record_with_proto_message_wrapper() { + let message = TestMessage { + name: "test".to_string(), + value: 42, + }; + let batch = + EncodedBatch::try_from_record(ProtoMessage(message.clone()), RecordType::Proto); + + assert!(batch.is_some()); + let batch = batch.unwrap(); + match batch { + EncodedBatch::Proto(records) => { + let decoded = TestMessage::decode(&records[0][..]).unwrap(); + assert_eq!(decoded, message); + } + _ => panic!("Expected Proto batch"), + } + } + + #[test] + fn test_try_from_batch_json_records() { + let records = vec![ + r#"{"id": 1}"#.to_string(), + r#"{"id": 2}"#.to_string(), + r#"{"id": 3}"#.to_string(), + ]; + let batch = EncodedBatch::try_from_batch(records.clone(), RecordType::Json); + + assert!(batch.is_some()); + let batch = batch.unwrap(); + assert_eq!(batch.get_record_count(), 3); + match batch { + EncodedBatch::Json(batch_records) => { + assert_eq!(batch_records.as_slice(), records.as_slice()); + } + _ => panic!("Expected Json batch"), + } + } + + #[test] + fn test_try_from_batch_proto_records() { + let records = vec![vec![1, 2], vec![3, 4], vec![5, 6]]; + let batch = EncodedBatch::try_from_batch(records.clone(), RecordType::Proto); + + assert!(batch.is_some()); + let batch = batch.unwrap(); + assert_eq!(batch.get_record_count(), 3); + match batch { + EncodedBatch::Proto(batch_records) => { + assert_eq!(batch_records.as_slice(), records.as_slice()); + } + _ => panic!("Expected Proto batch"), + } + } + + #[test] + fn test_try_from_batch_empty() { + let records: Vec = vec![]; + let batch = EncodedBatch::try_from_batch(records, RecordType::Json); + + assert!(batch.is_some()); + let batch = batch.unwrap(); + assert!(batch.is_empty()); + } + + #[test] + fn test_try_from_batch_json_with_proto_type_fails() { + let records = vec![r#"{"id": 1}"#.to_string(), r#"{"id": 2}"#.to_string()]; + let batch = EncodedBatch::try_from_batch(records, RecordType::Proto); + + assert!(batch.is_none()); + } + + #[test] + fn test_try_from_batch_proto_with_json_type_fails() { + let records = vec![vec![1, 2], vec![3, 4]]; + let batch = EncodedBatch::try_from_batch(records, RecordType::Json); + + assert!(batch.is_none()); + } + + #[test] + fn test_try_from_batch_with_json_string_wrappers() { + let records = vec![ + JsonString(r#"{"id": 1}"#.to_string()), + JsonString(r#"{"id": 2}"#.to_string()), + ]; + let batch = EncodedBatch::try_from_batch(records, RecordType::Json); + + assert!(batch.is_some()); + assert_eq!(batch.unwrap().get_record_count(), 2); + } + + #[test] + fn test_try_from_batch_with_proto_bytes_wrappers() { + let records = vec![ProtoBytes(vec![1, 2]), ProtoBytes(vec![3, 4])]; + let batch = EncodedBatch::try_from_batch(records, RecordType::Proto); + + assert!(batch.is_some()); + assert_eq!(batch.unwrap().get_record_count(), 2); + } + + #[test] + fn test_try_from_batch_with_json_value_wrappers() { + let records = vec![ + JsonValue(TestData { + name: "a".to_string(), + value: 1, + }), + JsonValue(TestData { + name: "b".to_string(), + value: 2, + }), + ]; + let batch = EncodedBatch::try_from_batch(records, RecordType::Json); + + assert!(batch.is_some()); + assert_eq!(batch.unwrap().get_record_count(), 2); + } + + #[test] + fn test_try_from_batch_with_proto_message_wrappers() { + let records = vec![ + ProtoMessage(TestMessage { + name: "a".to_string(), + value: 1, + }), + ProtoMessage(TestMessage { + name: "b".to_string(), + value: 2, + }), + ]; + let batch = EncodedBatch::try_from_batch(records, RecordType::Proto); + + assert!(batch.is_some()); + assert_eq!(batch.unwrap().get_record_count(), 2); + } + } + + mod encoded_batch_methods { + use super::*; + + #[test] + fn test_get_record_count() { + let proto_batch = EncodedBatch::Proto(smallvec![vec![1], vec![2], vec![3]]); + assert_eq!(proto_batch.get_record_count(), 3); + + let json_batch = EncodedBatch::Json(smallvec!["a".to_string(), "b".to_string()]); + assert_eq!(json_batch.get_record_count(), 2); + + let empty_batch = EncodedBatch::Proto(smallvec![]); + assert_eq!(empty_batch.get_record_count(), 0); + } + + #[test] + fn test_is_empty() { + let non_empty = EncodedBatch::Proto(smallvec![vec![1]]); + assert!(!non_empty.is_empty()); + + let empty = EncodedBatch::Json(smallvec![]); + assert!(empty.is_empty()); + } + + #[test] + fn test_into_request_payload_single_proto_record() { + let record = vec![1, 2, 3]; + let batch = EncodedBatch::Proto(smallvec![record.clone()]); + let payload = batch.into_request_payload(42); + + match payload { + RequestPayload::IngestRecord(req) => { + assert_eq!(req.offset_id, Some(42)); + match req.record { + Some(IngestRequestRecord::ProtoEncodedRecord(data)) => { + assert_eq!(data, record); + } + _ => panic!("Expected ProtoEncodedRecord"), + } + } + _ => panic!("Expected IngestRecord payload"), + } + } + + #[test] + fn test_into_request_payload_single_json_record() { + let record = r#"{"id": 1}"#.to_string(); + let batch = EncodedBatch::Json(smallvec![record.clone()]); + let payload = batch.into_request_payload(123); + + match payload { + RequestPayload::IngestRecord(req) => { + assert_eq!(req.offset_id, Some(123)); + match req.record { + Some(IngestRequestRecord::JsonRecord(data)) => { + assert_eq!(data, record); + } + _ => panic!("Expected JsonRecord"), + } + } + _ => panic!("Expected IngestRecord payload"), + } + } + + #[test] + fn test_into_request_payload_batch_proto() { + let records = vec![vec![1, 2, 3], vec![4, 5, 6]]; + let batch = EncodedBatch::Proto(SmallVec::from_vec(records.clone())); + let payload = batch.into_request_payload(99); + + match payload { + RequestPayload::IngestRecordBatch(req) => { + assert_eq!(req.offset_id, Some(99)); + match req.batch { + Some(IngestRequestBatch::ProtoEncodedBatch(proto_batch)) => { + assert_eq!(proto_batch.records, records); + } + _ => panic!("Expected ProtoEncodedBatch"), + } + } + _ => panic!("Expected IngestRecordBatch payload"), + } + } + + #[test] + fn test_into_request_payload_batch_json() { + let records = vec![r#"{"id": 1}"#.to_string(), r#"{"id": 2}"#.to_string()]; + let batch = EncodedBatch::Json(SmallVec::from_vec(records.clone())); + let payload = batch.into_request_payload(77); + + match payload { + RequestPayload::IngestRecordBatch(req) => { + assert_eq!(req.offset_id, Some(77)); + match req.batch { + Some(IngestRequestBatch::JsonBatch(json_batch)) => { + assert_eq!(json_batch.records, records); + } + _ => panic!("Expected JsonBatch"), + } + } + _ => panic!("Expected IngestRecordBatch payload"), + } + } + } + + mod encoded_batch_iter { + use super::*; + + #[test] + fn test_iter_proto_batch() { + let records = vec![vec![1, 2], vec![3, 4], vec![5, 6]]; + let batch = EncodedBatch::Proto(SmallVec::from_vec(records.clone())); + + let collected: Vec = batch.into_iter().collect(); + assert_eq!(collected.len(), 3); + + for (i, record) in collected.iter().enumerate() { + match record { + EncodedRecord::Proto(data) => assert_eq!(data, &records[i]), + _ => panic!("Expected Proto variant"), + } + } + } + + #[test] + fn test_iter_json_batch() { + let records = vec!["a".to_string(), "b".to_string(), "c".to_string()]; + let batch = EncodedBatch::Json(SmallVec::from_vec(records.clone())); + + let collected: Vec = batch.into_iter().collect(); + assert_eq!(collected.len(), 3); + + for (i, record) in collected.iter().enumerate() { + match record { + EncodedRecord::Json(data) => assert_eq!(data, &records[i]), + _ => panic!("Expected Json variant"), + } + } + } + + #[test] + fn test_iter_empty_batch() { + let batch = EncodedBatch::Proto(smallvec![]); + let collected: Vec = batch.into_iter().collect(); + assert!(collected.is_empty()); + } + + #[test] + fn test_iter_size_hint() { + let batch = EncodedBatch::Proto(smallvec![vec![1], vec![2], vec![3]]); + let iter = batch.into_iter(); + assert_eq!(iter.size_hint(), (3, Some(3))); + } + + #[test] + fn test_iter_size_hint_decreases() { + let batch = EncodedBatch::Json(smallvec!["a".to_string(), "b".to_string()]); + let mut iter = batch.into_iter(); + + assert_eq!(iter.size_hint(), (2, Some(2))); + iter.next(); + assert_eq!(iter.size_hint(), (1, Some(1))); + iter.next(); + assert_eq!(iter.size_hint(), (0, Some(0))); + } + } + + mod batch_conversions { + use super::*; + + #[test] + fn test_batch_with_proto_messages() { + let msg1 = TestMessage { + name: "msg1".to_string(), + value: 1, + }; + let msg2 = TestMessage { + name: "msg2".to_string(), + value: 2, + }; + + let batch: Vec> = + vec![ProtoMessage(msg1.clone()), ProtoMessage(msg2.clone())]; + + let records: Vec = batch.into_iter().map(|m| m.into()).collect(); + + assert_eq!(records.len(), 2); + for (i, record) in records.iter().enumerate() { + match record { + EncodedRecord::Proto(_) => {} + _ => panic!("Expected Proto variant at index {}", i), + } + } + } + + #[test] + fn test_batch_with_json_values() { + let data1 = TestData { + name: "data1".to_string(), + value: 1, + }; + let data2 = TestData { + name: "data2".to_string(), + value: 2, + }; + + let batch: Vec> = vec![JsonValue(data1), JsonValue(data2)]; + + let records: Vec = batch.into_iter().map(|m| m.into()).collect(); + + assert_eq!(records.len(), 2); + for (i, record) in records.iter().enumerate() { + match record { + EncodedRecord::Json(data) => { + let parsed: serde_json::Value = serde_json::from_str(data).unwrap(); + assert!(parsed["name"].as_str().unwrap().starts_with("data")); + } + _ => panic!("Expected Json variant at index {}", i), + } + } + } + + #[test] + fn test_batch_with_proto_bytes() { + let bytes1 = vec![1, 2, 3]; + let bytes2 = vec![4, 5, 6]; + + let batch = vec![ProtoBytes(bytes1.clone()), ProtoBytes(bytes2.clone())]; + let records: Vec = batch.into_iter().map(|b| b.into()).collect(); + + assert_eq!(records.len(), 2); + match &records[0] { + EncodedRecord::Proto(data) => assert_eq!(data, &bytes1), + _ => panic!("Expected Proto variant"), + } + match &records[1] { + EncodedRecord::Proto(data) => assert_eq!(data, &bytes2), + _ => panic!("Expected Proto variant"), + } + } + + #[test] + fn test_batch_with_json_strings() { + let json1 = r#"{"id":1}"#.to_string(); + let json2 = r#"{"id":2}"#.to_string(); + + let batch = vec![JsonString(json1.clone()), JsonString(json2.clone())]; + let records: Vec = batch.into_iter().map(|s| s.into()).collect(); + + assert_eq!(records.len(), 2); + match &records[0] { + EncodedRecord::Json(data) => assert_eq!(data, &json1), + _ => panic!("Expected Json variant"), + } + match &records[1] { + EncodedRecord::Json(data) => assert_eq!(data, &json2), + _ => panic!("Expected Json variant"), + } + } + + #[test] + fn test_batch_backward_compat_vec_u8() { + let bytes1 = vec![1, 2, 3]; + let bytes2 = vec![4, 5, 6]; + + let batch: Vec> = vec![bytes1.clone(), bytes2.clone()]; + let records: Vec = batch.into_iter().map(|b| b.into()).collect(); + + assert_eq!(records.len(), 2); + match &records[0] { + EncodedRecord::Proto(data) => assert_eq!(data, &bytes1), + _ => panic!("Expected Proto variant"), + } + match &records[1] { + EncodedRecord::Proto(data) => assert_eq!(data, &bytes2), + _ => panic!("Expected Proto variant"), + } + } + + #[test] + fn test_batch_backward_compat_string() { + let json1 = r#"{"id":1}"#.to_string(); + let json2 = r#"{"id":2}"#.to_string(); + + let batch: Vec = vec![json1.clone(), json2.clone()]; + let records: Vec = batch.into_iter().map(|s| s.into()).collect(); + + assert_eq!(records.len(), 2); + match &records[0] { + EncodedRecord::Json(data) => assert_eq!(data, &json1), + _ => panic!("Expected Json variant"), + } + match &records[1] { + EncodedRecord::Json(data) => assert_eq!(data, &json2), + _ => panic!("Expected Json variant"), + } + } + } +} diff --git a/lib/zerobus-ffi-1.3.0/rust/sdk/src/schema.rs b/lib/zerobus-ffi-1.3.0/rust/sdk/src/schema.rs new file mode 100644 index 00000000000..13c636266fe --- /dev/null +++ b/lib/zerobus-ffi-1.3.0/rust/sdk/src/schema.rs @@ -0,0 +1,1448 @@ +//! Convert a Unity Catalog table schema into a protobuf [`DescriptorProto`]. +//! +//! The Zerobus service accepts records described by a protobuf message descriptor. +//! Callers that already have the Unity Catalog metadata for a table can use +//! [`descriptor_from_uc_columns`] or [`descriptor_from_uc_schema`] to build that +//! descriptor on the fly instead of pre-generating a `.proto` file offline. +//! +//! # Example +//! +//! ```no_run +//! use databricks_zerobus_ingest_sdk::schema::{UcColumn, descriptor_from_uc_columns}; +//! +//! let columns = vec![ +//! UcColumn { +//! name: "id".into(), +//! type_name: "BIGINT".into(), +//! type_text: "BIGINT".into(), +//! type_json: String::new(), +//! nullable: false, +//! position: 0, +//! }, +//! UcColumn { +//! name: "payload".into(), +//! type_name: "STRING".into(), +//! type_text: "STRING".into(), +//! type_json: String::new(), +//! nullable: true, +//! position: 1, +//! }, +//! ]; +//! let descriptor = descriptor_from_uc_columns(&columns, "my_table").unwrap(); +//! assert_eq!(descriptor.name(), "my_table"); +//! ``` +//! +//! For `STRUCT`, `ARRAY`, and `MAP` columns the `type_json` field must be populated +//! with the JSON representation returned by the Unity Catalog REST API (the +//! `/api/2.1/unity-catalog/tables/{name}` response includes it per column). +//! +//! # Type mapping +//! +//! | Unity Catalog type | Proto type | Encoding contract | +//! |-------------------------------|------------|---------------------------------------------| +//! | `STRING`, `VARIANT`, `DECIMAL`| `string` | UTF-8 text | +//! | `INT`, `INTEGER` | `int32` | | +//! | `LONG`, `BIGINT` | `int64` | | +//! | `SHORT`, `SMALLINT`, `BYTE`, `TINYINT` | `int32` | zero-extended; range-checked by the server | +//! | `FLOAT` | `float` | | +//! | `DOUBLE` | `double` | | +//! | `BOOLEAN` | `bool` | | +//! | `BINARY` | `bytes` | | +//! | `DATE` | `int32` | **days since 1970-01-01** (Unix epoch) | +//! | `TIMESTAMP` | `int64` | **microseconds since 1970-01-01 00:00:00 UTC** | +//! | `TIMESTAMP_NTZ` | `int64` | **microseconds since 1970-01-01 00:00:00**, no timezone | +//! | `STRUCT<...>` | nested message | fields sanitized to valid proto identifiers | +//! | `ARRAY` | `repeated T` | elements are always present (protobuf repeated has no null elements) | +//! | `MAP` | synthetic map-entry message + `repeated` | K must be integral, bool, or string | +//! +//! ## Timestamp and date encoding +//! +//! `DATE` and `TIMESTAMP*` columns are encoded as integers, **not** as +//! `google.protobuf.Timestamp` or ISO-8601 strings. Clients must convert their +//! source values into the expected unit before writing them into the generated +//! proto message; otherwise every row will be silently off by a factor of 10³ +//! (milliseconds mistaken for microseconds) or 10⁶ (seconds mistaken for +//! microseconds), or will land on the wrong day (milliseconds-since-epoch written +//! into a `DATE` field). +//! +//! Quick reference: +//! +//! ```text +//! DATE → (chrono::NaiveDate - 1970-01-01).num_days() as i32 +//! TIMESTAMP → instant.timestamp_micros() as i64 // UTC micros +//! TIMESTAMP_NTZ → naive.and_utc().timestamp_micros() as i64 // local-wall-clock micros +//! ``` +//! +//! `TIMESTAMP` and `TIMESTAMP_NTZ` collapse to the same proto type (`int64`); the +//! descriptor alone does not preserve the timezone distinction. The server +//! recovers it from the Unity Catalog table schema on the write path, so the +//! caller only needs to ensure the integer value matches the column's declared +//! semantics. + +use std::collections::HashSet; + +use prost_types::field_descriptor_proto::{Label, Type as ProtoType}; +use prost_types::{DescriptorProto, FieldDescriptorProto, MessageOptions}; +use serde::Deserialize; +use serde_json::Value as JsonValue; +use thiserror::Error; + +/// A single column from a Unity Catalog table. +/// +/// Field names mirror the Unity Catalog REST API response so the struct can be +/// deserialized directly from it (see the `Deserialize` impl). +#[derive(Debug, Clone, Deserialize)] +pub struct UcColumn { + pub name: String, + /// Top-level type name, e.g. `"STRING"`, `"INT"`, `"STRUCT"`, `"ARRAY"`, `"MAP"`. + pub type_name: String, + /// Human-readable type (e.g. `"struct"`). Not used by the + /// conversion — kept so the struct round-trips cleanly against the UC API. + #[serde(default)] + pub type_text: String, + /// JSON representation of the type. Required for `STRUCT`, `ARRAY`, `MAP`. + #[serde(default)] + pub type_json: String, + /// Defaults to `true` when absent, matching Spark/Delta `StructField` + /// semantics (a missing `nullable` key means "unspecified, assume + /// nullable"). This also matches the default used for nested struct + /// fields in `type_json`. + #[serde(default = "default_true")] + pub nullable: bool, + #[serde(default)] + pub position: i32, +} + +fn default_true() -> bool { + true +} + +/// A Unity Catalog table schema, as returned by the REST API. +#[derive(Debug, Clone, Deserialize)] +pub struct UcTableSchema { + pub name: String, + pub catalog_name: String, + pub schema_name: String, + pub columns: Vec, +} + +/// Errors produced while converting a UC schema to a protobuf descriptor. +#[derive(Debug, Error)] +#[non_exhaustive] +pub enum SchemaError { + #[error("invalid field name '{name}': {reason}")] + InvalidFieldName { name: String, reason: String }, + #[error("unsupported Databricks type '{0}'")] + UnsupportedType(String), + #[error("missing type_json for complex column '{0}'")] + MissingTypeJson(String), + #[error("failed to parse type_json for column '{column}': {reason}")] + InvalidTypeJson { column: String, reason: String }, + #[error("{0}")] + Invalid(String), +} + +/// Build a [`DescriptorProto`] from a Unity Catalog table's columns. +/// +/// `message_name` becomes the top-level message name on the returned descriptor. +/// Columns with `type_name` of `STRUCT`, `ARRAY`, or `MAP` require `type_json` +/// to be populated; simple columns only need `type_name`. +pub fn descriptor_from_uc_columns( + columns: &[UcColumn], + message_name: &str, +) -> Result { + let mut collector = MessageCollector::new(); + let mut fields = Vec::with_capacity(columns.len()); + + let mut sorted: Vec<&UcColumn> = columns.iter().filter(|c| c.position >= 0).collect(); + sorted.sort_by_key(|c| c.position); + + // Protobuf field number = UC position + 1. Unity Catalog's `position` is + // 0-indexed; adding 1 produces a valid proto field number and preserves + // any gaps UC reports (e.g. after DROP COLUMN in column-mapping mode), + // keeping a one-to-one correspondence between field number and UC column. + for column in sorted.iter() { + validate_field_name(&column.name)?; + + let (field_type, type_name, is_repeated) = if is_complex(&column.type_name) { + if column.type_json.is_empty() { + return Err(SchemaError::MissingTypeJson(column.name.clone())); + } + let complex = parse_type_json(&column.type_json).map_err(|reason| { + SchemaError::InvalidTypeJson { + column: column.name.clone(), + reason, + } + })?; + let is_repeated = matches!(complex, ComplexType::Array(_) | ComplexType::Map { .. }); + let (ty, type_name) = + map_complex_type_to_protobuf(&complex, &column.name, &mut collector)?; + (ty, type_name, is_repeated) + } else { + let p = parse_uc_top_level_type(&column.type_name)?; + (map_primitive_to_protobuf(p), None, false) + }; + + fields.push(field_descriptor( + &column.name, + column.position + 1, + field_type, + type_name, + column.nullable, + is_repeated, + )); + } + + Ok(DescriptorProto { + name: Some(message_name.to_string()), + field: fields, + nested_type: collector.nested, + ..Default::default() + }) +} + +/// Build a [`DescriptorProto`] from a full [`UcTableSchema`]. +/// +/// The generated message name is `_` (sanitized to a +/// valid protobuf identifier). +pub fn descriptor_from_uc_schema(schema: &UcTableSchema) -> Result { + let message_name = sanitize_message_name(&format!("{}_{}", schema.schema_name, schema.name)); + descriptor_from_uc_columns(&schema.columns, &message_name) +} + +fn is_complex(type_name: &str) -> bool { + matches!(type_name, "STRUCT" | "ARRAY" | "MAP") +} + +fn field_descriptor( + name: &str, + number: i32, + field_type: ProtoType, + type_name: Option, + nullable: bool, + is_repeated: bool, +) -> FieldDescriptorProto { + let label = if is_repeated { + Label::Repeated + } else if nullable { + Label::Optional + } else { + Label::Required + }; + FieldDescriptorProto { + name: Some(name.to_string()), + number: Some(number), + label: Some(label as i32), + r#type: Some(field_type as i32), + type_name, + json_name: Some(name.to_string()), + proto3_optional: Some(nullable && !is_repeated), + ..Default::default() + } +} + +/// Parse a top-level UC `type_name` (e.g. `"BIGINT"`, `"TIMESTAMP_NTZ"`) into our +/// internal [`PrimitiveType`]. Single source of truth for the accepted set of +/// non-complex UC types; each backend (proto, Arrow) projects from `PrimitiveType` +/// onto its own target representation. +fn parse_uc_top_level_type(type_name: &str) -> Result { + Ok(match type_name { + "STRING" | "VARIANT" => PrimitiveType::String, + "INT" | "INTEGER" => PrimitiveType::Integer, + "LONG" | "BIGINT" => PrimitiveType::Long, + "SHORT" | "SMALLINT" => PrimitiveType::Short, + "BYTE" | "TINYINT" => PrimitiveType::Byte, + "BOOLEAN" | "BOOL" => PrimitiveType::Boolean, + "DOUBLE" => PrimitiveType::Double, + "FLOAT" => PrimitiveType::Float, + "TIMESTAMP" => PrimitiveType::Timestamp, + "TIMESTAMP_NTZ" => PrimitiveType::TimestampNtz, + "DATE" => PrimitiveType::Date, + "BINARY" => PrimitiveType::Binary, + "DECIMAL" => PrimitiveType::Decimal, + other => return Err(SchemaError::UnsupportedType(other.to_string())), + }) +} + +#[derive(Debug, Clone)] +enum ComplexType { + Primitive(PrimitiveType), + Struct(StructType), + Array(Box), + Map { + key: Box, + value: Box, + }, +} + +#[derive(Debug, Clone, Copy)] +enum PrimitiveType { + String, + Long, + Integer, + Short, + Byte, + Double, + Float, + Boolean, + Binary, + Timestamp, + TimestampNtz, + Date, + Decimal, +} + +#[derive(Debug, Clone)] +struct StructField { + name: String, + field_type: ComplexType, + nullable: bool, +} + +#[derive(Debug, Clone)] +struct StructType { + fields: Vec, +} + +/// Maximum nesting depth accepted in `type_json`. Bounds recursion into +/// user-controlled JSON so a pathological input cannot blow the stack. +const MAX_NESTING_DEPTH: usize = 100; + +/// Either a primitive name (`"integer"`) or a nested complex type object. +#[derive(Deserialize)] +#[serde(untagged)] +enum TypeRef { + Complex(ComplexTypeJson), + Primitive(String), +} + +#[derive(Deserialize)] +#[serde(tag = "type", rename_all = "lowercase")] +enum ComplexTypeJson { + Struct { + fields: Vec, + }, + Array { + #[serde(rename = "elementType")] + element_type: Box, + }, + // `valueContainsNull` is intentionally not captured: proto scalar maps + // cannot carry null values on the wire, so there is nothing for the + // encoder to coerce — only non-null values can be transmitted. + Map { + #[serde(rename = "keyType")] + key_type: Box, + #[serde(rename = "valueType")] + value_type: Box, + }, +} + +#[derive(Deserialize)] +struct StructFieldJson { + name: String, + #[serde(rename = "type")] + ty: TypeRef, + #[serde(default = "default_true")] + nullable: bool, +} + +fn parse_type_json(type_json: &str) -> Result { + if type_json.is_empty() || type_json == "{}" { + return Err("empty type_json".into()); + } + let raw: JsonValue = serde_json::from_str(type_json).map_err(|e| e.to_string())?; + // Unity Catalog sometimes wraps the type in {"name": ..., "type": ...}. + let inner = match raw.as_object() { + Some(obj) if obj.contains_key("name") && obj.contains_key("type") => { + obj.get("type").unwrap().clone() + } + _ => raw, + }; + let tref: TypeRef = serde_json::from_value(inner).map_err(|e| e.to_string())?; + type_ref_to_complex(&tref, 0) +} + +fn type_ref_to_complex(tref: &TypeRef, level: usize) -> Result { + if level > MAX_NESTING_DEPTH { + return Err(format!( + "nesting level exceeds maximum depth of {}", + MAX_NESTING_DEPTH + )); + } + match tref { + TypeRef::Primitive(s) => parse_primitive_type(s).map(ComplexType::Primitive), + TypeRef::Complex(ComplexTypeJson::Struct { fields }) => { + let mut out = Vec::with_capacity(fields.len()); + for f in fields { + out.push(StructField { + name: f.name.clone(), + field_type: type_ref_to_complex(&f.ty, level + 1)?, + nullable: f.nullable, + }); + } + Ok(ComplexType::Struct(StructType { fields: out })) + } + TypeRef::Complex(ComplexTypeJson::Array { element_type }) => Ok(ComplexType::Array( + Box::new(type_ref_to_complex(element_type, level + 1)?), + )), + TypeRef::Complex(ComplexTypeJson::Map { + key_type, + value_type, + }) => Ok(ComplexType::Map { + key: Box::new(type_ref_to_complex(key_type, level + 1)?), + value: Box::new(type_ref_to_complex(value_type, level + 1)?), + }), + } +} + +fn parse_primitive_type(s: &str) -> Result { + Ok(match s { + "string" => PrimitiveType::String, + "long" => PrimitiveType::Long, + "integer" => PrimitiveType::Integer, + "short" => PrimitiveType::Short, + "byte" => PrimitiveType::Byte, + "double" => PrimitiveType::Double, + "float" => PrimitiveType::Float, + "boolean" => PrimitiveType::Boolean, + "binary" => PrimitiveType::Binary, + "timestamp" => PrimitiveType::Timestamp, + "timestamp_ntz" => PrimitiveType::TimestampNtz, + "date" => PrimitiveType::Date, + s if s.starts_with("decimal") => PrimitiveType::Decimal, + other => return Err(format!("unknown primitive type '{}'", other)), + }) +} + +const fn map_primitive_to_protobuf(p: PrimitiveType) -> ProtoType { + match p { + PrimitiveType::String => ProtoType::String, + PrimitiveType::Long => ProtoType::Int64, + PrimitiveType::Integer => ProtoType::Int32, + PrimitiveType::Short | PrimitiveType::Byte => ProtoType::Int32, + PrimitiveType::Double => ProtoType::Double, + PrimitiveType::Float => ProtoType::Float, + PrimitiveType::Boolean => ProtoType::Bool, + PrimitiveType::Binary => ProtoType::Bytes, + PrimitiveType::Timestamp | PrimitiveType::TimestampNtz => ProtoType::Int64, + PrimitiveType::Date => ProtoType::Int32, + PrimitiveType::Decimal => ProtoType::String, + } +} + +const fn is_valid_map_key(p: PrimitiveType) -> bool { + !matches!( + p, + PrimitiveType::Double | PrimitiveType::Float | PrimitiveType::Binary + ) +} + +fn validate_map_key(key: &ComplexType, path: &str) -> Result { + match key { + ComplexType::Primitive(p) if is_valid_map_key(*p) => Ok(*p), + ComplexType::Primitive(p) => Err(SchemaError::Invalid(format!( + "unsupported map key type {:?} for field '{}' \ + (map keys must be integral, bool, or string)", + p, path + ))), + _ => Err(SchemaError::Invalid(format!( + "map keys must be primitive types (field '{}')", + path + ))), + } +} + +fn shape_unsupported(kind: &str, path: &str) -> SchemaError { + SchemaError::Invalid(format!("{} not supported for field '{}'", kind, path)) +} + +/// Accumulates nested message definitions during conversion and dedupes their names. +struct MessageCollector { + nested: Vec, + used: HashSet, +} + +impl MessageCollector { + fn new() -> Self { + Self { + nested: Vec::new(), + used: HashSet::new(), + } + } + + /// Return `base` if unused in this scope, otherwise append an incrementing + /// suffix (`base2`, `base3`, …). + /// + /// The suffix order tracks the order in which the caller registers names, + /// which is ultimately the order UC returns columns / struct fields. Callers + /// that regenerate descriptors and compare the output bit-for-bit need to + /// feed this function in the same order on every run — top-level columns + /// are already sorted by `position` in [`descriptor_from_uc_columns`], and + /// struct fields preserve the `type_json` order (which UC returns stably). + fn unique_name(&mut self, base: String) -> String { + if self.used.insert(base.clone()) { + return base; + } + let mut n = 2u32; + loop { + let candidate = format!("{}{}", base, n); + if self.used.insert(candidate.clone()) { + return candidate; + } + n += 1; + } + } + + fn push(&mut self, message: DescriptorProto) { + self.nested.push(message); + } +} + +fn map_complex_type_to_protobuf( + ct: &ComplexType, + path: &str, + collector: &mut MessageCollector, +) -> Result<(ProtoType, Option), SchemaError> { + match ct { + ComplexType::Primitive(p) => Ok((map_primitive_to_protobuf(*p), None)), + ComplexType::Struct(st) => { + let name = collector.unique_name(sanitize_message_name(path)); + // Each struct owns its own nested scope, so any messages it + // generates (inner structs, map entries on its own fields) land in + // that struct's `nested_type` rather than leaking up to the root. + let msg = generate_struct_message(&name, st)?; + collector.push(msg); + Ok((ProtoType::Message, Some(name))) + } + ComplexType::Array(element) => match element.as_ref() { + ComplexType::Primitive(p) => Ok((map_primitive_to_protobuf(*p), None)), + ComplexType::Struct(_) => { + let element_path = format!("{}_element", sanitize_message_name(path)); + map_complex_type_to_protobuf(element, &element_path, collector) + } + ComplexType::Array(_) => Err(shape_unsupported("nested arrays", path)), + ComplexType::Map { .. } => Err(shape_unsupported("arrays of maps", path)), + }, + ComplexType::Map { key, value } => { + let key_primitive = validate_map_key(key, path)?; + let base = sanitize_message_name(path); + let map_value = match value.as_ref() { + ComplexType::Primitive(v) => MapValue::Primitive(*v), + ComplexType::Struct(st) => { + let value_name = collector.unique_name(format!("{}Value", base)); + let value_msg = generate_struct_message(&value_name, st)?; + collector.push(value_msg); + MapValue::Message(value_name) + } + ComplexType::Array(_) | ComplexType::Map { .. } => { + return Err(shape_unsupported("maps with complex value types", path)); + } + }; + let entry_name = collector.unique_name(format!("{}Entry", base)); + let entry = generate_map_entry(&entry_name, key_primitive, map_value); + collector.push(entry); + Ok((ProtoType::Message, Some(entry_name))) + } + } +} + +fn generate_struct_message( + message_name: &str, + st: &StructType, +) -> Result { + let mut local = MessageCollector::new(); + let mut fields = Vec::with_capacity(st.fields.len()); + for (index, f) in st.fields.iter().enumerate() { + validate_field_name(&f.name)?; + let path = format!("{}_{}", message_name, f.name); + let (field_type, type_name) = + map_complex_type_to_protobuf(&f.field_type, &path, &mut local)?; + let is_repeated = matches!( + f.field_type, + ComplexType::Array(_) | ComplexType::Map { .. } + ); + fields.push(field_descriptor( + &f.name, + (index + 1) as i32, + field_type, + type_name, + f.nullable, + is_repeated, + )); + } + Ok(DescriptorProto { + name: Some(message_name.to_string()), + field: fields, + nested_type: local.nested, + ..Default::default() + }) +} + +enum MapValue { + Primitive(PrimitiveType), + Message(String), +} + +fn generate_map_entry(name: &str, key: PrimitiveType, value: MapValue) -> DescriptorProto { + let key_field = FieldDescriptorProto { + name: Some("key".into()), + number: Some(1), + label: Some(Label::Optional as i32), + r#type: Some(map_primitive_to_protobuf(key) as i32), + json_name: Some("key".into()), + proto3_optional: Some(false), + ..Default::default() + }; + let (value_type, value_type_name) = match value { + MapValue::Primitive(p) => (map_primitive_to_protobuf(p), None), + MapValue::Message(n) => (ProtoType::Message, Some(n)), + }; + let value_field = FieldDescriptorProto { + name: Some("value".into()), + number: Some(2), + label: Some(Label::Optional as i32), + r#type: Some(value_type as i32), + type_name: value_type_name, + json_name: Some("value".into()), + proto3_optional: Some(true), + ..Default::default() + }; + DescriptorProto { + name: Some(name.to_string()), + field: vec![key_field, value_field], + options: Some(MessageOptions { + map_entry: Some(true), + ..Default::default() + }), + ..Default::default() + } +} + +fn validate_field_name(name: &str) -> Result<(), SchemaError> { + if name.is_empty() { + return Err(SchemaError::InvalidFieldName { + name: name.to_string(), + reason: "empty".into(), + }); + } + if name.starts_with(|c: char| c.is_ascii_digit()) { + return Err(SchemaError::InvalidFieldName { + name: name.to_string(), + reason: "cannot start with a digit".into(), + }); + } + if !name.chars().all(|c| c.is_ascii_alphanumeric() || c == '_') { + return Err(SchemaError::InvalidFieldName { + name: name.to_string(), + reason: "only alphanumeric and '_' characters allowed".into(), + }); + } + Ok(()) +} + +/// Convert a Unity Catalog identifier to a valid PascalCase protobuf message name. +/// +/// Protobuf identifiers must be ASCII (`[A-Za-z_][A-Za-z0-9_]*`), so any +/// non-ASCII characters (e.g. `é`, `中`) are dropped even though they are +/// Unicode-alphanumeric — otherwise the generated descriptor would fail +/// to compile. +fn sanitize_message_name(name: &str) -> String { + let mut out = String::with_capacity(name.len()); + let mut capitalize = true; + for c in name.chars() { + if c.is_ascii_alphanumeric() { + if capitalize { + out.push(c.to_ascii_uppercase()); + capitalize = false; + } else { + out.push(c); + } + } else { + capitalize = true; + } + } + if out.is_empty() || !out.chars().next().unwrap().is_ascii_alphabetic() { + out.insert(0, 'M'); + } + out +} + +// --------------------------------------------------------------------------- +// Arrow schema conversion (feature = "arrow-flight") +// --------------------------------------------------------------------------- + +/// Build an [`arrow_schema::Schema`] from a Unity Catalog table's columns. +/// +/// Parallels [`descriptor_from_uc_columns`] but targets Arrow Flight callers. +/// Accepts the same set of UC types and applies the same structural rules +/// (nested arrays, arrays-of-maps, and maps with complex values are rejected; +/// map keys must be integral, bool, or string). +/// +/// Notable Arrow choices, all dictated by the Databricks Arrow Flight server: +/// `STRING` / `VARIANT` / `DECIMAL` → `LargeUtf8`, `BINARY` → `LargeBinary`, +/// `DATE` → `Date32`, `TIMESTAMP` → `Timestamp(Microsecond, Some("UTC"))`, +/// `TIMESTAMP_NTZ` → `Timestamp(Microsecond, None)`, `ARRAY` → `List` with +/// item field `"item"`, `MAP` → `Map` with entries field `"entries"` +/// containing `"keys"` and `"values"` (the canonical schema the Databricks +/// Arrow Flight server builds from Delta). +#[cfg(feature = "arrow-flight")] +pub fn arrow_schema_from_uc_columns( + columns: &[UcColumn], +) -> Result { + let mut sorted: Vec<&UcColumn> = columns.iter().filter(|c| c.position >= 0).collect(); + sorted.sort_by_key(|c| c.position); + + let mut fields = Vec::with_capacity(sorted.len()); + for column in sorted.iter() { + validate_field_name(&column.name)?; + fields.push(uc_column_to_arrow_field(column)?); + } + Ok(arrow_schema::Schema::new(fields)) +} + +/// Build an [`arrow_schema::Schema`] from a full [`UcTableSchema`]. +/// +/// See [`arrow_schema_from_uc_columns`] for the type mapping. The schema name +/// is not preserved in the returned Arrow schema (Arrow schemas do not carry a +/// top-level name); only fields are emitted. +#[cfg(feature = "arrow-flight")] +pub fn arrow_schema_from_uc_schema( + schema: &UcTableSchema, +) -> Result { + arrow_schema_from_uc_columns(&schema.columns) +} + +#[cfg(feature = "arrow-flight")] +fn uc_column_to_arrow_field(column: &UcColumn) -> Result { + if is_complex(&column.type_name) { + if column.type_json.is_empty() { + return Err(SchemaError::MissingTypeJson(column.name.clone())); + } + let complex = + parse_type_json(&column.type_json).map_err(|reason| SchemaError::InvalidTypeJson { + column: column.name.clone(), + reason, + })?; + complex_type_to_arrow_field(&column.name, &complex, column.nullable) + } else { + let p = parse_uc_top_level_type(&column.type_name)?; + Ok(arrow_schema::Field::new( + &column.name, + map_primitive_to_arrow(p), + column.nullable, + )) + } +} + +#[cfg(feature = "arrow-flight")] +fn map_primitive_to_arrow(p: PrimitiveType) -> arrow_schema::DataType { + use arrow_schema::{DataType, TimeUnit}; + match p { + PrimitiveType::String => DataType::LargeUtf8, + PrimitiveType::Long => DataType::Int64, + PrimitiveType::Integer => DataType::Int32, + PrimitiveType::Short => DataType::Int16, + PrimitiveType::Byte => DataType::Int8, + PrimitiveType::Double => DataType::Float64, + PrimitiveType::Float => DataType::Float32, + PrimitiveType::Boolean => DataType::Boolean, + PrimitiveType::Binary => DataType::LargeBinary, + PrimitiveType::Timestamp => DataType::Timestamp(TimeUnit::Microsecond, Some("UTC".into())), + PrimitiveType::TimestampNtz => DataType::Timestamp(TimeUnit::Microsecond, None), + PrimitiveType::Date => DataType::Date32, + // TODO: emit Decimal128(precision, scale) once the Databricks Arrow + // Flight server accepts native Decimal128. UC carries (p, s) in + // `type_text` ("decimal(10,2)") and the `type_json` primitive string, + // but `PrimitiveType::Decimal` discards them today. + PrimitiveType::Decimal => DataType::LargeUtf8, + } +} + +#[cfg(feature = "arrow-flight")] +fn complex_type_to_arrow_field( + name: &str, + ct: &ComplexType, + nullable: bool, +) -> Result { + use arrow_schema::{DataType, Field, Fields}; + use std::sync::Arc; + + match ct { + ComplexType::Primitive(p) => Ok(Field::new(name, map_primitive_to_arrow(*p), nullable)), + ComplexType::Struct(st) => { + let mut child_fields = Vec::with_capacity(st.fields.len()); + for f in &st.fields { + validate_field_name(&f.name)?; + child_fields.push(complex_type_to_arrow_field( + &f.name, + &f.field_type, + f.nullable, + )?); + } + Ok(Field::new( + name, + DataType::Struct(Fields::from(child_fields)), + nullable, + )) + } + ComplexType::Array(element) => { + // UC's `containsNull` is not surfaced in our AST; default to + // nullable elements (Spark/Delta semantics for an unspecified + // value). + let item_field = match element.as_ref() { + ComplexType::Primitive(p) => Field::new("item", map_primitive_to_arrow(*p), true), + ComplexType::Struct(_) => complex_type_to_arrow_field("item", element, true)?, + ComplexType::Array(_) => return Err(shape_unsupported("nested arrays", name)), + ComplexType::Map { .. } => return Err(shape_unsupported("arrays of maps", name)), + }; + Ok(Field::new( + name, + DataType::List(Arc::new(item_field)), + nullable, + )) + } + ComplexType::Map { key, value } => { + let key_primitive = validate_map_key(key, name)?; + let value_field = match value.as_ref() { + ComplexType::Primitive(p) => Field::new("values", map_primitive_to_arrow(*p), true), + ComplexType::Struct(_) => complex_type_to_arrow_field("values", value, true)?, + ComplexType::Array(_) | ComplexType::Map { .. } => { + return Err(shape_unsupported("maps with complex value types", name)); + } + }; + let entries = DataType::Struct(Fields::from(vec![ + Field::new("keys", map_primitive_to_arrow(key_primitive), false), + value_field, + ])); + // "entries" / "keys" / "values" matches the canonical Arrow schema + // the Databricks Arrow Flight server builds from Delta on its side. + // The server also accepts the Arrow-default "key_value"/"key"/"value" + // names and normalizes via `arrow::compute::cast`, but emitting the + // canonical names directly avoids that per-batch cast. + Ok(Field::new( + name, + DataType::Map(Arc::new(Field::new("entries", entries, false)), false), + nullable, + )) + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + fn col(name: &str, type_name: &str, nullable: bool, position: i32) -> UcColumn { + UcColumn { + name: name.into(), + type_name: type_name.into(), + type_text: type_name.to_lowercase(), + type_json: String::new(), + nullable, + position, + } + } + + fn complex_col(name: &str, type_name: &str, type_json: &str, position: i32) -> UcColumn { + UcColumn { + name: name.into(), + type_name: type_name.into(), + type_text: String::new(), + type_json: type_json.into(), + nullable: true, + position, + } + } + + fn field<'a>(desc: &'a DescriptorProto, name: &str) -> &'a FieldDescriptorProto { + desc.field + .iter() + .find(|f| f.name() == name) + .unwrap_or_else(|| panic!("field '{}' not found in {:?}", name, desc.name())) + } + + #[test] + fn scalars_round_trip() { + let cols = vec![ + col("id", "BIGINT", false, 0), + col("name", "STRING", true, 1), + col("score", "DOUBLE", true, 2), + col("created_at", "TIMESTAMP", true, 3), + col("d", "DATE", false, 4), + col("data", "BINARY", false, 5), + ]; + let d = descriptor_from_uc_columns(&cols, "m").unwrap(); + assert_eq!(d.name(), "m"); + assert_eq!(field(&d, "id").r#type(), ProtoType::Int64); + assert_eq!(field(&d, "id").label(), Label::Required); + assert_eq!(field(&d, "name").label(), Label::Optional); + assert_eq!(field(&d, "score").r#type(), ProtoType::Double); + assert_eq!(field(&d, "created_at").r#type(), ProtoType::Int64); + assert_eq!(field(&d, "d").r#type(), ProtoType::Int32); + assert_eq!(field(&d, "data").r#type(), ProtoType::Bytes); + // Field numbers are position + 1 and preserve UC ordering. + assert_eq!(field(&d, "id").number(), 1); + assert_eq!(field(&d, "data").number(), 6); + } + + #[test] + fn field_numbers_mirror_uc_position() { + // Unity Catalog `position` is 0-indexed; proto field number = position + 1. + // Gaps (e.g. from DROP COLUMN under Delta column-mapping) are preserved so + // that field number uniquely identifies a UC column even across schema edits. + let cols = vec![ + col("a", "STRING", true, 0), + col("b", "STRING", true, 4), + col("c", "STRING", true, 8), + ]; + let d = descriptor_from_uc_columns(&cols, "m").unwrap(); + assert_eq!(field(&d, "a").number(), 1); + assert_eq!(field(&d, "b").number(), 5); + assert_eq!(field(&d, "c").number(), 9); + } + + #[test] + fn struct_becomes_nested_message() { + let type_json = r#"{ + "type":"struct", + "fields":[ + {"name":"street","type":"string","nullable":true,"metadata":{}}, + {"name":"zip","type":"integer","nullable":false,"metadata":{}} + ] + }"#; + let cols = vec![complex_col("address", "STRUCT", type_json, 0)]; + let d = descriptor_from_uc_columns(&cols, "m").unwrap(); + + let f = field(&d, "address"); + assert_eq!(f.r#type(), ProtoType::Message); + assert_eq!(f.label(), Label::Optional); + let type_name = f.type_name.as_deref().unwrap(); + let nested = d + .nested_type + .iter() + .find(|n| n.name() == type_name) + .expect("nested struct message not emitted"); + assert_eq!(field(nested, "street").r#type(), ProtoType::String); + assert_eq!(field(nested, "zip").r#type(), ProtoType::Int32); + assert_eq!(field(nested, "zip").label(), Label::Required); + } + + #[test] + fn array_of_primitive_is_repeated_scalar() { + let type_json = r#"{"type":"array","elementType":"long","containsNull":true}"#; + let cols = vec![complex_col("tags", "ARRAY", type_json, 0)]; + let d = descriptor_from_uc_columns(&cols, "m").unwrap(); + let f = field(&d, "tags"); + assert_eq!(f.label(), Label::Repeated); + assert_eq!(f.r#type(), ProtoType::Int64); + assert!(f.type_name.is_none()); + } + + #[test] + fn array_of_struct_emits_nested_message() { + let type_json = r#"{ + "type":"array", + "elementType":{ + "type":"struct", + "fields":[{"name":"k","type":"string","nullable":true,"metadata":{}}] + }, + "containsNull":true + }"#; + let cols = vec![complex_col("items", "ARRAY", type_json, 0)]; + let d = descriptor_from_uc_columns(&cols, "m").unwrap(); + let f = field(&d, "items"); + assert_eq!(f.label(), Label::Repeated); + assert_eq!(f.r#type(), ProtoType::Message); + let name = f.type_name.as_deref().unwrap(); + assert!(d.nested_type.iter().any(|n| n.name() == name)); + } + + #[test] + fn map_of_primitive_generates_entry_message() { + let type_json = + r#"{"type":"map","keyType":"string","valueType":"integer","valueContainsNull":true}"#; + let cols = vec![complex_col("props", "MAP", type_json, 0)]; + let d = descriptor_from_uc_columns(&cols, "m").unwrap(); + let f = field(&d, "props"); + assert_eq!(f.label(), Label::Repeated); + assert_eq!(f.r#type(), ProtoType::Message); + let entry_name = f.type_name.as_deref().unwrap(); + let entry = d + .nested_type + .iter() + .find(|n| n.name() == entry_name) + .expect("map entry message missing"); + assert_eq!(entry.options.as_ref().and_then(|o| o.map_entry), Some(true)); + assert_eq!(field(entry, "key").r#type(), ProtoType::String); + assert_eq!(field(entry, "value").r#type(), ProtoType::Int32); + } + + #[test] + fn map_with_struct_value_emits_value_and_entry() { + let type_json = r#"{ + "type":"map", + "keyType":"string", + "valueType":{ + "type":"struct", + "fields":[{"name":"v","type":"long","nullable":true,"metadata":{}}] + }, + "valueContainsNull":true + }"#; + let cols = vec![complex_col("lookup", "MAP", type_json, 0)]; + let d = descriptor_from_uc_columns(&cols, "m").unwrap(); + let f = field(&d, "lookup"); + let entry_name = f.type_name.as_deref().unwrap(); + let entry = d + .nested_type + .iter() + .find(|n| n.name() == entry_name) + .unwrap(); + assert_eq!(entry.options.as_ref().and_then(|o| o.map_entry), Some(true)); + let value_type_name = field(entry, "value").type_name.as_deref().unwrap(); + // The referenced value message also exists as a nested type. + assert!(d.nested_type.iter().any(|n| n.name() == value_type_name)); + } + + #[test] + fn rejects_unsupported_map_key() { + let type_json = + r#"{"type":"map","keyType":"double","valueType":"integer","valueContainsNull":true}"#; + let cols = vec![complex_col("bad", "MAP", type_json, 0)]; + let err = descriptor_from_uc_columns(&cols, "m").unwrap_err(); + assert!(matches!(err, SchemaError::Invalid(_)), "got {:?}", err); + } + + #[test] + fn rejects_excessively_deep_nesting() { + // Build a chain of `MAX_NESTING_DEPTH + 2` nested arrays. The parser + // should bail out with an InvalidTypeJson rather than overflowing the stack. + let mut type_json = String::from("\"integer\""); + for _ in 0..MAX_NESTING_DEPTH + 2 { + type_json = format!( + r#"{{"type":"array","elementType":{},"containsNull":true}}"#, + type_json + ); + } + let cols = vec![complex_col("deep", "ARRAY", &type_json, 0)]; + let err = descriptor_from_uc_columns(&cols, "m").unwrap_err(); + match err { + SchemaError::InvalidTypeJson { reason, .. } => { + assert!( + reason.contains("maximum depth"), + "unexpected reason: {}", + reason + ); + } + other => panic!("expected InvalidTypeJson, got {:?}", other), + } + } + + #[test] + fn rejects_nested_arrays() { + let type_json = r#"{"type":"array","elementType":{"type":"array","elementType":"integer","containsNull":true},"containsNull":true}"#; + let cols = vec![complex_col("nested", "ARRAY", type_json, 0)]; + let err = descriptor_from_uc_columns(&cols, "m").unwrap_err(); + assert!(matches!(err, SchemaError::Invalid(_)), "got {:?}", err); + } + + #[test] + fn rejects_invalid_field_name() { + let cols = vec![col("1bad", "STRING", true, 0)]; + let err = descriptor_from_uc_columns(&cols, "m").unwrap_err(); + assert!(matches!(err, SchemaError::InvalidFieldName { .. })); + } + + #[test] + fn allows_proto_keywords_and_type_names_as_field_names() { + // protoc accepts every proto keyword and primitive type name as a + // field name (verified against protoc 30.2, proto2/proto3 + cpp codegen). + // The descriptor only carries the name as a byte string; ambiguity + // exists only when re-rendering to `.proto` text in declaration position. + let names = [ + "message", "enum", "service", "rpc", "option", "import", "package", "oneof", "map", + "reserved", "syntax", "double", "float", "int32", "int64", "uint32", "uint64", + "sint32", "sint64", "fixed32", "fixed64", "sfixed32", "sfixed64", "bool", "string", + "bytes", + ]; + let cols: Vec = names + .iter() + .enumerate() + .map(|(i, n)| col(n, "STRING", true, i as i32)) + .collect(); + let d = descriptor_from_uc_columns(&cols, "m").unwrap(); + assert_eq!(d.field.len(), names.len()); + } + + #[test] + fn complex_column_requires_type_json() { + let cols = vec![col("x", "STRUCT", true, 0)]; + let err = descriptor_from_uc_columns(&cols, "m").unwrap_err(); + assert!(matches!(err, SchemaError::MissingTypeJson(_))); + } + + #[test] + fn descriptor_from_uc_schema_derives_name() { + let schema = UcTableSchema { + name: "events".into(), + catalog_name: "main".into(), + schema_name: "analytics".into(), + columns: vec![col("id", "BIGINT", false, 0)], + }; + let d = descriptor_from_uc_schema(&schema).unwrap(); + assert_eq!(d.name(), "AnalyticsEvents"); + } + + #[test] + fn unique_name_disambiguates_collisions_in_input_order() { + // Two sibling struct fields whose path-derived message names collide + // under `sanitize_message_name`: both `foo` and `Foo` build path + // "Parent_foo" / "Parent_Foo" which PascalCase to the same `ParentFoo`. + // The first-registered field must keep the bare name; the second gets + // the `2` suffix. This pins the ordering contract documented on + // `MessageCollector::unique_name`. + let type_json = r#"{ + "type":"struct", + "fields":[ + {"name":"foo","type":{"type":"struct","fields":[ + {"name":"a","type":"string","nullable":true,"metadata":{}} + ]},"nullable":true,"metadata":{}}, + {"name":"Foo","type":{"type":"struct","fields":[ + {"name":"b","type":"string","nullable":true,"metadata":{}} + ]},"nullable":true,"metadata":{}} + ] + }"#; + let cols = vec![complex_col("parent", "STRUCT", type_json, 0)]; + let d = descriptor_from_uc_columns(&cols, "m").unwrap(); + + let parent = d + .nested_type + .iter() + .find(|n| n.name() == "Parent") + .expect("Parent message missing"); + let foo = field(parent, "foo"); + let foo_cap = field(parent, "Foo"); + assert_eq!(foo.type_name.as_deref(), Some("ParentFoo")); + assert_eq!(foo_cap.type_name.as_deref(), Some("ParentFoo2")); + } + + #[test] + fn sanitize_message_name_handles_invalid_chars() { + assert_eq!(sanitize_message_name("foo-bar"), "FooBar"); + assert_eq!(sanitize_message_name("1abc"), "M1abc"); + assert_eq!(sanitize_message_name("analytics.events"), "AnalyticsEvents"); + } + + #[test] + fn sanitize_message_name_drops_non_ascii() { + // Non-ASCII letters are Unicode-alphanumeric but invalid in protobuf + // identifiers; they must be stripped rather than passed through. + // Stripping a non-ASCII char triggers the same "capitalize next" behavior + // as any other non-alphanumeric separator, so "événements" → "VNements". + assert_eq!(sanitize_message_name("café"), "Caf"); + assert_eq!(sanitize_message_name("événements"), "VNements"); + assert_eq!(sanitize_message_name("中文_table"), "Table"); + // All-non-ASCII input must still produce a valid identifier start. + assert_eq!(sanitize_message_name("中文"), "M"); + // Leading non-ASCII yields an ASCII-only result that still starts with a letter. + let result = sanitize_message_name("éfoo"); + assert!(result.chars().next().unwrap().is_ascii_alphabetic()); + assert!(result + .chars() + .all(|c| c.is_ascii_alphanumeric() || c == '_')); + } + + #[test] + fn uc_column_deserializes_from_uc_api_shape() { + let json = r#"{ + "name":"id", + "type_name":"INT", + "type_text":"int", + "type_json":"{\"name\":\"id\",\"type\":\"integer\",\"nullable\":false,\"metadata\":{}}", + "nullable":false, + "position":0 + }"#; + let col: UcColumn = serde_json::from_str(json).unwrap(); + assert_eq!(col.name, "id"); + assert_eq!(col.type_name, "INT"); + assert!(!col.nullable); + } + + #[cfg(feature = "arrow-flight")] + mod arrow { + use super::*; + use arrow_schema::{DataType, TimeUnit}; + + fn arrow_field<'a>( + schema: &'a arrow_schema::Schema, + name: &str, + ) -> &'a arrow_schema::Field { + schema + .field_with_name(name) + .unwrap_or_else(|_| panic!("field '{}' not found", name)) + } + + #[test] + fn scalars_use_proper_arrow_types() { + let cols = vec![ + col("id", "BIGINT", false, 0), + col("name", "STRING", true, 1), + col("score", "DOUBLE", true, 2), + col("created_at", "TIMESTAMP", true, 3), + col("seen_at", "TIMESTAMP_NTZ", true, 4), + col("d", "DATE", false, 5), + col("data", "BINARY", false, 6), + col("flag", "BOOLEAN", true, 7), + col("price", "DECIMAL", true, 8), + ]; + let s = arrow_schema_from_uc_columns(&cols).unwrap(); + assert_eq!(arrow_field(&s, "id").data_type(), &DataType::Int64); + assert!(!arrow_field(&s, "id").is_nullable()); + assert_eq!(arrow_field(&s, "name").data_type(), &DataType::LargeUtf8); + assert!(arrow_field(&s, "name").is_nullable()); + assert_eq!(arrow_field(&s, "score").data_type(), &DataType::Float64); + assert_eq!( + arrow_field(&s, "created_at").data_type(), + &DataType::Timestamp(TimeUnit::Microsecond, Some("UTC".into())) + ); + assert_eq!( + arrow_field(&s, "seen_at").data_type(), + &DataType::Timestamp(TimeUnit::Microsecond, None) + ); + assert_eq!(arrow_field(&s, "d").data_type(), &DataType::Date32); + assert_eq!(arrow_field(&s, "data").data_type(), &DataType::LargeBinary); + assert_eq!(arrow_field(&s, "flag").data_type(), &DataType::Boolean); + // DECIMAL renders as LargeUtf8 (text encoding contract preserved). + assert_eq!(arrow_field(&s, "price").data_type(), &DataType::LargeUtf8); + } + + #[test] + fn columns_sorted_by_position() { + let cols = vec![ + col("b", "STRING", true, 1), + col("a", "STRING", true, 0), + col("c", "STRING", true, 2), + ]; + let s = arrow_schema_from_uc_columns(&cols).unwrap(); + assert_eq!(s.field(0).name(), "a"); + assert_eq!(s.field(1).name(), "b"); + assert_eq!(s.field(2).name(), "c"); + } + + #[test] + fn struct_becomes_arrow_struct() { + let type_json = r#"{ + "type":"struct", + "fields":[ + {"name":"street","type":"string","nullable":true,"metadata":{}}, + {"name":"zip","type":"integer","nullable":false,"metadata":{}} + ] + }"#; + let cols = vec![complex_col("address", "STRUCT", type_json, 0)]; + let s = arrow_schema_from_uc_columns(&cols).unwrap(); + let f = arrow_field(&s, "address"); + match f.data_type() { + DataType::Struct(fs) => { + assert_eq!(fs.len(), 2); + assert_eq!(fs[0].name(), "street"); + assert_eq!(fs[0].data_type(), &DataType::LargeUtf8); + assert!(fs[0].is_nullable()); + assert_eq!(fs[1].name(), "zip"); + assert_eq!(fs[1].data_type(), &DataType::Int32); + assert!(!fs[1].is_nullable()); + } + other => panic!("expected Struct, got {:?}", other), + } + } + + #[test] + fn array_of_primitive_is_list() { + let type_json = r#"{"type":"array","elementType":"long","containsNull":true}"#; + let cols = vec![complex_col("tags", "ARRAY", type_json, 0)]; + let s = arrow_schema_from_uc_columns(&cols).unwrap(); + let f = arrow_field(&s, "tags"); + match f.data_type() { + DataType::List(item) => { + assert_eq!(item.name(), "item"); + assert_eq!(item.data_type(), &DataType::Int64); + assert!(item.is_nullable()); + } + other => panic!("expected List, got {:?}", other), + } + } + + #[test] + fn array_of_struct_is_list_of_struct() { + let type_json = r#"{ + "type":"array", + "elementType":{ + "type":"struct", + "fields":[{"name":"k","type":"string","nullable":true,"metadata":{}}] + }, + "containsNull":true + }"#; + let cols = vec![complex_col("items", "ARRAY", type_json, 0)]; + let s = arrow_schema_from_uc_columns(&cols).unwrap(); + let f = arrow_field(&s, "items"); + match f.data_type() { + DataType::List(item) => match item.data_type() { + DataType::Struct(fs) => { + assert_eq!(fs.len(), 1); + assert_eq!(fs[0].name(), "k"); + } + other => panic!("expected Struct inside List, got {:?}", other), + }, + other => panic!("expected List, got {:?}", other), + } + } + + #[test] + fn map_uses_entries_keys_values_canonical_names() { + let type_json = r#"{"type":"map","keyType":"string","valueType":"integer","valueContainsNull":true}"#; + let cols = vec![complex_col("props", "MAP", type_json, 0)]; + let s = arrow_schema_from_uc_columns(&cols).unwrap(); + let f = arrow_field(&s, "props"); + match f.data_type() { + DataType::Map(entries, sorted) => { + assert!(!sorted); + assert_eq!(entries.name(), "entries"); + assert!(!entries.is_nullable()); + match entries.data_type() { + DataType::Struct(kv) => { + assert_eq!(kv[0].name(), "keys"); + assert_eq!(kv[0].data_type(), &DataType::LargeUtf8); + assert!(!kv[0].is_nullable(), "map keys must not be nullable"); + assert_eq!(kv[1].name(), "values"); + assert_eq!(kv[1].data_type(), &DataType::Int32); + assert!(kv[1].is_nullable()); + } + other => panic!("expected Struct inside Map, got {:?}", other), + } + } + other => panic!("expected Map, got {:?}", other), + } + } + + #[test] + fn map_with_struct_value() { + let type_json = r#"{ + "type":"map", + "keyType":"long", + "valueType":{ + "type":"struct", + "fields":[{"name":"v","type":"long","nullable":true,"metadata":{}}] + }, + "valueContainsNull":true + }"#; + let cols = vec![complex_col("lookup", "MAP", type_json, 0)]; + let s = arrow_schema_from_uc_columns(&cols).unwrap(); + let f = arrow_field(&s, "lookup"); + match f.data_type() { + DataType::Map(entries, _) => match entries.data_type() { + DataType::Struct(kv) => { + assert_eq!(kv[0].data_type(), &DataType::Int64); + match kv[1].data_type() { + DataType::Struct(inner) => { + assert_eq!(inner[0].name(), "v"); + assert_eq!(inner[0].data_type(), &DataType::Int64); + } + other => panic!("expected Struct value, got {:?}", other), + } + } + other => panic!("expected Struct, got {:?}", other), + }, + other => panic!("expected Map, got {:?}", other), + } + } + + #[test] + fn rejects_unsupported_map_key() { + let type_json = r#"{"type":"map","keyType":"double","valueType":"integer","valueContainsNull":true}"#; + let cols = vec![complex_col("bad", "MAP", type_json, 0)]; + let err = arrow_schema_from_uc_columns(&cols).unwrap_err(); + assert!(matches!(err, SchemaError::Invalid(_)), "got {:?}", err); + } + + #[test] + fn rejects_nested_arrays() { + let type_json = r#"{"type":"array","elementType":{"type":"array","elementType":"integer","containsNull":true},"containsNull":true}"#; + let cols = vec![complex_col("nested", "ARRAY", type_json, 0)]; + let err = arrow_schema_from_uc_columns(&cols).unwrap_err(); + assert!(matches!(err, SchemaError::Invalid(_)), "got {:?}", err); + } + + #[test] + fn rejects_invalid_field_name() { + let cols = vec![col("1bad", "STRING", true, 0)]; + let err = arrow_schema_from_uc_columns(&cols).unwrap_err(); + assert!(matches!(err, SchemaError::InvalidFieldName { .. })); + } + + #[test] + fn complex_column_requires_type_json() { + let cols = vec![col("x", "STRUCT", true, 0)]; + let err = arrow_schema_from_uc_columns(&cols).unwrap_err(); + assert!(matches!(err, SchemaError::MissingTypeJson(_))); + } + + #[test] + fn nested_timestamp_ntz_preserves_no_timezone() { + // Inside type_json, "timestamp" carries UTC and "timestamp_ntz" carries no tz; + // the AST must distinguish the two so the Arrow types are correct in nested + // positions, not just at the top level. + let type_json = r#"{ + "type":"struct", + "fields":[ + {"name":"utc","type":"timestamp","nullable":true,"metadata":{}}, + {"name":"local","type":"timestamp_ntz","nullable":true,"metadata":{}} + ] + }"#; + let cols = vec![complex_col("ts", "STRUCT", type_json, 0)]; + let s = arrow_schema_from_uc_columns(&cols).unwrap(); + match arrow_field(&s, "ts").data_type() { + DataType::Struct(fs) => { + assert_eq!( + fs[0].data_type(), + &DataType::Timestamp(TimeUnit::Microsecond, Some("UTC".into())) + ); + assert_eq!( + fs[1].data_type(), + &DataType::Timestamp(TimeUnit::Microsecond, None) + ); + } + other => panic!("expected Struct, got {:?}", other), + } + } + + #[test] + fn rejects_excessively_deep_nesting() { + let mut type_json = String::from("\"integer\""); + for _ in 0..MAX_NESTING_DEPTH + 2 { + type_json = format!( + r#"{{"type":"array","elementType":{},"containsNull":true}}"#, + type_json + ); + } + let cols = vec![complex_col("deep", "ARRAY", &type_json, 0)]; + let err = arrow_schema_from_uc_columns(&cols).unwrap_err(); + match err { + SchemaError::InvalidTypeJson { reason, .. } => { + assert!(reason.contains("maximum depth"), "unexpected: {}", reason); + } + other => panic!("expected InvalidTypeJson, got {:?}", other), + } + } + + #[test] + fn arrow_schema_from_uc_schema_delegates_to_columns() { + let schema = UcTableSchema { + name: "events".into(), + catalog_name: "main".into(), + schema_name: "analytics".into(), + columns: vec![col("id", "BIGINT", false, 0)], + }; + let s = arrow_schema_from_uc_schema(&schema).unwrap(); + assert_eq!(s.fields().len(), 1); + assert_eq!(arrow_field(&s, "id").data_type(), &DataType::Int64); + } + } +} diff --git a/lib/zerobus-ffi-1.3.0/rust/sdk/src/stream_configuration.rs b/lib/zerobus-ffi-1.3.0/rust/sdk/src/stream_configuration.rs new file mode 100644 index 00000000000..15c6f8c5751 --- /dev/null +++ b/lib/zerobus-ffi-1.3.0/rust/sdk/src/stream_configuration.rs @@ -0,0 +1,178 @@ +use std::sync::Arc; + +use crate::callbacks::AckCallback; +use crate::databricks::zerobus::RecordType; +use crate::stream_options::defaults; + +/// Configuration options for stream creation, recovery of broken streams and flushing. +/// +/// These options control the behavior of ingestion streams, including memory limits, +/// recovery policies, and timeout settings. +/// +/// **Do not construct this directly.** Configure streams via the builder API: +/// +/// ```rust,ignore +/// let stream = sdk +/// .stream_builder() +/// .table("catalog.schema.table") +/// .oauth("client-id", "client-secret") +/// .json() +/// .max_inflight_requests(1_000_000) +/// .recovery(true) +/// .recovery_timeout_ms(20_000) +/// .recovery_retries(5) +/// .build() +/// .await?; +/// ``` +#[derive(Clone)] +#[non_exhaustive] +pub struct StreamConfigurationOptions { + /// Maximum number of requests that can be sending or pending acknowledgement at any given time. + /// + /// This limit controls memory usage and backpressure. When this limit is reached, + /// `ingest_record()` and `ingest_records()` calls will block until acknowledgments free up space. + /// + /// Default: 1,000,000 + pub max_inflight_requests: usize, + + /// Whether to enable automatic stream recovery on failure. + /// + /// When enabled, the SDK will automatically attempt to reconnect and recover + /// the stream when encountering retryable errors. + /// + /// Default: `true` + pub recovery: bool, + + /// Timeout in milliseconds for each stream recovery attempt. + /// + /// If a recovery attempt takes longer than this, it will be retried. + /// + /// Default: 15,000 (15 seconds) + pub recovery_timeout_ms: u64, + + /// Backoff time in milliseconds between stream recovery retry attempts. + /// + /// The SDK will wait this duration before attempting another recovery after a failure. + /// + /// Default: 2,000 (2 seconds) + pub recovery_backoff_ms: u64, + + /// Maximum number of recovery retry attempts before giving up. + /// + /// After this many failed attempts, the stream will close and return an error. + /// + /// Default: 4 + pub recovery_retries: u32, + + /// Timeout in milliseconds for waiting for server acknowledgements. + /// + /// If no acknowledgement is received within this time (and there are pending records), + /// the stream will be considered failed and recovery will be triggered. + /// + /// Default: 60,000 (60 seconds) + pub server_lack_of_ack_timeout_ms: u64, + + /// Timeout in milliseconds for flush operations. + /// + /// If a flush() call cannot complete within this time, it will return a timeout error. + /// + /// Default: 300,000 (5 minutes) + pub flush_timeout_ms: u64, + + /// Type of record to ingest. + /// + /// Supported values: + /// - RecordType::Proto + /// - RecordType::Json + /// - RecordType::Unspecified + /// + /// Default: RecordType::Proto + pub record_type: RecordType, + + /// Maximum time in milliseconds to wait during graceful stream close. + /// + /// When the server sends a CloseStreamSignal indicating it will close the stream, + /// the SDK can enter a "paused" state where it: + /// - Continues accepting and buffering new ingest_record() calls + /// - Stops sending buffered records to the server + /// - Continues processing acknowledgments for in-flight records + /// - Waits for either all in-flight records to be acknowledged or the timeout to expire + /// + /// Configuration values: + /// - `None`: Wait for the full server-specified duration (most graceful) + /// - `Some(0)`: Immediate recovery, close stream right away (current behavior) + /// - `Some(x)`: Wait up to min(x, server_duration) milliseconds + /// + /// Default: `None` (wait for full server duration) + pub stream_paused_max_wait_time_ms: Option, + + /// Optional callback invoked when records are acknowledged or encounter errors. + /// + /// When set, this callback will be invoked: + /// - On successful acknowledgment: `on_ack(offset_id)` is called + /// - On error: `on_error(offset_id, error_message)` is called + /// + /// + /// Default: `None` (no callbacks) + /// + /// # Examples + /// + /// ```rust,ignore + /// use std::sync::Arc; + /// use databricks_zerobus_ingest_sdk::{AckCallback, OffsetId}; + /// + /// struct MyCallback; + /// + /// impl AckCallback for MyCallback { + /// fn on_ack(&self, offset_id: OffsetId) { + /// println!("Acknowledged: {}", offset_id); + /// } + /// + /// fn on_error(&self, offset_id: OffsetId, error_message: &str) { + /// eprintln!("Error {}: {}", offset_id, error_message); + /// } + /// } + /// + /// let stream = sdk + /// .stream_builder() + /// .table("catalog.schema.table") + /// .oauth("client-id", "client-secret") + /// .json() + /// .ack_callback(Arc::new(MyCallback)) + /// .build() + /// .await?; + /// ``` + pub ack_callback: Option>, + + /// Maximum time in milliseconds to wait for callbacks to finish after calling close() on the stream. + /// + /// When the stream is closed, all tasks are shut down and the callback handler task is + /// given a timeout to finish processing callbacks. After the timeout expires, or once all + /// callbacks have been processed, the callback handler task is aborted and the stream is + /// fully closed. + /// + /// Configuration values: + /// - `None`: Wait forever + /// - `Some(x)`: Wait up to x milliseconds + /// + /// Default: `Some(5000)` (wait 5 seconds) + pub callback_max_wait_time_ms: Option, +} + +impl Default for StreamConfigurationOptions { + fn default() -> Self { + Self { + max_inflight_requests: 1_000_000, + recovery: defaults::RECOVERY, + recovery_timeout_ms: defaults::RECOVERY_TIMEOUT_MS, + recovery_backoff_ms: defaults::RECOVERY_BACKOFF_MS, + recovery_retries: defaults::RECOVERY_RETRIES, + server_lack_of_ack_timeout_ms: defaults::SERVER_LACK_OF_ACK_TIMEOUT_MS, + flush_timeout_ms: defaults::FLUSH_TIMEOUT_MS, + record_type: RecordType::Proto, + stream_paused_max_wait_time_ms: None, + ack_callback: None, + callback_max_wait_time_ms: Some(defaults::CALLBACK_MAX_WAIT_TIME_MS), + } + } +} diff --git a/lib/zerobus-ffi-1.3.0/rust/sdk/src/stream_options.rs b/lib/zerobus-ffi-1.3.0/rust/sdk/src/stream_options.rs new file mode 100644 index 00000000000..0f09003c543 --- /dev/null +++ b/lib/zerobus-ffi-1.3.0/rust/sdk/src/stream_options.rs @@ -0,0 +1,25 @@ +//! Shared configuration options for stream creation and operation. +//! +//! This module provides common configuration constants shared between gRPC and Arrow Flight streams. + +/// Default values for stream configuration options. +/// These are shared between gRPC and Arrow Flight streams. +pub mod defaults { + /// Default: enable automatic stream recovery + pub const RECOVERY: bool = true; + /// Default: 15 seconds per recovery attempt + pub const RECOVERY_TIMEOUT_MS: u64 = 15_000; + /// Default: 2 seconds backoff between retries + pub const RECOVERY_BACKOFF_MS: u64 = 2_000; + /// Default: 4 retry attempts + pub const RECOVERY_RETRIES: u32 = 4; + /// Default: 60 seconds lack of ack timeout + pub const SERVER_LACK_OF_ACK_TIMEOUT_MS: u64 = 60_000; + /// Default: 5 minutes flush timeout + pub const FLUSH_TIMEOUT_MS: u64 = 300_000; + /// Default: 30 seconds connection timeout + #[cfg(feature = "arrow-flight")] + pub const CONNECTION_TIMEOUT_MS: u64 = 30_000; + /// Default: 5 seconds callback timeout + pub const CALLBACK_MAX_WAIT_TIME_MS: u64 = 5_000; +} diff --git a/lib/zerobus-ffi-1.3.0/rust/sdk/src/tls_config.rs b/lib/zerobus-ffi-1.3.0/rust/sdk/src/tls_config.rs new file mode 100644 index 00000000000..a65f39d69fc --- /dev/null +++ b/lib/zerobus-ffi-1.3.0/rust/sdk/src/tls_config.rs @@ -0,0 +1,100 @@ +//! TLS configuration for Zerobus connections. +//! +//! This module provides a strategy pattern for TLS configuration, +//! allowing different TLS setups (secure, custom CA, or no TLS for testing). + +use crate::errors::ZerobusError; +use crate::ZerobusResult; +use tonic::transport::{ClientTlsConfig, Endpoint}; + +/// Trait for TLS configuration strategies. +/// +/// Implementations define how to configure the gRPC channel's TLS settings. +/// This allows the SDK to support different TLS configurations: +/// - `SecureTlsConfig`: Production TLS with system CA certificates (default) +/// - `NoTlsConfig`: No TLS, for testing with local `http://` endpoints (requires `testing` feature) +/// - Custom implementations for special certificate requirements +/// +/// # Examples +/// +/// ```rust +/// use databricks_zerobus_ingest_sdk::{SecureTlsConfig, TlsConfig}; +/// use std::sync::Arc; +/// +/// // Secure TLS with system CAs (default) +/// let tls: Arc = Arc::new(SecureTlsConfig::new()); +/// ``` +#[allow(clippy::result_large_err)] +pub trait TlsConfig: Send + Sync { + /// Configure a gRPC endpoint with TLS settings. + /// + /// # Arguments + /// + /// * `endpoint` - The gRPC endpoint to configure + /// + /// # Returns + /// + /// The configured endpoint, ready to connect + /// + /// # Errors + /// + /// Returns an error if TLS configuration fails + fn configure_endpoint(&self, endpoint: Endpoint) -> ZerobusResult; +} + +/// Secure TLS configuration using system CA certificates. +/// +/// This is the default and recommended configuration for production use. +/// It enables TLS encryption using the operating system's trusted CA certificates. +/// +/// # Examples +/// +/// ```rust +/// use databricks_zerobus_ingest_sdk::SecureTlsConfig; +/// +/// let tls = SecureTlsConfig::new(); +/// ``` +#[derive(Clone, Debug, Default)] +pub struct SecureTlsConfig; + +impl SecureTlsConfig { + /// Create a new secure TLS configuration. + pub fn new() -> Self { + Self + } +} + +impl TlsConfig for SecureTlsConfig { + fn configure_endpoint(&self, endpoint: Endpoint) -> ZerobusResult { + // Use native OS certificate store (works on Windows, macOS, and Linux). + let tls_config = ClientTlsConfig::new().with_native_roots(); + + endpoint + .tls_config(tls_config) + .map_err(|_| ZerobusError::FailedToEstablishTlsConnectionError) + } +} + +/// No-op TLS configuration for testing with plaintext `http://` endpoints. +/// +/// This passes the endpoint through without any TLS configuration. +/// Only available when the `testing` feature is enabled. +/// +/// # Examples +/// +/// ```rust +/// use databricks_zerobus_ingest_sdk::{NoTlsConfig, TlsConfig}; +/// use std::sync::Arc; +/// +/// let tls: Arc = Arc::new(NoTlsConfig); +/// ``` +#[cfg(feature = "testing")] +#[derive(Clone, Debug, Default)] +pub struct NoTlsConfig; + +#[cfg(feature = "testing")] +impl TlsConfig for NoTlsConfig { + fn configure_endpoint(&self, endpoint: Endpoint) -> ZerobusResult { + Ok(endpoint) + } +} diff --git a/lib/zerobus-ffi-1.3.0/rust/sdk/src/token_cache.rs b/lib/zerobus-ffi-1.3.0/rust/sdk/src/token_cache.rs new file mode 100644 index 00000000000..8b99ce67dbe --- /dev/null +++ b/lib/zerobus-ffi-1.3.0/rust/sdk/src/token_cache.rs @@ -0,0 +1,540 @@ +//! Per-table OAuth token cache for the default OAuth authentication path. +//! +//! Unity Catalog sets each token's lifetime (currently one hour), while a single +//! stream lives at most ~15 minutes. Without caching, every stream creation (and +//! every recovery) mints a fresh token, putting the Unity Catalog token endpoint +//! under unnecessary load when a client churns through many streams. The cache +//! does not assume a fixed lifetime — it serves a token until it nears the +//! `expires_in` the server reported. +//! +//! [`TokenCache`] caches one token per `(client_id, secret, table_name)` key on +//! the [`ZerobusSdk`](crate::ZerobusSdk) instance and serves it until it nears +//! expiry, refreshing lazily on access. Tokens are downscoped to a single table +//! (the authorization details embed the catalog/schema/table), so the table +//! name is part of the cache key. + +use std::collections::HashMap; +use std::sync::Arc; +use std::time::Duration; + +use sha2::{Digest, Sha256}; +use tokio::sync::Mutex; +use tokio::time::Instant; +use tracing::{debug, warn}; + +use crate::default_token_factory::{FetchedToken, MintReason}; +use crate::ZerobusResult; + +/// Default lead time before expiry at which a cached token is refreshed. +pub(crate) const DEFAULT_REFRESH_BUFFER: Duration = Duration::from_secs(300); + +/// A cached token and the instant at which it expires. +struct CachedToken { + value: String, + expires_at: Instant, +} + +impl CachedToken { + fn is_expired(&self) -> bool { + Instant::now() >= self.expires_at + } +} + +/// Identifies a cache entry. The client secret is keyed by its SHA-256 digest, +/// not plaintext: the digest is collision-resistant (distinct secrets cannot in +/// practice share a token) and keeps the raw secret out of the cache map. A +/// rotated secret yields a different digest, hence a fresh entry. +#[derive(Clone, PartialEq, Eq, Hash)] +struct TokenKey { + client_id: String, + secret_digest: [u8; 32], + table_name: String, +} + +impl TokenKey { + fn new(client_id: &str, client_secret: &str, table_name: &str) -> Self { + let secret_digest = Sha256::digest(client_secret.as_bytes()).into(); + Self { + client_id: client_id.to_string(), + secret_digest, + table_name: table_name.to_string(), + } + } +} + +/// Per-entry slot. Each key has its own mutex so that a cold-cache burst of +/// concurrent stream creations for the same table mints a single token +/// (single-flight) while creations for different tables never block each other. +type Slot = Arc>>; + +/// Caches OAuth tokens per table for the lifetime of a [`ZerobusSdk`]. +/// +/// Safe for concurrent use across streams created from the same SDK instance. +pub(crate) struct TokenCache { + entries: Mutex>, + refresh_buffer: Duration, + enabled: bool, +} + +impl TokenCache { + pub(crate) fn new(enabled: bool, refresh_buffer: Duration) -> Self { + Self { + entries: Mutex::new(HashMap::new()), + refresh_buffer, + enabled, + } + } + + /// Returns a valid token for the given credentials and table, fetching a new + /// one only if the cache is empty, the token has entered the refresh window, + /// or caching is disabled. + /// + /// `fetch` is invoked to mint a fresh token. It is only ever called once per + /// key at a time thanks to the per-entry lock. + pub(crate) async fn get_or_fetch( + &self, + client_id: &str, + client_secret: &str, + table_name: &str, + fetch: F, + ) -> ZerobusResult + where + F: FnOnce(MintReason) -> Fut, + Fut: std::future::Future>, + { + if !self.enabled { + return fetch(MintReason::CacheDisabled) + .await + .map(|fetched| fetched.token); + } + + let key = TokenKey::new(client_id, client_secret, table_name); + + let slot = { + let mut entries = self.entries.lock().await; + // Sweep only on a miss, keeping the cost off the hot lookup path. + if !entries.contains_key(&key) { + Self::prune_expired(&mut entries); + } + Arc::clone(entries.entry(key).or_default()) + }; + + // Hold the per-entry lock across the fetch so concurrent callers for the + // same key reuse a single mint instead of stampeding the token endpoint. + let mut guard = slot.lock().await; + + if let Some(cached) = guard.as_ref() { + if !self.needs_refresh(cached) { + debug!(table = %table_name, "token cache hit, reusing cached token"); + return Ok(cached.value.clone()); + } + } + + // A present-but-stale token means we are refreshing; an empty slot is a + // cold miss. The reason is surfaced on the mint log. + let reason = if guard.is_some() { + MintReason::Refresh + } else { + MintReason::ColdMiss + }; + + let fetched = match fetch(reason).await { + Ok(fetched) => fetched, + Err(err) => { + // On a retryable failure, serve the still-valid cached token; + // let non-retryable errors (bad/revoked creds) surface. + if err.is_retryable() { + if let Some(cached) = guard.as_ref() { + if !cached.is_expired() { + warn!(table = %table_name, "token refresh failed (retryable); serving still-valid cached token"); + return Ok(cached.value.clone()); + } + } + } + return Err(err); + } + }; + + let token = fetched.token.clone(); + + // Cache only tokens with a usable TTL. `checked_add` also drops an absurd + // `expires_in` that would overflow the clock instead of panicking. + let expires_at = fetched + .expires_in + .and_then(|ttl| Instant::now().checked_add(ttl)); + match expires_at { + Some(expires_at) => { + *guard = Some(CachedToken { + value: fetched.token, + expires_at, + }); + } + None => { + // No usable TTL: keep an existing still-valid token rather than + // discarding it. + let keep_existing = guard.as_ref().is_some_and(|cached| !cached.is_expired()); + if !keep_existing { + *guard = None; + } + } + } + + Ok(token) + } + + /// Drops any cached token for the given credentials and table so the next + /// `get_or_fetch` re-mints. Called when the server rejects the token (e.g. + /// it was revoked at the IdP), so the re-mint re-checks grants at UC. No-op + /// when caching is disabled or no entry exists. + pub(crate) async fn invalidate(&self, client_id: &str, client_secret: &str, table_name: &str) { + if !self.enabled { + return; + } + let key = TokenKey::new(client_id, client_secret, table_name); + if self.entries.lock().await.remove(&key).is_some() { + debug!(table = %table_name, "token cache entry invalidated after auth rejection"); + } + } + + fn needs_refresh(&self, cached: &CachedToken) -> bool { + // `checked_add` avoids a panic on an absurd refresh buffer (e.g. + // `Duration::MAX`); an overflowing deadline means "always refresh". + match Instant::now().checked_add(self.refresh_buffer) { + Some(deadline) => deadline >= cached.expires_at, + None => true, + } + } + + /// Drops entries whose token has fully expired. Locked (in-flight) entries, + /// still-valid tokens, and empty slots are kept — keeping empty slots is + /// what preserves single-flight for a key being minted concurrently. + fn prune_expired(entries: &mut HashMap) { + entries.retain(|_, slot| match slot.try_lock() { + Ok(guard) => match guard.as_ref() { + Some(cached) => !cached.is_expired(), + None => true, + }, + Err(_) => true, + }); + } +} + +#[cfg(test)] +mod tests { + use super::*; + use std::sync::atomic::{AtomicUsize, Ordering}; + + fn fetched(token: &str, ttl_secs: Option) -> FetchedToken { + FetchedToken { + token: token.to_string(), + expires_in: ttl_secs.map(Duration::from_secs), + } + } + + #[tokio::test] + async fn caches_token_across_calls() { + let cache = TokenCache::new(true, Duration::from_secs(60)); + let calls = AtomicUsize::new(0); + + let make = |_reason| async { + calls.fetch_add(1, Ordering::SeqCst); + Ok(fetched("tok", Some(3600))) + }; + + let a = cache + .get_or_fetch("id", "secret", "c.s.t", make) + .await + .unwrap(); + let b = cache + .get_or_fetch("id", "secret", "c.s.t", make) + .await + .unwrap(); + + assert_eq!(a, "tok"); + assert_eq!(b, "tok"); + assert_eq!( + calls.load(Ordering::SeqCst), + 1, + "second call should hit cache" + ); + } + + #[tokio::test] + async fn refetches_when_within_refresh_buffer() { + // TTL (1s) is smaller than the refresh buffer (60s), so the token is + // always considered due for refresh and every call mints anew. + let cache = TokenCache::new(true, Duration::from_secs(60)); + let calls = AtomicUsize::new(0); + + let make = |_reason| async { + let n = calls.fetch_add(1, Ordering::SeqCst); + Ok(fetched(&format!("tok{n}"), Some(1))) + }; + + let a = cache + .get_or_fetch("id", "secret", "c.s.t", make) + .await + .unwrap(); + let b = cache + .get_or_fetch("id", "secret", "c.s.t", make) + .await + .unwrap(); + + assert_eq!(a, "tok0"); + assert_eq!(b, "tok1"); + assert_eq!(calls.load(Ordering::SeqCst), 2); + } + + #[tokio::test] + async fn separate_tables_get_separate_entries() { + let cache = TokenCache::new(true, Duration::from_secs(60)); + let calls = AtomicUsize::new(0); + + let make = |_reason| async { + let n = calls.fetch_add(1, Ordering::SeqCst); + Ok(fetched(&format!("tok{n}"), Some(3600))) + }; + + let a = cache + .get_or_fetch("id", "secret", "c.s.t1", make) + .await + .unwrap(); + let b = cache + .get_or_fetch("id", "secret", "c.s.t2", make) + .await + .unwrap(); + + assert_ne!(a, b); + assert_eq!(calls.load(Ordering::SeqCst), 2); + } + + #[tokio::test] + async fn rotated_secret_gets_new_entry() { + let cache = TokenCache::new(true, Duration::from_secs(60)); + let calls = AtomicUsize::new(0); + + let make = |_reason| async { + let n = calls.fetch_add(1, Ordering::SeqCst); + Ok(fetched(&format!("tok{n}"), Some(3600))) + }; + + cache + .get_or_fetch("id", "secret-v1", "c.s.t", make) + .await + .unwrap(); + cache + .get_or_fetch("id", "secret-v2", "c.s.t", make) + .await + .unwrap(); + + assert_eq!(calls.load(Ordering::SeqCst), 2); + } + + #[tokio::test] + async fn token_without_ttl_is_not_cached() { + let cache = TokenCache::new(true, Duration::from_secs(60)); + let calls = AtomicUsize::new(0); + + let make = |_reason| async { + calls.fetch_add(1, Ordering::SeqCst); + Ok(fetched("tok", None)) + }; + + cache + .get_or_fetch("id", "secret", "c.s.t", make) + .await + .unwrap(); + cache + .get_or_fetch("id", "secret", "c.s.t", make) + .await + .unwrap(); + + assert_eq!(calls.load(Ordering::SeqCst), 2, "no TTL means no caching"); + } + + #[tokio::test] + async fn invalidate_forces_remint_on_next_call() { + let cache = TokenCache::new(true, Duration::from_secs(60)); + let calls = AtomicUsize::new(0); + + let make = |_reason| async { + calls.fetch_add(1, Ordering::SeqCst); + Ok(fetched("tok", Some(3600))) + }; + + cache + .get_or_fetch("id", "secret", "c.s.t", make) + .await + .unwrap(); + // Without invalidation a second call would hit the cache; invalidating + // the entry forces the next call to re-mint. + cache.invalidate("id", "secret", "c.s.t").await; + cache + .get_or_fetch("id", "secret", "c.s.t", make) + .await + .unwrap(); + + assert_eq!(calls.load(Ordering::SeqCst), 2); + } + + #[tokio::test] + async fn disabled_cache_always_fetches() { + let cache = TokenCache::new(false, Duration::from_secs(60)); + let calls = AtomicUsize::new(0); + + let make = |_reason| async { + calls.fetch_add(1, Ordering::SeqCst); + Ok(fetched("tok", Some(3600))) + }; + + cache + .get_or_fetch("id", "secret", "c.s.t", make) + .await + .unwrap(); + cache + .get_or_fetch("id", "secret", "c.s.t", make) + .await + .unwrap(); + + assert_eq!(calls.load(Ordering::SeqCst), 2); + } + + #[tokio::test] + async fn fetch_error_leaves_no_cached_entry() { + let cache = TokenCache::new(true, Duration::from_secs(60)); + + let err = cache + .get_or_fetch("id", "secret", "c.s.t", |_reason| async { + Err(crate::ZerobusError::TokenFetchError("boom".to_string())) + }) + .await; + assert!(err.is_err()); + + // A subsequent successful fetch should still succeed and cache. + let ok = cache + .get_or_fetch("id", "secret", "c.s.t", |_reason| async { + Ok(fetched("tok", Some(3600))) + }) + .await + .unwrap(); + assert_eq!(ok, "tok"); + } + + #[tokio::test] + async fn refresh_failure_serves_still_valid_token() { + let cache = TokenCache::new(true, Duration::from_secs(60)); + + // Seed a token that is within the refresh buffer (ttl < buffer) but not + // yet expired, so the next call is due for a refresh. + let seeded = cache + .get_or_fetch("id", "secret", "c.s.t", |_reason| async { + Ok(fetched("valid", Some(30))) + }) + .await + .unwrap(); + assert_eq!(seeded, "valid"); + + // The refresh mint fails; the still-valid cached token is served instead + // of surfacing the error. + let served = cache + .get_or_fetch("id", "secret", "c.s.t", |_reason| async { + Err(crate::ZerobusError::TokenFetchError("blip".to_string())) + }) + .await + .unwrap(); + assert_eq!(served, "valid"); + } + + #[tokio::test] + async fn refresh_failure_propagates_non_retryable_error() { + let cache = TokenCache::new(true, Duration::from_secs(60)); + + // Seed a token that is within the refresh buffer but not yet expired. + cache + .get_or_fetch("id", "secret", "c.s.t", |_reason| async { + Ok(fetched("valid", Some(30))) + }) + .await + .unwrap(); + + // A non-retryable refresh error (e.g. revoked or invalid credentials) + // must surface rather than being masked by the still-valid cached token. + let result = cache + .get_or_fetch("id", "secret", "c.s.t", |_reason| async { + Err(crate::ZerobusError::InvalidUCTokenError( + "revoked".to_string(), + )) + }) + .await; + assert!(matches!( + result, + Err(crate::ZerobusError::InvalidUCTokenError(_)) + )); + } + + #[tokio::test] + async fn no_ttl_response_does_not_evict_valid_token() { + let cache = TokenCache::new(true, Duration::from_secs(60)); + + // Seed a still-valid (within-buffer) token. + cache + .get_or_fetch("id", "secret", "c.s.t", |_reason| async { + Ok(fetched("valid", Some(30))) + }) + .await + .unwrap(); + + // A refresh returns a token with no TTL: the caller gets the fresh token, + // but the cached valid token must not be discarded. + let fresh = cache + .get_or_fetch("id", "secret", "c.s.t", |_reason| async { + Ok(fetched("nottl", None)) + }) + .await + .unwrap(); + assert_eq!(fresh, "nottl"); + + // A later refresh failure still finds the original valid token, proving + // it was retained. + let served = cache + .get_or_fetch("id", "secret", "c.s.t", |_reason| async { + Err(crate::ZerobusError::TokenFetchError("blip".to_string())) + }) + .await + .unwrap(); + assert_eq!(served, "valid"); + } + + #[tokio::test] + async fn single_flight_mints_once_for_concurrent_callers() { + let cache = Arc::new(TokenCache::new(true, Duration::from_secs(60))); + let calls = Arc::new(AtomicUsize::new(0)); + + let mut handles = Vec::new(); + for _ in 0..16 { + let cache = Arc::clone(&cache); + let calls = Arc::clone(&calls); + handles.push(tokio::spawn(async move { + cache + .get_or_fetch("id", "secret", "c.s.t", |_reason| async { + calls.fetch_add(1, Ordering::SeqCst); + // Hold the slot briefly so the other callers pile up + // behind the single-flight lock rather than racing. + tokio::time::sleep(Duration::from_millis(20)).await; + Ok(fetched("tok", Some(3600))) + }) + .await + .unwrap() + })); + } + + for handle in handles { + assert_eq!(handle.await.unwrap(), "tok"); + } + assert_eq!( + calls.load(Ordering::SeqCst), + 1, + "single-flight must mint exactly once for concurrent same-key callers" + ); + } +} diff --git a/lib/zerobus-ffi-1.3.0/rust/sdk/src/zeroparser/README.md b/lib/zerobus-ffi-1.3.0/rust/sdk/src/zeroparser/README.md new file mode 100644 index 00000000000..9c44ef718e8 --- /dev/null +++ b/lib/zerobus-ffi-1.3.0/rust/sdk/src/zeroparser/README.md @@ -0,0 +1,141 @@ +# Zeroparser + +Zero-copy, single-pass protobuf parser driven by a `DescriptorProto`. Parses +nested messages in one O(N) traversal; all string and byte values borrow from +the input buffer. + +Ships as part of the [Zerobus SDK](https://github.com/databricks/zerobus-sdk) +behind the optional `zeroparser` feature flag — there is no standalone +`zeroparser` crate on crates.io. + +## Why + +`prost-reflect`'s `DynamicMessage` is convenient when you have a schema only at +runtime, but each decode allocates a tree of owned values. Zeroparser keeps the +same "schema known only at runtime" property while avoiding those allocations: +fields are stored in two pre-sized arrays indexed via a per-descriptor field +cache, and `&str`/`&[u8]` values point straight into the input. + +## Benchmark + +`cargo bench --features zeroparser --bench zeroparser_bench_plot` writes +`bench_plot.svg`: + +![Decode throughput](benches/bench_plot.svg) + +Five decoders on the same bytes, each parsing and walking every field once +(the streaming-ingestion case — no field skipping). Three carry a +runtime-only schema (`prost-reflect`, C++ Reflection, Zeroparser); two are +compile-time-typed (`prost`, C++ generated accessors). + +| Schema | Record size | prost-reflect | prost | C++ reflect | C++ typed | Zeroparser | +| ---------------------- | ----------- | ------------- | ---------- | ----------- | ---------- | ----------- | +| AirQuality | 32 B | ~214 MB/s | ~954 MB/s | ~593 MB/s | ~955 MB/s | ~1010 MB/s | +| AirQuality | ~200 B | ~1283 MB/s | ~4942 MB/s | ~3300 MB/s | ~4988 MB/s | ~5366 MB/s | +| AirQuality | 1 KB | ~5438 MB/s | ~13491 MB/s| ~9519 MB/s | ~11682 MB/s| ~18686 MB/s | +| SupportedNullableTypes | 1 KB | ~763 MB/s | ~1618 MB/s | ~1813 MB/s | ~2048 MB/s | ~2057 MB/s | +| WideSchema | 1 KB | ~176 MB/s | ~624 MB/s | ~608 MB/s | ~1042 MB/s | ~1621 MB/s | + +Vs runtime-schema peers, Zeroparser is 4–9x faster than `prost-reflect` and +1.1–2.8x faster than C++ Reflection. Vs compile-time-typed peers +(`prost`, C++ typed), it keeps pace on small messages and pulls ahead on +larger/wider ones — despite carrying a runtime descriptor that they don't. +The advantage widens with field count: the WideSchema row (100 device-telemetry +fields) makes per-message overhead the bottleneck for reflection-based decoders +— `prost-reflect` drops to ~176 MB/s while Zeroparser holds ~1.6 GB/s (~9x). It +is also the one schema where Zeroparser outruns even C++'s generated accessors +(~1621 vs ~1042 MB/s), because its pre-sized field cache and zero-copy +`&str`/`&[u8]` layout amortize best when there are many fields to touch. + +Each Rust measurement is averaged over 3 trials in-bench; C++ values come from +an out-of-tree harness against libprotobuf 32.1 — AirQuality and +SupportedNullableTypes are the mean of 6 runs, WideSchema the mean of 3 (see +`benches/README.md`). + +Measured on: Apple M4 Max (16-core, arm64), 64 GB RAM, macOS 26.4.1, rustc +1.90.0, libprotobuf 32.1, clang++ from Apple Xcode. + +## Quick start + +Enable the feature in your `Cargo.toml`: + +```toml +[dependencies] +databricks-zerobus-ingest-sdk = { version = "...", features = ["zeroparser"] } +``` + +Then: + +```rust +use prost_types::DescriptorProto; +use databricks_zerobus_ingest_sdk::zeroparser::{ + parser::ParsedMessage, types::FieldValueRef, MessageRegistry, ParseResult, +}; + +# fn run(descriptor: DescriptorProto, bytes: &[u8]) -> ParseResult<()> { +let registry = MessageRegistry::from_descriptor(&descriptor); +let parsed = ParsedMessage::parse(bytes, ®istry)?; + +if let Some(FieldValueRef::String(s)) = parsed.get_scalar(1) { + println!("field 1 = {s}"); +} +# Ok(()) } +``` + +### API + +| Method | Returns | +| ---------------------------------- | --------------------------------------------------- | +| `has_field(field_num)` | `bool` | +| `get_scalar(field_num)` | `Option<&FieldValueRef>` | +| `get_message(field_num)` | `Option<&ParsedMessage>` | +| `get_repeated_scalars(field_num)` | `&[FieldValueRef]` | +| `get_repeated_messages(field_num)` | `&[ParsedMessage]` | +| `get_map_entries(field_num)` | `impl Iterator` | + +## Test and bench commands + +From `rust/sdk/`: + +``` +cargo test --features zeroparser --lib # unit tests inline in src/zeroparser/*.rs +cargo test --features zeroparser --test zeroparser_e2e # integration tests in src/zeroparser/tests/e2e.rs +cargo test --features zeroparser # everything: lib + integration + doc tests +cargo bench --features zeroparser --bench zeroparser_parser_bench # full criterion sweep +cargo bench --features zeroparser --bench zeroparser_bench_plot # produces bench_plot.svg +``` + +Or from anywhere in the workspace, `cargo test --workspace` covers it — the +`rust/tests` member enables the `zeroparser` feature on the SDK so the moved +tests run as part of normal CI. + +Requires `protoc` on `PATH` for the build script (used to compile the test and +bench `.proto` files). On Debian/Ubuntu: `apt install protobuf-compiler`. + +## Layout + +``` +rust/sdk/src/zeroparser/ + mod.rs — module entry: doc, mod decls, re-exports + wire.rs — wire format, varint parsing, field decoding + registry.rs — MessageRegistry, field descriptor caching + parser.rs — single-pass recursive parser + types.rs — FieldValueRef, ComplexType, conversions + errors.rs — ParseError, ParseResult + sparse_field_map.rs — O(1) per-descriptor field lookup + owned.rs — owning wrapper + tests/ — integration tests (wired via [[test]] in sdk/Cargo.toml) + e2e.rs + common/mod.rs + proto/ — proto2/proto3 fixtures compiled by sdk/build.rs + benches/ — criterion sweep + MB/s bar plot + parser_bench.rs — full criterion sweep + bench_plot.rs — produces bench_plot.svg + bench_plot.svg — committed throughput plot (regenerated by bench_plot.rs) + common/mod.rs + proto/ — AirQuality, SupportedNullableTypes, WideSchema +``` + +The proto fixtures are compiled in `rust/sdk/build.rs` only when the +`zeroparser` feature is enabled (`CARGO_FEATURE_ZEROPARSER` env check), so +consumers without the feature don't pay for the proto build step. diff --git a/lib/zerobus-ffi-1.3.0/rust/sdk/src/zeroparser/benches/README.md b/lib/zerobus-ffi-1.3.0/rust/sdk/src/zeroparser/benches/README.md new file mode 100644 index 00000000000..7b0f6969d3a --- /dev/null +++ b/lib/zerobus-ffi-1.3.0/rust/sdk/src/zeroparser/benches/README.md @@ -0,0 +1,45 @@ +## Reproducing C++ numbers + +The plot's C++ bars are pinned constants in `bench_plot.rs` (`cpp_baselines()`), +measured by an out-of-tree harness so this crate keeps no C++ source or build +dependency. To re-measure: + +1. Dump the exact encoded bytes the Rust benches see: + ```bash + ZEROPARSER_CPP_DUMP_DIR=/tmp/zeroparser-cpp-bench/data \ + cargo bench --features zeroparser --bench zeroparser_bench_plot + ``` + This writes one `