server: the checkpoint budget's eviction gives up what a saved page gives up first, never oldest-first

.github/workflows/server.yml

@@ -166,7 +166,13 @@ jobs:
             -DLLAMA_BUILD_BORINGSSL=ON ^
             -DGGML_SCHED_NO_REALLOC=ON
           set /A NINJA_JOBS=%NUMBER_OF_PROCESSORS%-1
-          cmake --build build --config Release -j %NINJA_JOBS% --target llama-server
+          cmake --build build --config Release -j %NINJA_JOBS% --target llama-server test-server-pages
+
+      - name: Page publication tests
+        id: server_page_tests
+        shell: cmd
+        run: |
+          ctest --test-dir build -C Release -R test-server-pages --output-on-failure
 
       - name: Python setup
         id: setup_python

common/arg.cpp

@@ -330,6 +330,18 @@ static std::string get_all_kv_cache_types() {
     return msg.str();
 }
 
+// an exact byte count: the caller passes a measured number, so no unit suffixes are guessed
+static uint64_t parse_byte_count(const std::string & value, const char * flag) {
+    if (value.empty() || value.size() > 20 || value.find_first_not_of("0123456789") != std::string::npos) {
+        throw std::invalid_argument(std::string(flag) + " takes an unsigned byte count, got: " + value);
+    }
+    try {
+        return std::stoull(value);
+    } catch (const std::exception &) {
+        throw std::invalid_argument(std::string(flag) + " is out of range: " + value);
+    }
+}
+
 static bool parse_bool_value(const std::string & value) {
     if (is_truthy(value)) {
         return true;
@@ -1691,6 +1703,23 @@ common_params_context common_params_parser_init(common_params & params, llama_ex
             params.n_ctx_checkpoints = value;
         }
     ).set_env("LLAMA_ARG_CTX_CHECKPOINTS").set_examples({LLAMA_EXAMPLE_SERVER, LLAMA_EXAMPLE_CLI}));
+    add_opt(common_arg(
+        {"--ctx-checkpoints-bytes"}, "BYTES",
+        "byte budget for the context checkpoints of all slots together, in place of the per-slot count "
+        "(default: 0 = use --ctx-checkpoints); when a new checkpoint would exceed it, the slot holding the "
+        "most checkpoint bytes drops its oldest checkpoint after its first, so each slot keeps its head and its newest",
+        [](common_params & params, const std::string & value) {
+            params.ctx_checkpoints_bytes = parse_byte_count(value, "--ctx-checkpoints-bytes");
+        }
+    ).set_env("LLAMA_ARG_CTX_CHECKPOINTS_BYTES").set_examples({LLAMA_EXAMPLE_SERVER}));
+    add_opt(common_arg(
+        {"--page-ram-bytes"}, "BYTES",
+        "ceiling of the host-RAM page store that /slots page_out fills (default: 0 = no ceiling); "
+        "a page_out that would exceed it is refused, so the caller spills a page to disk first",
+        [](common_params & params, const std::string & value) {
+            params.page_ram_bytes = parse_byte_count(value, "--page-ram-bytes");
+        }
+    ).set_env("LLAMA_ARG_PAGE_RAM_BYTES").set_examples({LLAMA_EXAMPLE_SERVER}));
     add_opt(common_arg(
         {"-cms", "--checkpoint-min-step"}, "N",
         string_format("minimum spacing between context checkpoints in tokens (default: %d, 0 = no minimum)", params.checkpoint_min_step),

common/common.h

@@ -621,6 +621,8 @@ struct common_params {
     bool    cache_idle_slots    = true;  // save and clear idle slots upon starting a new task
     int32_t n_ctx_checkpoints   = 32;    // max number of context checkpoints per slot
     int32_t checkpoint_min_step = 8192;  // minimum spacing between context checkpoints
+    uint64_t ctx_checkpoints_bytes = 0;  // checkpoint byte budget across all slots; 0 = the count above governs
+    uint64_t page_ram_bytes        = 0;  // ceiling of the named host-RAM page store; 0 = no ceiling
     int32_t cache_ram_mib       = 8192;  // -1 = no limit, 0 - disable, 1 = 1 MiB, etc.
 
     std::string hostname      = "127.0.0.1";

src/llama-context.cpp

@@ -3995,6 +3995,36 @@ size_t llama_state_seq_save_file(llama_context * ctx, const char * filepath, lla
     }
 }
 
+size_t llama_state_seq_save_file_from_data(const char * filepath, const uint8_t * data, size_t size, const llama_token * tokens, size_t n_token_count) {
+    // get_data writes [io_magic][seq_id][state]; the file is [magic][version][count][tokens][state]
+    const size_t prefix = sizeof(io_magic) + sizeof(llama_seq_id);
+    if (data == nullptr || size <= prefix) {
+        LLAMA_LOG_ERROR("%s: no sequence state to write\n", __func__);
+        return 0;
+    }
+    uint32_t magic_read = 0;
+    std::memcpy(&magic_read, data, sizeof(magic_read));
+    if (magic_read != io_magic) {
+        LLAMA_LOG_ERROR("%s: the bytes are not a sequence state (magic %08x)\n", __func__, magic_read);
+        return 0;
+    }
+    try {
+        llama_file file(filepath, "wb");
+        const uint32_t magic   = LLAMA_STATE_SEQ_MAGIC;
+        const uint32_t version = LLAMA_STATE_SEQ_VERSION;
+        const uint32_t count   = (uint32_t) n_token_count;
+        file.write_raw(&magic,   sizeof(magic));
+        file.write_raw(&version, sizeof(version));
+        file.write_raw(&count,   sizeof(count));
+        file.write_raw(tokens,   sizeof(llama_token) * n_token_count);
+        file.write_raw(data + prefix, size - prefix);
+        return file.tell();
+    } catch (const std::exception & err) {
+        LLAMA_LOG_ERROR("%s: error writing sequence state file: %s\n", __func__, err.what());
+        return 0;
+    }
+}
+
 size_t llama_state_seq_load_file(llama_context * ctx, const char * filepath, llama_seq_id dest_seq_id, llama_token * tokens_out, size_t n_token_capacity, size_t * n_token_count_out) {
     return llama_state_seq_load_file_with_progress(ctx, filepath, dest_seq_id, tokens_out, n_token_capacity, n_token_count_out, nullptr, nullptr);
 }

src/llama-ext.h

@@ -130,3 +130,19 @@ LLAMA_API uint32_t        llama_model_target_layer_ids_n(const struct llama_mode
 // if out is nullptr, returns the number of tokens without writing to out
 // caller must allocate enough memory for out before calling
 LLAMA_API uint32_t llama_model_get_tok_embd(const struct llama_model * model, float * out);
+
+//
+// sequence state pages (Continuum fork)
+//
+
+// Writes a sequence-state file (the llama_state_seq_save_file format) from bytes that
+// llama_state_seq_get_data_ext returned with LLAMA_STATE_SEQ_FLAGS_NONE, without a context:
+// a page held in host RAM spills to disk off the engine thread, and llama_state_seq_load_file
+// reads it back. Returns the bytes written, or 0 when the bytes are not a sequence state or
+// the write fails.
+LLAMA_API size_t llama_state_seq_save_file_from_data(
+        const char        * filepath,
+        const uint8_t     * data,
+        size_t              size,
+        const llama_token * tokens,
+        size_t              n_token_count);

tests/CMakeLists.txt

@@ -156,6 +156,13 @@ llama_build_and_test(test-grad-leaf.cpp)
 # ggml-opt with per-step graphs (the llama training path) starts each optimizer period from zero accumulators
 llama_build_and_test(test-opt-dynamic-accum.cpp)
 
+# Continuum fork: the RAM page store's ceiling and the page files' whole-generation publication.
+# Built on every platform: it links only server-context, as llama-server does, and Windows is
+# where its durable-move adapter differs most.
+llama_build_and_test(test-server-pages.cpp)
+target_include_directories(test-server-pages PRIVATE ${PROJECT_SOURCE_DIR}/tools/server)
+target_link_libraries(test-server-pages PRIVATE server-context)
+
 if (NOT WIN32 OR NOT BUILD_SHARED_LIBS)
     # these tests are disabled on Windows because they use internal functions not exported with LLAMA_API (when building with shared libraries)
     llama_build_and_test(test-unicode.cpp)
@@ -172,6 +179,9 @@ if (NOT WIN32 OR NOT BUILD_SHARED_LIBS)
     llama_build_and_test(test-chat.cpp WORKING_DIRECTORY ${PROJECT_SOURCE_DIR})
     target_include_directories(test-chat PRIVATE ${PROJECT_SOURCE_DIR}/tools/server)
     target_link_libraries(test-chat PRIVATE server-context)
+    llama_build_and_test(test-page-checkpoints.cpp)
+    target_include_directories(test-page-checkpoints PRIVATE ${PROJECT_SOURCE_DIR}/tools/server)
+    target_link_libraries(test-page-checkpoints PRIVATE server-context)
     # TODO: disabled on loongarch64 because the ggml-ci node lacks Python 3.8
     if (NOT ${CMAKE_SYSTEM_PROCESSOR} MATCHES "loongarch64")
         llama_build_and_test(test-json-schema-to-grammar.cpp   WORKING_DIRECTORY ${PROJECT_SOURCE_DIR})

tests/test-page-checkpoints.cpp

@@ -0,0 +1,114 @@
+// What a saved page keeps of a slot's context checkpoints (Continuum fork, select_page_checkpoints).
+//
+// what this catches: the M5 on 2026-10-25 (a hybrid model, 4 residents paging over 3 lanes) re-
+// prefilled 20k+ tokens a turn and once fell to "full re-prefill: no checkpoint passed the search
+// (raw lcp 9710)": a page kept the oldest checkpoint and the newest four, all near the end of a
+// 35-40k prompt, so a restored slot had nothing under the point where its next prompt diverged
+// (about 11k). The page must keep, within the same budget, checkpoints just under the slot's last
+// raw common prefix.
+//
+// The byte budget's eviction (server_checkpoint_victim, server_checkpoints_fit) dropped each
+// slot's oldest checkpoint after its first: exactly the ones under the last divergence. It now
+// gives up what the page subset gives up first, so a slot keeps its head, the newest at or below
+// its last raw common prefix and its newest longest.
+
+#include "server-checkpoints.h"
+#include "server-common.h"
+
+#include <cstdio>
+#include <cstdlib>
+#include <vector>
+
+#define CHECK(cond) do { if (!(cond)) { std::fprintf(stderr, "%s:%d: CHECK failed: %s\n", __FILE__, __LINE__, #cond); std::exit(1); } } while (0)
+
+int main() {
+    // A 35-40k prompt's checkpoints at user-message starts, about 1,576 apart.
+    std::vector<int64_t> pos;
+    for (int64_t p = 300; p < 38000; p += 1576) {
+        pos.push_back(p);
+    }
+    const size_t n = pos.size();
+
+    // keep < 0: every checkpoint, as before the subset existed.
+    CHECK(select_page_checkpoints(pos, -1, 11000).size() == n);
+
+    // No divergence known (a fresh slot, no cache_prompt): the oldest and the newest four, as today.
+    {
+        auto k = select_page_checkpoints(pos, 4, -1);
+        CHECK(k.size() == 5);
+        CHECK(k[0] == 0);
+        for (size_t j = 1; j < 5; ++j) {
+            CHECK(k[j] == n - 5 + j);
+        }
+    }
+
+    // Divergence at 11,000: the oldest, the two newest at or below 11,000, the newest two.
+    {
+        auto k = select_page_checkpoints(pos, 4, 11000);
+        CHECK(k.size() == 5);
+        CHECK(k[0] == 0);
+        CHECK(pos[k[1]] <= 11000 && pos[k[2]] <= 11000);
+        CHECK(pos[k[2]] + 1576 > 11000);              // the nearest under the break is kept
+        CHECK(k[3] == n - 2 && k[4] == n - 1);         // and the end of the prompt
+    }
+
+    // The budget never grows: at most 1 + keep, whatever the divergence.
+    for (int32_t keep = 0; keep <= 6; ++keep) {
+        CHECK(select_page_checkpoints(pos, keep, 11000).size() <= (size_t) keep + 1);
+    }
+
+    // A divergence under every checkpoint but the first keeps no extra "below" points.
+    {
+        auto k = select_page_checkpoints(pos, 4, 100);
+        CHECK(k.size() == 5 && k[0] == 0 && k[1] == n - 4);
+    }
+
+    // keep 0: the oldest only. An empty list: nothing.
+    CHECK(select_page_checkpoints(pos, 0, 11000).size() == 1);
+    CHECK(select_page_checkpoints({}, 4, 11000).empty());
+
+    // Eviction gives up what the subset gives up first: never the head, never the two newest at
+    // or below the divergence, never the newest; with no divergence known, the oldest after the
+    // head, as before.
+    {
+        auto list_of = [&](size_t bytes) {
+            server_checkpoint_list list;
+            for (int64_t p : pos) {
+                auto & c = list.emplace_back();
+                c.n_tokens = p;
+                c.data_tgt.resize(bytes);
+            }
+            return list;
+        };
+        const auto list = list_of(1);
+        const size_t victim = server_checkpoint_victim(list, 11000);
+        const auto keep = select_page_checkpoints(pos, (int32_t) n - 2, 11000);
+        CHECK(keep.size() == n - 1);
+        for (size_t i : keep) {
+            CHECK(i != victim);
+        }
+        CHECK(victim != 0 && victim != n - 1);
+        CHECK(!(pos[victim] <= 11000 && pos[victim] + 2 * 1576 > 11000)); // not one of the two under the break
+        CHECK(server_checkpoint_victim(list, -1) == 1);                   // unknown divergence: oldest after the head
+
+        // The budget across slots: the fuller list drops its victims until the new one fits, and
+        // the checkpoints just under its divergence survive.
+        auto a = list_of(10);
+        auto b = list_of(1);
+        std::vector<server_checkpoint_list *> lists = { &a, &b };
+        const uint64_t budget = server_checkpoints_bytes(a) + server_checkpoints_bytes(b) - 25;
+        size_t evicted = 0;
+        CHECK(server_checkpoints_fit(lists, { 11000, -1 }, 1, budget, 10, [&](size_t i, const common_prompt_checkpoint &) { CHECK(i == 0); ++evicted; }));
+        CHECK(evicted == 4);
+        CHECK(a.front().n_tokens == pos[0] && a.back().n_tokens == pos[n - 1]);
+        size_t under = 0;
+        for (const auto & c : a) {
+            under += (c.n_tokens <= 11000 && c.n_tokens + 2 * 1576 > 11000) ? 1 : 0;
+        }
+        CHECK(under == 2);
+        CHECK(!server_checkpoints_fit(lists, { 11000, -1 }, 1, 5, 10, [&](size_t, const common_prompt_checkpoint &) { CHECK(false); }));
+    }
+
+    std::printf("test-page-checkpoints: ok\n");
+    return 0;
+}