From 0d245d2b0618fc30364b50accdb0b072a8dc63eb Mon Sep 17 00:00:00 2001 From: bdchatham Date: Tue, 18 Aug 2026 12:38:04 -0700 Subject: [PATCH 01/24] config/seitoml: read, edit and write a node's sei.toml The file holds only what an operator decided, and editing it preserves the document, so the comments they wrote to explain a choice survive a later set or unset. Every save is atomic, since a node cannot boot from a file a crash truncated mid-write. Three top-level keys describe the file rather than configure the node, and Values leaves all three out so a check comparing written keys against a declared set never reports them as keys no section owns. Four defects found while covering the value decoder, each verified by reverting the fix and watching the covering test fail: - a multi-line basic string could not be decoded at all. Unquote reads Go's single-line syntax, so the literal newlines that make the form multi-line have to be escaped before it sees them. Left raw, a value an operator wrote was refused for having more than one line in it. - an integral float was written as "1", which reads back as an integer. TOML tells a float from an integer by the fractional part, so a key declared as a float resolved as one type from a node's own files and another from its sei.toml, and which of the two an operator got depended on the value they chose: 0.5 survived and 1.0 did not. - an infinity or a NaN could be read but not written. TOML spells them as words and ParseFloat accepts them, so a file could hold one, and any later edit of any other key failed on a value this package had handed back. Both directions now refuse. - writing back a list read from a file failed, because reading an array produces a list of any and only a list of string could be written. Narrower integer widths are refused rather than rendered. The cases are the widths configuration structs in this tree declare, so int8 through uint16 and float32 are a named refusal until a field needs one. Coverage is 94.2% of statements. The sixteen uncovered statements are all error propagations from operations that cannot fail on valid input: a mapping insert the caller already proved absent, a document render, six syscalls on a temporary file just created, and three decodings the parser already validated. Reaching the syscalls means an injectable filesystem, and the atomicity those lines serve is held by three tests that drive it end to end. Co-Authored-By: Claude Opus 5 (1M context) --- config/seitoml/doc.go | 48 ++ config/seitoml/edit.go | 193 +++++ config/seitoml/file.go | 274 +++++++ config/seitoml/guards_test.go | 116 +++ config/seitoml/seitoml_test.go | 1225 ++++++++++++++++++++++++++++++++ config/seitoml/values.go | 211 ++++++ 6 files changed, 2067 insertions(+) create mode 100644 config/seitoml/doc.go create mode 100644 config/seitoml/edit.go create mode 100644 config/seitoml/file.go create mode 100644 config/seitoml/guards_test.go create mode 100644 config/seitoml/seitoml_test.go create mode 100644 config/seitoml/values.go diff --git a/config/seitoml/doc.go b/config/seitoml/doc.go new file mode 100644 index 0000000000..7ab2418b14 --- /dev/null +++ b/config/seitoml/doc.go @@ -0,0 +1,48 @@ +// Package seitoml reads, edits and writes the node's sei.toml. +// +// The file holds only what an operator decided. A key present in it is authoritative; a key absent +// from it resolves to the running binary's baseline for the node's mode. Nothing here writes a +// baseline into the file, because a value the binary put there reads exactly like one an operator +// chose. +// +// Three keys at the top level describe the file rather than configure the node, and Values leaves +// all three out so a check comparing written keys against the declared set never reports them as +// keys no section owns. +// +// schema_version which migration the file has reached +// node_mode which mode's baselines its values were chosen against +// generated_by which release last produced or transformed it +// +// The first two are machinery and the third is not, and the difference matters enough to state. +// +// schema_version is a counter that rises by exactly one per migration, and a migration chain reads it +// to decide which steps a file still needs. It is deliberately not a release version. Most releases +// change no schema, so a release version could not answer whether the schema moved between two of +// them without a release-to-schema table, which is this counter reintroduced as an indirection. +// Releases also do not form the total order a chain needs: a hotfix can ship after a later minor, so +// ordering steps by release would run them in an order nobody intended. +// +// Nothing here migrates a file. This package reads the counter and writes it; the chain that acts on it +// arrives with the migrations themselves. +// +// generated_by is provenance. Nothing reads it to decide anything, which is what lets it be absent +// without consequence: the release reaches the binary through a linker flag the release build sets, +// so a binary built any other way knows none and a file it writes simply omits the key. Anything +// branching on it would turn every development build into a node that cannot read its own +// configuration, and a test drives every reader over a file recording a release, no release, and a +// release no build ever was, requiring identical answers. +// +// Editing preserves the document. An operator may hand-edit the file, and comments are how they +// explain a choice to whoever reads it next, so set and unset change the one line they name and +// leave the rest byte for byte. That is why this package edits a parsed document rather than +// re-rendering a decoded map. +// +// Every write is atomic. A node cannot boot from a configuration file a crash truncated mid-write, +// so a save lands in full or not at all. +// +// A value has to survive a round trip as its own type. TOML tells a float from an integer by the +// fractional part, so an integral float needs one written explicitly or it reads back as an integer, +// and a key declared as a float would resolve as one type from a node's own files and another from its +// sei.toml. Infinities and NaN have no form here at all, and are refused in both directions rather than +// written as a line no reader can load. +package seitoml diff --git a/config/seitoml/edit.go b/config/seitoml/edit.go new file mode 100644 index 0000000000..f8c71ab2c5 --- /dev/null +++ b/config/seitoml/edit.go @@ -0,0 +1,193 @@ +package seitoml + +import ( + "fmt" + "math" + "strconv" + "strings" + "time" + + "github.com/creachadair/tomledit" + "github.com/creachadair/tomledit/parser" + "github.com/creachadair/tomledit/transform" +) + +// Set writes one key's value, replacing it in place when the key is already present. +// +// Replacing the value on the existing line preserves the comment an operator wrote above or beside +// the key. Rewriting the file from a decoded map drops every comment in it, leaving the operator no +// way to recover the reasoning they recorded. +func (f *File) Set(key string, v any) error { + path, err := keyOf(key) + if err != nil { + return err + } + value, err := tomlValue(v) + if err != nil { + return fmt.Errorf("%s: %w", key, err) + } + + if e := f.doc.First(path...); e != nil && e.KeyValue != nil { + e.Value = value + return nil + } + return f.insert(path, value) +} + +// insert adds a key the document does not have yet. +// +// A key with no dots belongs at the top level. Otherwise it goes in the table its prefix names, +// which is created when it is absent so writing the first key of a section works without the +// operator having to add the heading by hand. +func (f *File) insert(path parser.Key, value parser.Value) error { + leaf := parser.Key{path[len(path)-1]} + kv := &parser.KeyValue{Name: leaf, Value: value} + + if len(path) == 1 { + return f.insertGlobal(kv) + } + + table := path[:len(path)-1] + if e := transform.FindTable(f.doc, table...); e != nil { + if !transform.InsertMapping(e.Section, kv, true) { + return fmt.Errorf("could not write %s into the existing [%s] table", leaf, table.String()) + } + return nil + } + f.doc.Sections = append(f.doc.Sections, &tomledit.Section{ + Heading: &parser.Heading{Name: table}, + Items: []parser.Item{kv}, + }) + return nil +} + +// insertGlobal adds a top-level key, creating the global section when the document has none. +func (f *File) insertGlobal(kv *parser.KeyValue) error { + if f.doc.Global == nil { + f.doc.Global = &tomledit.Section{} + } + if !transform.InsertMapping(f.doc.Global, kv, true) { + return fmt.Errorf("could not write %s at the top level", kv.Name.String()) + } + return nil +} + +// SetPreamble puts a comment block at the top of the document, above everything else. +// +// Comments rather than keys, so nothing a reader needs in order to understand the file becomes +// configuration the node has to recognize. This replaces any block it put there before, so +// regenerating does not stack one preamble on the last. +func (f *File) SetPreamble(lines []string) { + if f.doc.Global == nil { + f.doc.Global = &tomledit.Section{} + } + items := f.doc.Global.Items + if len(items) > 0 { + if _, leading := items[0].(parser.Comments); leading { + items = items[1:] + } + } + if len(lines) == 0 { + f.doc.Global.Items = items + return + } + f.doc.Global.Items = append([]parser.Item{parser.Comments(lines)}, items...) +} + +// Unset removes a key and reports whether the file carried one. +// +// This removes the key rather than writing a zero, because an absent key resolves to the running +// binary's baseline. A key set to its baseline value looks identical in the file but is a commitment +// that survives a release changing that baseline, which is the opposite of what unset means. +func (f *File) Unset(key string) (bool, error) { + path, err := keyOf(key) + if err != nil { + return false, err + } + e := f.doc.First(path...) + if e == nil || e.KeyValue == nil { + return false, nil + } + return e.Remove(), nil +} + +// tomlValue renders a Go value as the TOML literal that parses back to it. +// +// One case per type rather than a general formatter, so an unsupported type errors here instead of +// becoming a plausible-looking line in an operator's file. The cases are the widths configuration +// structs in this tree actually declare, which is why a narrower integer is a named refusal rather +// than a case: adding one is what you do when a field needs it. +// +// A duration goes in as its string form, since a bare number of nanoseconds is unreadable and reads +// back as an integer. +func tomlValue(v any) (parser.Value, error) { + switch x := v.(type) { + case bool: + return parser.ParseValue(strconv.FormatBool(x)) + case string: + return parser.ParseValue(strconv.Quote(x)) + case time.Duration: + return parser.ParseValue(strconv.Quote(x.String())) + case int: + return parser.ParseValue(quoteInt(int64(x))) + case int32: + return parser.ParseValue(quoteInt(int64(x))) + case int64: + return parser.ParseValue(quoteInt(x)) + case uint: + return parser.ParseValue(strconv.FormatUint(uint64(x), 10)) + case uint32: + return parser.ParseValue(strconv.FormatUint(uint64(x), 10)) + case uint64: + return parser.ParseValue(strconv.FormatUint(x, 10)) + case float64: + return floatValue(x) + case []string: + return parser.ParseValue("[" + strings.Join(quoteEach(x), ", ") + "]") + case []any: + // The shape reading an array back produces. Without this, anything that reads a list and writes + // it again fails on a value this package handed it. An element this cannot render still fails, so + // the asymmetry closes without the writer accepting more than the reader can produce. + rendered := make([]string, 0, len(x)) + for i, item := range x { + element, err := tomlValue(item) + if err != nil { + return parser.Value{}, fmt.Errorf("element %d: %w", i, err) + } + rendered = append(rendered, element.String()) + } + return parser.ParseValue("[" + strings.Join(rendered, ", ") + "]") + default: + return parser.Value{}, fmt.Errorf("cannot write a %T to a configuration file", v) + } +} + +// floatValue renders a float as a TOML float, which an integral one is not by default. +// +// TOML tells a float from an integer by the fractional part or the exponent, and the shortest form of +// 1.0 is "1", which reads back as an integer. A key declared as a float would then resolve as one type +// from a node's own files and as another from its sei.toml, and which of the two an operator gets +// depends on the value they chose: 0.5 survives and 1.0 does not. +// +// Infinities and NaN are refused, because this file format has no form for either. The alternative is +// a line no reader can load, written into an operator's file with nothing said. +func floatValue(x float64) (parser.Value, error) { + if math.IsInf(x, 0) || math.IsNaN(x) { + return parser.Value{}, fmt.Errorf("%v cannot be written to a configuration file, which holds "+ + "finite numbers", x) + } + text := strconv.FormatFloat(x, 'g', -1, 64) + if !strings.ContainsAny(text, ".eE") { + text += ".0" + } + return parser.ParseValue(text) +} + +// quoteEach quotes every element of a string list. +func quoteEach(ss []string) []string { + out := make([]string, len(ss)) + for i, s := range ss { + out[i] = strconv.Quote(s) + } + return out +} diff --git a/config/seitoml/file.go b/config/seitoml/file.go new file mode 100644 index 0000000000..fab7b5139c --- /dev/null +++ b/config/seitoml/file.go @@ -0,0 +1,274 @@ +package seitoml + +import ( + "bytes" + "fmt" + "io" + "os" + "path/filepath" + "strconv" + "strings" + + "github.com/creachadair/tomledit" + "github.com/creachadair/tomledit/parser" +) + +// SchemaVersion is the schema this binary writes and reads. +const SchemaVersion = 1 + +// VersionKey records which schema the file follows. +// +// At the document's top level rather than inside a table, so reading it never depends on knowing +// the shape of the file it describes. +const VersionKey = "schema_version" + +// ModeKey records which node mode the file's values resolve for. +// +// At the top level beside VersionKey, not inside a section, because the mode selects which baselines +// apply and so cannot itself have a per-mode baseline. It is also the only durable record of an +// archive node: seid init writes config.toml's mode as "full" for one, since Tendermint has no +// archive mode, so nothing else on disk distinguishes the two. +const ModeKey = "node_mode" + +// GeneratedByKey records the release that last produced or transformed the file. +// +// Provenance, never machinery. Nothing reads it to decide anything, which is what lets it be absent +// without consequence: a binary built outside the release process carries no version, and a file +// written by one simply omits the key. Anything branching on it would turn every development build +// into a node that cannot read its own configuration. +const GeneratedByKey = "generated_by" + +// newFileMode is the permission a file created here gets. +// +// Narrow rather than the usual 0644, because a configuration may name a private endpoint or an +// authentication token. An existing file keeps the mode it already has, since widening one an +// operator deliberately narrowed is worse than a default nobody wanted. +const newFileMode os.FileMode = 0o600 + +// File is a parsed sei.toml that survives editing with its comments and layout intact. +type File struct { + doc *tomledit.Document +} + +// Parse reads a document from r. +func Parse(r io.Reader) (*File, error) { + doc, err := tomledit.Parse(r) + if err != nil { + return nil, fmt.Errorf("parse sei.toml: %w", err) + } + return &File{doc: doc}, nil +} + +// Load reads the document at path. +func Load(path string) (*File, error) { + raw, err := os.ReadFile(path) //nolint:gosec // the caller's configured path is the subject + if err != nil { + return nil, err + } + f, err := Parse(bytes.NewReader(raw)) + if err != nil { + return nil, fmt.Errorf("%s: %w", path, err) + } + return f, nil +} + +// New returns an empty document carrying this binary's schema version, the given node mode, and the +// release that produced it. +// +// The mode is required rather than optional. Every value a caller goes on to write resolves for one +// mode, and a file that does not say which cannot be compared against a binary's defaults or checked +// against the mode the node actually runs. +// +// generatedBy is recorded when the caller has one and omitted when it is empty, because a binary built +// outside the release process knows no version and an empty string says less than no key at all. +func New(mode, generatedBy string) (*File, error) { + if mode == "" { + return nil, fmt.Errorf("a sei.toml needs a node mode: every value in it resolves for one, and " + + "a file that omits it cannot be compared against this binary's defaults") + } + f := &File{doc: &tomledit.Document{Global: &tomledit.Section{}}} + if err := f.setVersion(SchemaVersion); err != nil { + return nil, err + } + if err := f.Set(ModeKey, mode); err != nil { + return nil, err + } + if err := f.SetGeneratedBy(generatedBy); err != nil { + return nil, err + } + return f, nil +} + +// SetGeneratedBy records the release producing the file, and removes the key when given nothing. +func (f *File) SetGeneratedBy(release string) error { + if release == "" { + _, err := f.Unset(GeneratedByKey) + return err + } + return f.Set(GeneratedByKey, release) +} + +// GeneratedBy returns the release the file records, and whether it records one at all. +// +// No error for an absent key. Absence is ordinary, and a caller forced to handle it as a failure +// would be a caller whose behaviour depends on the field. +func (f *File) GeneratedBy() (string, bool) { + e := f.doc.First(GeneratedByKey) + if e == nil || e.KeyValue == nil { + return "", false + } + v, err := goValue(e.Value) + if err != nil { + return "", false + } + release, ok := v.(string) + return release, ok && release != "" +} + +// Mode returns the node mode the file's values resolve for. +// +// An absent mode is an error rather than a guess. Guessing picks one binary's idea of a default and +// silently compares an archive node's file against a validator's baselines, which is the mistake +// this key exists to make impossible. +func (f *File) Mode() (string, error) { + e := f.doc.First(ModeKey) + if e == nil || e.KeyValue == nil { + return "", fmt.Errorf("sei.toml has no %s. Every value in it resolves for one node mode, so "+ + "without it nothing can tell an archive node's file from a validator's", ModeKey) + } + v, err := goValue(e.Value) + if err != nil { + return "", fmt.Errorf("%s: %w", ModeKey, err) + } + mode, ok := v.(string) + if !ok { + return "", fmt.Errorf("%s is %T (%v), want a mode name", ModeKey, v, v) + } + if mode == "" { + return "", fmt.Errorf("%s is empty", ModeKey) + } + return mode, nil +} + +// Version returns the schema version the file records. +// +// An absent or unparsable version is an error, never a zero. A migration chain reads this to decide +// which steps to run, so guessing here transforms a file whose shape nobody established. +func (f *File) Version() (int, error) { + e := f.doc.First(VersionKey) + if e == nil || e.KeyValue == nil { + return 0, fmt.Errorf("sei.toml has no %s. Its shape cannot be established, so no migration "+ + "can safely run against it and no reader can know which keys it is expected to carry", + VersionKey) + } + v, err := goValue(e.Value) + if err != nil { + return 0, fmt.Errorf("%s: %w", VersionKey, err) + } + n, ok := v.(int64) + if !ok { + return 0, fmt.Errorf("%s is %T (%v), want an integer", VersionKey, v, v) + } + return int(n), nil +} + +// setVersion writes the schema version at the document's top level. +func (f *File) setVersion(n int) error { + return f.Set(VersionKey, n) +} + +// Bytes renders the document. +func (f *File) Bytes() ([]byte, error) { + var buf bytes.Buffer + if err := tomledit.Format(&buf, f.doc); err != nil { + return nil, fmt.Errorf("render sei.toml: %w", err) + } + return buf.Bytes(), nil +} + +// Save writes the document to path, atomically. +// +// The rename makes it atomic, and the temporary file sits in the destination's own directory so the +// rename stays within one filesystem. A crash at any point leaves either the previous file or the +// new one, never a truncated file a node cannot parse. +func (f *File) Save(path string) error { + raw, err := f.Bytes() + if err != nil { + return err + } + + mode := newFileMode + if info, err := os.Stat(path); err == nil { + // An existing file keeps its own mode, so a save never widens what an operator narrowed. + mode = info.Mode().Perm() + } + + dir := filepath.Dir(path) + tmp, err := os.CreateTemp(dir, "."+filepath.Base(path)+".*") + if err != nil { + return fmt.Errorf("create temporary file beside %s: %w", path, err) + } + tmpName := tmp.Name() + defer func() { + // Removing a temporary file that was already renamed fails harmlessly; leaving one behind + // after a failed write does not, since the next save would find the directory littered. + _ = os.Remove(tmpName) + }() + + if err := writeAndSync(tmp, raw, mode); err != nil { + return fmt.Errorf("write %s: %w", tmpName, err) + } + if err := os.Rename(tmpName, path); err != nil { + return fmt.Errorf("install %s: %w", path, err) + } + return syncDir(dir) +} + +// writeAndSync writes the whole payload, sets the mode, and flushes to the device. +// +// The sync makes the rename meaningful: without it the rename can land before the contents, leaving +// a file whose name is new and whose bytes are absent. +func writeAndSync(tmp *os.File, raw []byte, mode os.FileMode) error { + defer func() { _ = tmp.Close() }() + + if _, err := tmp.Write(raw); err != nil { + return err + } + if err := tmp.Chmod(mode); err != nil { + return err + } + if err := tmp.Sync(); err != nil { + return err + } + return tmp.Close() +} + +// syncDir flushes the directory entry so the rename itself survives a crash. +func syncDir(dir string) error { + d, err := os.Open(dir) //nolint:gosec // the destination's own directory + if err != nil { + return fmt.Errorf("open %s: %w", dir, err) + } + defer func() { _ = d.Close() }() + if err := d.Sync(); err != nil { + return fmt.Errorf("sync %s: %w", dir, err) + } + return nil +} + +// keyOf splits a dotted key into its parser path. +func keyOf(key string) (parser.Key, error) { + if key == "" { + return nil, fmt.Errorf("empty key") + } + parts := strings.Split(strings.ToLower(key), ".") + for _, p := range parts { + if p == "" { + return nil, fmt.Errorf("key %q has an empty segment", key) + } + } + return parser.Key(parts), nil +} + +// quoteInt renders an integer the way TOML spells one. +func quoteInt(n int64) string { return strconv.FormatInt(n, 10) } diff --git a/config/seitoml/guards_test.go b/config/seitoml/guards_test.go new file mode 100644 index 0000000000..90fad4a61d --- /dev/null +++ b/config/seitoml/guards_test.go @@ -0,0 +1,116 @@ +package seitoml + +import ( + "strings" + "testing" + + "github.com/creachadair/tomledit" + "github.com/creachadair/tomledit/parser" + "github.com/creachadair/tomledit/scanner" +) + +// The guards below cannot be reached by parsing a file, because the parser refuses the input that +// would reach them: a bare word, a malformed escape, an unterminated table. They exist because the +// parser is a dependency, and a version of it that accepted more would otherwise turn an unknown token +// into a plausible Go value rather than a refusal. Driving them here is what keeps that refusal real +// instead of assumed, so this test is in the package rather than beside it. + +// TestAnUnknownTokenIsRefusedRatherThanGuessed holds the value decoder's own vocabulary. +func TestAnUnknownTokenIsRefusedRatherThanGuessed(t *testing.T) { + for _, tc := range []struct { + name string + tok parser.Token + want string + }{ + {"a word that is neither true nor false", retyped(t, "3", scanner.Word), + "not a value TOML recognizes"}, + {"a token type that is not a value", retyped(t, "3", scanner.LBracket), + "is not a value"}, + } { + t.Run(tc.name, func(t *testing.T) { + got, err := tokenValue(tc.tok) + if err == nil { + t.Fatalf("%s decoded to %#v; an unknown token becoming a value is how a node runs a "+ + "setting nobody wrote", tc.name, got) + } + if !strings.Contains(err.Error(), tc.want) { + t.Errorf("the refusal reads %q, which does not mention %q", err, tc.want) + } + }) + } +} + +// TestAValueShapeWithNoDecoderIsRefused covers the outer switch over a parsed value. +func TestAValueShapeWithNoDecoderIsRefused(t *testing.T) { + if _, err := goValue(parser.Value{}); err == nil { + t.Error("a value carrying no shape this package knows decoded without complaint") + } +} + +// TestUnquoteRefusesAKindThatIsNotAString covers the string decoder's own guard. +// +// unquote picks the escaping rules from the token kind, so a kind it does not know has no rules to +// apply. Returning the text as written would decode a basic string's escapes as literal backslashes. +func TestUnquoteRefusesAKindThatIsNotAString(t *testing.T) { + if _, err := unquote(scanner.Integer, "3"); err == nil { + t.Error("unquote accepted a kind that is not a string") + } +} + +// TestAnInlineTableSkipsAnEntryCarryingNothing covers the nil entry a malformed inline table produces. +func TestAnInlineTableSkipsAnEntryCarryingNothing(t *testing.T) { + got, err := inlineValue(parser.Inline{nil}) + if err != nil { + t.Fatalf("inlineValue: %v", err) + } + if len(got.(map[string]any)) != 0 { + t.Errorf("an inline table of one empty entry decoded to %#v, want nothing", got) + } +} + +// TestATopLevelKeyReachesADocumentWithNoGlobalSection covers a document built rather than parsed. +// +// Parsing always produces a global section, even for a file whose first line is a table heading, so +// this shape only arises from a document assembled in code. Writing the schema version into one has to +// create the space rather than panic. +func TestATopLevelKeyReachesADocumentWithNoGlobalSection(t *testing.T) { + f := &File{doc: &tomledit.Document{}} + if err := f.Set(ModeKey, "seed"); err != nil { + t.Fatalf("Set on a document with no global section: %v", err) + } + mode, err := f.Mode() + if err != nil || mode != "seed" { + t.Errorf("Mode = (%q, %v), want seed", mode, err) + } +} + +// TestAPreambleReachesADocumentWithNoGlobalSection is the preamble's half of the same shape. +func TestAPreambleReachesADocumentWithNoGlobalSection(t *testing.T) { + f := &File{doc: &tomledit.Document{}} + f.SetPreamble([]string{" a header"}) + + raw, err := f.Bytes() + if err != nil { + t.Fatalf("Bytes: %v", err) + } + if !strings.Contains(string(raw), "# a header") { + t.Errorf("the preamble is not in the rendered document: %q", raw) + } +} + +// retyped parses a literal and relabels the token's type, which is how a decoder is driven over a +// token the parser will not produce from any file. A token's text is not settable from outside the +// parser, so the text stays whatever the literal scanned as and only the label changes. +func retyped(t *testing.T, literal string, kind scanner.Token) parser.Token { + t.Helper() + v, err := parser.ParseValue(literal) + if err != nil { + t.Fatalf("ParseValue(%q): %v", literal, err) + } + tok, ok := v.X.(parser.Token) + if !ok { + t.Fatalf("ParseValue(%q) is a %T, want a token", literal, v.X) + } + tok.Type = kind + return tok +} diff --git a/config/seitoml/seitoml_test.go b/config/seitoml/seitoml_test.go new file mode 100644 index 0000000000..afcdead593 --- /dev/null +++ b/config/seitoml/seitoml_test.go @@ -0,0 +1,1225 @@ +package seitoml_test + +import ( + "fmt" + "math" + "os" + "path/filepath" + "reflect" + "strings" + "testing" + "time" + + "github.com/sei-protocol/sei-chain/config/seitoml" +) + +// commented is a file written the way an operator writes one: a heading comment, a reason beside a +// value, and a blank line for legibility. +const commented = `schema_version = 1 +node_mode = "validator" + +# The giga executor. Turned on after the load test in March. +[giga_executor] +enabled = true +# Off deliberately: this node serves historical queries and OCC cost us more than it saved. +occ_enabled = false +` + +func parse(t *testing.T, body string) *seitoml.File { + t.Helper() + f, err := seitoml.Parse(strings.NewReader(body)) + if err != nil { + t.Fatalf("Parse: %v", err) + } + return f +} + +// render returns the file's current text. +func render(t *testing.T, f *seitoml.File) string { + t.Helper() + raw, err := f.Bytes() + if err != nil { + t.Fatalf("Bytes: %v", err) + } + return string(raw) +} + +// TestEditingPreservesAnOperatorsComments is the property that decides how this package is built. +// +// An operator's comments are how they explain a choice to whoever reads the file next. Rewriting +// the file from a decoded map would drop all of them, and the operator would have no way to get +// that reasoning back. Held by editing a value that has a comment explaining it. +func TestEditingPreservesAnOperatorsComments(t *testing.T) { + f := parse(t, commented) + + if err := f.Set("giga_executor.occ_enabled", true); err != nil { + t.Fatalf("Set: %v", err) + } + + got := render(t, f) + for _, comment := range []string{ + "# The giga executor. Turned on after the load test in March.", + "# Off deliberately: this node serves historical queries and OCC cost us more than it saved.", + } { + if !strings.Contains(got, comment) { + t.Errorf("editing one value dropped a comment:\n %s\n\nThe file now reads:\n%s\n\n"+ + "An operator cannot recover the reasoning they recorded, and nothing warned them", + comment, got) + } + } + if !strings.Contains(got, "occ_enabled = true") { + t.Errorf("the value was not written. The file reads:\n%s", got) + } + if strings.Contains(got, "occ_enabled = false") { + t.Errorf("the old value is still present, so the key is written twice:\n%s", got) + } +} + +// TestEditingChangesOnlyTheValueItWasAsked holds the rest of that property. +// +// Preserving comments is not enough if the content moves. A save that rewrote other values, or +// reordered them, would make every change unreviewable, because the diff would show the whole file +// rather than the one value that moved. +// +// Compared on the lines that carry content. Blank lines are excluded because the formatter +// normalizes vertical spacing, which the test below pins as a one-time change. +func TestEditingChangesOnlyTheValueItWasAsked(t *testing.T) { + f := parse(t, commented) + + if err := f.Set("giga_executor.occ_enabled", true); err != nil { + t.Fatalf("Set: %v", err) + } + + before, after := contentLines(commented), contentLines(render(t, f)) + if len(before) != len(after) { + t.Fatalf("the file went from %d lines of content to %d:\n%s", + len(before), len(after), strings.Join(after, "\n")) + } + var moved []string + for i := range before { + if before[i] != after[i] { + moved = append(moved, " -"+before[i]+"\n +"+after[i]) + } + } + if len(moved) != 1 { + t.Errorf("setting one value changed %d lines of content, want 1:\n%s", + len(moved), strings.Join(moved, "\n")) + } +} + +// contentLines returns the lines that carry content, in order. +func contentLines(body string) []string { + var out []string + for _, l := range strings.Split(body, "\n") { + if strings.TrimSpace(l) != "" { + out = append(out, l) + } + } + return out +} + +// TestFormattingNormalizesOnceAndThenHoldsSteady is what makes the normalization safe to accept. +// +// Rendering a hand-written file adjusts its vertical spacing, so the first save of a file nobody +// has saved before shows a blank line the operator did not add. That is tolerable only if it does +// not repeat: a file that gained a line on every save would grow without bound, and every diff +// after the first would carry noise nobody chose. +func TestFormattingNormalizesOnceAndThenHoldsSteady(t *testing.T) { + first := render(t, parse(t, commented)) + second := render(t, parse(t, first)) + + if second != first { + t.Errorf("rendering a rendered file changed it again, so each save moves the file:\n"+ + "first:\n%s\nsecond:\n%s", first, second) + } + // The normalization is spacing only, so nothing that carries content may differ. + if a, b := contentLines(commented), contentLines(first); strings.Join(a, "\n") != strings.Join(b, "\n") { + t.Errorf("rendering changed the file's content, not just its spacing:\n%s", first) + } +} + +// TestAnAbsentSchemaVersionIsAnError holds that the file's shape is never guessed. +// +// A migration chain reads the version to decide which steps to run. Defaulting an absent one to +// zero would run every step in history against a file nobody established the shape of, and the +// result would look like a successful upgrade. +func TestAnAbsentSchemaVersionIsAnError(t *testing.T) { + for _, tc := range []struct{ name, body string }{ + {"absent", "[giga_executor]\nenabled = true\n"}, + {"not an integer", "schema_version = \"1\"\n"}, + {"a float", "schema_version = 1.0\n"}, + } { + t.Run(tc.name, func(t *testing.T) { + if _, err := parse(t, tc.body).Version(); err == nil { + t.Errorf("a %s schema version was accepted. A migration would then run against a file "+ + "whose shape nobody established, and report success", tc.name) + } + }) + } + if v, err := parse(t, commented).Version(); err != nil || v != 1 { + t.Errorf("a well-formed version read (%d, %v), want (1, nil)", v, err) + } +} + +// TestTheSchemaVersionIsNotAConfigurationKey keeps file metadata out of the key space. +// +// It describes the file rather than configuring the node, so a check comparing written keys against +// the declared set would report it as a key no section owns, on every node, forever. +func TestTheSchemaVersionIsNotAConfigurationKey(t *testing.T) { + values, err := parse(t, commented).Values() + if err != nil { + t.Fatalf("Values: %v", err) + } + + if _, present := values[seitoml.VersionKey]; present { + t.Errorf("%s appears in the written key space: %v. Every node would then be told it has a "+ + "key no section declares", seitoml.VersionKey, values) + } + if len(values) != 2 { + t.Errorf("read %d keys, want the section's 2: %v", len(values), values) + } + if values["giga_executor.occ_enabled"] != false { + t.Errorf("occ_enabled read %#v, want false", values["giga_executor.occ_enabled"]) + } +} + +// TestSetRoundTripsEveryTypeItAccepts is what keeps the writer and the reader from disagreeing. +// +// Writing renders a Go value as TOML text and reading parses it back, and the two are separate +// enumerations. Without this, a type could be written in a form that reads back as something else, +// and the file would look correct while the node ran a different value. +func TestSetRoundTripsEveryTypeItAccepts(t *testing.T) { + for _, tc := range []struct { + name string + set any + want any + }{ + {"bool", true, true}, + {"bool false", false, false}, + {"int", 16, int64(16)}, + {"negative int", -3, int64(-3)}, + {"int64", int64(1 << 40), int64(1 << 40)}, + {"uint", uint(7), int64(7)}, + {"int32", int32(1 << 20), int64(1 << 20)}, + {"uint32", uint32(4294967295), int64(4294967295)}, + {"uint64", uint64(1 << 40), int64(1 << 40)}, + {"float", 1.5, 1.5}, + // An integral float has to keep its type. TOML tells a float from an integer by the fractional + // part, and the shortest form of 1.0 is "1", which reads back as an integer. + {"integral float", float64(1), float64(1)}, + {"negative integral float", float64(-2), float64(-2)}, + {"float needing an exponent", 1e21, 1e21}, + // The shape reading an array back produces, which anything that reads a list and writes it again + // hands straight back to Set. + {"list read back as any", []any{"a", int64(2), true}, []any{"a", int64(2), true}}, + {"string", "hello", "hello"}, + {"string with a quote", `say "hi"`, `say "hi"`}, + {"string with a backslash", `C:\sei\data`, `C:\sei\data`}, + {"empty string", "", ""}, + {"duration", 90 * time.Second, "1m30s"}, + {"string list", []string{"a", "b"}, []any{"a", "b"}}, + } { + t.Run(tc.name, func(t *testing.T) { + f, err := seitoml.New("validator", "v6.7.0") + if err != nil { + t.Fatalf("New: %v", err) + } + if err := f.Set("probe.value", tc.set); err != nil { + t.Fatalf("Set(%#v): %v", tc.set, err) + } + + // Re-parsed rather than read back from the same document, so this measures what a + // later process reads off disk rather than what is still in memory. + reread := parse(t, render(t, f)) + got, ok, err := reread.Get("probe.value") + if err != nil || !ok { + t.Fatalf("Get after a round trip: (%#v, %v, %v)\nfile:\n%s", got, ok, err, render(t, f)) + } + + if !equal(got, tc.want) { + t.Errorf("wrote %#v and read back %#v, want %#v.\nfile:\n%s\n\nA value that does not "+ + "survive a round trip means the file looks correct while the node runs something "+ + "else", tc.set, got, tc.want, render(t, f)) + } + }) + } +} + +// equal compares two read values, including lists. +func equal(a, b any) bool { + as, aok := a.([]any) + bs, bok := b.([]any) + if aok || bok { + if !aok || !bok || len(as) != len(bs) { + return false + } + for i := range as { + if as[i] != bs[i] { + return false + } + } + return true + } + return a == b +} + +// TestALiteralStringIsTakenAsWritten holds the difference between TOML's two string forms. +// +// A basic string carries escapes and a literal string does not, which is why TOML has both. +// Decoding a literal string as though it had escapes turns a Windows path's separators into +// control characters, and the value the node runs is not the one in the file. +func TestALiteralStringIsTakenAsWritten(t *testing.T) { + f := parse(t, "schema_version = 1\n[probe]\nliteral = 'C:\\sei\\data'\nbasic = \"a\\tb\"\n") + + values, err := f.Values() + if err != nil { + t.Fatalf("Values: %v", err) + } + + if got := values["probe.literal"]; got != `C:\sei\data` { + t.Errorf("a literal string read %#v, want the text as written. Escapes were processed in the "+ + "form that does not have them", got) + } + if got := values["probe.basic"]; got != "a\tb" { + t.Errorf("a basic string read %#v, want its escape decoded to a tab", got) + } +} + +// TestUnsetRemovesTheKeyRatherThanWritingItsBaseline holds what unset means. +// +// An absent key resolves to the running binary's baseline. Writing the baseline value instead +// looks identical in the file but is a commitment that survives a release changing that baseline, +// which is the opposite of what the operator asked for. +func TestUnsetRemovesTheKeyRatherThanWritingItsBaseline(t *testing.T) { + f := parse(t, commented) + + removed, err := f.Unset("giga_executor.occ_enabled") + if err != nil { + t.Fatalf("Unset: %v", err) + } + if !removed { + t.Fatal("Unset reported no change for a key the file carries") + } + + values, err := f.Values() + if err != nil { + t.Fatalf("Values: %v", err) + } + if _, present := values["giga_executor.occ_enabled"]; present { + t.Errorf("the key is still written after unset: %v. It would keep overriding the baseline the "+ + "operator asked to fall back to", values) + } + if strings.Contains(render(t, f), "occ_enabled") { + t.Errorf("the key is still in the file text:\n%s", render(t, f)) + } + // The other key is untouched, or this would pass for an unset that emptied the section. + if values["giga_executor.enabled"] != true { + t.Errorf("unsetting one key disturbed another: %v", values) + } + + again, err := f.Unset("giga_executor.occ_enabled") + if err != nil { + t.Fatalf("Unset on an absent key: %v", err) + } + if again { + t.Error("Unset reported a change for a key that was already gone, so a caller cannot tell " + + "whether it had anything to remove") + } +} + +// TestSetCreatesTheTableWhenTheSectionIsNew holds the first-key case. +// +// Without it, writing the first key of a section would need the operator to add the heading by +// hand, and set would fail on exactly the file a new node starts from. +func TestSetCreatesTheTableWhenTheSectionIsNew(t *testing.T) { + f, err := seitoml.New("validator", "v6.7.0") + if err != nil { + t.Fatalf("New: %v", err) + } + + if err := f.Set("state-store.ss-keep-recent", 100000); err != nil { + t.Fatalf("Set into a section that does not exist: %v", err) + } + + got := render(t, f) + if !strings.Contains(got, "[state-store]") { + t.Errorf("no table heading was written:\n%s", got) + } + values, err := parse(t, got).Values() + if err != nil { + t.Fatalf("Values after a round trip: %v", err) + } + if values["state-store.ss-keep-recent"] != int64(100000) { + t.Errorf("the key read %#v after a round trip, want 100000. file:\n%s", + values["state-store.ss-keep-recent"], got) + } +} + +// TestValuesFlattensAnInlineTable keeps an inline table's leaves visible. +// +// An inline table is one written line holding several keys. Left nested, its leaves are invisible +// to any check that walks declared keys, so an operator could write a setting nothing validates. +func TestValuesFlattensAnInlineTable(t *testing.T) { + f := parse(t, "schema_version = 1\n[state-commit]\nflatkv = { enable = true, dir = \"/data\" }\n") + + values, err := f.Values() + if err != nil { + t.Fatalf("Values: %v", err) + } + + if values["state-commit.flatkv.enable"] != true { + t.Errorf("the inline table's leaf is not reachable as a dotted key: %v.\n\nA key nothing can "+ + "see is a setting nothing validates", values) + } + if values["state-commit.flatkv.dir"] != "/data" { + t.Errorf("the second leaf read %#v, want /data", values["state-commit.flatkv.dir"]) + } + if _, nested := values["state-commit.flatkv"]; nested { + t.Errorf("the table itself is also reported as a value, so a check would see a key whose "+ + "value is a map: %v", values) + } +} + +// TestAnUnsupportedTypeIsRefused keeps a wrong guess out of an operator's file. +// +// A formatter that rendered anything would put a plausible-looking line in the file that reads +// back as something else. Refusing is what makes the round-trip guarantee above hold for every +// type this accepts. +func TestAnUnsupportedTypeIsRefused(t *testing.T) { + f, err := seitoml.New("validator", "v6.7.0") + if err != nil { + t.Fatalf("New: %v", err) + } + + if err := f.Set("probe.value", map[string]string{"a": "b"}); err == nil { + t.Error("a map was written to the file. Whatever line that produced, nothing guarantees it " + + "reads back as the value the caller meant") + } + if err := f.Set("", true); err == nil { + t.Error("an empty key was accepted") + } + if err := f.Set("probe..value", true); err == nil { + t.Error("a key with an empty segment was accepted") + } +} + +// TestSaveLandsInFullOrNotAtAll holds the atomicity the boot depends on. +// +// A configuration file truncated by a crash mid-write is one the node cannot parse, so a save that +// cannot complete must leave the previous file exactly as it was. Driven by making the install step +// fail, which is the part that would otherwise have already replaced the file. +func TestSaveLandsInFullOrNotAtAll(t *testing.T) { + dir := t.TempDir() + path := filepath.Join(dir, "sei.toml") + if err := os.WriteFile(path, []byte(commented), 0o600); err != nil { + t.Fatalf("seed the file: %v", err) + } + + f := parse(t, commented) + if err := f.Set("giga_executor.occ_enabled", true); err != nil { + t.Fatalf("Set: %v", err) + } + // A directory the process cannot write is how the temporary file, and therefore the install, + // is made to fail without the previous file being touched. + if err := os.Chmod(dir, 0o500); err != nil { + t.Fatalf("chmod: %v", err) + } + t.Cleanup(func() { _ = os.Chmod(dir, 0o700) }) + + if err := f.Save(path); err == nil { + t.Fatal("Save reported success in a directory it cannot write, so a caller would believe " + + "the new configuration is on disk") + } + + raw, err := os.ReadFile(path) //nolint:gosec // a path this test created under t.TempDir + if err != nil { + t.Fatalf("the previous file is unreadable after a failed save: %v", err) + } + if string(raw) != commented { + t.Errorf("a failed save changed the file on disk. It now reads:\n%s\n\nThe node would boot "+ + "from something nobody wrote", raw) + } +} + +// TestSaveLeavesNoTemporaryFileBehind keeps the directory clean on both paths. +// +// The temporary file has to sit beside the destination so the rename stays on one filesystem. +// Left behind, a partial configuration accumulates next to the real one, and a reader globbing the +// directory can find it. +func TestSaveLeavesNoTemporaryFileBehind(t *testing.T) { + dir := t.TempDir() + path := filepath.Join(dir, "sei.toml") + f := parse(t, commented) + + if err := f.Save(path); err != nil { + t.Fatalf("Save: %v", err) + } + + entries, err := os.ReadDir(dir) + if err != nil { + t.Fatalf("ReadDir: %v", err) + } + for _, e := range entries { + if e.Name() != "sei.toml" { + t.Errorf("a save left %q beside the configuration. A partial file next to the real one is "+ + "something a reader can find", e.Name()) + } + } +} + +// TestSaveKeepsAnExistingFilesPermissions holds that a save never widens access. +// +// A configuration may name a private endpoint or carry a token. An operator who narrowed the file +// deliberately would have that undone by a save, silently, and nothing about the change is visible +// in the file's contents. +func TestSaveKeepsAnExistingFilesPermissions(t *testing.T) { + dir := t.TempDir() + path := filepath.Join(dir, "sei.toml") + if err := os.WriteFile(path, []byte(commented), 0o640); err != nil { + t.Fatalf("seed: %v", err) + } + if err := os.Chmod(path, 0o600); err != nil { + t.Fatalf("chmod: %v", err) + } + + f := parse(t, commented) + if err := f.Set("giga_executor.enabled", false); err != nil { + t.Fatalf("Set: %v", err) + } + if err := f.Save(path); err != nil { + t.Fatalf("Save: %v", err) + } + + info, err := os.Stat(path) + if err != nil { + t.Fatalf("Stat: %v", err) + } + if got := info.Mode().Perm(); got != 0o600 { + t.Errorf("the file's mode moved from 0600 to %#o. A save that widens access undoes a "+ + "restriction an operator chose, and the file's contents do not show it", got) + } +} + +// TestANewFileIsNotWorldReadable holds the mode a file created here gets. +// +// The usual 0644 would be wrong for a file that may name a private endpoint, and a new file has no +// previous mode to inherit, so the choice has to be made here. +func TestANewFileIsNotWorldReadable(t *testing.T) { + path := filepath.Join(t.TempDir(), "sei.toml") + f, err := seitoml.New("validator", "v6.7.0") + if err != nil { + t.Fatalf("New: %v", err) + } + + if err := f.Save(path); err != nil { + t.Fatalf("Save: %v", err) + } + + info, err := os.Stat(path) + if err != nil { + t.Fatalf("Stat: %v", err) + } + if got := info.Mode().Perm(); got&0o077 != 0 { + t.Errorf("a new configuration file was created with mode %#o, readable beyond its owner", got) + } +} + +// TestNewCarriesThisBinarysSchemaVersion holds that a generated file is never version-less. +// +// A file written without one cannot be migrated later, and the failure appears at the first +// upgrade rather than at the write that caused it. +func TestNewCarriesThisBinarysSchemaVersion(t *testing.T) { + f, err := seitoml.New("validator", "v6.7.0") + if err != nil { + t.Fatalf("New: %v", err) + } + + v, err := parse(t, render(t, f)).Version() + if err != nil { + t.Fatalf("a new file has no readable schema version: %v\nfile:\n%s", err, render(t, f)) + } + if v != seitoml.SchemaVersion { + t.Errorf("a new file records version %d, want %d", v, seitoml.SchemaVersion) + } +} + +// TestANewFileRecordsItsNodeMode holds the field every value in the file depends on. +// +// The mode selects which defaults the values were chosen against, so a file that omits it cannot be +// compared against a binary or checked against the mode the node runs. +func TestANewFileRecordsItsNodeMode(t *testing.T) { + f, err := seitoml.New("archive", "v6.7.0") + if err != nil { + t.Fatalf("New: %v", err) + } + + mode, err := parse(t, render(t, f)).Mode() + if err != nil { + t.Fatalf("a new file has no readable node mode: %v\nfile:\n%s", err, render(t, f)) + } + if mode != "archive" { + t.Errorf("the file records mode %q, want archive", mode) + } + if _, err := seitoml.New("", "v6.7.0"); err == nil { + t.Error("a file was created with no mode. Every value written into it resolves for one, so " + + "nothing could later tell an archive node's file from a validator's") + } +} + +// TestAnAbsentOrUnreadableNodeModeIsAnError keeps a comparison from guessing. +// +// Guessing picks one binary's idea of a default and silently measures an archive node's file against +// a validator's baselines, which is the mistake this key exists to make impossible. +func TestAnAbsentOrUnreadableNodeModeIsAnError(t *testing.T) { + for _, tc := range []struct{ name, body string }{ + {"absent", "schema_version = 1\n"}, + {"not text", "schema_version = 1\nnode_mode = 3\n"}, + {"empty", "schema_version = 1\nnode_mode = \"\"\n"}, + } { + t.Run(tc.name, func(t *testing.T) { + if _, err := parse(t, tc.body).Mode(); err == nil { + t.Errorf("a %s node mode was accepted, so a reader would compare the file against "+ + "whichever defaults it happened to pick", tc.name) + } + }) + } + if mode, err := parse(t, commented).Mode(); err != nil || mode != "validator" { + t.Errorf("a well-formed mode read (%q, %v), want validator", mode, err) + } +} + +// TestTheNodeModeIsNotAConfigurationKey keeps file metadata out of the key space. +// +// It describes the file rather than configuring the node, so a check comparing written keys against +// the declared set would otherwise report it as a key no section owns, on every node, forever. +func TestTheNodeModeIsNotAConfigurationKey(t *testing.T) { + values, err := parse(t, commented).Values() + if err != nil { + t.Fatalf("Values: %v", err) + } + + if _, present := values[seitoml.ModeKey]; present { + t.Errorf("%s appears in the written key space: %v", seitoml.ModeKey, values) + } + if len(values) != 2 { + t.Errorf("read %d keys, want the section's 2: %v", len(values), values) + } +} + +// TestAMigrationCarriesTheNodeModeForward holds the field across an upgrade. +// +// A migration that dropped it would leave a file nothing can compare, and the failure would appear +// at the next diff rather than at the upgrade that caused it. +func TestAMigrationCarriesTheNodeModeForward(t *testing.T) { + f := parse(t, commented) + + if err := f.Set("giga_executor.enabled", false); err != nil { + t.Fatalf("Set: %v", err) + } + + mode, err := parse(t, render(t, f)).Mode() + if err != nil || mode != "validator" { + t.Errorf("editing the file lost its node mode: (%q, %v)", mode, err) + } +} + +// TestTheReleaseThatWroteTheFileIsRecorded holds the provenance the file carries. +// +// Nobody could otherwise tell which binary produced a file, which is the first thing worth knowing +// when its values look wrong and the first thing that says whether regenerating would change +// anything. +func TestTheReleaseThatWroteTheFileIsRecorded(t *testing.T) { + f, err := seitoml.New("validator", "v6.7.0") + if err != nil { + t.Fatalf("New: %v", err) + } + + release, recorded := parse(t, render(t, f)).GeneratedBy() + if !recorded || release != "v6.7.0" { + t.Errorf("the file records (%q, %v), want v6.7.0\nfile:\n%s", release, recorded, render(t, f)) + } +} + +// TestABuildWithNoReleaseOmitsTheKey holds the case every developer hits. +// +// The release comes from a linker flag the release build sets, so a binary built any other way knows +// none. Writing an empty string would put a key in an operator's file that says less than no key, and +// refusing would leave a development build unable to produce a file at all. +func TestABuildWithNoReleaseOmitsTheKey(t *testing.T) { + f, err := seitoml.New("validator", "") + if err != nil { + t.Fatalf("New refused a build that carries no release: %v", err) + } + + body := render(t, f) + if strings.Contains(body, seitoml.GeneratedByKey) { + t.Errorf("the file carries %s with nothing behind it:\n%s", seitoml.GeneratedByKey, body) + } + if _, recorded := parse(t, body).GeneratedBy(); recorded { + t.Error("GeneratedBy reports a release for a file that records none") + } + // A file already on disk carrying an empty value reads the same way, since an empty release says + // nothing and a caller told one is present would print it as though it meant something. + onDisk := parse(t, "schema_version = 1\nnode_mode = \"validator\"\ngenerated_by = \"\"\n") + if release, recorded := onDisk.GeneratedBy(); recorded { + t.Errorf("a file recording an empty release reports (%q, %v), want it treated as absent", + release, recorded) + } + // And the file is otherwise complete, or omitting the key would have cost something. + if v, err := parse(t, body).Version(); err != nil || v != seitoml.SchemaVersion { + t.Errorf("the file lost its schema version: (%d, %v)", v, err) + } + if mode, err := parse(t, body).Mode(); err != nil || mode != "validator" { + t.Errorf("the file lost its node mode: (%q, %v)", mode, err) + } +} + +// TestTheReleaseKeyIsNotAConfigurationKey keeps provenance out of the key space. +// +// Only the key at the document's top level. A key of the same name inside a table is an ordinary +// setting called section.generated_by, and doctor should report it as one, so the exclusion is on the +// exact path rather than on the name. +func TestTheReleaseKeyIsNotAConfigurationKey(t *testing.T) { + f := parse(t, commented) + if err := f.SetGeneratedBy("v6.7.0"); err != nil { + t.Fatalf("SetGeneratedBy: %v", err) + } + + values, err := f.Values() + if err != nil { + t.Fatalf("Values: %v", err) + } + if _, present := values[seitoml.GeneratedByKey]; present { + t.Errorf("%s appears in the written key space: %v", seitoml.GeneratedByKey, values) + } + if len(values) != 2 { + t.Errorf("read %d keys, want the section's 2: %v", len(values), values) + } + + // The same name inside a table stays a configuration key, since it is one. + inTable := parse(t, "schema_version = 1\nnode_mode = \"validator\"\n\n[probe]\ngenerated_by = \"x\"\n") + nested, err := inTable.Values() + if err != nil { + t.Fatalf("Values: %v", err) + } + if _, present := nested["probe.generated_by"]; !present { + t.Errorf("probe.generated_by was excluded from the key space: %v. Only the top-level key is "+ + "provenance; inside a table it is a setting like any other and doctor should say so", nested) + } +} + +// TestNothingBehavesDifferentlyForTheReleaseKey is the constraint that makes the field safe. +// +// It is provenance, so no answer anywhere may depend on it. The moment something branches on it, a +// development build writing no release becomes a node that cannot read its own configuration, and a +// file hand-edited to a nonsense release becomes one nothing will touch. +// +// Held by driving every reader over the same file three ways: recording a release, recording none, +// and recording something no release ever was. Every answer has to match. +func TestNothingBehavesDifferentlyForTheReleaseKey(t *testing.T) { + const body = "schema_version = 1\nnode_mode = \"validator\"\n\n[probe]\nworkers = 4\n" + + answers := map[string][]string{} + for _, release := range []string{"", "v6.7.0", "not-a-release-anyone-shipped"} { + f := parse(t, body) + if err := f.SetGeneratedBy(release); err != nil { + t.Fatalf("SetGeneratedBy(%q): %v", release, err) + } + + var got []string + version, err := f.Version() + got = append(got, fmt.Sprintf("version=%d err=%v", version, err)) + mode, err := f.Mode() + got = append(got, fmt.Sprintf("mode=%q err=%v", mode, err)) + values, err := f.Values() + got = append(got, fmt.Sprintf("values=%v err=%v", values, err)) + value, present, err := f.Get("probe.workers") + got = append(got, fmt.Sprintf("get=%#v present=%v err=%v", value, present, err)) + + // Save too, since a round trip through the disk is where a reader could pick the key up again. + path := filepath.Join(t.TempDir(), "sei.toml") + if err := f.Save(path); err != nil { + t.Fatalf("Save: %v", err) + } + reread, err := seitoml.Load(path) + if err != nil { + t.Fatalf("Load: %v", err) + } + rereadValues, err := reread.Values() + got = append(got, fmt.Sprintf("reread=%v err=%v", rereadValues, err)) + + answers[release] = got + } + + base := answers["v6.7.0"] + for release, got := range answers { + for i := range got { + if got[i] != base[i] { + t.Errorf("with %s = %q a reader answered\n %s\nand with a recorded release it answered\n"+ + " %s\n\nThe field is provenance and nothing may depend on it: a development build "+ + "writes none, so anything branching on it makes such a build unable to read its own "+ + "configuration", seitoml.GeneratedByKey, release, got[i], base[i]) + } + } + } +} + +// TestEveryValueShapeTomlAllowsReadsBack drives the value forms an operator's file can hold. +// +// The file is hand-written, and TOML gives an operator more ways to write a value than a generated +// file would ever use: two string quotings and their multi-line forms, an integer in hex or with +// separators, a date, an array with a comment inside it, an inline table. Each has to come back as the +// Go value a reader compares against a default, because a shape that decodes wrongly is a value an +// operator wrote and the node silently disagrees about. +func TestEveryValueShapeTomlAllowsReadsBack(t *testing.T) { + f := parse(t, `schema_version = 1 +node_mode = "validator" + +[probe] +flag = true +basic = "a\ttab" +literal = 'C:\Users\node' +folded = """ +first line +second line""" +verbatim = ''' +kept \as \written''' +grouped = 1_000_000 +hex = 0x1f +ratio = 2.5 +stamped = 2026-08-18 +peers = ["a", "b"] +commented = [ + # the first one is the seed + "a", + "b", +] +inline = { host = "h", port = 26657 } +nested = { outer = { inner = 3 } } +`) + + values, err := f.Values() + if err != nil { + t.Fatalf("Values: %v", err) + } + for key, want := range map[string]any{ + "probe.flag": true, + "probe.basic": "a\ttab", + "probe.literal": `C:\Users\node`, + "probe.folded": "first line\nsecond line", + "probe.verbatim": `kept \as \written`, + "probe.grouped": int64(1000000), + "probe.hex": int64(31), + "probe.ratio": 2.5, + "probe.stamped": "2026-08-18", + "probe.inline.host": "h", + "probe.inline.port": int64(26657), + "probe.nested.outer.inner": int64(3), + } { + got, ok := values[key] + if !ok { + t.Errorf("%s is written in the file and Values left it out", key) + continue + } + if got != want { + t.Errorf("%s read back as %#v, want %#v", key, got, want) + } + } + + // Arrays compare element by element, and the commented one proves a comment between items is not + // mistaken for an item. + for key, want := range map[string][]any{ + "probe.peers": {"a", "b"}, + "probe.commented": {"a", "b"}, + } { + got, ok := values[key].([]any) + if !ok { + t.Errorf("%s read back as %#v, want a list", key, values[key]) + continue + } + if !reflect.DeepEqual(got, want) { + t.Errorf("%s read back as %#v, want %#v", key, got, want) + } + } + + // Get answers for one key the same way Values does for all of them, since a caller reading a single + // key must not get a different decoding from one reading the file. + for _, key := range []string{"probe.literal", "probe.folded", "probe.hex", "probe.inline.port"} { + got, present, err := f.Get(key) + if err != nil || !present { + // An inline table's leaf is reachable through Values and not through Get, which walks the + // document rather than the flattened space. + if strings.Contains(key, "inline") { + continue + } + t.Errorf("Get(%q) = (%#v, %v, %v)", key, got, present, err) + continue + } + if got != values[key] { + t.Errorf("Get(%q) read %#v and Values read %#v; one reader disagrees with the other", + key, got, values[key]) + } + } +} + +// TestAValueTomlDoesNotRecognizeIsNamedNotGuessed covers what a hand-edited file can go wrong as. +// +// A bare word never reaches here, because the parser refuses one before any value is decoded, in a +// table and inside an array or an inline table alike. What does reach here is a value TOML accepts and +// this package cannot use: a number past int64, and an infinity, which TOML spells as a word and +// ParseFloat accepts. +// +// Each has to name the key and what is wrong with it. Read as a zero, the node would boot on a value +// nobody wrote; dropped, the operator's line would be silently ignored. +func TestAValueTomlDoesNotRecognizeIsNamedNotGuessed(t *testing.T) { + for _, tc := range []struct { + name string + body string + key string + want string + }{ + {"an integer past int64", "[probe]\nn = 99999999999999999999\n", "probe.n", "not an integer"}, + {"an infinity", "[probe]\nn = inf\n", "probe.n", "not a finite number"}, + {"a negative infinity", "[probe]\nn = -inf\n", "probe.n", "not a finite number"}, + {"a NaN", "[probe]\nn = nan\n", "probe.n", "not a finite number"}, + {"an infinity inside an array", "[probe]\nlist = [1.5, inf]\n", "probe.list", "not a finite number"}, + {"an infinity inside an inline table", "[probe]\nt = { a = inf }\n", "probe.t", "not a finite number"}, + } { + t.Run(tc.name, func(t *testing.T) { + f := parse(t, tc.body) + + _, err := f.Values() + if err == nil { + t.Fatalf("Values accepted %s, so the node would run on a value nothing produced", tc.name) + } + if !strings.Contains(err.Error(), tc.want) { + t.Errorf("the error reads %q, which does not mention %q", err, tc.want) + } + + // Get has to refuse the same value, or a caller reading one key would see what a caller + // reading the whole file cannot. + if _, _, err := f.Get(tc.key); err == nil { + t.Errorf("Get(%q) accepted the value Values refused", tc.key) + } + }) + } +} + +// TestParseRefusesAFileTomlCannotRead covers the boundary before any value is read. +// +// A truncated or corrupt file has to fail as a file rather than as an empty one, since an empty +// document reads as a node that chose nothing and resolves every key to a default. +func TestParseRefusesAFileTomlCannotRead(t *testing.T) { + if _, err := seitoml.Parse(strings.NewReader("[unterminated\nkey = 1\n")); err == nil { + t.Error("a malformed document parsed, and an empty one reads as a node that chose nothing") + } + if _, err := seitoml.Load(filepath.Join(t.TempDir(), "absent.toml")); err == nil { + t.Error("loading a file that does not exist succeeded") + } + bad := filepath.Join(t.TempDir(), "sei.toml") + if err := os.WriteFile(bad, []byte("[unterminated\n"), 0o600); err != nil { + t.Fatalf("seed: %v", err) + } + _, err := seitoml.Load(bad) + if err == nil { + t.Fatal("loading a malformed file succeeded") + } + if !strings.Contains(err.Error(), bad) { + t.Errorf("the error reads %q and does not name the file, so an operator reading it cannot tell "+ + "which one to fix", err) + } +} + +// TestSetWritesIntoATableTheFileAlreadyHas covers the branch that does not create a heading. +// +// A file an operator wrote already has its sections, so most writes land in an existing table rather +// than making one. Appending a second heading for a table that is already there produces a file with +// the section twice, which is not what the operator wrote and not what a reader expects. +func TestSetWritesIntoATableTheFileAlreadyHas(t *testing.T) { + f := parse(t, "schema_version = 1\nnode_mode = \"validator\"\n\n[probe]\nfirst = 1\n") + if err := f.Set("probe.second", 2); err != nil { + t.Fatalf("Set: %v", err) + } + + out := render(t, f) + if n := strings.Count(out, "[probe]"); n != 1 { + t.Errorf("the file carries the [probe] heading %d times, want once:\n%s", n, out) + } + reread := parse(t, out) + for key, want := range map[string]any{"probe.first": int64(1), "probe.second": int64(2)} { + got, ok, err := reread.Get(key) + if err != nil || !ok || got != want { + t.Errorf("%s = (%#v, %v, %v), want %#v", key, got, ok, err, want) + } + } +} + +// TestSetWritesATopLevelKeyIntoAFileThatStartsWithATable covers a document with no global section. +// +// A file whose first line is a heading has nothing above it, so writing the schema version or the node +// mode into one has to create that space rather than fail or land inside the first table. Landing +// inside it would make the key read as section.schema_version, which no reader asks for. +func TestSetWritesATopLevelKeyIntoAFileThatStartsWithATable(t *testing.T) { + f := parse(t, "[probe]\nfirst = 1\n") + if err := f.Set(seitoml.ModeKey, "archive"); err != nil { + t.Fatalf("Set: %v", err) + } + + reread := parse(t, render(t, f)) + mode, err := reread.Mode() + if err != nil { + t.Fatalf("Mode after writing it into a file that had no top level: %v\n%s", err, render(t, f)) + } + if mode != "archive" { + t.Errorf("mode read back as %q, want archive", mode) + } + if _, ok, _ := reread.Get("probe." + seitoml.ModeKey); ok { + t.Error("the key landed inside the first table, so it reads as one that section owns") + } +} + +// TestThePreambleIsReplacedRatherThanStacked holds what regenerating a file does to its header. +// +// The preamble explains the file to whoever opens it, and a generate or adopt run writes one. Stacking +// a new block on the last would grow the header on every run until the explanation is buried in copies +// of itself. An empty list removes it, which is how a caller drops a header it no longer wants. +func TestThePreambleIsReplacedRatherThanStacked(t *testing.T) { + f := parse(t, "schema_version = 1\nnode_mode = \"validator\"\n\n[probe]\nfirst = 1\n") + + f.SetPreamble([]string{" written by the first run"}) + first := render(t, f) + if !strings.Contains(first, "# written by the first run") { + t.Fatalf("the preamble is not in the file:\n%s", first) + } + + f.SetPreamble([]string{" written by the second run"}) + second := render(t, f) + if strings.Contains(second, "first run") { + t.Errorf("the second preamble stacked on the first, so a header grows on every run:\n%s", second) + } + if !strings.Contains(second, "# written by the second run") { + t.Errorf("the second preamble is not in the file:\n%s", second) + } + + // The keys are untouched throughout, since a header is not configuration. + if got, ok, err := parse(t, second).Get("probe.first"); err != nil || !ok || got != int64(1) { + t.Errorf("probe.first = (%#v, %v, %v) after two preambles, want 1", got, ok, err) + } + + f.SetPreamble(nil) + if bare := render(t, f); strings.Contains(bare, "second run") { + t.Errorf("an empty preamble left the old one in place:\n%s", bare) + } +} + +// TestThePreambleGoesAboveEverythingInAFileThatStartsWithATable covers the no-global-section case. +func TestThePreambleGoesAboveEverythingInAFileThatStartsWithATable(t *testing.T) { + f := parse(t, "[probe]\nfirst = 1\n") + f.SetPreamble([]string{" a header"}) + + out := render(t, f) + if !strings.HasPrefix(strings.TrimSpace(out), "# a header") { + t.Errorf("the preamble is not the first thing in the file:\n%s", out) + } +} + +// TestAMalformedKeyIsRefusedByEveryVerbThatTakesOne holds the four entry points to one answer. +// +// Set, Unset and Get each take a dotted key from a caller, and a key TOML cannot express has to be +// refused rather than written as something else. Held together because a verb that accepted one would +// put a key in the file that no other verb can address. +func TestAMalformedKeyIsRefusedByEveryVerbThatTakesOne(t *testing.T) { + for _, key := range []string{"", "probe.", ".value", "probe..value"} { + t.Run("key "+key, func(t *testing.T) { + f := parse(t, "schema_version = 1\nnode_mode = \"validator\"\n") + if err := f.Set(key, 1); err == nil { + t.Errorf("Set(%q) was accepted", key) + } + if _, err := f.Unset(key); err == nil { + t.Errorf("Unset(%q) was accepted", key) + } + if _, _, err := f.Get(key); err == nil { + t.Errorf("Get(%q) was accepted", key) + } + }) + } +} + +// TestGetAnswersAbsentForAKeyTheFileDoesNotCarry separates absence from failure. +// +// An absent key is ordinary: it means the operator chose nothing and the value resolves to a default. A +// caller told this as an error would treat every unset key as a broken file. +func TestGetAnswersAbsentForAKeyTheFileDoesNotCarry(t *testing.T) { + f := parse(t, "schema_version = 1\nnode_mode = \"validator\"\n\n[probe]\nfirst = 1\n") + for _, key := range []string{"probe.absent", "absent.key", "absent"} { + got, present, err := f.Get(key) + if err != nil { + t.Errorf("Get(%q) failed with %v, and an unwritten key is ordinary", key, err) + } + if present || got != nil { + t.Errorf("Get(%q) = (%#v, %v), want absent", key, got, present) + } + } +} + +// TestAnInfinityCannotBeWritten holds the writer to the same rule as the reader. +// +// This file format has no form for an infinity or a NaN, so writing one produces a line no reader can +// load. Refusing names the value; the alternative writes it into an operator's file with nothing said. +func TestAnInfinityCannotBeWritten(t *testing.T) { + for _, tc := range []struct { + name string + v float64 + }{ + {"positive infinity", math.Inf(1)}, + {"negative infinity", math.Inf(-1)}, + {"NaN", math.NaN()}, + } { + t.Run(tc.name, func(t *testing.T) { + f, err := seitoml.New("validator", "v6.7.0") + if err != nil { + t.Fatalf("New: %v", err) + } + err = f.Set("probe.value", tc.v) + if err == nil { + t.Fatalf("%s was written, and no reader can load the line it produces", tc.name) + } + if !strings.Contains(err.Error(), "finite numbers") { + t.Errorf("the refusal reads %q and does not say why", err) + } + }) + } +} + +// TestAFileDescribingItselfWithANonValueIsRefusedPerReader covers the three keys about the file. +// +// schema_version, node_mode and generated_by are read before anything else, so a value this package +// cannot decode has to fail there rather than further in. The three differ in what failure means: +// version and mode are machinery a reader cannot proceed without, and the release is provenance whose +// absence is ordinary, so an undecodable one reads as absent rather than as an error. +func TestAFileDescribingItselfWithANonValueIsRefusedPerReader(t *testing.T) { + t.Run("an undecodable schema version", func(t *testing.T) { + _, err := parse(t, "schema_version = inf\n").Version() + if err == nil { + t.Fatal("an undecodable version was accepted, so a migration would run against a file whose " + + "shape nobody established") + } + if !strings.Contains(err.Error(), seitoml.VersionKey) { + t.Errorf("the error reads %q and does not name the key", err) + } + }) + + t.Run("an undecodable node mode", func(t *testing.T) { + _, err := parse(t, "node_mode = inf\n").Mode() + if err == nil { + t.Fatal("an undecodable mode was accepted, so an archive node's file would be compared " + + "against a validator's defaults") + } + if !strings.Contains(err.Error(), seitoml.ModeKey) { + t.Errorf("the error reads %q and does not name the key", err) + } + }) + + t.Run("an undecodable release", func(t *testing.T) { + release, ok := parse(t, "generated_by = inf\n").GeneratedBy() + if ok || release != "" { + t.Errorf("GeneratedBy = (%q, %v), want absent. The field is provenance, so a value nothing "+ + "can decode is the same as no value rather than a failure a caller has to handle", + release, ok) + } + }) + + t.Run("a node mode that is not a string", func(t *testing.T) { + _, err := parse(t, "node_mode = 3\n").Mode() + if err == nil || !strings.Contains(err.Error(), "want a mode name") { + t.Errorf("Mode on a numeric mode returned %v, want a refusal naming what it wanted", err) + } + }) + + t.Run("a schema version that is not an integer", func(t *testing.T) { + _, err := parse(t, "schema_version = \"one\"\n").Version() + if err == nil || !strings.Contains(err.Error(), "want an integer") { + t.Errorf("Version on a string version returned %v, want a refusal naming what it wanted", err) + } + }) +} + +// TestSaveNamesThePathWhenItCannotWriteThere holds the failure an operator is most likely to hit. +// +// A configured directory that does not exist is an ordinary mistake, and the error has to name the path +// so the operator knows which one to create. Failing without it leaves them guessing which of a +// configured data directory, home directory or flag was wrong. +func TestSaveNamesThePathWhenItCannotWriteThere(t *testing.T) { + f, err := seitoml.New("validator", "") + if err != nil { + t.Fatalf("New: %v", err) + } + path := filepath.Join(t.TempDir(), "absent-directory", "sei.toml") + + err = f.Save(path) + if err == nil { + t.Fatal("saving into a directory that does not exist succeeded") + } + if !strings.Contains(err.Error(), path) { + t.Errorf("the error reads %q and does not name the path, so an operator cannot tell which "+ + "directory to create", err) + } +} + +// TestSaveRefusesAPathThatIsADirectory covers the install step of the atomic write. +// +// A configured path pointing at a directory is an ordinary mistake, and the temporary file is written +// before anything notices. The error has to name the destination, and the directory it wrote beside has +// to be left clean, or the next save finds it littered with the leavings of this one. +func TestSaveRefusesAPathThatIsADirectory(t *testing.T) { + f, err := seitoml.New("validator", "") + if err != nil { + t.Fatalf("New: %v", err) + } + dir := t.TempDir() + target := filepath.Join(dir, "sei.toml") + if err := os.Mkdir(target, 0o750); err != nil { + t.Fatalf("seed: %v", err) + } + + err = f.Save(target) + if err == nil { + t.Fatal("saving over a directory succeeded") + } + if !strings.Contains(err.Error(), target) { + t.Errorf("the error reads %q and does not name the destination", err) + } + + entries, err := os.ReadDir(dir) + if err != nil { + t.Fatalf("ReadDir: %v", err) + } + if len(entries) != 1 { + var names []string + for _, e := range entries { + names = append(names, e.Name()) + } + t.Errorf("the directory holds %v after a failed save, want only the destination. A temporary "+ + "file left behind accumulates on every retry", names) + } +} + +// TestAListCarryingAValueThatCannotBeWrittenNamesTheElement covers writing a list back. +// +// Reading an array produces a list of any, which anything that reads a list and writes it again hands +// straight back. An element this package cannot render has to name its position, since a list of ten +// values with one bad element is otherwise a refusal an operator cannot act on. +func TestAListCarryingAValueThatCannotBeWrittenNamesTheElement(t *testing.T) { + f, err := seitoml.New("validator", "") + if err != nil { + t.Fatalf("New: %v", err) + } + + err = f.Set("probe.list", []any{"fine", struct{}{}}) + if err == nil { + t.Fatal("a list carrying a value with no TOML form was written") + } + if !strings.Contains(err.Error(), "element 1") { + t.Errorf("the refusal reads %q and does not say which element is at fault", err) + } +} diff --git a/config/seitoml/values.go b/config/seitoml/values.go new file mode 100644 index 0000000000..49ab7c0e0e --- /dev/null +++ b/config/seitoml/values.go @@ -0,0 +1,211 @@ +package seitoml + +import ( + "fmt" + "math" + "strconv" + "strings" + + "github.com/creachadair/tomledit" + "github.com/creachadair/tomledit/parser" + "github.com/creachadair/tomledit/scanner" +) + +// Values returns every key the file writes, as dotted paths to Go values. +// +// This leaves out the schema version, the node mode and the release that produced the file. All +// three describe the file rather than configuring the node, so a reader checking written keys against +// the declared set would otherwise report them as keys no section owns, on every node, forever. +func (f *File) Values() (map[string]any, error) { + out := map[string]any{} + var bad error + + f.doc.Scan(func(full parser.Key, e *tomledit.Entry) bool { + if e.KeyValue == nil { + return true // a table heading carries no value of its own + } + key := strings.ToLower(full.String()) + if key == VersionKey || key == ModeKey || key == GeneratedByKey { + return true + } + v, err := goValue(e.Value) + if err != nil { + bad = fmt.Errorf("%s: %w", key, err) + return false + } + // An inline table is one written value holding several keys, so it flattens into the same + // dotted space as a table would. Left nested, its leaves would be invisible to any check + // that walks declared keys. + if inline, ok := v.(map[string]any); ok { + for sub, sv := range flatten(key, inline) { + out[sub] = sv + } + return true + } + out[key] = v + return true + }) + if bad != nil { + return nil, bad + } + return out, nil +} + +// Get returns one key's written value. +func (f *File) Get(key string) (any, bool, error) { + path, err := keyOf(key) + if err != nil { + return nil, false, err + } + e := f.doc.First(path...) + if e == nil || e.KeyValue == nil { + return nil, false, nil + } + v, err := goValue(e.Value) + if err != nil { + return nil, false, fmt.Errorf("%s: %w", key, err) + } + return v, true, nil +} + +// flatten expands an inline table into dotted keys under prefix. +func flatten(prefix string, m map[string]any) map[string]any { + out := map[string]any{} + for k, v := range m { + key := prefix + "." + k + if nested, ok := v.(map[string]any); ok { + for sub, sv := range flatten(key, nested) { + out[sub] = sv + } + continue + } + out[key] = v + } + return out +} + +// goValue converts a parsed value to the Go value a reader sees. +// +// Integers arrive as int64 and floats as float64, matching what a TOML decoder produces, so a +// comparison against a baseline does not have to know which parser read the file. A date or time +// comes back as its text: nothing configures a node with one, and the text keeps such a key visible +// to a check rather than dropping it. +func goValue(v parser.Value) (any, error) { + switch d := v.X.(type) { + case parser.Token: + return tokenValue(d) + case parser.Array: + return arrayValue(d) + case parser.Inline: + return inlineValue(d) + default: + return nil, fmt.Errorf("unsupported value %T", v.X) + } +} + +// tokenValue converts a single literal. +func tokenValue(t parser.Token) (any, error) { + text := t.String() + switch t.Type { + case scanner.Word: + switch text { + case "true": + return true, nil + case "false": + return false, nil + } + return nil, fmt.Errorf("%q is not a value TOML recognizes", text) + case scanner.String, scanner.MString, scanner.LString, scanner.MLString: + return unquote(t.Type, text) + case scanner.Integer: + n, err := strconv.ParseInt(strings.ReplaceAll(text, "_", ""), 0, 64) + if err != nil { + return nil, fmt.Errorf("%q is not an integer: %w", text, err) + } + return n, nil + case scanner.Float: + x, err := strconv.ParseFloat(strings.ReplaceAll(text, "_", ""), 64) + if err != nil { + return nil, fmt.Errorf("%q is not a number: %w", text, err) + } + if math.IsInf(x, 0) || math.IsNaN(x) { + // TOML spells these as words, and ParseFloat accepts them, so a file can hold one. Refused + // here because writing one is refused: read but not writable, a rename or an edit of any + // other key in the file would fail on a value this package handed back. + return nil, fmt.Errorf("%q is not a finite number, and a configuration value has to be one", + text) + } + return x, nil + case scanner.DateTime, scanner.LocalDate, scanner.LocalTime, scanner.LocalDateTime: + return text, nil + default: + return nil, fmt.Errorf("%q is not a value (%v)", text, t.Type) + } +} + +// unquote strips a string literal's quoting. +// +// A basic string carries escapes and needs them decoded; a literal string reads exactly as written, +// which is the whole reason TOML has both. Getting this backwards turns a Windows path's +// backslashes into control characters. +func unquote(kind scanner.Token, text string) (string, error) { + switch kind { + case scanner.String: + s, err := strconv.Unquote(text) + if err != nil { + return "", fmt.Errorf("%s is not a well-formed string: %w", text, err) + } + return s, nil + case scanner.MString: + // Unquote reads Go's single-line syntax, so the literal newlines and tabs that make this form + // multi-line have to become escapes before it sees them. Left raw, Unquote rejects the whole + // string and a value an operator wrote is refused for having more than one line in it. + inner := strings.TrimPrefix(strings.TrimSuffix(strings.TrimPrefix(text, `"""`), `"""`), "\n") + escaped := strings.NewReplacer("\n", `\n`, "\r", `\r`, "\t", `\t`, `"`, `\"`).Replace(inner) + s, err := strconv.Unquote(`"` + escaped + `"`) + if err != nil { + return "", fmt.Errorf("%s is not a well-formed string: %w", text, err) + } + return s, nil + case scanner.LString: + return strings.TrimSuffix(strings.TrimPrefix(text, `'`), `'`), nil + case scanner.MLString: + inner := strings.TrimSuffix(strings.TrimPrefix(text, `'''`), `'''`) + return strings.TrimPrefix(inner, "\n"), nil + default: + return "", fmt.Errorf("%v is not a string", kind) + } +} + +// arrayValue converts an array, skipping the comment lines written between its items. +func arrayValue(a parser.Array) (any, error) { + out := make([]any, 0, len(a)) + for _, item := range a { + v, ok := item.(parser.Value) + if !ok { + continue // a comment between items + } + gv, err := goValue(v) + if err != nil { + return nil, err + } + out = append(out, gv) + } + return out, nil +} + +// inlineValue converts an inline table to a map its caller flattens. +func inlineValue(in parser.Inline) (any, error) { + out := map[string]any{} + for _, kv := range in { + if kv == nil { + continue + } + v, err := goValue(kv.Value) + if err != nil { + return nil, err + } + out[strings.ToLower(kv.Name.String())] = v + } + return out, nil +} From 8b24818355a2d1e4fdc20b2ccd69eb2c45b29409 Mon Sep 17 00:00:00 2001 From: bdchatham Date: Tue, 18 Aug 2026 13:26:31 -0700 Subject: [PATCH 02/24] config/seitoml: decode with TOML's grammar, and refuse the shapes it cannot hold Two independent reviews of this package found a class of defect this commit closes at its cause. Every fix is verified by reverting it and watching the covering test fail. The decoder used Go's string grammar where TOML's was needed. strconv.Unquote and strconv.Quote implement a different language from the one the file is written in, and the same dependency exports scanner.Unescape and scanner.Escape for the right one. Four defects followed from that single substitution: - a multi-line basic string containing an escaped quote made Values return nothing for the entire file, because the decoder escaped a quote the operator had already escaped - a backslash ending a line decoded to the characters backslash and n, where TOML folds the line break away - a file saved on Windows carried its carriage returns into every multi-line value, so a value differed from the default it matched and a diff reported a change nobody could see - a control character could be read and not written, because Go writes one as \x07 and TOML defines no such escape TOML also permits more shapes than a node's configuration uses, and each was accepted and then lost or corrupted further in. Parse now refuses an inline table, an array of tables, a repeated table heading, a key or heading that is not lower case, and a quoted key carrying a dot or a space. The inline-table case was the worst of them: Set on a new leaf defined the table a second time and produced a file BurntSushi/toml, a direct dependency of this repo, will not load, while Set returned nil and Values reported the file healthy. Refusing an inline table makes the reader unable to produce a map, which closes the last gap between what it reads and what the writer accepts. The flattening machinery those values needed is gone. Version now refuses a file whose schema counter is ahead of this binary. A release migrates the file on the node's own disk, so rolling the binary back does not roll the file back with it, and the older binary would otherwise apply only the keys it still recognises. Save no longer reports a failure after the new file is installed. Past the rename the new values are what the node reads, so a directory entry that has not been flushed is reported as ErrNotDurable rather than as a failed write. It also refuses a symbolic link, which it used to replace with a regular file while the link's target kept the old values. Three tests could not fail: - the mode-preservation test asserted 0600 stays 0600, and 0600 is the default for a new file, so it passed with the whole inheritance deleted. It now drives 0600, 0640 and 0644. - the atomicity test made the temporary file fail to create, so it never reached the install step its name and comment described. It now drives two failures that each leave a previous file to compare, and says plainly that a rename failing after the write is held by the ordering in Save rather than by a test. - one loop skipped its assertions behind a comment claiming an inline leaf was unreachable through Get. It was reachable. Removes setVersion and the error returns from insert and insertGlobal. transform.InsertMapping reports false only for a collision it was told not to replace, and both call sites tell it to replace, so four functions plumbed a failure that cannot occur. Coverage is 95.7% of statements. The fourteen uncovered statements are error propagations that need a filesystem fault or an input the parser rejects. Co-Authored-By: Claude Opus 5 (1M context) --- config/seitoml/doc.go | 28 ++- config/seitoml/edit.go | 47 ++-- config/seitoml/file.go | 168 +++++++++++-- config/seitoml/guards_test.go | 27 ++- config/seitoml/seitoml_test.go | 417 ++++++++++++++++++++++++++------- config/seitoml/values.go | 155 +++++++----- 6 files changed, 639 insertions(+), 203 deletions(-) diff --git a/config/seitoml/doc.go b/config/seitoml/doc.go index 7ab2418b14..bc3d804039 100644 --- a/config/seitoml/doc.go +++ b/config/seitoml/doc.go @@ -23,7 +23,9 @@ // ordering steps by release would run them in an order nobody intended. // // Nothing here migrates a file. This package reads the counter and writes it; the chain that acts on it -// arrives with the migrations themselves. +// arrives with the migrations themselves. A file whose counter is ahead of this binary's is refused, +// because a release migrates the file on the node's own disk and rolling the binary back does not roll +// the file back with it. Read anyway, the older binary would apply only the keys it still recognises. // // generated_by is provenance. Nothing reads it to decide anything, which is what lets it be absent // without consequence: the release reaches the binary through a linker flag the release build sets, @@ -45,4 +47,28 @@ // and a key declared as a float would resolve as one type from a node's own files and another from its // sei.toml. Infinities and NaN have no form here at all, and are refused in both directions rather than // written as a line no reader can load. +// +// # Shapes This File Does Not Carry +// +// TOML permits more shapes than a node's configuration uses, and Parse refuses these rather than +// reading them into something a later verb cannot write back: +// +// - an inline table, whose keys flatten into the same space a table's do, so editing one of them +// defines the table a second time and produces a file a conforming reader will not load +// - an array of tables, where every entry but the last disappears from the flattened key space +// - a table heading that appears twice, where an edit reaches the first and a read answers from the +// last +// - a key or heading segment that is not lower case, which is read under a name that is not the one +// written +// - a quoted key carrying a dot or a space, which no dotted spelling splits back into +// +// Each was previously accepted and then lost or corrupted further in. Refusing at the door is what +// lets every verb below assume the document holds only shapes it can read and write back, and it is +// only free while no operator has written a file that uses them. +// +// Escapes are decoded with the scanner's own rules rather than Go's. The two grammars differ in three +// places that each reach an operator's file: TOML has no \x escape, Go has no line-ending +// continuation, and Go's decoder rejects the literal newline that makes a multi-line string +// multi-line. A carriage return and newline pair inside a multi-line string reads as a newline, so a +// file saved on Windows holds the same values as the same file saved anywhere else. package seitoml diff --git a/config/seitoml/edit.go b/config/seitoml/edit.go index f8c71ab2c5..8f8716460c 100644 --- a/config/seitoml/edit.go +++ b/config/seitoml/edit.go @@ -9,6 +9,7 @@ import ( "github.com/creachadair/tomledit" "github.com/creachadair/tomledit/parser" + "github.com/creachadair/tomledit/scanner" "github.com/creachadair/tomledit/transform" ) @@ -31,7 +32,8 @@ func (f *File) Set(key string, v any) error { e.Value = value return nil } - return f.insert(path, value) + f.insert(path, value) + return nil } // insert adds a key the document does not have yet. @@ -39,37 +41,35 @@ func (f *File) Set(key string, v any) error { // A key with no dots belongs at the top level. Otherwise it goes in the table its prefix names, // which is created when it is absent so writing the first key of a section works without the // operator having to add the heading by hand. -func (f *File) insert(path parser.Key, value parser.Value) error { +func (f *File) insert(path parser.Key, value parser.Value) { leaf := parser.Key{path[len(path)-1]} kv := &parser.KeyValue{Name: leaf, Value: value} if len(path) == 1 { - return f.insertGlobal(kv) + f.insertGlobal(kv) + return } table := path[:len(path)-1] if e := transform.FindTable(f.doc, table...); e != nil { - if !transform.InsertMapping(e.Section, kv, true) { - return fmt.Errorf("could not write %s into the existing [%s] table", leaf, table.String()) - } - return nil + transform.InsertMapping(e.Section, kv, true) + return } f.doc.Sections = append(f.doc.Sections, &tomledit.Section{ Heading: &parser.Heading{Name: table}, Items: []parser.Item{kv}, }) - return nil } // insertGlobal adds a top-level key, creating the global section when the document has none. -func (f *File) insertGlobal(kv *parser.KeyValue) error { +// +// InsertMapping's result is not checked because it only reports a collision it was told not to +// replace, and it is told to replace. +func (f *File) insertGlobal(kv *parser.KeyValue) { if f.doc.Global == nil { f.doc.Global = &tomledit.Section{} } - if !transform.InsertMapping(f.doc.Global, kv, true) { - return fmt.Errorf("could not write %s at the top level", kv.Name.String()) - } - return nil + transform.InsertMapping(f.doc.Global, kv, true) } // SetPreamble puts a comment block at the top of the document, above everything else. @@ -125,9 +125,9 @@ func tomlValue(v any) (parser.Value, error) { case bool: return parser.ParseValue(strconv.FormatBool(x)) case string: - return parser.ParseValue(strconv.Quote(x)) + return parser.ParseValue(basicString(x)) case time.Duration: - return parser.ParseValue(strconv.Quote(x.String())) + return parser.ParseValue(basicString(x.String())) case int: return parser.ParseValue(quoteInt(int64(x))) case int32: @@ -146,8 +146,8 @@ func tomlValue(v any) (parser.Value, error) { return parser.ParseValue("[" + strings.Join(quoteEach(x), ", ") + "]") case []any: // The shape reading an array back produces. Without this, anything that reads a list and writes - // it again fails on a value this package handed it. An element this cannot render still fails, so - // the asymmetry closes without the writer accepting more than the reader can produce. + // it again fails on a value this package handed it. Every element is a value the reader can + // produce, and every one of those has a case above, so the reader and the writer agree. rendered := make([]string, 0, len(x)) for i, item := range x { element, err := tomlValue(item) @@ -183,11 +183,20 @@ func floatValue(x float64) (parser.Value, error) { return parser.ParseValue(text) } -// quoteEach quotes every element of a string list. +// basicString renders a Go string as a quoted TOML basic string. +// +// The escaping is the scanner's own rather than Go's. Go's quoter writes a control character as \x07 +// or \a and TOML defines neither, so such a value was refused with a diagnostic naming an offset into +// a string the operator never saw. +func basicString(s string) string { + return `"` + string(scanner.Escape(s)) + `"` +} + +// quoteEach renders every element of a string list. func quoteEach(ss []string) []string { out := make([]string, len(ss)) for i, s := range ss { - out[i] = strconv.Quote(s) + out[i] = basicString(s) } return out } diff --git a/config/seitoml/file.go b/config/seitoml/file.go index fab7b5139c..d981f6036f 100644 --- a/config/seitoml/file.go +++ b/config/seitoml/file.go @@ -2,6 +2,7 @@ package seitoml import ( "bytes" + "errors" "fmt" "io" "os" @@ -38,11 +39,13 @@ const ModeKey = "node_mode" // into a node that cannot read its own configuration. const GeneratedByKey = "generated_by" -// newFileMode is the permission a file created here gets. +// newFileMode is the permission a file created here gets, and only that. // -// Narrow rather than the usual 0644, because a configuration may name a private endpoint or an -// authentication token. An existing file keeps the mode it already has, since widening one an -// operator deliberately narrowed is worse than a default nobody wanted. +// A save onto an existing file inherits whatever mode that file already has, so this value describes +// the first save and nothing after it. Narrow rather than the usual 0644 because a configuration names +// the paths of a node's key files and its peers, and because it is the narrower of the two modes used +// by the files it consolidates. Widening one an operator deliberately narrowed is worse than a default +// nobody wanted, which is why the existing mode wins. const newFileMode os.FileMode = 0o600 // File is a parsed sei.toml that survives editing with its comments and layout intact. @@ -56,7 +59,99 @@ func Parse(r io.Reader) (*File, error) { if err != nil { return nil, fmt.Errorf("parse sei.toml: %w", err) } - return &File{doc: doc}, nil + f := &File{doc: doc} + if err := f.refuseUnsupportedShapes(); err != nil { + return nil, err + } + return f, nil +} + +// refuseUnsupportedShapes rejects TOML this format does not carry. +// +// TOML permits more shapes than a node's configuration uses, and each of these was previously accepted +// and then lost or corrupted somewhere downstream: a mixed-case key read back under a different name, +// an inline table that Set split into a second definition of the same table, an array of tables whose +// earlier entries vanished from Values. Refusing at the door is what keeps one answer per key, and it +// is only free while no operator has written a file that uses them. +func (f *File) refuseUnsupportedShapes() error { + headings := map[string]bool{} + // Every entry in Sections is a named table, so each carries a heading; the global section is a field + // of its own and is not in here. + for _, s := range f.doc.Sections { + if s.IsArray { + return fmt.Errorf("[[%s]] is an array of tables, which this file does not carry; every key "+ + "holds one value, so a repeated section has no reading", s.Name) + } + if err := keyIsAddressable(s.Name); err != nil { + return fmt.Errorf("table [%s]: %w", s.Name, err) + } + name := s.Name.String() + if headings[name] { + return fmt.Errorf("[%s] appears more than once, and an edit reaches only the first, so a "+ + "value written into this file would not be the one read back", name) + } + headings[name] = true + } + + var bad error + f.doc.Scan(func(full parser.Key, e *tomledit.Entry) bool { + if e.KeyValue == nil { + return true + } + if err := keyIsAddressable(full); err != nil { + bad = err + return false + } + if err := valueIsAddressable(full, e.Value); err != nil { + bad = err + return false + } + return true + }) + return bad +} + +// keyIsAddressable reports whether every segment of a key can be read back as written. +// +// A source enumerates lower-cased, so an upper-case segment is read under a name that is not the one +// in the file, and a segment carrying a dot or a space cannot be split back into the segments it came +// from. +func keyIsAddressable(key parser.Key) error { + for _, segment := range key { + if segment != strings.ToLower(segment) { + return fmt.Errorf("%q is not lower case, and this file's keys are read lower-cased, so it "+ + "would be read under a name that is not the one written here", segment) + } + if strings.ContainsAny(segment, ". ") { + return fmt.Errorf("%q carries a dot or a space, so it cannot be addressed: a key is split "+ + "on dots, and no spelling of this one splits back into it", segment) + } + } + return nil +} + +// valueIsAddressable rejects an inline table, at the top level of a value or inside an array. +// +// An inline table holds several keys in one written value. Its leaves flatten into the same dotted +// space a table's do, so a caller works in that space and an edit there defines the table a second +// time, producing a file a conforming reader refuses to load. +func valueIsAddressable(key parser.Key, v parser.Value) error { + switch x := v.X.(type) { + case parser.Inline: + return fmt.Errorf("%s is an inline table, which this file does not carry; write it as a [%s] "+ + "table so each key it holds can be edited on its own line", key, key) + case parser.Array: + for _, item := range x { + element, ok := item.(parser.Value) + if !ok { + continue + } + if err := valueIsAddressable(key, element); err != nil { + return err + } + } + } + return nil } // Load reads the document at path. @@ -87,7 +182,7 @@ func New(mode, generatedBy string) (*File, error) { "a file that omits it cannot be compared against this binary's defaults") } f := &File{doc: &tomledit.Document{Global: &tomledit.Section{}}} - if err := f.setVersion(SchemaVersion); err != nil { + if err := f.Set(VersionKey, SchemaVersion); err != nil { return nil, err } if err := f.Set(ModeKey, mode); err != nil { @@ -169,14 +264,18 @@ func (f *File) Version() (int, error) { if !ok { return 0, fmt.Errorf("%s is %T (%v), want an integer", VersionKey, v, v) } + if int(n) > SchemaVersion { + // The rollback case, and the reason the counter exists. A release migrates the file forward on + // the node's own disk, so rolling the binary back does not roll the file back with it. Read + // anyway, this binary would silently ignore every key the newer schema added or renamed and boot + // on a configuration neither release produced. + return 0, fmt.Errorf("sei.toml is at %s %d and this binary understands %d. It was written by a "+ + "newer release, so reading it would apply only the keys this binary still recognises", + VersionKey, n, SchemaVersion) + } return int(n), nil } -// setVersion writes the schema version at the document's top level. -func (f *File) setVersion(n int) error { - return f.Set(VersionKey, n) -} - // Bytes renders the document. func (f *File) Bytes() ([]byte, error) { var buf bytes.Buffer @@ -197,10 +296,9 @@ func (f *File) Save(path string) error { return err } - mode := newFileMode - if info, err := os.Stat(path); err == nil { - // An existing file keeps its own mode, so a save never widens what an operator narrowed. - mode = info.Mode().Perm() + mode, err := modeToWrite(path) + if err != nil { + return err } dir := filepath.Dir(path) @@ -216,12 +314,48 @@ func (f *File) Save(path string) error { }() if err := writeAndSync(tmp, raw, mode); err != nil { - return fmt.Errorf("write %s: %w", tmpName, err) + return fmt.Errorf("write %s: %w", path, err) } if err := os.Rename(tmpName, path); err != nil { return fmt.Errorf("install %s: %w", path, err) } - return syncDir(dir) + // Past this point the new file is what the node will read, so a failure to flush the directory + // entry is not a failure of the save. Returning one would tell a caller their change did not land + // when it did, and the next thing they do is write it again or open an incident. + if err := syncDir(dir); err != nil { + return fmt.Errorf("%w: %w", ErrNotDurable, err) + } + return nil +} + +// ErrNotDurable reports that a save installed the file and could not flush the directory entry. +// +// The values are in place and a reader sees them. Only their survival of a machine losing power before +// the filesystem flushes on its own is unproven, so a caller that treats this as a failed write is +// wrong about what happened. +var ErrNotDurable = errors.New("the file is installed and its directory entry is not yet flushed") + +// modeToWrite returns the permission a save should use, and refuses a destination it must not replace. +// +// An existing file keeps its own mode, so a save never widens what an operator narrowed. A symbolic +// link is refused: renaming onto one replaces the link with a regular file, leaving whatever it pointed +// at holding the old values, and nothing about the result says the link is gone. +func modeToWrite(path string) (os.FileMode, error) { + info, err := os.Lstat(path) + switch { + case err != nil: + return newFileMode, nil // no file there yet, which is the ordinary first save + case info.Mode()&os.ModeSymlink != 0: + target, err := os.Readlink(path) + if err != nil { + target = "somewhere this process cannot read" + } + return 0, fmt.Errorf("%s is a symbolic link to %s. Writing here would replace the link with a "+ + "regular file and leave %s holding the old values; edit the target directly", path, target, + target) + default: + return info.Mode().Perm(), nil + } } // writeAndSync writes the whole payload, sets the mode, and flushes to the device. diff --git a/config/seitoml/guards_test.go b/config/seitoml/guards_test.go index 90fad4a61d..ecb2b951f0 100644 --- a/config/seitoml/guards_test.go +++ b/config/seitoml/guards_test.go @@ -47,6 +47,22 @@ func TestAValueShapeWithNoDecoderIsRefused(t *testing.T) { } } +// TestAnUndefinedEscapeIsRefusedRatherThanReplaced covers the scanner's substitution behaviour. +// +// The scanner writes a replacement rune for an escape TOML does not define rather than failing, so +// without this check a typo such as \q would reach a node as U+FFFD inside a configuration value. The +// parser rejects the escape before a file can carry one, which is why this drives the decoder directly. +func TestAnUndefinedEscapeIsRefusedRatherThanReplaced(t *testing.T) { + if _, err := unescape(`"a\qc"`, `a\qc`); err == nil { + t.Error("an escape TOML does not define decoded without complaint") + } + // A replacement rune the operator actually wrote is theirs to keep. + got, err := unescape("\"a\ufffdc\"", "a\ufffdc") + if err != nil || got != "a\ufffdc" { + t.Errorf("a written replacement rune came back as (%q, %v), want it preserved", got, err) + } +} + // TestUnquoteRefusesAKindThatIsNotAString covers the string decoder's own guard. // // unquote picks the escaping rules from the token kind, so a kind it does not know has no rules to @@ -57,17 +73,6 @@ func TestUnquoteRefusesAKindThatIsNotAString(t *testing.T) { } } -// TestAnInlineTableSkipsAnEntryCarryingNothing covers the nil entry a malformed inline table produces. -func TestAnInlineTableSkipsAnEntryCarryingNothing(t *testing.T) { - got, err := inlineValue(parser.Inline{nil}) - if err != nil { - t.Fatalf("inlineValue: %v", err) - } - if len(got.(map[string]any)) != 0 { - t.Errorf("an inline table of one empty entry decoded to %#v, want nothing", got) - } -} - // TestATopLevelKeyReachesADocumentWithNoGlobalSection covers a document built rather than parsed. // // Parsing always produces a global section, even for a file whose first line is a table heading, so diff --git a/config/seitoml/seitoml_test.go b/config/seitoml/seitoml_test.go index afcdead593..a4e7096917 100644 --- a/config/seitoml/seitoml_test.go +++ b/config/seitoml/seitoml_test.go @@ -1,6 +1,7 @@ package seitoml_test import ( + "errors" "fmt" "math" "os" @@ -355,28 +356,108 @@ func TestSetCreatesTheTableWhenTheSectionIsNew(t *testing.T) { } } -// TestValuesFlattensAnInlineTable keeps an inline table's leaves visible. +// TestAShapeThisFileDoesNotCarryIsRefusedAtTheDoor drives the shapes Parse rejects. // -// An inline table is one written line holding several keys. Left nested, its leaves are invisible -// to any check that walks declared keys, so an operator could write a setting nothing validates. -func TestValuesFlattensAnInlineTable(t *testing.T) { - f := parse(t, "schema_version = 1\n[state-commit]\nflatkv = { enable = true, dir = \"/data\" }\n") +// TOML permits more shapes than a node's configuration uses, and each of these was accepted and then +// lost or corrupted further in. Refusing at Parse is what keeps one answer per key: every later verb +// can then assume the document holds only shapes it can read and write back. +// +// Each case is a shape an operator could reasonably write, so each refusal has to name what is wrong +// and what to write instead. +func TestAShapeThisFileDoesNotCarryIsRefusedAtTheDoor(t *testing.T) { + for _, tc := range []struct { + name string + body string + want string + }{ + { + "an inline table", + "[state-commit]\nflatkv = { enable = true, dir = \"/data\" }\n", + "is an inline table", + }, + { + "an inline table inside an array", + "[p]\npeers = [{ host = \"a\" }]\n", + "is an inline table", + }, + { + "an array of tables", + "[[peer]]\nhost = \"a\"\n\n[[peer]]\nhost = \"b\"\n", + "is an array of tables", + }, + { + "a repeated table heading", + "[probe]\nn = 1\n\n[probe]\nn = 2\n", + "appears more than once", + }, + { + "an upper-case key", + "[probe]\nEnabled = true\n", + "is not lower case", + }, + { + "an upper-case table heading", + "[Probe]\nenabled = true\n", + "is not lower case", + }, + { + "a quoted key carrying a dot", + "[probe]\n\"a.b\" = 1\n", + "carries a dot or a space", + }, + { + "a quoted key carrying a space", + "[probe]\n\"a b\" = 1\n", + "carries a dot or a space", + }, + } { + t.Run(tc.name, func(t *testing.T) { + _, err := seitoml.Parse(strings.NewReader(tc.body)) + if err == nil { + t.Fatalf("%s parsed; every verb below Parse assumes it cannot appear", tc.name) + } + if !strings.Contains(err.Error(), tc.want) { + t.Errorf("the refusal reads %q, which does not mention %q", err, tc.want) + } + }) + } +} + +// TestTheShapesThisFileDoesCarryStillParse is the other half, so the refusals cannot pass by refusing +// everything. +// +// A table, a dotted key inside one, a lower-case hyphenated name and an array of scalars are what a +// node's configuration is written with, and each has to survive the check above. +func TestTheShapesThisFileDoesCarryStillParse(t *testing.T) { + f := parse(t, `schema_version = 1 +node_mode = "validator" + +[state-commit] +sc-async-commit-buffer = 100 + +[state-commit.flatkv] +enable = true +dir = "/data" + +[p2p] +persistent-peers = ["a", "b"] +`) values, err := f.Values() if err != nil { t.Fatalf("Values: %v", err) } - - if values["state-commit.flatkv.enable"] != true { - t.Errorf("the inline table's leaf is not reachable as a dotted key: %v.\n\nA key nothing can "+ - "see is a setting nothing validates", values) - } - if values["state-commit.flatkv.dir"] != "/data" { - t.Errorf("the second leaf read %#v, want /data", values["state-commit.flatkv.dir"]) + for key, want := range map[string]any{ + "state-commit.sc-async-commit-buffer": int64(100), + "state-commit.flatkv.enable": true, + "state-commit.flatkv.dir": "/data", + } { + if values[key] != want { + t.Errorf("%s read back as %#v, want %#v", key, values[key], want) + } } - if _, nested := values["state-commit.flatkv"]; nested { - t.Errorf("the table itself is also reported as a value, so a check would see a key whose "+ - "value is a map: %v", values) + if got, ok := values["p2p.persistent-peers"].([]any); !ok || len(got) != 2 { + t.Errorf("the peer list read back as %#v, want two elements", values["p2p.persistent-peers"]) } } @@ -403,41 +484,82 @@ func TestAnUnsupportedTypeIsRefused(t *testing.T) { } } -// TestSaveLandsInFullOrNotAtAll holds the atomicity the boot depends on. +// TestAFailedSaveLeavesThePreviousFileExactlyAsItWas holds what a caller relies on after an error. // // A configuration file truncated by a crash mid-write is one the node cannot parse, so a save that -// cannot complete must leave the previous file exactly as it was. Driven by making the install step -// fail, which is the part that would otherwise have already replaced the file. -func TestSaveLandsInFullOrNotAtAll(t *testing.T) { - dir := t.TempDir() - path := filepath.Join(dir, "sei.toml") - if err := os.WriteFile(path, []byte(commented), 0o600); err != nil { - t.Fatalf("seed the file: %v", err) +// cannot complete must leave the previous file byte for byte. Two ways to fail are driven, because +// they fail at different points and only one of them creates a temporary file to clean up. +// +// Neither reaches a rename that fails after the temporary file is written. That path needs the rename +// itself to fail with the destination writable, which no input to this package produces, so what holds +// it is the ordering in Save rather than a test. +func TestAFailedSaveLeavesThePreviousFileExactlyAsItWas(t *testing.T) { + if os.Geteuid() == 0 { + // A mode of 0500 does not stop uid 0, so the save would succeed and the failure would look + // like a defect in Save rather than a test that cannot run as root. + t.Skip("this drives failure through directory permissions, which do not apply to uid 0") } - f := parse(t, commented) - if err := f.Set("giga_executor.occ_enabled", true); err != nil { - t.Fatalf("Set: %v", err) - } - // A directory the process cannot write is how the temporary file, and therefore the install, - // is made to fail without the previous file being touched. - if err := os.Chmod(dir, 0o500); err != nil { - t.Fatalf("chmod: %v", err) - } - t.Cleanup(func() { _ = os.Chmod(dir, 0o700) }) + for _, tc := range []struct { + name string + arrange func(t *testing.T, dir, path string) + }{ + { + "a directory the process cannot write", + func(t *testing.T, dir, _ string) { + if err := os.Chmod(dir, 0o500); err != nil { + t.Fatalf("chmod: %v", err) + } + t.Cleanup(func() { _ = os.Chmod(dir, 0o700) }) + }, + }, + { + "a destination that is a symbolic link", + func(t *testing.T, dir, path string) { + target := filepath.Join(dir, "managed.toml") + if err := os.WriteFile(target, []byte("schema_version = 1\n"), 0o600); err != nil { + t.Fatalf("seed the target: %v", err) + } + if err := os.Remove(path); err != nil { + t.Fatalf("clear the seeded file: %v", err) + } + if err := os.Symlink(target, path); err != nil { + t.Fatalf("link: %v", err) + } + }, + }, + } { + t.Run(tc.name, func(t *testing.T) { + dir := t.TempDir() + path := filepath.Join(dir, "sei.toml") + if err := os.WriteFile(path, []byte(commented), 0o600); err != nil { + t.Fatalf("seed the file: %v", err) + } + tc.arrange(t, dir, path) + // Read after arranging, so this is what is on disk immediately before the save rather than + // what the seed wrote. The symlink case deliberately points somewhere else. + before, err := os.ReadFile(path) //nolint:gosec // a path this test created under t.TempDir + if err != nil { + t.Fatalf("read what is on disk before the save: %v", err) + } - if err := f.Save(path); err == nil { - t.Fatal("Save reported success in a directory it cannot write, so a caller would believe " + - "the new configuration is on disk") - } + f := parse(t, commented) + if err := f.Set("giga_executor.occ_enabled", true); err != nil { + t.Fatalf("Set: %v", err) + } + if err := f.Save(path); err == nil { + t.Fatal("Save reported success, so a caller would believe the new configuration is on disk") + } - raw, err := os.ReadFile(path) //nolint:gosec // a path this test created under t.TempDir - if err != nil { - t.Fatalf("the previous file is unreadable after a failed save: %v", err) - } - if string(raw) != commented { - t.Errorf("a failed save changed the file on disk. It now reads:\n%s\n\nThe node would boot "+ - "from something nobody wrote", raw) + raw, err := os.ReadFile(path) //nolint:gosec // a path this test created under t.TempDir + if err != nil { + t.Fatalf("the previous file is unreadable after a failed save: %v", err) + } + if string(raw) != string(before) { + t.Errorf("a failed save changed what is on disk. It now reads:\n%s\n\nThe node would "+ + "boot from something nobody wrote", raw) + } + }) } } @@ -467,36 +589,44 @@ func TestSaveLeavesNoTemporaryFileBehind(t *testing.T) { } } -// TestSaveKeepsAnExistingFilesPermissions holds that a save never widens access. +// TestSaveKeepsAnExistingFilesPermissions holds that a save carries the mode it found. // -// A configuration may name a private endpoint or carry a token. An operator who narrowed the file -// deliberately would have that undone by a save, silently, and nothing about the change is visible -// in the file's contents. +// A configuration names the paths of a node's key files and its peers. An operator who narrowed the +// file deliberately would have that undone by a save, silently, and nothing about the change is +// visible in the file's contents. The reverse matters as much: a save is not the place to impose a +// mode, so a file an operator or an init step left wider stays as they left it. +// +// Both directions are driven, because a mode equal to newFileMode proves nothing. Asserting only that +// a 0600 file stays 0600 passes with the whole inheritance removed. func TestSaveKeepsAnExistingFilesPermissions(t *testing.T) { - dir := t.TempDir() - path := filepath.Join(dir, "sei.toml") - if err := os.WriteFile(path, []byte(commented), 0o640); err != nil { - t.Fatalf("seed: %v", err) - } - if err := os.Chmod(path, 0o600); err != nil { - t.Fatalf("chmod: %v", err) - } + for _, mode := range []os.FileMode{0o600, 0o640, 0o644} { + t.Run(fmt.Sprintf("%#o", mode), func(t *testing.T) { + path := filepath.Join(t.TempDir(), "sei.toml") + if err := os.WriteFile(path, []byte(commented), mode); err != nil { + t.Fatalf("seed: %v", err) + } + if err := os.Chmod(path, mode); err != nil { + t.Fatalf("chmod past the umask: %v", err) + } - f := parse(t, commented) - if err := f.Set("giga_executor.enabled", false); err != nil { - t.Fatalf("Set: %v", err) - } - if err := f.Save(path); err != nil { - t.Fatalf("Save: %v", err) - } + f := parse(t, commented) + if err := f.Set("giga_executor.enabled", false); err != nil { + t.Fatalf("Set: %v", err) + } + if err := f.Save(path); err != nil { + t.Fatalf("Save: %v", err) + } - info, err := os.Stat(path) - if err != nil { - t.Fatalf("Stat: %v", err) - } - if got := info.Mode().Perm(); got != 0o600 { - t.Errorf("the file's mode moved from 0600 to %#o. A save that widens access undoes a "+ - "restriction an operator chose, and the file's contents do not show it", got) + info, err := os.Stat(path) + if err != nil { + t.Fatalf("Stat: %v", err) + } + if got := info.Mode().Perm(); got != mode { + t.Errorf("the file's mode moved from %#o to %#o. A save that changes access either "+ + "undoes a restriction an operator chose or imposes one they did not, and the "+ + "file's contents do not show it", mode, got) + } + }) } } @@ -784,6 +914,12 @@ first line second line""" verbatim = ''' kept \as \written''' +escaped = """say \"hi\" here""" +folded_onto_one_line = """a\ + b""" +literal_backslash = """C:\\ +next""" +coded = "a\u0062c" grouped = 1_000_000 hex = 0x1f ratio = 2.5 @@ -794,8 +930,6 @@ commented = [ "a", "b", ] -inline = { host = "h", port = 26657 } -nested = { outer = { inner = 3 } } `) values, err := f.Values() @@ -803,18 +937,21 @@ nested = { outer = { inner = 3 } } t.Fatalf("Values: %v", err) } for key, want := range map[string]any{ - "probe.flag": true, - "probe.basic": "a\ttab", - "probe.literal": `C:\Users\node`, - "probe.folded": "first line\nsecond line", - "probe.verbatim": `kept \as \written`, - "probe.grouped": int64(1000000), - "probe.hex": int64(31), - "probe.ratio": 2.5, - "probe.stamped": "2026-08-18", - "probe.inline.host": "h", - "probe.inline.port": int64(26657), - "probe.nested.outer.inner": int64(3), + "probe.flag": true, + "probe.basic": "a\ttab", + "probe.literal": `C:\Users\node`, + "probe.folded": "first line\nsecond line", + "probe.verbatim": `kept \as \written`, + "probe.grouped": int64(1000000), + "probe.hex": int64(31), + "probe.ratio": 2.5, + "probe.stamped": "2026-08-18", + "probe.escaped": `say "hi" here`, + "probe.coded": "abc", + // A backslash ending a line folds the line break away; a doubled one is an escaped backslash + // and keeps the break, so the two cannot be handled by the same rule. + "probe.folded_onto_one_line": "ab", + "probe.literal_backslash": "C:\\\nnext", } { got, ok := values[key] if !ok { @@ -844,14 +981,9 @@ nested = { outer = { inner = 3 } } // Get answers for one key the same way Values does for all of them, since a caller reading a single // key must not get a different decoding from one reading the file. - for _, key := range []string{"probe.literal", "probe.folded", "probe.hex", "probe.inline.port"} { + for _, key := range []string{"probe.literal", "probe.folded", "probe.verbatim", "probe.hex"} { got, present, err := f.Get(key) if err != nil || !present { - // An inline table's leaf is reachable through Values and not through Get, which walks the - // document rather than the flattened space. - if strings.Contains(key, "inline") { - continue - } t.Errorf("Get(%q) = (%#v, %v, %v)", key, got, present, err) continue } @@ -883,7 +1015,6 @@ func TestAValueTomlDoesNotRecognizeIsNamedNotGuessed(t *testing.T) { {"a negative infinity", "[probe]\nn = -inf\n", "probe.n", "not a finite number"}, {"a NaN", "[probe]\nn = nan\n", "probe.n", "not a finite number"}, {"an infinity inside an array", "[probe]\nlist = [1.5, inf]\n", "probe.list", "not a finite number"}, - {"an infinity inside an inline table", "[probe]\nt = { a = inf }\n", "probe.t", "not a finite number"}, } { t.Run(tc.name, func(t *testing.T) { f := parse(t, tc.body) @@ -1223,3 +1354,107 @@ func TestAListCarryingAValueThatCannotBeWrittenNamesTheElement(t *testing.T) { t.Errorf("the refusal reads %q and does not say which element is at fault", err) } } + +// TestAFileFromANewerReleaseIsRefused holds the guard the schema counter exists for. +// +// A release migrates the file forward on the node's own disk, so rolling the binary back does not roll +// the file back with it. Read anyway, the older binary applies only the keys it still recognises and +// boots on a configuration neither release produced, with nothing reporting it. +func TestAFileFromANewerReleaseIsRefused(t *testing.T) { + ahead := fmt.Sprintf("schema_version = %d\nnode_mode = \"validator\"\n", seitoml.SchemaVersion+1) + + _, err := parse(t, ahead).Version() + if err == nil { + t.Fatal("a file from a newer release was read, so this binary would apply only the keys it " + + "still recognises and boot on a configuration neither release produced") + } + for _, want := range []string{ + fmt.Sprint(seitoml.SchemaVersion + 1), + fmt.Sprint(seitoml.SchemaVersion), + } { + if !strings.Contains(err.Error(), want) { + t.Errorf("the refusal reads %q and does not mention %q; an operator cannot tell how far "+ + "ahead the file is", err, want) + } + } + + // The current version and every version behind it still read, so the guard cannot pass by refusing + // everything. Behind is what a migration exists to move forward. + for v := 1; v <= seitoml.SchemaVersion; v++ { + body := fmt.Sprintf("schema_version = %d\nnode_mode = \"validator\"\n", v) + if got, err := parse(t, body).Version(); err != nil || got != v { + t.Errorf("a file at version %d read as (%d, %v), want it accepted", v, got, err) + } + } +} + +// TestAnUnflushedDirectoryEntryIsNotAFailedSave separates two outcomes a caller must not confuse. +// +// After the rename the new values are what the node reads. A directory entry that has not been flushed +// only leaves their survival of a power loss unproven, so reporting it the same way as a failed write +// tells an operator their change did not land when it did. The next thing they do is write it again or +// open an incident. +func TestAnUnflushedDirectoryEntryIsNotAFailedSave(t *testing.T) { + if os.Geteuid() == 0 { + t.Skip("this drives failure through directory permissions, which do not apply to uid 0") + } + + dir := t.TempDir() + path := filepath.Join(dir, "sei.toml") + f, err := seitoml.New("validator", "") + if err != nil { + t.Fatalf("New: %v", err) + } + if err := f.Set("probe.value", 7); err != nil { + t.Fatalf("Set: %v", err) + } + + // Write and traverse but not read, which is enough for the temporary file and the rename and not + // enough to open the directory afterwards. + if err := os.Chmod(dir, 0o300); err != nil { + t.Fatalf("chmod: %v", err) + } + t.Cleanup(func() { _ = os.Chmod(dir, 0o700) }) + + saveErr := f.Save(path) + if err := os.Chmod(dir, 0o700); err != nil { + t.Fatalf("restore: %v", err) + } + + if saveErr != nil && !errors.Is(saveErr, seitoml.ErrNotDurable) { + t.Fatalf("Save reported %v, which a caller reads as the file not being written", saveErr) + } + reread, err := seitoml.Load(path) + if err != nil { + t.Fatalf("the file is not readable after the save: %v", err) + } + got, present, err := reread.Get("probe.value") + if err != nil || !present || got != int64(7) { + t.Errorf("the value on disk is (%#v, %v, %v), want 7. The rename completed, so the new "+ + "configuration is what the node reads whatever the sync reported", got, present, err) + } +} + +// TestAFileWrittenOnWindowsReadsTheSameValues covers the line ending an editor leaves behind. +// +// An operator editing on Windows produces a file whose lines end with a carriage return. A multi-line +// string's value begins after the delimiter's own newline, and matching only the Unix form leaves the +// carriage return inside the value, so it differs from the default it matches and a diff reports a +// change nobody can see. +func TestAFileWrittenOnWindowsReadsTheSameValues(t *testing.T) { + const unix = "schema_version = 1\nnode_mode = \"validator\"\n\n[probe]\nfolded = \"\"\"\nfirst\nsecond\"\"\"\nflag = true\n" + windows := strings.ReplaceAll(unix, "\n", "\r\n") + + want, err := parse(t, unix).Values() + if err != nil { + t.Fatalf("the Unix file: %v", err) + } + got, err := parse(t, windows).Values() + if err != nil { + t.Fatalf("the Windows file: %v", err) + } + if !reflect.DeepEqual(got, want) { + t.Errorf("the same file read\n %#v\nwith carriage returns and\n %#v\nwithout. A value that "+ + "differs by line ending differs from the default it matches", got, want) + } +} diff --git a/config/seitoml/values.go b/config/seitoml/values.go index 49ab7c0e0e..f45513a2ae 100644 --- a/config/seitoml/values.go +++ b/config/seitoml/values.go @@ -1,10 +1,12 @@ package seitoml import ( + "bytes" "fmt" "math" "strconv" "strings" + "unicode/utf8" "github.com/creachadair/tomledit" "github.com/creachadair/tomledit/parser" @@ -33,15 +35,6 @@ func (f *File) Values() (map[string]any, error) { bad = fmt.Errorf("%s: %w", key, err) return false } - // An inline table is one written value holding several keys, so it flattens into the same - // dotted space as a table would. Left nested, its leaves would be invisible to any check - // that walks declared keys. - if inline, ok := v.(map[string]any); ok { - for sub, sv := range flatten(key, inline) { - out[sub] = sv - } - return true - } out[key] = v return true }) @@ -68,36 +61,21 @@ func (f *File) Get(key string) (any, bool, error) { return v, true, nil } -// flatten expands an inline table into dotted keys under prefix. -func flatten(prefix string, m map[string]any) map[string]any { - out := map[string]any{} - for k, v := range m { - key := prefix + "." + k - if nested, ok := v.(map[string]any); ok { - for sub, sv := range flatten(key, nested) { - out[sub] = sv - } - continue - } - out[key] = v - } - return out -} - // goValue converts a parsed value to the Go value a reader sees. // // Integers arrive as int64 and floats as float64, matching what a TOML decoder produces, so a -// comparison against a baseline does not have to know which parser read the file. A date or time -// comes back as its text: nothing configures a node with one, and the text keeps such a key visible -// to a check rather than dropping it. +// comparison against a default does not have to know which parser read the file. A date or time comes +// back as its text: nothing configures a node with one, and the text keeps such a key visible to a +// check rather than dropping it. +// +// There is no case for an inline table, because Parse refuses one. Every value a reader sees is +// therefore one this package can also write back. func goValue(v parser.Value) (any, error) { switch d := v.X.(type) { case parser.Token: return tokenValue(d) case parser.Array: return arrayValue(d) - case parser.Inline: - return inlineValue(d) default: return nil, fmt.Errorf("unsupported value %T", v.X) } @@ -143,40 +121,105 @@ func tokenValue(t parser.Token) (any, error) { } } -// unquote strips a string literal's quoting. +// unquote strips a string literal's quoting and decodes its escapes. // // A basic string carries escapes and needs them decoded; a literal string reads exactly as written, // which is the whole reason TOML has both. Getting this backwards turns a Windows path's // backslashes into control characters. +// +// The decoding is the scanner's own rather than Go's. The two grammars differ in three places that +// each reach an operator's file: TOML has no \x escape, Go has no line-ending continuation, and Go's +// decoder rejects the literal newline that makes a multi-line string multi-line. func unquote(kind scanner.Token, text string) (string, error) { switch kind { case scanner.String: - s, err := strconv.Unquote(text) - if err != nil { - return "", fmt.Errorf("%s is not a well-formed string: %w", text, err) - } - return s, nil + return unescape(text, strings.TrimSuffix(strings.TrimPrefix(text, `"`), `"`)) case scanner.MString: - // Unquote reads Go's single-line syntax, so the literal newlines and tabs that make this form - // multi-line have to become escapes before it sees them. Left raw, Unquote rejects the whole - // string and a value an operator wrote is refused for having more than one line in it. - inner := strings.TrimPrefix(strings.TrimSuffix(strings.TrimPrefix(text, `"""`), `"""`), "\n") - escaped := strings.NewReplacer("\n", `\n`, "\r", `\r`, "\t", `\t`, `"`, `\"`).Replace(inner) - s, err := strconv.Unquote(`"` + escaped + `"`) - if err != nil { - return "", fmt.Errorf("%s is not a well-formed string: %w", text, err) - } - return s, nil + inner := unixNewlines(strings.TrimSuffix(strings.TrimPrefix(text, `"""`), `"""`)) + return unescape(text, foldContinuations(trimOpeningNewline(inner))) case scanner.LString: return strings.TrimSuffix(strings.TrimPrefix(text, `'`), `'`), nil case scanner.MLString: - inner := strings.TrimSuffix(strings.TrimPrefix(text, `'''`), `'''`) - return strings.TrimPrefix(inner, "\n"), nil + inner := unixNewlines(strings.TrimSuffix(strings.TrimPrefix(text, `'''`), `'''`)) + return trimOpeningNewline(inner), nil default: return "", fmt.Errorf("%v is not a string", kind) } } +// unescape decodes a basic string's escapes, refusing one TOML does not define. +// +// The scanner substitutes a replacement rune for an undefined escape rather than failing, so a typo +// such as \q would otherwise reach a node as U+FFFD inside its configuration. text is the literal as +// written, so a refusal quotes what the operator typed rather than the decoded form. +func unescape(text, inner string) (string, error) { + out, err := scanner.Unescape([]byte(inner)) + if err != nil { + return "", fmt.Errorf("%s is not a well-formed string: %w", text, err) + } + if bytes.ContainsRune(out, utf8.RuneError) && !strings.ContainsRune(inner, utf8.RuneError) { + return "", fmt.Errorf("%s carries an escape TOML does not define", text) + } + return string(out), nil +} + +// unixNewlines rewrites a carriage return and newline pair as a newline alone. +// +// The line ending an editor chose is not part of the value. A default in the binary carries a bare +// newline, so a file saved on Windows would differ from a default it matches, on every line, and a diff +// would report a change nobody can see. Rendering already writes bare newlines, so this is what makes +// reading agree with writing. +// +// A carriage return the operator wrote as an escape is two characters here and survives, since escapes +// are decoded after this runs. +func unixNewlines(s string) string { return strings.ReplaceAll(s, "\r\n", "\n") } + +// trimOpeningNewline drops the newline TOML allows immediately after a multi-line delimiter, so the +// value starts at the operator's first line of content. +func trimOpeningNewline(s string) string { return strings.TrimPrefix(s, "\n") } + +// foldContinuations removes a backslash that ends a line together with the whitespace following it. +// +// This is how TOML lets one value span several lines without carrying the newlines into it. An escape +// consumes the character after it, so a doubled backslash is written through rather than read as a +// continuation; otherwise a value ending in a path separator would swallow the next line. +func foldContinuations(s string) string { + var out strings.Builder + for i := 0; i < len(s); { + if s[i] != '\\' { + out.WriteByte(s[i]) + i++ + continue + } + if end, folded := continuationEnd(s, i); folded { + i = end + continue + } + out.WriteByte(s[i]) + i++ + if i < len(s) { + out.WriteByte(s[i]) + i++ + } + } + return out.String() +} + +// continuationEnd reports where a continuation starting at the backslash in s[i] ends, if it is one. +func continuationEnd(s string, i int) (int, bool) { + j := i + 1 + for j < len(s) && (s[j] == ' ' || s[j] == '\t') { + j++ + } + if j >= len(s) || (s[j] != '\n' && s[j] != '\r') { + return 0, false + } + for j < len(s) && (s[j] == ' ' || s[j] == '\t' || s[j] == '\r' || s[j] == '\n') { + j++ + } + return j, true +} + // arrayValue converts an array, skipping the comment lines written between its items. func arrayValue(a parser.Array) (any, error) { out := make([]any, 0, len(a)) @@ -193,19 +236,3 @@ func arrayValue(a parser.Array) (any, error) { } return out, nil } - -// inlineValue converts an inline table to a map its caller flattens. -func inlineValue(in parser.Inline) (any, error) { - out := map[string]any{} - for _, kv := range in { - if kv == nil { - continue - } - v, err := goValue(kv.Value) - if err != nil { - return nil, err - } - out[strings.ToLower(kv.Name.String())] = v - } - return out, nil -} From f70ad19b037f8f93aa098262349a31144a5ac761 Mon Sep 17 00:00:00 2001 From: bdchatham Date: Tue, 18 Aug 2026 13:40:20 -0700 Subject: [PATCH 03/24] config/seitoml: call a default a default The comments called the value a binary carries for an absent key a baseline. It is a default, which is the word the registry uses for the same thing, and one word for one thing is what keeps the two packages readable together. floatValue's godoc used "by default" in the other sense, so with the rename the word would have carried two meanings in one file. Co-Authored-By: Claude Opus 5 (1M context) --- config/seitoml/doc.go | 6 +++--- config/seitoml/edit.go | 6 +++--- config/seitoml/file.go | 6 +++--- config/seitoml/seitoml_test.go | 12 ++++++------ 4 files changed, 15 insertions(+), 15 deletions(-) diff --git a/config/seitoml/doc.go b/config/seitoml/doc.go index bc3d804039..8cc5ec0fcb 100644 --- a/config/seitoml/doc.go +++ b/config/seitoml/doc.go @@ -1,8 +1,8 @@ // Package seitoml reads, edits and writes the node's sei.toml. // // The file holds only what an operator decided. A key present in it is authoritative; a key absent -// from it resolves to the running binary's baseline for the node's mode. Nothing here writes a -// baseline into the file, because a value the binary put there reads exactly like one an operator +// from it resolves to the running binary's default for the node's mode. Nothing here writes a +// default into the file, because a value the binary put there reads exactly like one an operator // chose. // // Three keys at the top level describe the file rather than configure the node, and Values leaves @@ -10,7 +10,7 @@ // keys no section owns. // // schema_version which migration the file has reached -// node_mode which mode's baselines its values were chosen against +// node_mode which mode's defaults its values were chosen against // generated_by which release last produced or transformed it // // The first two are machinery and the third is not, and the difference matters enough to state. diff --git a/config/seitoml/edit.go b/config/seitoml/edit.go index 8f8716460c..44593d1933 100644 --- a/config/seitoml/edit.go +++ b/config/seitoml/edit.go @@ -97,8 +97,8 @@ func (f *File) SetPreamble(lines []string) { // Unset removes a key and reports whether the file carried one. // // This removes the key rather than writing a zero, because an absent key resolves to the running -// binary's baseline. A key set to its baseline value looks identical in the file but is a commitment -// that survives a release changing that baseline, which is the opposite of what unset means. +// binary's default. A key set to its default value looks identical in the file but is a commitment +// that survives a release changing that default, which is the opposite of what unset means. func (f *File) Unset(key string) (bool, error) { path, err := keyOf(key) if err != nil { @@ -162,7 +162,7 @@ func tomlValue(v any) (parser.Value, error) { } } -// floatValue renders a float as a TOML float, which an integral one is not by default. +// floatValue renders a float as a TOML float, which the shortest form of an integral one is not. // // TOML tells a float from an integer by the fractional part or the exponent, and the shortest form of // 1.0 is "1", which reads back as an integer. A key declared as a float would then resolve as one type diff --git a/config/seitoml/file.go b/config/seitoml/file.go index d981f6036f..8200dc1193 100644 --- a/config/seitoml/file.go +++ b/config/seitoml/file.go @@ -25,8 +25,8 @@ const VersionKey = "schema_version" // ModeKey records which node mode the file's values resolve for. // -// At the top level beside VersionKey, not inside a section, because the mode selects which baselines -// apply and so cannot itself have a per-mode baseline. It is also the only durable record of an +// At the top level beside VersionKey, not inside a section, because the mode selects which defaults +// apply and so cannot itself have a per-mode default. It is also the only durable record of an // archive node: seid init writes config.toml's mode as "full" for one, since Tendermint has no // archive mode, so nothing else on disk distinguishes the two. const ModeKey = "node_mode" @@ -223,7 +223,7 @@ func (f *File) GeneratedBy() (string, bool) { // Mode returns the node mode the file's values resolve for. // // An absent mode is an error rather than a guess. Guessing picks one binary's idea of a default and -// silently compares an archive node's file against a validator's baselines, which is the mistake +// silently compares an archive node's file against a validator's defaults, which is the mistake // this key exists to make impossible. func (f *File) Mode() (string, error) { e := f.doc.First(ModeKey) diff --git a/config/seitoml/seitoml_test.go b/config/seitoml/seitoml_test.go index a4e7096917..44858599bc 100644 --- a/config/seitoml/seitoml_test.go +++ b/config/seitoml/seitoml_test.go @@ -286,12 +286,12 @@ func TestALiteralStringIsTakenAsWritten(t *testing.T) { } } -// TestUnsetRemovesTheKeyRatherThanWritingItsBaseline holds what unset means. +// TestUnsetRemovesTheKeyRatherThanWritingItsDefault holds what unset means. // -// An absent key resolves to the running binary's baseline. Writing the baseline value instead -// looks identical in the file but is a commitment that survives a release changing that baseline, +// An absent key resolves to the running binary's default. Writing the default value instead +// looks identical in the file but is a commitment that survives a release changing that default, // which is the opposite of what the operator asked for. -func TestUnsetRemovesTheKeyRatherThanWritingItsBaseline(t *testing.T) { +func TestUnsetRemovesTheKeyRatherThanWritingItsDefault(t *testing.T) { f := parse(t, commented) removed, err := f.Unset("giga_executor.occ_enabled") @@ -307,7 +307,7 @@ func TestUnsetRemovesTheKeyRatherThanWritingItsBaseline(t *testing.T) { t.Fatalf("Values: %v", err) } if _, present := values["giga_executor.occ_enabled"]; present { - t.Errorf("the key is still written after unset: %v. It would keep overriding the baseline the "+ + t.Errorf("the key is still written after unset: %v. It would keep overriding the default the "+ "operator asked to fall back to", values) } if strings.Contains(render(t, f), "occ_enabled") { @@ -699,7 +699,7 @@ func TestANewFileRecordsItsNodeMode(t *testing.T) { // TestAnAbsentOrUnreadableNodeModeIsAnError keeps a comparison from guessing. // // Guessing picks one binary's idea of a default and silently measures an archive node's file against -// a validator's baselines, which is the mistake this key exists to make impossible. +// a validator's defaults, which is the mistake this key exists to make impossible. func TestAnAbsentOrUnreadableNodeModeIsAnError(t *testing.T) { for _, tc := range []struct{ name, body string }{ {"absent", "schema_version = 1\n"}, From 7e4746ca0ea59c053fd687c44bdcdab380439795 Mon Sep 17 00:00:00 2001 From: bdchatham Date: Tue, 18 Aug 2026 13:50:36 -0700 Subject: [PATCH 04/24] config/seitoml: drop generated_by, which nothing reads Three callers wrote it and nothing read it. Its own documentation said no answer may depend on it and a test enforced that none did, so it was write-only by design. The release is knowable from the deployment more reliably than from the file. A static configuration moves in lock-step with the container it is mounted into, and a node that writes its own file is running the release in question. A copy in the file is a second source that can disagree with the one the platform already knows. Removing it makes New take one argument. It took two adjacent strings, so New("v6.7.0", "validator") compiled, passed, and wrote a node mode no mode matches, which Mode then returned without complaint. The swap is now unrepresentable rather than caught by a type. Absence was already permanent, since a build outside the release process omitted the key, so a file written before this returns is indistinguishable from one the design already tolerated. That is what makes adding it back cost nothing. Coverage is 95.8% of statements. Co-Authored-By: Claude Opus 5 (1M context) --- config/seitoml/doc.go | 16 +-- config/seitoml/file.go | 45 +------- config/seitoml/seitoml_test.go | 185 +++------------------------------ config/seitoml/values.go | 8 +- 4 files changed, 26 insertions(+), 228 deletions(-) diff --git a/config/seitoml/doc.go b/config/seitoml/doc.go index 8cc5ec0fcb..4eeb45c0ea 100644 --- a/config/seitoml/doc.go +++ b/config/seitoml/doc.go @@ -5,15 +5,12 @@ // default into the file, because a value the binary put there reads exactly like one an operator // chose. // -// Three keys at the top level describe the file rather than configure the node, and Values leaves -// all three out so a check comparing written keys against the declared set never reports them as -// keys no section owns. +// Two keys at the top level describe the file rather than configure the node, and Values leaves both +// out so a check comparing written keys against the declared set never reports them as keys no section +// owns. // // schema_version which migration the file has reached // node_mode which mode's defaults its values were chosen against -// generated_by which release last produced or transformed it -// -// The first two are machinery and the third is not, and the difference matters enough to state. // // schema_version is a counter that rises by exactly one per migration, and a migration chain reads it // to decide which steps a file still needs. It is deliberately not a release version. Most releases @@ -27,13 +24,6 @@ // because a release migrates the file on the node's own disk and rolling the binary back does not roll // the file back with it. Read anyway, the older binary would apply only the keys it still recognises. // -// generated_by is provenance. Nothing reads it to decide anything, which is what lets it be absent -// without consequence: the release reaches the binary through a linker flag the release build sets, -// so a binary built any other way knows none and a file it writes simply omits the key. Anything -// branching on it would turn every development build into a node that cannot read its own -// configuration, and a test drives every reader over a file recording a release, no release, and a -// release no build ever was, requiring identical answers. -// // Editing preserves the document. An operator may hand-edit the file, and comments are how they // explain a choice to whoever reads it next, so set and unset change the one line they name and // leave the rest byte for byte. That is why this package edits a parsed document rather than diff --git a/config/seitoml/file.go b/config/seitoml/file.go index 8200dc1193..5da7f83e03 100644 --- a/config/seitoml/file.go +++ b/config/seitoml/file.go @@ -31,14 +31,6 @@ const VersionKey = "schema_version" // archive mode, so nothing else on disk distinguishes the two. const ModeKey = "node_mode" -// GeneratedByKey records the release that last produced or transformed the file. -// -// Provenance, never machinery. Nothing reads it to decide anything, which is what lets it be absent -// without consequence: a binary built outside the release process carries no version, and a file -// written by one simply omits the key. Anything branching on it would turn every development build -// into a node that cannot read its own configuration. -const GeneratedByKey = "generated_by" - // newFileMode is the permission a file created here gets, and only that. // // A save onto an existing file inherits whatever mode that file already has, so this value describes @@ -167,16 +159,12 @@ func Load(path string) (*File, error) { return f, nil } -// New returns an empty document carrying this binary's schema version, the given node mode, and the -// release that produced it. +// New returns an empty document carrying this binary's schema version and the given node mode. // // The mode is required rather than optional. Every value a caller goes on to write resolves for one // mode, and a file that does not say which cannot be compared against a binary's defaults or checked // against the mode the node actually runs. -// -// generatedBy is recorded when the caller has one and omitted when it is empty, because a binary built -// outside the release process knows no version and an empty string says less than no key at all. -func New(mode, generatedBy string) (*File, error) { +func New(mode string) (*File, error) { if mode == "" { return nil, fmt.Errorf("a sei.toml needs a node mode: every value in it resolves for one, and " + "a file that omits it cannot be compared against this binary's defaults") @@ -188,38 +176,9 @@ func New(mode, generatedBy string) (*File, error) { if err := f.Set(ModeKey, mode); err != nil { return nil, err } - if err := f.SetGeneratedBy(generatedBy); err != nil { - return nil, err - } return f, nil } -// SetGeneratedBy records the release producing the file, and removes the key when given nothing. -func (f *File) SetGeneratedBy(release string) error { - if release == "" { - _, err := f.Unset(GeneratedByKey) - return err - } - return f.Set(GeneratedByKey, release) -} - -// GeneratedBy returns the release the file records, and whether it records one at all. -// -// No error for an absent key. Absence is ordinary, and a caller forced to handle it as a failure -// would be a caller whose behaviour depends on the field. -func (f *File) GeneratedBy() (string, bool) { - e := f.doc.First(GeneratedByKey) - if e == nil || e.KeyValue == nil { - return "", false - } - v, err := goValue(e.Value) - if err != nil { - return "", false - } - release, ok := v.(string) - return release, ok && release != "" -} - // Mode returns the node mode the file's values resolve for. // // An absent mode is an error rather than a guess. Guessing picks one binary's idea of a default and diff --git a/config/seitoml/seitoml_test.go b/config/seitoml/seitoml_test.go index 44858599bc..c9297e3108 100644 --- a/config/seitoml/seitoml_test.go +++ b/config/seitoml/seitoml_test.go @@ -221,7 +221,7 @@ func TestSetRoundTripsEveryTypeItAccepts(t *testing.T) { {"string list", []string{"a", "b"}, []any{"a", "b"}}, } { t.Run(tc.name, func(t *testing.T) { - f, err := seitoml.New("validator", "v6.7.0") + f, err := seitoml.New("validator") if err != nil { t.Fatalf("New: %v", err) } @@ -333,7 +333,7 @@ func TestUnsetRemovesTheKeyRatherThanWritingItsDefault(t *testing.T) { // Without it, writing the first key of a section would need the operator to add the heading by // hand, and set would fail on exactly the file a new node starts from. func TestSetCreatesTheTableWhenTheSectionIsNew(t *testing.T) { - f, err := seitoml.New("validator", "v6.7.0") + f, err := seitoml.New("validator") if err != nil { t.Fatalf("New: %v", err) } @@ -467,7 +467,7 @@ persistent-peers = ["a", "b"] // back as something else. Refusing is what makes the round-trip guarantee above hold for every // type this accepts. func TestAnUnsupportedTypeIsRefused(t *testing.T) { - f, err := seitoml.New("validator", "v6.7.0") + f, err := seitoml.New("validator") if err != nil { t.Fatalf("New: %v", err) } @@ -636,7 +636,7 @@ func TestSaveKeepsAnExistingFilesPermissions(t *testing.T) { // previous mode to inherit, so the choice has to be made here. func TestANewFileIsNotWorldReadable(t *testing.T) { path := filepath.Join(t.TempDir(), "sei.toml") - f, err := seitoml.New("validator", "v6.7.0") + f, err := seitoml.New("validator") if err != nil { t.Fatalf("New: %v", err) } @@ -659,7 +659,7 @@ func TestANewFileIsNotWorldReadable(t *testing.T) { // A file written without one cannot be migrated later, and the failure appears at the first // upgrade rather than at the write that caused it. func TestNewCarriesThisBinarysSchemaVersion(t *testing.T) { - f, err := seitoml.New("validator", "v6.7.0") + f, err := seitoml.New("validator") if err != nil { t.Fatalf("New: %v", err) } @@ -678,7 +678,7 @@ func TestNewCarriesThisBinarysSchemaVersion(t *testing.T) { // The mode selects which defaults the values were chosen against, so a file that omits it cannot be // compared against a binary or checked against the mode the node runs. func TestANewFileRecordsItsNodeMode(t *testing.T) { - f, err := seitoml.New("archive", "v6.7.0") + f, err := seitoml.New("archive") if err != nil { t.Fatalf("New: %v", err) } @@ -690,7 +690,7 @@ func TestANewFileRecordsItsNodeMode(t *testing.T) { if mode != "archive" { t.Errorf("the file records mode %q, want archive", mode) } - if _, err := seitoml.New("", "v6.7.0"); err == nil { + if _, err := seitoml.New(""); err == nil { t.Error("a file was created with no mode. Every value written into it resolves for one, so " + "nothing could later tell an archive node's file from a validator's") } @@ -753,147 +753,6 @@ func TestAMigrationCarriesTheNodeModeForward(t *testing.T) { } } -// TestTheReleaseThatWroteTheFileIsRecorded holds the provenance the file carries. -// -// Nobody could otherwise tell which binary produced a file, which is the first thing worth knowing -// when its values look wrong and the first thing that says whether regenerating would change -// anything. -func TestTheReleaseThatWroteTheFileIsRecorded(t *testing.T) { - f, err := seitoml.New("validator", "v6.7.0") - if err != nil { - t.Fatalf("New: %v", err) - } - - release, recorded := parse(t, render(t, f)).GeneratedBy() - if !recorded || release != "v6.7.0" { - t.Errorf("the file records (%q, %v), want v6.7.0\nfile:\n%s", release, recorded, render(t, f)) - } -} - -// TestABuildWithNoReleaseOmitsTheKey holds the case every developer hits. -// -// The release comes from a linker flag the release build sets, so a binary built any other way knows -// none. Writing an empty string would put a key in an operator's file that says less than no key, and -// refusing would leave a development build unable to produce a file at all. -func TestABuildWithNoReleaseOmitsTheKey(t *testing.T) { - f, err := seitoml.New("validator", "") - if err != nil { - t.Fatalf("New refused a build that carries no release: %v", err) - } - - body := render(t, f) - if strings.Contains(body, seitoml.GeneratedByKey) { - t.Errorf("the file carries %s with nothing behind it:\n%s", seitoml.GeneratedByKey, body) - } - if _, recorded := parse(t, body).GeneratedBy(); recorded { - t.Error("GeneratedBy reports a release for a file that records none") - } - // A file already on disk carrying an empty value reads the same way, since an empty release says - // nothing and a caller told one is present would print it as though it meant something. - onDisk := parse(t, "schema_version = 1\nnode_mode = \"validator\"\ngenerated_by = \"\"\n") - if release, recorded := onDisk.GeneratedBy(); recorded { - t.Errorf("a file recording an empty release reports (%q, %v), want it treated as absent", - release, recorded) - } - // And the file is otherwise complete, or omitting the key would have cost something. - if v, err := parse(t, body).Version(); err != nil || v != seitoml.SchemaVersion { - t.Errorf("the file lost its schema version: (%d, %v)", v, err) - } - if mode, err := parse(t, body).Mode(); err != nil || mode != "validator" { - t.Errorf("the file lost its node mode: (%q, %v)", mode, err) - } -} - -// TestTheReleaseKeyIsNotAConfigurationKey keeps provenance out of the key space. -// -// Only the key at the document's top level. A key of the same name inside a table is an ordinary -// setting called section.generated_by, and doctor should report it as one, so the exclusion is on the -// exact path rather than on the name. -func TestTheReleaseKeyIsNotAConfigurationKey(t *testing.T) { - f := parse(t, commented) - if err := f.SetGeneratedBy("v6.7.0"); err != nil { - t.Fatalf("SetGeneratedBy: %v", err) - } - - values, err := f.Values() - if err != nil { - t.Fatalf("Values: %v", err) - } - if _, present := values[seitoml.GeneratedByKey]; present { - t.Errorf("%s appears in the written key space: %v", seitoml.GeneratedByKey, values) - } - if len(values) != 2 { - t.Errorf("read %d keys, want the section's 2: %v", len(values), values) - } - - // The same name inside a table stays a configuration key, since it is one. - inTable := parse(t, "schema_version = 1\nnode_mode = \"validator\"\n\n[probe]\ngenerated_by = \"x\"\n") - nested, err := inTable.Values() - if err != nil { - t.Fatalf("Values: %v", err) - } - if _, present := nested["probe.generated_by"]; !present { - t.Errorf("probe.generated_by was excluded from the key space: %v. Only the top-level key is "+ - "provenance; inside a table it is a setting like any other and doctor should say so", nested) - } -} - -// TestNothingBehavesDifferentlyForTheReleaseKey is the constraint that makes the field safe. -// -// It is provenance, so no answer anywhere may depend on it. The moment something branches on it, a -// development build writing no release becomes a node that cannot read its own configuration, and a -// file hand-edited to a nonsense release becomes one nothing will touch. -// -// Held by driving every reader over the same file three ways: recording a release, recording none, -// and recording something no release ever was. Every answer has to match. -func TestNothingBehavesDifferentlyForTheReleaseKey(t *testing.T) { - const body = "schema_version = 1\nnode_mode = \"validator\"\n\n[probe]\nworkers = 4\n" - - answers := map[string][]string{} - for _, release := range []string{"", "v6.7.0", "not-a-release-anyone-shipped"} { - f := parse(t, body) - if err := f.SetGeneratedBy(release); err != nil { - t.Fatalf("SetGeneratedBy(%q): %v", release, err) - } - - var got []string - version, err := f.Version() - got = append(got, fmt.Sprintf("version=%d err=%v", version, err)) - mode, err := f.Mode() - got = append(got, fmt.Sprintf("mode=%q err=%v", mode, err)) - values, err := f.Values() - got = append(got, fmt.Sprintf("values=%v err=%v", values, err)) - value, present, err := f.Get("probe.workers") - got = append(got, fmt.Sprintf("get=%#v present=%v err=%v", value, present, err)) - - // Save too, since a round trip through the disk is where a reader could pick the key up again. - path := filepath.Join(t.TempDir(), "sei.toml") - if err := f.Save(path); err != nil { - t.Fatalf("Save: %v", err) - } - reread, err := seitoml.Load(path) - if err != nil { - t.Fatalf("Load: %v", err) - } - rereadValues, err := reread.Values() - got = append(got, fmt.Sprintf("reread=%v err=%v", rereadValues, err)) - - answers[release] = got - } - - base := answers["v6.7.0"] - for release, got := range answers { - for i := range got { - if got[i] != base[i] { - t.Errorf("with %s = %q a reader answered\n %s\nand with a recorded release it answered\n"+ - " %s\n\nThe field is provenance and nothing may depend on it: a development build "+ - "writes none, so anything branching on it makes such a build unable to read its own "+ - "configuration", seitoml.GeneratedByKey, release, got[i], base[i]) - } - } - } -} - // TestEveryValueShapeTomlAllowsReadsBack drives the value forms an operator's file can hold. // // The file is hand-written, and TOML gives an operator more ways to write a value than a generated @@ -1207,7 +1066,7 @@ func TestAnInfinityCannotBeWritten(t *testing.T) { {"NaN", math.NaN()}, } { t.Run(tc.name, func(t *testing.T) { - f, err := seitoml.New("validator", "v6.7.0") + f, err := seitoml.New("validator") if err != nil { t.Fatalf("New: %v", err) } @@ -1222,13 +1081,12 @@ func TestAnInfinityCannotBeWritten(t *testing.T) { } } -// TestAFileDescribingItselfWithANonValueIsRefusedPerReader covers the three keys about the file. +// TestAFileDescribingItselfWithANonValueIsRefused covers the two keys about the file. // -// schema_version, node_mode and generated_by are read before anything else, so a value this package -// cannot decode has to fail there rather than further in. The three differ in what failure means: -// version and mode are machinery a reader cannot proceed without, and the release is provenance whose -// absence is ordinary, so an undecodable one reads as absent rather than as an error. -func TestAFileDescribingItselfWithANonValueIsRefusedPerReader(t *testing.T) { +// schema_version and node_mode are read before anything else, and both are machinery a reader cannot +// proceed without, so a value this package cannot decode has to fail there rather than further in. Each +// refusal names its key, because the two have different fixes. +func TestAFileDescribingItselfWithANonValueIsRefused(t *testing.T) { t.Run("an undecodable schema version", func(t *testing.T) { _, err := parse(t, "schema_version = inf\n").Version() if err == nil { @@ -1251,15 +1109,6 @@ func TestAFileDescribingItselfWithANonValueIsRefusedPerReader(t *testing.T) { } }) - t.Run("an undecodable release", func(t *testing.T) { - release, ok := parse(t, "generated_by = inf\n").GeneratedBy() - if ok || release != "" { - t.Errorf("GeneratedBy = (%q, %v), want absent. The field is provenance, so a value nothing "+ - "can decode is the same as no value rather than a failure a caller has to handle", - release, ok) - } - }) - t.Run("a node mode that is not a string", func(t *testing.T) { _, err := parse(t, "node_mode = 3\n").Mode() if err == nil || !strings.Contains(err.Error(), "want a mode name") { @@ -1281,7 +1130,7 @@ func TestAFileDescribingItselfWithANonValueIsRefusedPerReader(t *testing.T) { // so the operator knows which one to create. Failing without it leaves them guessing which of a // configured data directory, home directory or flag was wrong. func TestSaveNamesThePathWhenItCannotWriteThere(t *testing.T) { - f, err := seitoml.New("validator", "") + f, err := seitoml.New("validator") if err != nil { t.Fatalf("New: %v", err) } @@ -1303,7 +1152,7 @@ func TestSaveNamesThePathWhenItCannotWriteThere(t *testing.T) { // before anything notices. The error has to name the destination, and the directory it wrote beside has // to be left clean, or the next save finds it littered with the leavings of this one. func TestSaveRefusesAPathThatIsADirectory(t *testing.T) { - f, err := seitoml.New("validator", "") + f, err := seitoml.New("validator") if err != nil { t.Fatalf("New: %v", err) } @@ -1341,7 +1190,7 @@ func TestSaveRefusesAPathThatIsADirectory(t *testing.T) { // straight back. An element this package cannot render has to name its position, since a list of ten // values with one bad element is otherwise a refusal an operator cannot act on. func TestAListCarryingAValueThatCannotBeWrittenNamesTheElement(t *testing.T) { - f, err := seitoml.New("validator", "") + f, err := seitoml.New("validator") if err != nil { t.Fatalf("New: %v", err) } @@ -1401,7 +1250,7 @@ func TestAnUnflushedDirectoryEntryIsNotAFailedSave(t *testing.T) { dir := t.TempDir() path := filepath.Join(dir, "sei.toml") - f, err := seitoml.New("validator", "") + f, err := seitoml.New("validator") if err != nil { t.Fatalf("New: %v", err) } diff --git a/config/seitoml/values.go b/config/seitoml/values.go index f45513a2ae..692b02657f 100644 --- a/config/seitoml/values.go +++ b/config/seitoml/values.go @@ -15,9 +15,9 @@ import ( // Values returns every key the file writes, as dotted paths to Go values. // -// This leaves out the schema version, the node mode and the release that produced the file. All -// three describe the file rather than configuring the node, so a reader checking written keys against -// the declared set would otherwise report them as keys no section owns, on every node, forever. +// This leaves out the schema version and the node mode. Both describe the file rather than configuring +// the node, so a reader checking written keys against the declared set would otherwise report them as +// keys no section owns, on every node, forever. func (f *File) Values() (map[string]any, error) { out := map[string]any{} var bad error @@ -27,7 +27,7 @@ func (f *File) Values() (map[string]any, error) { return true // a table heading carries no value of its own } key := strings.ToLower(full.String()) - if key == VersionKey || key == ModeKey || key == GeneratedByKey { + if key == VersionKey || key == ModeKey { return true } v, err := goValue(e.Value) From 0bfb84b04d5ba19158af302d6b47ea874e45a3bd Mon Sep 17 00:00:00 2001 From: bdchatham Date: Tue, 18 Aug 2026 14:02:37 -0700 Subject: [PATCH 05/24] config/seitoml: read values with a conforming decoder instead of by hand The editing parser locates lines and preserves comments, which is why it is here, and it stops short of interpreting a literal. Deciding what 1_000 or a multi-line string means was a second implementation of the TOML specification, and two reviews found five defects in it. BurntSushi/toml, already a direct dependency of this repo, is that implementation maintained by somebody else. Compared against the old decoder on every value shape, twelve of fifteen keys were already byte-identical, including every shape the hand-written one got wrong. Removes goValue, tokenValue, unquote, unescape, unixNewlines, trimOpeningNewline, foldContinuations, continuationEnd and arrayValue, and the four in-package tests that existed only because those functions were ours. values.go goes from 238 lines to 151, and guards_test.go from 121 to 41. Three consequences of adopting it, each carried rather than absorbed: - the decoder reads an infinity and a NaN, which the old one refused, so the refusal moved to the decoded values. It recurses into a list, because that is where the old check missed one. - a date decodes to a time this package has no way to write back. Nothing configures a node with a date, so Parse refuses one rather than adding a writer for a type no field uses. - a carriage return inside a multi-line string is kept, as the specification says. The old decoder normalised it, which is a deviation this no longer makes; a multi-line string in a node's configuration is a shape no field has. Parse also refuses a key written twice in one table. That is the one shape the editing parser accepts and a conforming decoder rejects, so without it a file parsed and then every read of it failed. Coverage is 96.0% of statements. Each new refusal is verified by reverting it and watching the covering test fail. Co-Authored-By: Claude Opus 5 (1M context) --- config/seitoml/doc.go | 17 +- config/seitoml/file.go | 52 +++--- config/seitoml/guards_test.go | 84 +--------- config/seitoml/seitoml_test.go | 51 +++--- config/seitoml/values.go | 281 ++++++++++++--------------------- 5 files changed, 161 insertions(+), 324 deletions(-) diff --git a/config/seitoml/doc.go b/config/seitoml/doc.go index 4eeb45c0ea..c1eca63f6b 100644 --- a/config/seitoml/doc.go +++ b/config/seitoml/doc.go @@ -51,14 +51,21 @@ // - a key or heading segment that is not lower case, which is read under a name that is not the one // written // - a quoted key carrying a dot or a space, which no dotted spelling splits back into +// - a key written twice in one table, which an edit reaches only the first of +// - a date or a time, which nothing configures a node with, and which cannot be written back as the +// type it was read as // // Each was previously accepted and then lost or corrupted further in. Refusing at the door is what // lets every verb below assume the document holds only shapes it can read and write back, and it is // only free while no operator has written a file that uses them. // -// Escapes are decoded with the scanner's own rules rather than Go's. The two grammars differ in three -// places that each reach an operator's file: TOML has no \x escape, Go has no line-ending -// continuation, and Go's decoder rejects the literal newline that makes a multi-line string -// multi-line. A carriage return and newline pair inside a multi-line string reads as a newline, so a -// file saved on Windows holds the same values as the same file saved anywhere else. +// Reading and editing use different libraries, on purpose. The editing parser locates lines and +// preserves comments, and stops short of interpreting a literal; deciding what an underscore-separated +// integer or a multi-line string means is a second implementation of the specification. Values come +// from a conforming decoder instead, so the shape of a literal is somebody else's problem and the file +// reads the way every other TOML reader reads it. +// +// Two things that decoder allows are still refused here, because this package has to write back what it +// reads: an infinity or a NaN, which have no form to write, and a date, which would come back as a time +// this package cannot render. package seitoml diff --git a/config/seitoml/file.go b/config/seitoml/file.go index 5da7f83e03..98c0d57b76 100644 --- a/config/seitoml/file.go +++ b/config/seitoml/file.go @@ -12,6 +12,7 @@ import ( "github.com/creachadair/tomledit" "github.com/creachadair/tomledit/parser" + "github.com/creachadair/tomledit/scanner" ) // SchemaVersion is the schema this binary writes and reads. @@ -86,6 +87,7 @@ func (f *File) refuseUnsupportedShapes() error { } var bad error + written := map[string]bool{} f.doc.Scan(func(full parser.Key, e *tomledit.Entry) bool { if e.KeyValue == nil { return true @@ -98,6 +100,16 @@ func (f *File) refuseUnsupportedShapes() error { bad = err return false } + // One value per key, checked here rather than left to the decoder. A duplicate is the one shape + // the editing parser accepts and a conforming decoder rejects, so without this the file parses + // and then every read of it fails. + key := full.String() + if written[key] { + bad = fmt.Errorf("%s is written more than once, and an edit reaches only the first, so a "+ + "value written into this file would not be the one read back", key) + return false + } + written[key] = true return true }) return bad @@ -129,6 +141,13 @@ func keyIsAddressable(key parser.Key) error { // time, producing a file a conforming reader refuses to load. func valueIsAddressable(key parser.Key, v parser.Value) error { switch x := v.X.(type) { + case parser.Token: + switch x.Type { + case scanner.DateTime, scanner.LocalDate, scanner.LocalTime, scanner.LocalDateTime: + return fmt.Errorf("%s is a date or a time, which this file does not carry; nothing "+ + "configures a node with one, and it cannot be written back as the type it was read as", + key) + } case parser.Inline: return fmt.Errorf("%s is an inline table, which this file does not carry; write it as a [%s] "+ "table so each key it holds can be edited on its own line", key, key) @@ -185,20 +204,14 @@ func New(mode string) (*File, error) { // silently compares an archive node's file against a validator's defaults, which is the mistake // this key exists to make impossible. func (f *File) Mode() (string, error) { - e := f.doc.First(ModeKey) - if e == nil || e.KeyValue == nil { + mode, present, err := f.stringValue(ModeKey) + switch { + case err != nil: + return "", err + case !present: return "", fmt.Errorf("sei.toml has no %s. Every value in it resolves for one node mode, so "+ "without it nothing can tell an archive node's file from a validator's", ModeKey) - } - v, err := goValue(e.Value) - if err != nil { - return "", fmt.Errorf("%s: %w", ModeKey, err) - } - mode, ok := v.(string) - if !ok { - return "", fmt.Errorf("%s is %T (%v), want a mode name", ModeKey, v, v) - } - if mode == "" { + case mode == "": return "", fmt.Errorf("%s is empty", ModeKey) } return mode, nil @@ -209,20 +222,15 @@ func (f *File) Mode() (string, error) { // An absent or unparsable version is an error, never a zero. A migration chain reads this to decide // which steps to run, so guessing here transforms a file whose shape nobody established. func (f *File) Version() (int, error) { - e := f.doc.First(VersionKey) - if e == nil || e.KeyValue == nil { + n, present, err := f.intValue(VersionKey) + switch { + case err != nil: + return 0, err + case !present: return 0, fmt.Errorf("sei.toml has no %s. Its shape cannot be established, so no migration "+ "can safely run against it and no reader can know which keys it is expected to carry", VersionKey) } - v, err := goValue(e.Value) - if err != nil { - return 0, fmt.Errorf("%s: %w", VersionKey, err) - } - n, ok := v.(int64) - if !ok { - return 0, fmt.Errorf("%s is %T (%v), want an integer", VersionKey, v, v) - } if int(n) > SchemaVersion { // The rollback case, and the reason the counter exists. A release migrates the file forward on // the node's own disk, so rolling the binary back does not roll the file back with it. Read diff --git a/config/seitoml/guards_test.go b/config/seitoml/guards_test.go index ecb2b951f0..963d48414f 100644 --- a/config/seitoml/guards_test.go +++ b/config/seitoml/guards_test.go @@ -5,73 +5,10 @@ import ( "testing" "github.com/creachadair/tomledit" - "github.com/creachadair/tomledit/parser" - "github.com/creachadair/tomledit/scanner" ) -// The guards below cannot be reached by parsing a file, because the parser refuses the input that -// would reach them: a bare word, a malformed escape, an unterminated table. They exist because the -// parser is a dependency, and a version of it that accepted more would otherwise turn an unknown token -// into a plausible Go value rather than a refusal. Driving them here is what keeps that refusal real -// instead of assumed, so this test is in the package rather than beside it. - -// TestAnUnknownTokenIsRefusedRatherThanGuessed holds the value decoder's own vocabulary. -func TestAnUnknownTokenIsRefusedRatherThanGuessed(t *testing.T) { - for _, tc := range []struct { - name string - tok parser.Token - want string - }{ - {"a word that is neither true nor false", retyped(t, "3", scanner.Word), - "not a value TOML recognizes"}, - {"a token type that is not a value", retyped(t, "3", scanner.LBracket), - "is not a value"}, - } { - t.Run(tc.name, func(t *testing.T) { - got, err := tokenValue(tc.tok) - if err == nil { - t.Fatalf("%s decoded to %#v; an unknown token becoming a value is how a node runs a "+ - "setting nobody wrote", tc.name, got) - } - if !strings.Contains(err.Error(), tc.want) { - t.Errorf("the refusal reads %q, which does not mention %q", err, tc.want) - } - }) - } -} - -// TestAValueShapeWithNoDecoderIsRefused covers the outer switch over a parsed value. -func TestAValueShapeWithNoDecoderIsRefused(t *testing.T) { - if _, err := goValue(parser.Value{}); err == nil { - t.Error("a value carrying no shape this package knows decoded without complaint") - } -} - -// TestAnUndefinedEscapeIsRefusedRatherThanReplaced covers the scanner's substitution behaviour. -// -// The scanner writes a replacement rune for an escape TOML does not define rather than failing, so -// without this check a typo such as \q would reach a node as U+FFFD inside a configuration value. The -// parser rejects the escape before a file can carry one, which is why this drives the decoder directly. -func TestAnUndefinedEscapeIsRefusedRatherThanReplaced(t *testing.T) { - if _, err := unescape(`"a\qc"`, `a\qc`); err == nil { - t.Error("an escape TOML does not define decoded without complaint") - } - // A replacement rune the operator actually wrote is theirs to keep. - got, err := unescape("\"a\ufffdc\"", "a\ufffdc") - if err != nil || got != "a\ufffdc" { - t.Errorf("a written replacement rune came back as (%q, %v), want it preserved", got, err) - } -} - -// TestUnquoteRefusesAKindThatIsNotAString covers the string decoder's own guard. -// -// unquote picks the escaping rules from the token kind, so a kind it does not know has no rules to -// apply. Returning the text as written would decode a basic string's escapes as literal backslashes. -func TestUnquoteRefusesAKindThatIsNotAString(t *testing.T) { - if _, err := unquote(scanner.Integer, "3"); err == nil { - t.Error("unquote accepted a kind that is not a string") - } -} +// The shapes below arise only from a document assembled in code rather than parsed from a file, so +// they are driven from inside the package. // TestATopLevelKeyReachesADocumentWithNoGlobalSection covers a document built rather than parsed. // @@ -102,20 +39,3 @@ func TestAPreambleReachesADocumentWithNoGlobalSection(t *testing.T) { t.Errorf("the preamble is not in the rendered document: %q", raw) } } - -// retyped parses a literal and relabels the token's type, which is how a decoder is driven over a -// token the parser will not produce from any file. A token's text is not settable from outside the -// parser, so the text stays whatever the literal scanned as and only the label changes. -func retyped(t *testing.T, literal string, kind scanner.Token) parser.Token { - t.Helper() - v, err := parser.ParseValue(literal) - if err != nil { - t.Fatalf("ParseValue(%q): %v", literal, err) - } - tok, ok := v.X.(parser.Token) - if !ok { - t.Fatalf("ParseValue(%q) is a %T, want a token", literal, v.X) - } - tok.Type = kind - return tok -} diff --git a/config/seitoml/seitoml_test.go b/config/seitoml/seitoml_test.go index c9297e3108..e8b9de4bc2 100644 --- a/config/seitoml/seitoml_test.go +++ b/config/seitoml/seitoml_test.go @@ -385,6 +385,11 @@ func TestAShapeThisFileDoesNotCarryIsRefusedAtTheDoor(t *testing.T) { "[[peer]]\nhost = \"a\"\n\n[[peer]]\nhost = \"b\"\n", "is an array of tables", }, + { + "a key written twice in one table", + "[probe]\nn = 1\nn = 2\n", + "is written more than once", + }, { "a repeated table heading", "[probe]\nn = 1\n\n[probe]\nn = 2\n", @@ -410,6 +415,16 @@ func TestAShapeThisFileDoesNotCarryIsRefusedAtTheDoor(t *testing.T) { "[probe]\n\"a b\" = 1\n", "carries a dot or a space", }, + { + "a date", + "[probe]\nstamped = 2026-08-18\n", + "is a date or a time", + }, + { + "a time", + "[probe]\nat = 07:32:00\n", + "is a date or a time", + }, } { t.Run(tc.name, func(t *testing.T) { _, err := seitoml.Parse(strings.NewReader(tc.body)) @@ -782,7 +797,6 @@ coded = "a\u0062c" grouped = 1_000_000 hex = 0x1f ratio = 2.5 -stamped = 2026-08-18 peers = ["a", "b"] commented = [ # the first one is the seed @@ -804,7 +818,6 @@ commented = [ "probe.grouped": int64(1000000), "probe.hex": int64(31), "probe.ratio": 2.5, - "probe.stamped": "2026-08-18", "probe.escaped": `say "hi" here`, "probe.coded": "abc", // A backslash ending a line folds the line break away; a doubled one is an escaped backslash @@ -869,11 +882,11 @@ func TestAValueTomlDoesNotRecognizeIsNamedNotGuessed(t *testing.T) { key string want string }{ - {"an integer past int64", "[probe]\nn = 99999999999999999999\n", "probe.n", "not an integer"}, - {"an infinity", "[probe]\nn = inf\n", "probe.n", "not a finite number"}, - {"a negative infinity", "[probe]\nn = -inf\n", "probe.n", "not a finite number"}, - {"a NaN", "[probe]\nn = nan\n", "probe.n", "not a finite number"}, - {"an infinity inside an array", "[probe]\nlist = [1.5, inf]\n", "probe.list", "not a finite number"}, + {"an integer past int64", "[probe]\nn = 99999999999999999999\n", "probe.n", "out of range for int64"}, + {"an infinity", "[probe]\nn = inf\n", "probe.n", "has to be a finite number"}, + {"a negative infinity", "[probe]\nn = -inf\n", "probe.n", "has to be a finite number"}, + {"a NaN", "[probe]\nn = nan\n", "probe.n", "has to be a finite number"}, + {"an infinity inside an array", "[probe]\nlist = [1.5, inf]\n", "probe.list", "has to be a finite number"}, } { t.Run(tc.name, func(t *testing.T) { f := parse(t, tc.body) @@ -1283,27 +1296,3 @@ func TestAnUnflushedDirectoryEntryIsNotAFailedSave(t *testing.T) { "configuration is what the node reads whatever the sync reported", got, present, err) } } - -// TestAFileWrittenOnWindowsReadsTheSameValues covers the line ending an editor leaves behind. -// -// An operator editing on Windows produces a file whose lines end with a carriage return. A multi-line -// string's value begins after the delimiter's own newline, and matching only the Unix form leaves the -// carriage return inside the value, so it differs from the default it matches and a diff reports a -// change nobody can see. -func TestAFileWrittenOnWindowsReadsTheSameValues(t *testing.T) { - const unix = "schema_version = 1\nnode_mode = \"validator\"\n\n[probe]\nfolded = \"\"\"\nfirst\nsecond\"\"\"\nflag = true\n" - windows := strings.ReplaceAll(unix, "\n", "\r\n") - - want, err := parse(t, unix).Values() - if err != nil { - t.Fatalf("the Unix file: %v", err) - } - got, err := parse(t, windows).Values() - if err != nil { - t.Fatalf("the Windows file: %v", err) - } - if !reflect.DeepEqual(got, want) { - t.Errorf("the same file read\n %#v\nwith carriage returns and\n %#v\nwithout. A value that "+ - "differs by line ending differs from the default it matches", got, want) - } -} diff --git a/config/seitoml/values.go b/config/seitoml/values.go index 692b02657f..29421825b6 100644 --- a/config/seitoml/values.go +++ b/config/seitoml/values.go @@ -1,16 +1,10 @@ package seitoml import ( - "bytes" "fmt" "math" - "strconv" - "strings" - "unicode/utf8" - "github.com/creachadair/tomledit" - "github.com/creachadair/tomledit/parser" - "github.com/creachadair/tomledit/scanner" + "github.com/BurntSushi/toml" ) // Values returns every key the file writes, as dotted paths to Go values. @@ -19,29 +13,13 @@ import ( // the node, so a reader checking written keys against the declared set would otherwise report them as // keys no section owns, on every node, forever. func (f *File) Values() (map[string]any, error) { - out := map[string]any{} - var bad error - - f.doc.Scan(func(full parser.Key, e *tomledit.Entry) bool { - if e.KeyValue == nil { - return true // a table heading carries no value of its own - } - key := strings.ToLower(full.String()) - if key == VersionKey || key == ModeKey { - return true - } - v, err := goValue(e.Value) - if err != nil { - bad = fmt.Errorf("%s: %w", key, err) - return false - } - out[key] = v - return true - }) - if bad != nil { - return nil, bad + all, err := f.decoded() + if err != nil { + return nil, err } - return out, nil + delete(all, VersionKey) + delete(all, ModeKey) + return all, nil } // Get returns one key's written value. @@ -50,189 +28,124 @@ func (f *File) Get(key string) (any, bool, error) { if err != nil { return nil, false, err } - e := f.doc.First(path...) - if e == nil || e.KeyValue == nil { - return nil, false, nil - } - v, err := goValue(e.Value) + all, err := f.decoded() if err != nil { - return nil, false, fmt.Errorf("%s: %w", key, err) + return nil, false, err } - return v, true, nil + v, ok := all[path.String()] + return v, ok, nil } -// goValue converts a parsed value to the Go value a reader sees. +// decoded renders the document and reads it back as Go values, keyed by dotted path. // -// Integers arrive as int64 and floats as float64, matching what a TOML decoder produces, so a -// comparison against a default does not have to know which parser read the file. A date or time comes -// back as its text: nothing configures a node with one, and the text keeps such a key visible to a -// check rather than dropping it. +// The values come from a TOML decoder rather than from the editing parser's tokens. The editing parser +// locates lines and preserves comments, which is why it is here, and it deliberately stops short of +// interpreting a literal. Deciding what "1_000" or a multi-line string means is a second implementation +// of the TOML specification, and the difference between Go's string grammar and TOML's is where a +// hand-written one goes wrong. // -// There is no case for an inline table, because Parse refuses one. Every value a reader sees is -// therefore one this package can also write back. -func goValue(v parser.Value) (any, error) { - switch d := v.X.(type) { - case parser.Token: - return tokenValue(d) - case parser.Array: - return arrayValue(d) - default: - return nil, fmt.Errorf("unsupported value %T", v.X) +// Rendering first rather than holding the source means an unsaved edit is read back through the same +// path a later process would use, so a value this package cannot express fails here rather than on a +// node. +func (f *File) decoded() (map[string]any, error) { + raw, err := f.Bytes() + if err != nil { + return nil, err } -} - -// tokenValue converts a single literal. -func tokenValue(t parser.Token) (any, error) { - text := t.String() - switch t.Type { - case scanner.Word: - switch text { - case "true": - return true, nil - case "false": - return false, nil - } - return nil, fmt.Errorf("%q is not a value TOML recognizes", text) - case scanner.String, scanner.MString, scanner.LString, scanner.MLString: - return unquote(t.Type, text) - case scanner.Integer: - n, err := strconv.ParseInt(strings.ReplaceAll(text, "_", ""), 0, 64) - if err != nil { - return nil, fmt.Errorf("%q is not an integer: %w", text, err) - } - return n, nil - case scanner.Float: - x, err := strconv.ParseFloat(strings.ReplaceAll(text, "_", ""), 64) - if err != nil { - return nil, fmt.Errorf("%q is not a number: %w", text, err) - } - if math.IsInf(x, 0) || math.IsNaN(x) { - // TOML spells these as words, and ParseFloat accepts them, so a file can hold one. Refused - // here because writing one is refused: read but not writable, a rename or an edit of any - // other key in the file would fail on a value this package handed back. - return nil, fmt.Errorf("%q is not a finite number, and a configuration value has to be one", - text) - } - return x, nil - case scanner.DateTime, scanner.LocalDate, scanner.LocalTime, scanner.LocalDateTime: - return text, nil - default: - return nil, fmt.Errorf("%q is not a value (%v)", text, t.Type) + var nested map[string]any + if _, err := toml.Decode(string(raw), &nested); err != nil { + return nil, fmt.Errorf("read sei.toml: %w", err) + } + out := make(map[string]any, len(nested)) + flatten("", nested, out) + if err := refuseNonFiniteNumbers(out); err != nil { + return nil, err } + return out, nil } -// unquote strips a string literal's quoting and decodes its escapes. +// refuseNonFiniteNumbers rejects an infinity or a NaN a decoder accepted. // -// A basic string carries escapes and needs them decoded; a literal string reads exactly as written, -// which is the whole reason TOML has both. Getting this backwards turns a Windows path's -// backslashes into control characters. -// -// The decoding is the scanner's own rather than Go's. The two grammars differ in three places that -// each reach an operator's file: TOML has no \x escape, Go has no line-ending continuation, and Go's -// decoder rejects the literal newline that makes a multi-line string multi-line. -func unquote(kind scanner.Token, text string) (string, error) { - switch kind { - case scanner.String: - return unescape(text, strings.TrimSuffix(strings.TrimPrefix(text, `"`), `"`)) - case scanner.MString: - inner := unixNewlines(strings.TrimSuffix(strings.TrimPrefix(text, `"""`), `"""`)) - return unescape(text, foldContinuations(trimOpeningNewline(inner))) - case scanner.LString: - return strings.TrimSuffix(strings.TrimPrefix(text, `'`), `'`), nil - case scanner.MLString: - inner := unixNewlines(strings.TrimSuffix(strings.TrimPrefix(text, `'''`), `'''`)) - return trimOpeningNewline(inner), nil - default: - return "", fmt.Errorf("%v is not a string", kind) +// TOML spells both as words and a conforming decoder reads them, so a file can hold one. This file +// cannot write one back, because rendering it produces a line no reader loads, so accepting one here +// would mean any later edit of any other key failed on a value this package had handed out. +func refuseNonFiniteNumbers(values map[string]any) error { + for key, v := range values { + if err := finite(key, v); err != nil { + return err + } } + return nil } -// unescape decodes a basic string's escapes, refusing one TOML does not define. -// -// The scanner substitutes a replacement rune for an undefined escape rather than failing, so a typo -// such as \q would otherwise reach a node as U+FFFD inside its configuration. text is the literal as -// written, so a refusal quotes what the operator typed rather than the decoded form. -func unescape(text, inner string) (string, error) { - out, err := scanner.Unescape([]byte(inner)) - if err != nil { - return "", fmt.Errorf("%s is not a well-formed string: %w", text, err) - } - if bytes.ContainsRune(out, utf8.RuneError) && !strings.ContainsRune(inner, utf8.RuneError) { - return "", fmt.Errorf("%s carries an escape TOML does not define", text) +// finite reports whether a value, or any element of a list, is a number this file can write back. +func finite(key string, v any) error { + switch x := v.(type) { + case float64: + if math.IsInf(x, 0) || math.IsNaN(x) { + return fmt.Errorf("%s is %v, and a configuration value has to be a finite number", key, x) + } + case []any: + for i, element := range x { + if err := finite(fmt.Sprintf("%s element %d", key, i), element); err != nil { + return err + } + } } - return string(out), nil + return nil } -// unixNewlines rewrites a carriage return and newline pair as a newline alone. +// flatten expands a decoded table into dotted keys, keeping only the leaves. // -// The line ending an editor chose is not part of the value. A default in the binary carries a bare -// newline, so a file saved on Windows would differ from a default it matches, on every line, and a diff -// would report a change nobody can see. Rendering already writes bare newlines, so this is what makes -// reading agree with writing. -// -// A carriage return the operator wrote as an escape is two characters here and survives, since escapes -// are decoded after this runs. -func unixNewlines(s string) string { return strings.ReplaceAll(s, "\r\n", "\n") } - -// trimOpeningNewline drops the newline TOML allows immediately after a multi-line delimiter, so the -// value starts at the operator's first line of content. -func trimOpeningNewline(s string) string { return strings.TrimPrefix(s, "\n") } - -// foldContinuations removes a backslash that ends a line together with the whitespace following it. -// -// This is how TOML lets one value span several lines without carrying the newlines into it. An escape -// consumes the character after it, so a doubled backslash is written through rather than read as a -// continuation; otherwise a value ending in a path separator would swallow the next line. -func foldContinuations(s string) string { - var out strings.Builder - for i := 0; i < len(s); { - if s[i] != '\\' { - out.WriteByte(s[i]) - i++ - continue +// A table contributes its name as a prefix and no value of its own, which is what makes the result one +// entry per written key and comparable against a set of declared keys. +func flatten(prefix string, in, out map[string]any) { + for name, v := range in { + key := name + if prefix != "" { + key = prefix + "." + name } - if end, folded := continuationEnd(s, i); folded { - i = end + if table, ok := v.(map[string]any); ok { + flatten(key, table, out) continue } - out.WriteByte(s[i]) - i++ - if i < len(s) { - out.WriteByte(s[i]) - i++ - } + out[key] = v } - return out.String() } -// continuationEnd reports where a continuation starting at the backslash in s[i] ends, if it is one. -func continuationEnd(s string, i int) (int, bool) { - j := i + 1 - for j < len(s) && (s[j] == ' ' || s[j] == '\t') { - j++ +// stringValue reads one of the keys that describe the file. +// +// Both are read before anything else, and neither has a sensible reading when it is absent or holds +// something other than a string, so each caller states its own consequence rather than sharing one. +func (f *File) stringValue(key string) (string, bool, error) { + all, err := f.decoded() + if err != nil { + return "", false, err } - if j >= len(s) || (s[j] != '\n' && s[j] != '\r') { - return 0, false + v, ok := all[key] + if !ok { + return "", false, nil } - for j < len(s) && (s[j] == ' ' || s[j] == '\t' || s[j] == '\r' || s[j] == '\n') { - j++ + s, ok := v.(string) + if !ok { + return "", true, fmt.Errorf("%s is %T (%v), want a mode name", key, v, v) } - return j, true + return s, true, nil } -// arrayValue converts an array, skipping the comment lines written between its items. -func arrayValue(a parser.Array) (any, error) { - out := make([]any, 0, len(a)) - for _, item := range a { - v, ok := item.(parser.Value) - if !ok { - continue // a comment between items - } - gv, err := goValue(v) - if err != nil { - return nil, err - } - out = append(out, gv) +// intValue reads one of the keys that describe the file as a whole number. +func (f *File) intValue(key string) (int64, bool, error) { + all, err := f.decoded() + if err != nil { + return 0, false, err } - return out, nil + v, ok := all[key] + if !ok { + return 0, false, nil + } + n, ok := v.(int64) + if !ok { + return 0, true, fmt.Errorf("%s is %T (%v), want an integer", key, v, v) + } + return n, true, nil } From 832751ac3377a8b6f1106c346c3a9678e3fb7c5d Mon Sep 17 00:00:00 2001 From: bdchatham Date: Tue, 18 Aug 2026 14:05:32 -0700 Subject: [PATCH 06/24] config/seitoml: say why the atomic write is by hand creachadair/atomicfile is already in this module and sei-tendermint's confix uses it for this exact job, so the next reader of writeAndSync will reasonably ask why it is not used here. It renames on Close and never syncs, and its temporary file is unexported, so the flush cannot be added from outside. Recorded at the function rather than left to be rediscovered. Co-Authored-By: Claude Opus 5 (1M context) --- config/seitoml/file.go | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/config/seitoml/file.go b/config/seitoml/file.go index 98c0d57b76..6994cab090 100644 --- a/config/seitoml/file.go +++ b/config/seitoml/file.go @@ -327,8 +327,14 @@ func modeToWrite(path string) (os.FileMode, error) { // writeAndSync writes the whole payload, sets the mode, and flushes to the device. // -// The sync makes the rename meaningful: without it the rename can land before the contents, leaving -// a file whose name is new and whose bytes are absent. +// The sync is what makes the rename meaningful: without it the rename can land before the contents, +// leaving a file whose name is new and whose bytes are absent, which is the one outcome a node cannot +// boot from. +// +// This is the reason the write is by hand rather than through creachadair/atomicfile, which this module +// already depends on and which sei-tendermint's confix uses for the same job. That package renames on +// Close and never syncs, and its temporary file is unexported, so the flush cannot be added from +// outside. Fewer lines are not worth the flush here. func writeAndSync(tmp *os.File, raw []byte, mode os.FileMode) error { defer func() { _ = tmp.Close() }() From 67ee9b54ac2eca334b982271ecc7785dc86868d8 Mon Sep 17 00:00:00 2001 From: bdchatham Date: Tue, 18 Aug 2026 14:43:34 -0700 Subject: [PATCH 07/24] config/seitoml: apply one key rule everywhere, and refuse a shadowed name Two gaps of the same kind, both found by review: the writer accepted keys the reader refuses, so a file could be saved and then never read. keyOf, which Set, Unset and Get share, rejected an empty segment and nothing else, while Parse also rejects a segment carrying a space. Set("foo bar", 1) therefore succeeded and the next Parse of the saved file failed. keyOf now applies keyIsAddressable, so the rule is stated once. TOML gives a name to a value or to a table, never both, and neither Parse nor Set refused a file using one name for each. Three shapes reached it: - a file already holding a scalar flatkv beside a [state-commit.flatkv] heading parsed, and every read of it then failed - Set writing state-commit.flatkv.enable where flatkv was already a value succeeded, and what it wrote re-parsed cleanly while no read of it could succeed, so nothing on the way in or out reported the damage - Set writing state-commit.flatkv where that heading already existed did the same in the other direction keysDoNotShadowEachOther holds the rule for both, over the document's own paths at Parse and over those plus the candidate at Set. It sorts and compares neighbours, so a key that merely shares a prefix, such as flatkvx beside flatkv, still writes; a test drives that, because refusing it would be the easy mistake. Coverage is 96.2% of statements. Each guard is verified by removing it and watching the covering test fail, including the prefix comparison, which is loosened rather than removed. Co-Authored-By: Claude Opus 5 (1M context) --- config/seitoml/doc.go | 5 ++ config/seitoml/edit.go | 6 ++ config/seitoml/file.go | 46 ++++++++++++++- config/seitoml/seitoml_test.go | 103 +++++++++++++++++++++++++++++++++ 4 files changed, 158 insertions(+), 2 deletions(-) diff --git a/config/seitoml/doc.go b/config/seitoml/doc.go index c1eca63f6b..3b51cc2902 100644 --- a/config/seitoml/doc.go +++ b/config/seitoml/doc.go @@ -54,6 +54,11 @@ // - a key written twice in one table, which an edit reaches only the first of // - a date or a time, which nothing configures a node with, and which cannot be written back as the // type it was read as +// - one name used for both a value and a table, which names the same thing twice +// +// Set, Unset and Get apply the same rule to the key a caller hands them, so a key one of them writes is +// a key the file reads back. Set also refuses a key that would name a value where a table already is, +// or the reverse, because the file that produces parses cleanly and can never be read. // // Each was previously accepted and then lost or corrupted further in. Refusing at the door is what // lets every verb below assume the document holds only shapes it can read and write back, and it is diff --git a/config/seitoml/edit.go b/config/seitoml/edit.go index 44593d1933..2a9ba479e6 100644 --- a/config/seitoml/edit.go +++ b/config/seitoml/edit.go @@ -32,6 +32,12 @@ func (f *File) Set(key string, v any) error { e.Value = value return nil } + // A key the document does not hold yet, so it may collide with one it does. Writing a value under a + // name that already names a table, or a table under one that already names a value, produces a file + // this package can save and no conforming reader can load. + if err := keysDoNotShadowEachOther(append(f.writtenPaths(), path.String())); err != nil { + return fmt.Errorf("%s: %w", key, err) + } f.insert(path, value) return nil } diff --git a/config/seitoml/file.go b/config/seitoml/file.go index 6994cab090..1b9b32cbca 100644 --- a/config/seitoml/file.go +++ b/config/seitoml/file.go @@ -7,6 +7,7 @@ import ( "io" "os" "path/filepath" + "sort" "strconv" "strings" @@ -112,7 +113,10 @@ func (f *File) refuseUnsupportedShapes() error { written[key] = true return true }) - return bad + if bad != nil { + return bad + } + return keysDoNotShadowEachOther(f.writtenPaths()) } // keyIsAddressable reports whether every segment of a key can be read back as written. @@ -374,7 +378,45 @@ func keyOf(key string) (parser.Key, error) { return nil, fmt.Errorf("key %q has an empty segment", key) } } - return parser.Key(parts), nil + out := parser.Key(parts) + // The same rule Parse applies to a key it reads. Stated once, so a key a caller writes is a key + // the file reads back; checked only here, Set could write a key the next Parse refuses. + if err := keyIsAddressable(out); err != nil { + return nil, fmt.Errorf("key %q: %w", key, err) + } + return out, nil +} + +// keysDoNotShadowEachOther refuses a key whose path is a prefix of another key's. +// +// TOML gives a name to a value or to a table, never to both, so a scalar flatkv beside a +// [state-commit.flatkv] heading defines one name twice. The editing parser accepts that and a +// conforming decoder rejects it, so a file carrying one parses and then every read of it fails. +// +// Sorting puts a path immediately before anything nested under it, which is what makes one pass over +// the neighbours enough. +func keysDoNotShadowEachOther(paths []string) error { + sorted := append([]string(nil), paths...) + sort.Strings(sorted) + for i := 1; i < len(sorted); i++ { + if strings.HasPrefix(sorted[i], sorted[i-1]+".") { + return fmt.Errorf("%s is a value and %s is a table under the same name, so a reader cannot "+ + "decide which one this file means", sorted[i-1], sorted[i]) + } + } + return nil +} + +// writtenPaths returns the dotted path of every value the document holds. +func (f *File) writtenPaths() []string { + var out []string + f.doc.Scan(func(full parser.Key, e *tomledit.Entry) bool { + if e.KeyValue != nil { + out = append(out, full.String()) + } + return true + }) + return out } // quoteInt renders an integer the way TOML spells one. diff --git a/config/seitoml/seitoml_test.go b/config/seitoml/seitoml_test.go index e8b9de4bc2..fcbeeb5b64 100644 --- a/config/seitoml/seitoml_test.go +++ b/config/seitoml/seitoml_test.go @@ -1296,3 +1296,106 @@ func TestAnUnflushedDirectoryEntryIsNotAFailedSave(t *testing.T) { "configuration is what the node reads whatever the sync reported", got, present, err) } } + +// TestANameCannotBeAValueAndATableAtOnce covers the shape both entry points can produce. +// +// TOML gives a name to a value or to a table, never both. The editing parser accepts a file holding +// each under one name and a conforming decoder rejects it, so such a file parses and then every read of +// it fails. Set can produce it too, in both directions, and the file it writes re-parses cleanly, which +// makes it the worse of the two: nothing on the way in or out reports the damage. +func TestANameCannotBeAValueAndATableAtOnce(t *testing.T) { + t.Run("a file already carrying both", func(t *testing.T) { + _, err := seitoml.Parse(strings.NewReader( + "[state-commit]\nflatkv = true\n\n[state-commit.flatkv]\nenable = false\n")) + if err == nil { + t.Fatal("a file naming one thing a value and a table parsed; every read of it then fails") + } + if !strings.Contains(err.Error(), "state-commit.flatkv") { + t.Errorf("the refusal reads %q and does not name the key at fault", err) + } + }) + + t.Run("a table written under a name a value already has", func(t *testing.T) { + f := parse(t, "schema_version = 1\nnode_mode = \"validator\"\n\n[state-commit]\nflatkv = true\n") + err := f.Set("state-commit.flatkv.enable", false) + if err == nil { + t.Fatal("Set wrote a table under a name a value already had, so Save would produce a file " + + "no reader can load") + } + if !strings.Contains(err.Error(), "state-commit.flatkv") { + t.Errorf("the refusal reads %q and does not name the key at fault", err) + } + requireStillReadable(t, f) + }) + + t.Run("a value written under a name a table already has", func(t *testing.T) { + f := parse(t, "schema_version = 1\nnode_mode = \"validator\"\n\n[state-commit.flatkv]\nenable = false\n") + err := f.Set("state-commit.flatkv", true) + if err == nil { + t.Fatal("Set wrote a value under a name a table already had") + } + requireStillReadable(t, f) + }) + + t.Run("a sibling that merely shares a prefix still writes", func(t *testing.T) { + // state-commit.flatkvx is not nested under state-commit.flatkv, so the check must not refuse it. + f := parse(t, "schema_version = 1\nnode_mode = \"validator\"\n\n[state-commit]\nflatkv = true\n") + if err := f.Set("state-commit.flatkvx", false); err != nil { + t.Fatalf("a key sharing only a prefix was refused: %v", err) + } + requireStillReadable(t, f) + }) +} + +// requireStillReadable holds that a refused edit left the document readable. +// +// A refusal that half-applied would leave the file in the state the refusal exists to prevent. +func requireStillReadable(t *testing.T, f *seitoml.File) { + t.Helper() + if _, err := f.Values(); err != nil { + t.Errorf("the document is unreadable after the edit: %v", err) + } + raw, err := f.Bytes() + if err != nil { + t.Fatalf("Bytes: %v", err) + } + if _, err := seitoml.Parse(strings.NewReader(string(raw))); err != nil { + t.Errorf("what the document renders to no longer parses: %v", err) + } +} + +// TestEveryVerbTakingAKeyAppliesOneRule holds Set, Unset and Get to the rule Parse applies. +// +// A key a verb accepts and Parse refuses is a key that can be written and then never read: the save +// succeeds, and the node cannot load its own configuration afterwards. +func TestEveryVerbTakingAKeyAppliesOneRule(t *testing.T) { + for _, key := range []string{"foo bar", "probe.a b", " leading", "trailing "} { + t.Run(fmt.Sprintf("key %q", key), func(t *testing.T) { + f, err := seitoml.New("validator") + if err != nil { + t.Fatalf("New: %v", err) + } + if err := f.Set(key, 1); err == nil { + t.Errorf("Set(%q) was accepted, so the file it saves cannot be parsed again", key) + } + if _, err := f.Unset(key); err == nil { + t.Errorf("Unset(%q) was accepted", key) + } + if _, _, err := f.Get(key); err == nil { + t.Errorf("Get(%q) was accepted", key) + } + }) + } + + // A caller's upper case is folded rather than refused, since a key read lower-cased is the same key. + f, err := seitoml.New("validator") + if err != nil { + t.Fatalf("New: %v", err) + } + if err := f.Set("Probe.Enabled", true); err != nil { + t.Fatalf("an upper-case key was refused rather than folded: %v", err) + } + if got, ok, err := f.Get("probe.enabled"); err != nil || !ok || got != true { + t.Errorf("the folded key reads back as (%#v, %v, %v), want true", got, ok, err) + } +} From 54db0a6d5bda859e684eb5dc72e5d719304c4401 Mon Sep 17 00:00:00 2001 From: bdchatham Date: Tue, 18 Aug 2026 14:47:58 -0700 Subject: [PATCH 08/24] config/seitoml: admit only a bare key, and close three round-trip gaps A quoted key is spelled one way by the decoder and another by a lookup, so Values reported a key Get answered absent for. Reproduced with "a#b" = 1, which parsed, appeared as p.a#b, and could not be fetched. keyIsAddressable now admits only a bare TOML key, meaning lower-case letters, digits, underscores and hyphens, and refuses an empty segment, which "" = 1 previously slipped through. Every mapstructure tag in this tree is already a bare key; the only two that are not are refusal fixtures in config/registry's own tests. Two more values could be written and not read back: - an unsigned integer above int64, which renders and then decodes as an error, so every later read of the file failed including the two keys that describe it - schema_version 0 or below, which Version returned as-is while its own documentation says an absent or unreadable counter is an error rather than a zero, leaving a caller unable to tell that zero from the error's Rewords the editing promise. It said set and unset leave the rest byte for byte, which overstates what tomledit.Format does: it re-renders and normalises vertical spacing once. The doc now says what the tests actually hold, that every other line of content is untouched and spacing settles on the first save. Adds the coverage two review findings pointed at without either failure reproducing. A dotted key inside a table renders beside a later nested heading and BurntSushi accepts the result, so the reported unreadable file did not occur, but the fixture claiming to cover that shape used a heading instead and now uses a dotted key. A preamble replaces its predecessor across a save and reload, and leaves an operator's own top-of-file comment in place, so both halves of the reported failure are absent, but the round trip is the flow that runs and was only exercised in memory. Coverage is 96.1% of statements. Each guard is verified by removing it and watching the covering test fail. Co-Authored-By: Claude Opus 5 (1M context) --- config/seitoml/doc.go | 10 ++- config/seitoml/edit.go | 19 ++++- config/seitoml/file.go | 32 +++++++- config/seitoml/seitoml_test.go | 141 ++++++++++++++++++++++++++++++++- 4 files changed, 190 insertions(+), 12 deletions(-) diff --git a/config/seitoml/doc.go b/config/seitoml/doc.go index 3b51cc2902..cbbd954661 100644 --- a/config/seitoml/doc.go +++ b/config/seitoml/doc.go @@ -25,9 +25,10 @@ // the file back with it. Read anyway, the older binary would apply only the keys it still recognises. // // Editing preserves the document. An operator may hand-edit the file, and comments are how they -// explain a choice to whoever reads it next, so set and unset change the one line they name and -// leave the rest byte for byte. That is why this package edits a parsed document rather than -// re-rendering a decoded map. +// explain a choice to whoever reads it next, so set and unset change the one line they name and leave +// every other line of content untouched. Vertical spacing is normalised once, on the first save of a +// file nothing has saved before, and holds steady after that. This is why the package edits a parsed +// document rather than re-rendering a decoded map, which would drop every comment in the file. // // Every write is atomic. A node cannot boot from a configuration file a crash truncated mid-write, // so a save lands in full or not at all. @@ -50,7 +51,8 @@ // last // - a key or heading segment that is not lower case, which is read under a name that is not the one // written -// - a quoted key carrying a dot or a space, which no dotted spelling splits back into +// - a key outside a bare TOML key, meaning anything but lower-case letters, digits, underscores and +// hyphens, because a quoted key is spelled one way by the decoder and another by a lookup // - a key written twice in one table, which an edit reaches only the first of // - a date or a time, which nothing configures a node with, and which cannot be written back as the // type it was read as diff --git a/config/seitoml/edit.go b/config/seitoml/edit.go index 2a9ba479e6..b421b220a1 100644 --- a/config/seitoml/edit.go +++ b/config/seitoml/edit.go @@ -141,11 +141,11 @@ func tomlValue(v any) (parser.Value, error) { case int64: return parser.ParseValue(quoteInt(x)) case uint: - return parser.ParseValue(strconv.FormatUint(uint64(x), 10)) + return unsignedValue(uint64(x)) case uint32: - return parser.ParseValue(strconv.FormatUint(uint64(x), 10)) + return unsignedValue(uint64(x)) case uint64: - return parser.ParseValue(strconv.FormatUint(x, 10)) + return unsignedValue(x) case float64: return floatValue(x) case []string: @@ -168,6 +168,19 @@ func tomlValue(v any) (parser.Value, error) { } } +// unsignedValue renders an unsigned integer, refusing one no reader can hand back. +// +// A TOML integer is signed and decodes into an int64, so a value above its maximum renders as a line +// that reads back as an error rather than a number. Refused here for the same reason an infinity is: +// this package does not write what it cannot read. +func unsignedValue(x uint64) (parser.Value, error) { + if x > math.MaxInt64 { + return parser.Value{}, fmt.Errorf("%d is larger than a configuration file's integers go, which "+ + "reach %d", x, int64(math.MaxInt64)) + } + return parser.ParseValue(strconv.FormatUint(x, 10)) +} + // floatValue renders a float as a TOML float, which the shortest form of an integral one is not. // // TOML tells a float from an integer by the fractional part or the exponent, and the shortest form of diff --git a/config/seitoml/file.go b/config/seitoml/file.go index 1b9b32cbca..b84596aaa1 100644 --- a/config/seitoml/file.go +++ b/config/seitoml/file.go @@ -125,19 +125,41 @@ func (f *File) refuseUnsupportedShapes() error { // in the file, and a segment carrying a dot or a space cannot be split back into the segments it came // from. func keyIsAddressable(key parser.Key) error { + if len(key) == 0 { + return fmt.Errorf("a key names nothing") + } for _, segment := range key { + if segment == "" { + return fmt.Errorf("%s has an empty segment, which names nothing", key) + } if segment != strings.ToLower(segment) { return fmt.Errorf("%q is not lower case, and this file's keys are read lower-cased, so it "+ "would be read under a name that is not the one written here", segment) } - if strings.ContainsAny(segment, ". ") { - return fmt.Errorf("%q carries a dot or a space, so it cannot be addressed: a key is split "+ - "on dots, and no spelling of this one splits back into it", segment) + if bad := strings.IndexFunc(segment, notBareKeyRune); bad >= 0 { + return fmt.Errorf("%q carries %q, so it is not a bare key. A bare key holds lower-case "+ + "letters, digits, underscores and hyphens; anything else has to be quoted in the file "+ + "and a dotted spelling of it does not split back into the segments it came from", + segment, segment[bad:bad+1]) } } return nil } +// notBareKeyRune reports whether a character cannot appear in a bare TOML key. +// +// A key outside this set has to be quoted where it is written, and the two readers of this file spell a +// quoted key differently: the decoder hands back the name itself, while looking one up rebuilds the +// quoting. Values would then report a key Get answers absent for. +func notBareKeyRune(r rune) bool { + switch { + case r >= 'a' && r <= 'z', r >= '0' && r <= '9', r == '_', r == '-': + return false + default: + return true + } +} + // valueIsAddressable rejects an inline table, at the top level of a value or inside an array. // // An inline table holds several keys in one written value. Its leaves flatten into the same dotted @@ -235,6 +257,10 @@ func (f *File) Version() (int, error) { "can safely run against it and no reader can know which keys it is expected to carry", VersionKey) } + if n < 1 { + return 0, fmt.Errorf("sei.toml is at %s %d, and the first schema this format had is 1. Its shape "+ + "cannot be established, so no migration can safely run against it", VersionKey, n) + } if int(n) > SchemaVersion { // The rollback case, and the reason the counter exists. A release migrates the file forward on // the node's own disk, so rolling the binary back does not roll the file back with it. Read diff --git a/config/seitoml/seitoml_test.go b/config/seitoml/seitoml_test.go index fcbeeb5b64..51a83d5b94 100644 --- a/config/seitoml/seitoml_test.go +++ b/config/seitoml/seitoml_test.go @@ -408,12 +408,27 @@ func TestAShapeThisFileDoesNotCarryIsRefusedAtTheDoor(t *testing.T) { { "a quoted key carrying a dot", "[probe]\n\"a.b\" = 1\n", - "carries a dot or a space", + "is not a bare key", }, { "a quoted key carrying a space", "[probe]\n\"a b\" = 1\n", - "carries a dot or a space", + "is not a bare key", + }, + { + "a quoted key carrying punctuation", + "[probe]\n\"a#b\" = 1\n", + "is not a bare key", + }, + { + "a quoted key carrying a plus", + "[probe]\n\"a+b\" = 1\n", + "is not a bare key", + }, + { + "an empty quoted key", + "\"\" = 1\n", + "empty segment", }, { "a date", @@ -454,6 +469,9 @@ sc-async-commit-buffer = 100 enable = true dir = "/data" +[pruning] +memiavl.snapshot-interval = 100 + [p2p] persistent-peers = ["a", "b"] `) @@ -466,6 +484,9 @@ persistent-peers = ["a", "b"] "state-commit.sc-async-commit-buffer": int64(100), "state-commit.flatkv.enable": true, "state-commit.flatkv.dir": "/data", + // A dotted key inside a table, which is a different shape from a nested heading and reads to the + // same flattened key. + "pruning.memiavl.snapshot-interval": int64(100), } { if values[key] != want { t.Errorf("%s read back as %#v, want %#v", key, values[key], want) @@ -1399,3 +1420,119 @@ func TestEveryVerbTakingAKeyAppliesOneRule(t *testing.T) { t.Errorf("the folded key reads back as (%#v, %v, %v), want true", got, ok, err) } } + +// TestThePreambleIsReplacedAcrossASaveAndReload holds the property over the flow that actually runs. +// +// Regenerating is Load, SetPreamble, Save, and the same again on the next release, so the block this +// method must recognise is one that has been through the parser rather than one it just inserted. Held +// in memory only, the test cannot see a parser that reattaches a leading comment to whatever follows it, +// and the header would grow on every run. +func TestThePreambleIsReplacedAcrossASaveAndReload(t *testing.T) { + dir := t.TempDir() + path := filepath.Join(dir, "sei.toml") + + f, err := seitoml.New("validator") + if err != nil { + t.Fatalf("New: %v", err) + } + if err := f.Set("probe.n", 1); err != nil { + t.Fatalf("Set: %v", err) + } + f.SetPreamble([]string{" written by run one"}) + if err := f.Save(path); err != nil { + t.Fatalf("Save: %v", err) + } + + reread, err := seitoml.Load(path) + if err != nil { + t.Fatalf("Load: %v", err) + } + reread.SetPreamble([]string{" written by run two"}) + if err := reread.Save(path); err != nil { + t.Fatalf("Save: %v", err) + } + + raw, err := os.ReadFile(path) //nolint:gosec // a path this test created under t.TempDir + if err != nil { + t.Fatalf("ReadFile: %v", err) + } + if strings.Contains(string(raw), "run one") { + t.Errorf("the first preamble survived the second, so a header grows on every regenerate:\n%s", raw) + } + if !strings.Contains(string(raw), "# written by run two") { + t.Errorf("the second preamble is not in the file:\n%s", raw) + } + // The values are untouched by either header, since a preamble is not configuration. + final, err := seitoml.Load(path) + if err != nil { + t.Fatalf("Load after two preambles: %v", err) + } + if got, ok, err := final.Get("probe.n"); err != nil || !ok || got != int64(1) { + t.Errorf("probe.n = (%#v, %v, %v) after two preambles, want 1", got, ok, err) + } +} + +// TestAPreambleLeavesAnOperatorsOwnTopCommentAlone covers the comment this method must not claim. +// +// SetPreamble replaces the block it put there before, and an operator may have written their own +// explanation at the top of the file. Removing that would be the comment loss this package exists to +// prevent, so it has to survive a header being written above it. +func TestAPreambleLeavesAnOperatorsOwnTopCommentAlone(t *testing.T) { + f := parse(t, "# I wrote this and it explains the file\n# do not delete it\nschema_version = 1\n"+ + "node_mode = \"validator\"\n\n[probe]\nn = 1\n") + + f.SetPreamble([]string{" generated header"}) + + out := render(t, f) + for _, want := range []string{"I wrote this", "do not delete it", "# generated header"} { + if !strings.Contains(out, want) { + t.Errorf("%q is not in the file after a preamble was written:\n%s", want, out) + } + } +} + +// TestAnUnsignedValueTooLargeToReadBackIsRefused holds the writer to what a reader can return. +// +// A TOML integer is signed and decodes into an int64, so a larger unsigned value renders as a line that +// reads back as an error. Accepted, it would make every later read of the file fail, including the two +// keys that describe the file itself. +func TestAnUnsignedValueTooLargeToReadBackIsRefused(t *testing.T) { + f, err := seitoml.New("validator") + if err != nil { + t.Fatalf("New: %v", err) + } + + err = f.Set("probe.n", uint64(math.MaxUint64)) + if err == nil { + t.Fatal("a value past int64 was written; every later read of the file would then fail") + } + if !strings.Contains(err.Error(), "integers go") { + t.Errorf("the refusal reads %q and does not say what the limit is", err) + } + + // The largest value that does read back is still accepted, so the bound is not off by one. + if err := f.Set("probe.big", uint64(math.MaxInt64)); err != nil { + t.Fatalf("the largest readable unsigned value was refused: %v", err) + } + reread := parse(t, render(t, f)) + if got, ok, err := reread.Get("probe.big"); err != nil || !ok || got != int64(math.MaxInt64) { + t.Errorf("probe.big = (%#v, %v, %v), want %d", got, ok, err, int64(math.MaxInt64)) + } +} + +// TestASchemaVersionBelowTheFirstOneIsRefused closes the counter's lower end. +// +// Version documents that an absent or unreadable counter is an error rather than a zero, so an explicit +// zero must not return the very value that sentence rules out. A caller cannot tell that zero from the +// one it gets alongside an error. +func TestASchemaVersionBelowTheFirstOneIsRefused(t *testing.T) { + for _, body := range []string{"schema_version = 0\n", "schema_version = -5\n"} { + got, err := parse(t, body).Version() + if err == nil { + t.Errorf("%q read as version %d; a counter below the first schema names no shape", + strings.TrimSpace(body), got) + } else if !strings.Contains(err.Error(), "first schema") { + t.Errorf("the refusal reads %q and does not say what the floor is", err) + } + } +} From a438899cf0bd56e71bfc2a610ec0af3e60de3d06 Mon Sep 17 00:00:00 2001 From: bdchatham Date: Tue, 18 Aug 2026 15:18:59 -0700 Subject: [PATCH 09/24] config/seitoml: decode with the library a node reads its configuration with A systems review found that Parse and Set both accepted files viper refuses, so a save could produce a sei.toml no node can boot from, having atomically replaced the good one. I had reported the first of those as a non-issue because BurntSushi accepts it; BurntSushi is the only decoder in this module graph that does, and the TOML specification gives that exact shape as an invalid example. viper decodes TOML with pelletier/go-toml/v2, so this package now does too. "This file parses here" and "the node can boot from it" become one statement rather than two that drift. That closes the class rather than three instances of it. The guard added last commit was written against leaf value paths when the thing that collides is the table namespace, so it could not see a table an ancestor's dotted key created, could not see a section holding nothing, and stepped over any collision a hyphenated sibling sorted between, since a hyphen orders before a dot and is the word separator every key in this tree uses. That guard is deleted. The decoder answers instead, at Parse and again after every edit: Set writes, renders, offers the result to the decoder, and undoes the write if the document no longer reads. insert also stops appending a heading for a table an ancestor's dotted key already created, and extends the dotted name instead, so that edit now succeeds and viper reads the result. Two more from the same review: - SetPreamble deleted a comment an operator wrote at the top of their file. It had no way to tell that block from one it wrote, so it now writes a mark and looks for it, and leaves an unrecognised block alone. - Set silently changed a string that was not valid UTF-8, because the escaper substitutes a replacement rune. Refused now, in both the scalar and list paths. Three test fixtures were shaped to agree with a conclusion rather than to attack it, and each is now the variant that can fail: flatkv-mode rather than flatkvx, since only a hyphen hides the ordering bug; a blank line after the operator's comment, since without one the parser attaches it to the next key and the branch under test is unreachable; and a#b in the verb loop, since every key there was caught by the rule the commit replaced. That last one left the previous commit's headline untested. Coverage is 94.8% of statements. Seven guards verified by removing each and watching a named test fail, including the three reshaped fixtures. Co-Authored-By: Claude Opus 5 (1M context) --- config/seitoml/doc.go | 60 +++++------ config/seitoml/edit.go | 143 ++++++++++++++++++++----- config/seitoml/file.go | 41 +------- config/seitoml/seitoml_test.go | 184 +++++++++++++++++++++++++-------- config/seitoml/values.go | 17 +-- go.mod | 2 +- 6 files changed, 303 insertions(+), 144 deletions(-) diff --git a/config/seitoml/doc.go b/config/seitoml/doc.go index cbbd954661..373ed7a78e 100644 --- a/config/seitoml/doc.go +++ b/config/seitoml/doc.go @@ -39,40 +39,40 @@ // sei.toml. Infinities and NaN have no form here at all, and are refused in both directions rather than // written as a line no reader can load. // -// # Shapes This File Does Not Carry +// # What This File May Hold // -// TOML permits more shapes than a node's configuration uses, and Parse refuses these rather than -// reading them into something a later verb cannot write back: +// Values come from the decoder a node reads its configuration with. viper decodes TOML with +// pelletier/go-toml/v2, so this package does too, which makes "this file parses here" and "the node can +// boot from it" the same statement rather than two that can drift apart. // -// - an inline table, whose keys flatten into the same space a table's do, so editing one of them -// defines the table a second time and produces a file a conforming reader will not load -// - an array of tables, where every entry but the last disappears from the flattened key space -// - a table heading that appears twice, where an edit reaches the first and a read answers from the -// last -// - a key or heading segment that is not lower case, which is read under a name that is not the one -// written -// - a key outside a bare TOML key, meaning anything but lower-case letters, digits, underscores and -// hyphens, because a quoted key is spelled one way by the decoder and another by a lookup -// - a key written twice in one table, which an edit reaches only the first of -// - a date or a time, which nothing configures a node with, and which cannot be written back as the -// type it was read as -// - one name used for both a value and a table, which names the same thing twice +// That decoder is therefore the authority on shape, and Parse asks it rather than keeping a list. A +// name used for both a value and a table, a table defined twice, a key written twice, a table given a +// heading after an ancestor's dotted key already created it: all of it is one question with one answer. +// A list kept here instead missed three of those, and every one of them produced a file this package +// would save and no node could read. +// +// The editing parser is a second library and does a different job: it locates lines and preserves +// comments, and stops short of interpreting a literal, because deciding what an underscore-separated +// integer or a multi-line string means is a second implementation of the specification and a +// hand-written one went wrong in four places. // -// Set, Unset and Get apply the same rule to the key a caller hands them, so a key one of them writes is -// a key the file reads back. Set also refuses a key that would name a value where a table already is, -// or the reverse, because the file that produces parses cleanly and can never be read. +// Every edit is asked the same question. Set writes the key, renders, and offers the result to the +// decoder; if the document no longer reads, the write is undone and the key is named. So a shape nobody +// anticipated is refused as surely as one somebody did, which is the property enumerating shapes by hand +// could not give. // -// Each was previously accepted and then lost or corrupted further in. Refusing at the door is what -// lets every verb below assume the document holds only shapes it can read and write back, and it is -// only free while no operator has written a file that uses them. +// Four things that decoder allows are refused anyway, because this package has to write back what it +// reads: // -// Reading and editing use different libraries, on purpose. The editing parser locates lines and -// preserves comments, and stops short of interpreting a literal; deciding what an underscore-separated -// integer or a multi-line string means is a second implementation of the specification. Values come -// from a conforming decoder instead, so the shape of a literal is somebody else's problem and the file -// reads the way every other TOML reader reads it. +// - an infinity or a NaN, which have no form to write +// - a date or a time, which nothing configures a node with and which cannot be written back as the +// type it was read as +// - an inline table, whose keys flatten into the same space a table's do, so an edit to one of them +// has no line of its own to change +// - an array of tables, where every entry but the last disappears from the flattened key space // -// Two things that decoder allows are still refused here, because this package has to write back what it -// reads: an infinity or a NaN, which have no form to write, and a date, which would come back as a time -// this package cannot render. +// A key is a bare TOML key: lower-case letters, digits, underscores and hyphens. Anything else has to be +// quoted where it is written, and a quoted key is spelled one way by the decoder and another by a +// lookup, so Values would report a key Get answers absent for. Set, Unset and Get apply that same rule +// to the key a caller hands them, so a key one of them writes is a key the file reads back. package seitoml diff --git a/config/seitoml/edit.go b/config/seitoml/edit.go index b421b220a1..6992e92c61 100644 --- a/config/seitoml/edit.go +++ b/config/seitoml/edit.go @@ -6,6 +6,7 @@ import ( "strconv" "strings" "time" + "unicode/utf8" "github.com/creachadair/tomledit" "github.com/creachadair/tomledit/parser" @@ -32,64 +33,124 @@ func (f *File) Set(key string, v any) error { e.Value = value return nil } - // A key the document does not hold yet, so it may collide with one it does. Writing a value under a - // name that already names a table, or a table under one that already names a value, produces a file - // this package can save and no conforming reader can load. - if err := keysDoNotShadowEachOther(append(f.writtenPaths(), path.String())); err != nil { + + // A key the document does not hold yet lands in a namespace that may already use its name for a + // table, or use a table's name for it. Rather than enumerate the shapes that collide, insert and then + // ask the decoder, which is the same one the node reads with: if the document no longer decodes, the + // insert is undone and the key is named. Enumerating them by hand missed three. + undo := f.insert(path, value) + if err := f.decodable(); err != nil { + undo() return fmt.Errorf("%s: %w", key, err) } - f.insert(path, value) return nil } +// decodable reports whether the document still renders to something the node's decoder can read. +// +// The one check every edit passes through, so a shape no caller anticipated is refused as surely as one +// somebody did. Rendering is what a later process reads, so this asks the question in the form the +// answer matters in. +func (f *File) decodable() error { + _, err := f.decoded() + return err +} + // insert adds a key the document does not have yet. // // A key with no dots belongs at the top level. Otherwise it goes in the table its prefix names, // which is created when it is absent so writing the first key of a section works without the // operator having to add the heading by hand. -func (f *File) insert(path parser.Key, value parser.Value) { +func (f *File) insert(path parser.Key, value parser.Value) func() { leaf := parser.Key{path[len(path)-1]} kv := &parser.KeyValue{Name: leaf, Value: value} if len(path) == 1 { - f.insertGlobal(kv) - return + return f.insertGlobal(kv) } table := path[:len(path)-1] if e := transform.FindTable(f.doc, table...); e != nil { - transform.InsertMapping(e.Section, kv, true) - return + return appendItem(e.Section, kv) } + // No section carries this name. It may still be a table the document created by writing a dotted key + // inside its parent, and a heading for one of those defines it twice, so the leaf goes in as a dotted + // key under the nearest section instead. Where there is no such section either, the heading is new + // and correct. + if owner, rest := f.sectionOwning(table); owner != nil { + return appendItem(owner, &parser.KeyValue{Name: append(rest, leaf...), Value: value}) + } + before := len(f.doc.Sections) f.doc.Sections = append(f.doc.Sections, &tomledit.Section{ Heading: &parser.Heading{Name: table}, Items: []parser.Item{kv}, }) + return func() { f.doc.Sections = f.doc.Sections[:before] } +} + +// sectionOwning returns the section whose heading is the longest prefix of table, and the rest of the +// path below it. +// +// A table can exist without a heading of its own, written as a dotted key inside an ancestor. Adding a +// key to one has to extend that dotted name rather than introduce a heading the decoder reads as a +// second definition. +func (f *File) sectionOwning(table parser.Key) (*tomledit.Section, parser.Key) { + var best *tomledit.Section + var rest parser.Key + for _, s := range f.doc.Sections { + if s.Heading == nil || !s.Name.IsPrefixOf(table) { + continue + } + if best == nil || len(s.Name) > len(best.Name) { + best, rest = s, table[len(s.Name):] + } + } + return best, rest +} + +// appendItem adds an item to a section and reports how to remove it again. +func appendItem(s *tomledit.Section, kv *parser.KeyValue) func() { + transform.InsertMapping(s, kv, true) + return func() { + for i, item := range s.Items { + if item == parser.Item(kv) { + s.Items = append(s.Items[:i], s.Items[i+1:]...) + return + } + } + } } // insertGlobal adds a top-level key, creating the global section when the document has none. // // InsertMapping's result is not checked because it only reports a collision it was told not to // replace, and it is told to replace. -func (f *File) insertGlobal(kv *parser.KeyValue) { +func (f *File) insertGlobal(kv *parser.KeyValue) func() { if f.doc.Global == nil { f.doc.Global = &tomledit.Section{} } - transform.InsertMapping(f.doc.Global, kv, true) + return appendItem(f.doc.Global, kv) } +// preambleMark is the last line of a block SetPreamble owns. +// +// A comment block at the top of a file may be one an operator wrote, and this method has to replace its +// own without touching theirs. Nothing distinguishes the two but a mark it writes and recognises. +const preambleMark = " -- above this line is generated; edit below --" + // SetPreamble puts a comment block at the top of the document, above everything else. // // Comments rather than keys, so nothing a reader needs in order to understand the file becomes -// configuration the node has to recognize. This replaces any block it put there before, so -// regenerating does not stack one preamble on the last. +// configuration the node has to recognize. This replaces a block it wrote before, so regenerating does +// not stack one preamble on the last, and leaves any other leading comment alone: an operator's +// explanation at the top of the file is exactly what this package exists to preserve. func (f *File) SetPreamble(lines []string) { if f.doc.Global == nil { f.doc.Global = &tomledit.Section{} } items := f.doc.Global.Items if len(items) > 0 { - if _, leading := items[0].(parser.Comments); leading { + if block, leading := items[0].(parser.Comments); leading && ownedPreamble(block) { items = items[1:] } } @@ -97,7 +158,20 @@ func (f *File) SetPreamble(lines []string) { f.doc.Global.Items = items return } - f.doc.Global.Items = append([]parser.Item{parser.Comments(lines)}, items...) + marked := append(append([]string(nil), lines...), preambleMark) + f.doc.Global.Items = append([]parser.Item{parser.Comments(marked)}, items...) +} + +// ownedPreamble reports whether a leading comment block is one SetPreamble wrote. +// +// Compared on the line's content, because a block this method inserts holds the text alone while the +// same block read back from a file carries the comment character the renderer added. +func ownedPreamble(block parser.Comments) bool { + if len(block) == 0 { + return false + } + last := strings.TrimSpace(strings.TrimPrefix(strings.TrimSpace(block[len(block)-1]), "#")) + return last == strings.TrimSpace(preambleMark) } // Unset removes a key and reports whether the file carried one. @@ -131,9 +205,17 @@ func tomlValue(v any) (parser.Value, error) { case bool: return parser.ParseValue(strconv.FormatBool(x)) case string: - return parser.ParseValue(basicString(x)) + text, err := basicString(x) + if err != nil { + return parser.Value{}, err + } + return parser.ParseValue(text) case time.Duration: - return parser.ParseValue(basicString(x.String())) + text, err := basicString(x.String()) + if err != nil { + return parser.Value{}, err + } + return parser.ParseValue(text) case int: return parser.ParseValue(quoteInt(int64(x))) case int32: @@ -149,7 +231,11 @@ func tomlValue(v any) (parser.Value, error) { case float64: return floatValue(x) case []string: - return parser.ParseValue("[" + strings.Join(quoteEach(x), ", ") + "]") + quoted, err := quoteEach(x) + if err != nil { + return parser.Value{}, err + } + return parser.ParseValue("[" + strings.Join(quoted, ", ") + "]") case []any: // The shape reading an array back produces. Without this, anything that reads a list and writes // it again fails on a value this package handed it. Every element is a value the reader can @@ -207,15 +293,24 @@ func floatValue(x float64) (parser.Value, error) { // The escaping is the scanner's own rather than Go's. Go's quoter writes a control character as \x07 // or \a and TOML defines neither, so such a value was refused with a diagnostic naming an offset into // a string the operator never saw. -func basicString(s string) string { - return `"` + string(scanner.Escape(s)) + `"` +func basicString(s string) (string, error) { + if !utf8.ValidString(s) { + // The escaper substitutes a replacement rune for a byte that is not valid UTF-8, so writing one + // would store a different value than the caller passed and no error would say so. + return "", fmt.Errorf("the value is not valid UTF-8, and a configuration file holds text") + } + return `"` + string(scanner.Escape(s)) + `"`, nil } // quoteEach renders every element of a string list. -func quoteEach(ss []string) []string { +func quoteEach(ss []string) ([]string, error) { out := make([]string, len(ss)) for i, s := range ss { - out[i] = basicString(s) + text, err := basicString(s) + if err != nil { + return nil, fmt.Errorf("element %d: %w", i, err) + } + out[i] = text } - return out + return out, nil } diff --git a/config/seitoml/file.go b/config/seitoml/file.go index b84596aaa1..2a9af71fbc 100644 --- a/config/seitoml/file.go +++ b/config/seitoml/file.go @@ -7,7 +7,6 @@ import ( "io" "os" "path/filepath" - "sort" "strconv" "strings" @@ -116,7 +115,10 @@ func (f *File) refuseUnsupportedShapes() error { if bad != nil { return bad } - return keysDoNotShadowEachOther(f.writtenPaths()) + // One name used for both a value and a table, a table defined twice, a key written twice: all of it + // is the decoder's answer rather than a list kept here. A hand-written list missed an implicitly + // created table, an empty section, and any collision a hyphen sorted between. + return f.decodable() } // keyIsAddressable reports whether every segment of a key can be read back as written. @@ -125,9 +127,6 @@ func (f *File) refuseUnsupportedShapes() error { // in the file, and a segment carrying a dot or a space cannot be split back into the segments it came // from. func keyIsAddressable(key parser.Key) error { - if len(key) == 0 { - return fmt.Errorf("a key names nothing") - } for _, segment := range key { if segment == "" { return fmt.Errorf("%s has an empty segment, which names nothing", key) @@ -413,37 +412,5 @@ func keyOf(key string) (parser.Key, error) { return out, nil } -// keysDoNotShadowEachOther refuses a key whose path is a prefix of another key's. -// -// TOML gives a name to a value or to a table, never to both, so a scalar flatkv beside a -// [state-commit.flatkv] heading defines one name twice. The editing parser accepts that and a -// conforming decoder rejects it, so a file carrying one parses and then every read of it fails. -// -// Sorting puts a path immediately before anything nested under it, which is what makes one pass over -// the neighbours enough. -func keysDoNotShadowEachOther(paths []string) error { - sorted := append([]string(nil), paths...) - sort.Strings(sorted) - for i := 1; i < len(sorted); i++ { - if strings.HasPrefix(sorted[i], sorted[i-1]+".") { - return fmt.Errorf("%s is a value and %s is a table under the same name, so a reader cannot "+ - "decide which one this file means", sorted[i-1], sorted[i]) - } - } - return nil -} - -// writtenPaths returns the dotted path of every value the document holds. -func (f *File) writtenPaths() []string { - var out []string - f.doc.Scan(func(full parser.Key, e *tomledit.Entry) bool { - if e.KeyValue != nil { - out = append(out, full.String()) - } - return true - }) - return out -} - // quoteInt renders an integer the way TOML spells one. func quoteInt(n int64) string { return strconv.FormatInt(n, 10) } diff --git a/config/seitoml/seitoml_test.go b/config/seitoml/seitoml_test.go index 51a83d5b94..0624344d58 100644 --- a/config/seitoml/seitoml_test.go +++ b/config/seitoml/seitoml_test.go @@ -887,7 +887,7 @@ commented = [ } } -// TestAValueTomlDoesNotRecognizeIsNamedNotGuessed covers what a hand-edited file can go wrong as. +// TestAValueTomlDoesNotRecognizeIsRefusedAtTheDoor covers what a hand-edited file can go wrong as. // // A bare word never reaches here, because the parser refuses one before any value is decoded, in a // table and inside an array or an inline table alike. What does reach here is a value TOML accepts and @@ -896,34 +896,26 @@ commented = [ // // Each has to name the key and what is wrong with it. Read as a zero, the node would boot on a value // nobody wrote; dropped, the operator's line would be silently ignored. -func TestAValueTomlDoesNotRecognizeIsNamedNotGuessed(t *testing.T) { +func TestAValueTomlDoesNotRecognizeIsRefusedAtTheDoor(t *testing.T) { for _, tc := range []struct { name string body string - key string want string }{ - {"an integer past int64", "[probe]\nn = 99999999999999999999\n", "probe.n", "out of range for int64"}, - {"an infinity", "[probe]\nn = inf\n", "probe.n", "has to be a finite number"}, - {"a negative infinity", "[probe]\nn = -inf\n", "probe.n", "has to be a finite number"}, - {"a NaN", "[probe]\nn = nan\n", "probe.n", "has to be a finite number"}, - {"an infinity inside an array", "[probe]\nlist = [1.5, inf]\n", "probe.list", "has to be a finite number"}, + {"an integer past int64", "[probe]\nn = 99999999999999999999\n", "value out of range"}, + {"an infinity", "[probe]\nn = inf\n", "has to be a finite number"}, + {"a negative infinity", "[probe]\nn = -inf\n", "has to be a finite number"}, + {"a NaN", "[probe]\nn = nan\n", "has to be a finite number"}, + {"an infinity inside an array", "[probe]\nlist = [1.5, inf]\n", "has to be a finite number"}, } { t.Run(tc.name, func(t *testing.T) { - f := parse(t, tc.body) - - _, err := f.Values() + _, err := seitoml.Parse(strings.NewReader(tc.body)) if err == nil { - t.Fatalf("Values accepted %s, so the node would run on a value nothing produced", tc.name) + t.Fatalf("%s parsed; a value no reader can use has to fail at the door rather than on "+ + "the first read of it", tc.name) } if !strings.Contains(err.Error(), tc.want) { - t.Errorf("the error reads %q, which does not mention %q", err, tc.want) - } - - // Get has to refuse the same value, or a caller reading one key would see what a caller - // reading the whole file cannot. - if _, _, err := f.Get(tc.key); err == nil { - t.Errorf("Get(%q) accepted the value Values refused", tc.key) + t.Errorf("the refusal reads %q, which does not mention %q", err, tc.want) } }) } @@ -1121,25 +1113,13 @@ func TestAnInfinityCannotBeWritten(t *testing.T) { // proceed without, so a value this package cannot decode has to fail there rather than further in. Each // refusal names its key, because the two have different fixes. func TestAFileDescribingItselfWithANonValueIsRefused(t *testing.T) { - t.Run("an undecodable schema version", func(t *testing.T) { - _, err := parse(t, "schema_version = inf\n").Version() - if err == nil { - t.Fatal("an undecodable version was accepted, so a migration would run against a file whose " + - "shape nobody established") - } - if !strings.Contains(err.Error(), seitoml.VersionKey) { - t.Errorf("the error reads %q and does not name the key", err) - } - }) - - t.Run("an undecodable node mode", func(t *testing.T) { - _, err := parse(t, "node_mode = inf\n").Mode() - if err == nil { - t.Fatal("an undecodable mode was accepted, so an archive node's file would be compared " + - "against a validator's defaults") - } - if !strings.Contains(err.Error(), seitoml.ModeKey) { - t.Errorf("the error reads %q and does not name the key", err) + t.Run("a describing key holding a value no reader can use", func(t *testing.T) { + // Refused at Parse along with every other value, so neither reader has to handle it. + for _, body := range []string{"schema_version = inf\n", "node_mode = inf\n"} { + if _, err := seitoml.Parse(strings.NewReader(body)); err == nil { + t.Errorf("%q parsed, so a migration or a mode comparison would run against it", + strings.TrimSpace(body)) + } } }) @@ -1331,7 +1311,7 @@ func TestANameCannotBeAValueAndATableAtOnce(t *testing.T) { if err == nil { t.Fatal("a file naming one thing a value and a table parsed; every read of it then fails") } - if !strings.Contains(err.Error(), "state-commit.flatkv") { + if !strings.Contains(err.Error(), "flatkv") { t.Errorf("the refusal reads %q and does not name the key at fault", err) } }) @@ -1343,7 +1323,7 @@ func TestANameCannotBeAValueAndATableAtOnce(t *testing.T) { t.Fatal("Set wrote a table under a name a value already had, so Save would produce a file " + "no reader can load") } - if !strings.Contains(err.Error(), "state-commit.flatkv") { + if !strings.Contains(err.Error(), "flatkv") { t.Errorf("the refusal reads %q and does not name the key at fault", err) } requireStillReadable(t, f) @@ -1359,10 +1339,57 @@ func TestANameCannotBeAValueAndATableAtOnce(t *testing.T) { }) t.Run("a sibling that merely shares a prefix still writes", func(t *testing.T) { - // state-commit.flatkvx is not nested under state-commit.flatkv, so the check must not refuse it. + // A hyphen is this tree's word separator and sorts before a dot, so flatkv-mode is the sibling + // that a string-ordered check would step over. flatkvx would not: x sorts after the dot, which is + // the half of the comparison that cannot go wrong. f := parse(t, "schema_version = 1\nnode_mode = \"validator\"\n\n[state-commit]\nflatkv = true\n") - if err := f.Set("state-commit.flatkvx", false); err != nil { - t.Fatalf("a key sharing only a prefix was refused: %v", err) + for _, key := range []string{"state-commit.flatkv-mode", "state-commit.flatkvx"} { + if err := f.Set(key, false); err != nil { + t.Errorf("%s shares only a prefix with state-commit.flatkv and was refused: %v", key, err) + } + } + requireStillReadable(t, f) + }) + + t.Run("a conflict a hyphenated sibling sits between", func(t *testing.T) { + // The shape a sorted comparison misses: flatkv-mode orders between flatkv and flatkv.enable. + _, err := seitoml.Parse(strings.NewReader("schema_version = 1\nnode_mode = \"validator\"\n\n" + + "[state-commit]\nflatkv = true\nflatkv-mode = \"sync\"\n\n[state-commit.flatkv]\nenable = false\n")) + if err == nil { + t.Fatal("a conflict separated by a hyphenated sibling parsed; the node's own decoder refuses it") + } + }) + + t.Run("a table an ancestor's dotted key created", func(t *testing.T) { + // The table exists without a heading of its own, so a heading for it would define it twice. The + // key has to join the dotted name instead, and the result has to satisfy the node's decoder. + f := parse(t, "schema_version = 1\nnode_mode = \"validator\"\n\n[state-commit]\nflatkv.enable = true\n") + if err := f.Set("state-commit.flatkv.dir", "/data"); err != nil { + t.Fatalf("writing a sibling into an implicitly created table was refused: %v", err) + } + requireStillReadable(t, f) + values, err := f.Values() + if err != nil { + t.Fatalf("Values: %v", err) + } + for key, want := range map[string]any{ + "state-commit.flatkv.enable": true, + "state-commit.flatkv.dir": "/data", + } { + if values[key] != want { + t.Errorf("%s = %#v, want %#v", key, values[key], want) + } + } + }) + + t.Run("a value named like a section holding nothing", func(t *testing.T) { + // An empty section contributes no value, so a check over written values cannot see it. + f := parse(t, "schema_version = 1\nnode_mode = \"validator\"\n\n[probe]\nn = 1\n") + if _, err := f.Unset("probe.n"); err != nil { + t.Fatalf("Unset: %v", err) + } + if err := f.Set("probe", 1); err == nil { + t.Fatal("a value took the name of a section that still exists") } requireStillReadable(t, f) }) @@ -1390,7 +1417,9 @@ func requireStillReadable(t *testing.T, f *seitoml.File) { // A key a verb accepts and Parse refuses is a key that can be written and then never read: the save // succeeds, and the node cannot load its own configuration afterwards. func TestEveryVerbTakingAKeyAppliesOneRule(t *testing.T) { - for _, key := range []string{"foo bar", "probe.a b", " leading", "trailing "} { + // The first four are refused by a dot-or-space rule as well, so the last two are what hold the bare-key + // rule these verbs now share with Parse. + for _, key := range []string{"foo bar", "probe.a b", " leading", "trailing ", "probe.a#b", "probe.a+b"} { t.Run(fmt.Sprintf("key %q", key), func(t *testing.T) { f, err := seitoml.New("validator") if err != nil { @@ -1478,7 +1507,10 @@ func TestThePreambleIsReplacedAcrossASaveAndReload(t *testing.T) { // explanation at the top of the file. Removing that would be the comment loss this package exists to // prevent, so it has to survive a header being written above it. func TestAPreambleLeavesAnOperatorsOwnTopCommentAlone(t *testing.T) { - f := parse(t, "# I wrote this and it explains the file\n# do not delete it\nschema_version = 1\n"+ + // The blank line matters: the parser keeps a comment block standalone only when one follows it, and a + // block attached to the next key is one SetPreamble never looks at. Without it this test passes + // whatever the code does. + f := parse(t, "# I wrote this and it explains the file\n# do not delete it\n\nschema_version = 1\n"+ "node_mode = \"validator\"\n\n[probe]\nn = 1\n") f.SetPreamble([]string{" generated header"}) @@ -1536,3 +1568,65 @@ func TestASchemaVersionBelowTheFirstOneIsRefused(t *testing.T) { } } } + +// TestAValueThatIsNotTextIsRefused covers what the escaper would otherwise change silently. +// +// The escaper substitutes a replacement rune for a byte that is not valid UTF-8, so writing one stored a +// different value than the caller passed with nothing reporting it. A configuration file holds text, so +// the refusal is the honest answer. +func TestAValueThatIsNotTextIsRefused(t *testing.T) { + f, err := seitoml.New("validator") + if err != nil { + t.Fatalf("New: %v", err) + } + invalid := string([]byte{'a', 0xff, 'b'}) + + if err := f.Set("probe.v", invalid); err == nil { + t.Fatal("a value that is not valid UTF-8 was written, and it reads back as something else") + } + if err := f.Set("probe.list", []string{"fine", invalid}); err == nil { + t.Error("a list carrying one was written") + } + + // Text that merely looks unusual still writes and survives, so the guard is not refusing breadth. + for _, ok := range []string{"héllo", "日本語", "a\tb", `C:\sei`} { + if err := f.Set("probe.v", ok); err != nil { + t.Errorf("Set(%q) was refused: %v", ok, err) + continue + } + if got, _, err := parse(t, render(t, f)).Get("probe.v"); err != nil || got != ok { + t.Errorf("%q read back as (%#v, %v)", ok, got, err) + } + } +} + +// TestAPreambleOnlyReplacesOneItWrote holds the block this method may claim. +// +// A comment block at the top of a file is an operator's explanation unless this method put it there, and +// it has no way to tell the two apart but a mark it writes and looks for. Deleting theirs would be the +// comment loss the package exists to prevent; leaving its own would grow a header on every regenerate. +func TestAPreambleOnlyReplacesOneItWrote(t *testing.T) { + const operator = "# ops: do not raise flatkv without asking" + f := parse(t, operator+"\n\nschema_version = 1\nnode_mode = \"validator\"\n\n[probe]\nn = 1\n") + + f.SetPreamble([]string{" run one"}) + first := render(t, f) + if !strings.Contains(first, "do not raise flatkv") { + t.Fatalf("the operator's comment was deleted by the first preamble:\n%s", first) + } + + // Through a parse, which is the state the next release's run sees. + again := parse(t, first) + again.SetPreamble([]string{" run two"}) + second := render(t, again) + + if strings.Contains(second, "run one") { + t.Errorf("the first preamble survived the second, so a header grows on every run:\n%s", second) + } + if !strings.Contains(second, "run two") { + t.Errorf("the second preamble is not in the file:\n%s", second) + } + if !strings.Contains(second, "do not raise flatkv") { + t.Errorf("the operator's comment was deleted by the second preamble:\n%s", second) + } +} diff --git a/config/seitoml/values.go b/config/seitoml/values.go index 29421825b6..e561157549 100644 --- a/config/seitoml/values.go +++ b/config/seitoml/values.go @@ -4,7 +4,7 @@ import ( "fmt" "math" - "github.com/BurntSushi/toml" + toml "github.com/pelletier/go-toml/v2" ) // Values returns every key the file writes, as dotted paths to Go values. @@ -38,11 +38,14 @@ func (f *File) Get(key string) (any, bool, error) { // decoded renders the document and reads it back as Go values, keyed by dotted path. // -// The values come from a TOML decoder rather than from the editing parser's tokens. The editing parser -// locates lines and preserves comments, which is why it is here, and it deliberately stops short of -// interpreting a literal. Deciding what "1_000" or a multi-line string means is a second implementation -// of the TOML specification, and the difference between Go's string grammar and TOML's is where a -// hand-written one goes wrong. +// The values come from the decoder a node reads its configuration with, which is what makes "this file +// parses" and "this node can boot from it" the same statement. viper decodes TOML with +// pelletier/go-toml/v2, so this does too, and a shape that library refuses is refused here rather than +// discovered on a node. +// +// The editing parser locates lines and preserves comments, which is why it is also here, and it stops +// short of interpreting a literal. Deciding what "1_000" or a multi-line string means is a second +// implementation of the specification, and a hand-written one went wrong in four places. // // Rendering first rather than holding the source means an unsaved edit is read back through the same // path a later process would use, so a value this package cannot express fails here rather than on a @@ -53,7 +56,7 @@ func (f *File) decoded() (map[string]any, error) { return nil, err } var nested map[string]any - if _, err := toml.Decode(string(raw), &nested); err != nil { + if err := toml.Unmarshal(raw, &nested); err != nil { return nil, fmt.Errorf("read sei.toml: %w", err) } out := make(map[string]any, len(nested)) diff --git a/go.mod b/go.mod index f098f760ad..e46ab4e48e 100644 --- a/go.mod +++ b/go.mod @@ -224,7 +224,7 @@ require ( github.com/opencontainers/go-digest v1.0.0 // indirect github.com/opencontainers/image-spec v1.1.0-rc2 // indirect github.com/opencontainers/runc v1.1.14 // indirect - github.com/pelletier/go-toml/v2 v2.2.4 // indirect + github.com/pelletier/go-toml/v2 v2.2.4 github.com/petermattis/goid v0.0.0-20260113132338-7c7de50cc741 // indirect github.com/pierrec/lz4/v4 v4.1.22 // indirect github.com/pion/dtls/v2 v2.2.7 // indirect From 75182439d9bdc536000f3586846b1c67f350f3b6 Mon Sep 17 00:00:00 2001 From: bdchatham Date: Tue, 18 Aug 2026 15:35:18 -0700 Subject: [PATCH 10/24] config/seitoml: let the global section own a table its dotted keys created MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A table can exist without a heading anywhere in a file, including at the top level: giga.enabled = true creates the table giga, and a.b.c = 1 creates both a and a.b. insert only looked for an ancestor among the named sections, and the global section carries no heading, so no prefix could find it. Set then wrote a heading, the decoder refused the second definition, and the write was undone — correctly, but the write should have succeeded, and the form it would have written is one viper reads without complaint. ancestorOf now falls back to the global section when a top-level dotted key has already created the table. globalCreated is what decides, and it checks every proper prefix of every top-level key, because each one names a table. The other direction matters as much and is tested: a table nothing has created is new, and gets a heading, because that is the form an operator expects to read. Treating the global section as everything's ancestor would turn every new section into top-level dotted keys. Both mutations fail their test. Also copies the key paths where they are stored or extended. They are slices of one another, so appending to a shorter one wrote into the longer one's storage; it happened to write the same byte it read, which is luck rather than a property. Coverage is 94.5% of statements. Co-Authored-By: Claude Opus 5 (1M context) --- config/seitoml/edit.go | 69 ++++++++++++++++++++++++++-------- config/seitoml/seitoml_test.go | 50 ++++++++++++++++++++++++ 2 files changed, 103 insertions(+), 16 deletions(-) diff --git a/config/seitoml/edit.go b/config/seitoml/edit.go index 6992e92c61..7ace0a1dd5 100644 --- a/config/seitoml/edit.go +++ b/config/seitoml/edit.go @@ -73,39 +73,76 @@ func (f *File) insert(path parser.Key, value parser.Value) func() { if e := transform.FindTable(f.doc, table...); e != nil { return appendItem(e.Section, kv) } - // No section carries this name. It may still be a table the document created by writing a dotted key - // inside its parent, and a heading for one of those defines it twice, so the leaf goes in as a dotted - // key under the nearest section instead. Where there is no such section either, the heading is new - // and correct. - if owner, rest := f.sectionOwning(table); owner != nil { - return appendItem(owner, &parser.KeyValue{Name: append(rest, leaf...), Value: value}) + // No section carries this name. It may still be a table the document created by writing a dotted key, + // and a heading for one of those defines it twice, so the leaf joins that dotted name instead. Where + // nothing has created it, the heading is new and correct, which is the form an operator expects to + // read. + if owner, under := f.ancestorOf(table); owner != nil { + return appendItem(owner, &parser.KeyValue{Name: dottedName(under, leaf), Value: value}) } before := len(f.doc.Sections) f.doc.Sections = append(f.doc.Sections, &tomledit.Section{ - Heading: &parser.Heading{Name: table}, + Heading: &parser.Heading{Name: copyKey(table)}, Items: []parser.Item{kv}, }) return func() { f.doc.Sections = f.doc.Sections[:before] } } -// sectionOwning returns the section whose heading is the longest prefix of table, and the rest of the -// path below it. +// ancestorOf returns the section a table's keys belong in when the table has no heading of its own, and +// the path from that section down to the table. // -// A table can exist without a heading of its own, written as a dotted key inside an ancestor. Adding a -// key to one has to extend that dotted name rather than introduce a heading the decoder reads as a -// second definition. -func (f *File) sectionOwning(table parser.Key) (*tomledit.Section, parser.Key) { +// A section whose heading is a prefix of the table owns it, and the longest such heading is the nearest +// ancestor. The global section owns it when a top-level dotted key has already created it: that section +// has no heading, so no prefix can find it, and a heading written for the table would be the second +// definition the decoder refuses. +func (f *File) ancestorOf(table parser.Key) (*tomledit.Section, parser.Key) { var best *tomledit.Section - var rest parser.Key + var under parser.Key for _, s := range f.doc.Sections { if s.Heading == nil || !s.Name.IsPrefixOf(table) { continue } if best == nil || len(s.Name) > len(best.Name) { - best, rest = s, table[len(s.Name):] + best, under = s, copyKey(table[len(s.Name):]) } } - return best, rest + if best == nil && f.globalCreated(table) { + return f.doc.Global, copyKey(table) + } + return best, under +} + +// globalCreated reports whether a top-level dotted key has already created this table. +// +// Every proper prefix of a dotted key names a table, so a.b.c = 1 creates both a and a.b without either +// carrying a heading. +func (f *File) globalCreated(table parser.Key) bool { + if f.doc.Global == nil { + return false + } + for _, item := range f.doc.Global.Items { + kv, ok := item.(*parser.KeyValue) + if !ok { + continue + } + for i := 1; i < len(kv.Name); i++ { + if kv.Name[:i].Equals(table) { + return true + } + } + } + return false +} + +// copyKey returns a key that shares no storage with its argument. +// +// The paths here are slices of one another, so appending to a shorter one would write into the longer +// one's storage. +func copyKey(k parser.Key) parser.Key { return append(parser.Key(nil), k...) } + +// dottedName joins the path down to a table with the key inside it. +func dottedName(under parser.Key, leaf parser.Key) parser.Key { + return append(copyKey(under), leaf...) } // appendItem adds an item to a section and reports how to remove it again. diff --git a/config/seitoml/seitoml_test.go b/config/seitoml/seitoml_test.go index 0624344d58..cbd0c949d6 100644 --- a/config/seitoml/seitoml_test.go +++ b/config/seitoml/seitoml_test.go @@ -1382,6 +1382,56 @@ func TestANameCannotBeAValueAndATableAtOnce(t *testing.T) { } }) + t.Run("a table a top-level dotted key created", func(t *testing.T) { + // The same shape as the headed case below it, except the table's ancestor is the file itself. The + // global section carries no heading, so no prefix can find it, and a heading written for the table + // would be the second definition the decoder refuses. + f := parse(t, "schema_version = 1\nnode_mode = \"validator\"\ngiga.enabled = true\n") + if err := f.Set("giga.workers", 4); err != nil { + t.Fatalf("writing a sibling under a top-level dotted table was refused: %v", err) + } + requireStillReadable(t, f) + values, err := f.Values() + if err != nil { + t.Fatalf("Values: %v", err) + } + for key, want := range map[string]any{"giga.enabled": true, "giga.workers": int64(4)} { + if values[key] != want { + t.Errorf("%s = %#v, want %#v", key, values[key], want) + } + } + }) + + t.Run("a table two levels below a top-level dotted key", func(t *testing.T) { + f := parse(t, "schema_version = 1\nnode_mode = \"validator\"\na.b.c = 1\n") + for _, key := range []string{"a.b.d", "a.e"} { + if err := f.Set(key, 2); err != nil { + t.Errorf("Set(%q) was refused: %v", key, err) + } + } + requireStillReadable(t, f) + }) + + t.Run("a section nothing has created still gets a heading", func(t *testing.T) { + // The other half: a table no key has named is new, and a heading is the form an operator expects + // to read. Treating the global section as everything's ancestor would write dotted keys instead. + f, err := seitoml.New("validator") + if err != nil { + t.Fatalf("New: %v", err) + } + for _, key := range []string{"state-commit.enable", "p2p.laddr"} { + if err := f.Set(key, "x"); err != nil { + t.Fatalf("Set(%q): %v", key, err) + } + } + out := render(t, f) + for _, heading := range []string{"[state-commit]", "[p2p]"} { + if !strings.Contains(out, heading) { + t.Errorf("a brand new section lost its %s heading:\n%s", heading, out) + } + } + }) + t.Run("a value named like a section holding nothing", func(t *testing.T) { // An empty section contributes no value, so a check over written values cannot see it. f := parse(t, "schema_version = 1\nnode_mode = \"validator\"\n\n[probe]\nn = 1\n") From 3078f55a7faafca3e8d6ad4003d32fd2e736be3d Mon Sep 17 00:00:00 2001 From: bdchatham Date: Tue, 18 Aug 2026 17:32:43 -0700 Subject: [PATCH 11/24] config/seitoml: delimit the generated preamble, and cache the decode an edit invalidates A re-review cleared the previous blockers and found that the preamble mark moved the failure rather than closing it, plus four smaller things and answers to two questions. All of it is here. SetPreamble owned a block, and a block's extent is chosen by the parser's blank-line grouping, which an operator controls by how they type. So the header still stacked when anything was written above it, since the search was anchored at the first item, and it still deleted an operator's line when the parser grouped that line into the same block. Reproduced both. The region is now delimited at both ends. One mark cannot tell a line before the region from a line inside it; two can, so everything between them belongs to this package and everything outside them belongs to whoever wrote it. Comments are also taken from wherever the parser put them: a block with a blank line after it is an item of its own, and without one it is carried by the item below, and the earlier fix only looked at the former. Every generated region is dropped rather than the first, so a stale header cannot survive, and an unpaired delimiter leaves the lines alone rather than guessing where a region ended. The three fixtures that mattered are now the shapes that fail: an operator block above the region, and an operator line immediately below and immediately above it with no blank line. A fourth pins the contract the delimiters state, that a note written between them is inside the part a regenerate replaces. Also from the re-review: - ancestorOf extended a dotted name whenever any prefix section existed, so a table was spelled heading-or-dotted depending on the order its keys were set. It now joins a dotted name only where one already created the table, and a table nothing created gets a heading, which is the form an operator reads. - appendItem asked InsertMapping to replace while its undo removed by identity, so a replacement would have left the undo deleting an entry that predated the edit. Unreachable, because Set looks the key up first, and now an assertion instead of an assumption. - Save checks the rendering it is about to write. Unreachable today for the same reason, and it is the one function every write to disk passes through, so a verb added later cannot forget it. - The duplicate-key and duplicate-heading refusals are subsumed by the decoder. They stay because they name the key and say what an edit would reach, and their comments now say that rather than claiming the decoder would miss them. Landed(err) answers the question Save's error cannot: one outcome it reports is not a failure, so the plain err != nil check reads a landed save as a failed one. The correct check now ships beside the sentinel that needs it. Reading no longer renders and decodes every time. A caller walking the declared key set paid that per key, and building a file was quadratic in its size because every edit checks the result. The cache is dropped by every edit, including an undone one, which is the half the test drives. Coverage is 94.5% of statements over 137 cases. Nine guards verified by removing each and watching a named test fail. Co-Authored-By: Claude Opus 5 (1M context) --- config/seitoml/edit.go | 172 ++++++++++++++++++++------ config/seitoml/file.go | 30 ++++- config/seitoml/seitoml_test.go | 220 ++++++++++++++++++++++++++++++++- config/seitoml/values.go | 16 +++ 4 files changed, 398 insertions(+), 40 deletions(-) diff --git a/config/seitoml/edit.go b/config/seitoml/edit.go index 7ace0a1dd5..95185efd5a 100644 --- a/config/seitoml/edit.go +++ b/config/seitoml/edit.go @@ -30,9 +30,11 @@ func (f *File) Set(key string, v any) error { } if e := f.doc.First(path...); e != nil && e.KeyValue != nil { + f.changed() e.Value = value return nil } + f.changed() // A key the document does not hold yet lands in a namespace that may already use its name for a // table, or use a table's name for it. Rather than enumerate the shapes that collide, insert and then @@ -41,6 +43,7 @@ func (f *File) Set(key string, v any) error { undo := f.insert(path, value) if err := f.decodable(); err != nil { undo() + f.changed() return fmt.Errorf("%s: %w", key, err) } return nil @@ -102,25 +105,33 @@ func (f *File) ancestorOf(table parser.Key) (*tomledit.Section, parser.Key) { if s.Heading == nil || !s.Name.IsPrefixOf(table) { continue } + below := table[len(s.Name):] + if !createsTable(s, below) { + continue + } if best == nil || len(s.Name) > len(best.Name) { - best, under = s, copyKey(table[len(s.Name):]) + best, under = s, copyKey(below) } } - if best == nil && f.globalCreated(table) { + if best != nil { + return best, under + } + if f.doc.Global != nil && createsTable(f.doc.Global, table) { return f.doc.Global, copyKey(table) } - return best, under + return nil, nil } -// globalCreated reports whether a top-level dotted key has already created this table. +// createsTable reports whether a dotted key in this section already names the given table. // -// Every proper prefix of a dotted key names a table, so a.b.c = 1 creates both a and a.b without either -// carrying a heading. -func (f *File) globalCreated(table parser.Key) bool { - if f.doc.Global == nil { +// Every proper prefix of a dotted key names a table, so flatkv.enable creates flatkv without giving it a +// heading. Only such a table is joined by extending a dotted name; one nothing has created gets a +// heading of its own, so a table is spelled the same way whatever order its keys were written in. +func createsTable(s *tomledit.Section, table parser.Key) bool { + if len(table) == 0 { return false } - for _, item := range f.doc.Global.Items { + for _, item := range s.Items { kv, ok := item.(*parser.KeyValue) if !ok { continue @@ -146,8 +157,14 @@ func dottedName(under parser.Key, leaf parser.Key) parser.Key { } // appendItem adds an item to a section and reports how to remove it again. +// +// Told not to replace, so a key already present is reported rather than overwritten. Set rules that out +// by looking the key up first, and the distinction matters for the undo: replacing would leave it +// deleting an entry that was there before the edit. func appendItem(s *tomledit.Section, kv *parser.KeyValue) func() { - transform.InsertMapping(s, kv, true) + if !transform.InsertMapping(s, kv, false) { + return func() {} // nothing was inserted, so nothing needs undoing + } return func() { for i, item := range s.Items { if item == parser.Item(kv) { @@ -159,9 +176,6 @@ func appendItem(s *tomledit.Section, kv *parser.KeyValue) func() { } // insertGlobal adds a top-level key, creating the global section when the document has none. -// -// InsertMapping's result is not checked because it only reports a collision it was told not to -// replace, and it is told to replace. func (f *File) insertGlobal(kv *parser.KeyValue) func() { if f.doc.Global == nil { f.doc.Global = &tomledit.Section{} @@ -169,46 +183,131 @@ func (f *File) insertGlobal(kv *parser.KeyValue) func() { return appendItem(f.doc.Global, kv) } -// preambleMark is the last line of a block SetPreamble owns. +// The lines delimiting the region SetPreamble owns. // -// A comment block at the top of a file may be one an operator wrote, and this method has to replace its -// own without touching theirs. Nothing distinguishes the two but a mark it writes and recognises. -const preambleMark = " -- above this line is generated; edit below --" +// A comment at the top of a file may be one an operator wrote, and this method has to replace its own +// without touching theirs. One mark cannot tell a line before the region from a line inside it, so the +// region is delimited at both ends: everything between these two lines belongs to this method, and +// everything outside them belongs to whoever wrote it. +const ( + preambleBegin = " ---- generated by seid ----" + preambleEnd = " ---- end generated; your notes are safe below ----" +) // SetPreamble puts a comment block at the top of the document, above everything else. // // Comments rather than keys, so nothing a reader needs in order to understand the file becomes -// configuration the node has to recognize. This replaces a block it wrote before, so regenerating does -// not stack one preamble on the last, and leaves any other leading comment alone: an operator's -// explanation at the top of the file is exactly what this package exists to preserve. +// configuration the node has to recognize. This replaces a region it wrote before, so regenerating does +// not stack one preamble on the last, and leaves every other comment alone: an operator's explanation at +// the top of the file is exactly what this package exists to preserve. func (f *File) SetPreamble(lines []string) { if f.doc.Global == nil { f.doc.Global = &tomledit.Section{} } - items := f.doc.Global.Items - if len(items) > 0 { - if block, leading := items[0].(parser.Comments); leading && ownedPreamble(block) { - items = items[1:] + f.changed() + + kept := withoutGeneratedLines(f.takeLeadingComments()) + var leading []parser.Item + if len(lines) > 0 { + region := append([]string{preambleBegin}, lines...) + leading = append(leading, parser.Comments(append(region, preambleEnd))) + } + if len(kept) > 0 { + leading = append(leading, parser.Comments(kept)) + } + f.doc.Global.Items = append(leading, f.doc.Global.Items...) +} + +// takeLeadingComments removes and returns every comment line standing above the document's first value. +// +// A comment block is an item of its own when a blank line follows it, and part of the item below it +// otherwise, and an operator decides which by how they type. Taking both means a region is found wherever +// the parser put it, and returning what is kept as one block puts it somewhere a later run still looks. +func (f *File) takeLeadingComments() []string { + var lines []string + + cut := 0 + for _, item := range f.doc.Global.Items { + block, comments := item.(parser.Comments) + if !comments { + break } + lines = append(lines, block...) + cut++ } - if len(lines) == 0 { - f.doc.Global.Items = items - return + f.doc.Global.Items = f.doc.Global.Items[cut:] + + if attached := f.firstBlock(); attached != nil { + lines = append(lines, *attached...) + *attached = nil } - marked := append(append([]string(nil), lines...), preambleMark) - f.doc.Global.Items = append([]parser.Item{parser.Comments(marked)}, items...) + return lines } -// ownedPreamble reports whether a leading comment block is one SetPreamble wrote. +// firstBlock returns the comment block carried by whatever the document holds first, or nil. +func (f *File) firstBlock() *parser.Comments { + if len(f.doc.Global.Items) > 0 { + switch first := f.doc.Global.Items[0].(type) { + case *parser.KeyValue: + return &first.Block + case *parser.Heading: + return &first.Block + } + return nil + } + // Nothing at the top level, so the first thing in the file is a section heading. + for _, s := range f.doc.Sections { + if s.Heading != nil { + return &s.Block + } + } + return nil +} + +// withoutGeneratedLines drops every generated region from a set of comment lines. // -// Compared on the line's content, because a block this method inserts holds the text alone while the +// Every region rather than the first, so the result does not depend on how many a previous run left +// behind: dropping one and keeping another would leave a stale header in the file forever. +func withoutGeneratedLines(lines []string) []string { + var out []string + for i := 0; i < len(lines); { + from, to := generatedRegion(lines[i:]) + if from < 0 { + return append(out, lines[i:]...) + } + out = append(out, lines[i:i+from]...) + i += to + 1 + } + return out +} + +// generatedRegion returns the first and last line of the earliest generated region, or -1 for none. +// +// Both delimiters have to be present. A file holding only one of them was hand-edited into a shape this +// cannot reason about, and leaving those lines alone keeps an operator's writing over its own tidiness. +// +// Compared on each line's content, because a block this package inserts holds the text alone while the // same block read back from a file carries the comment character the renderer added. -func ownedPreamble(block parser.Comments) bool { - if len(block) == 0 { - return false +func generatedRegion(lines []string) (int, int) { + from := -1 + for i, line := range lines { + switch content(line) { + case content(preambleBegin): + if from < 0 { + from = i + } + case content(preambleEnd): + if from >= 0 { + return from, i + } + } } - last := strings.TrimSpace(strings.TrimPrefix(strings.TrimSpace(block[len(block)-1]), "#")) - return last == strings.TrimSpace(preambleMark) + return -1, -1 +} + +// content is a comment line without its marker or surrounding space. +func content(line string) string { + return strings.TrimSpace(strings.TrimPrefix(strings.TrimSpace(line), "#")) } // Unset removes a key and reports whether the file carried one. @@ -225,6 +324,7 @@ func (f *File) Unset(key string) (bool, error) { if e == nil || e.KeyValue == nil { return false, nil } + f.changed() return e.Remove(), nil } diff --git a/config/seitoml/file.go b/config/seitoml/file.go index 2a9af71fbc..e28a95f656 100644 --- a/config/seitoml/file.go +++ b/config/seitoml/file.go @@ -44,8 +44,17 @@ const newFileMode os.FileMode = 0o600 // File is a parsed sei.toml that survives editing with its comments and layout intact. type File struct { doc *tomledit.Document + // values caches the last decode, and is nil whenever the document has changed since. + // + // Reading asks the decoder rather than the editing parser, which means rendering the document, so a + // caller walking every declared key would otherwise render and decode once per key. Building a file + // would be quadratic in its size for the same reason, since every edit checks the result. + values map[string]any } +// changed drops the decode a read would otherwise reuse. Every edit calls it before mutating. +func (f *File) changed() { f.values = nil } + // Parse reads a document from r. func Parse(r io.Reader) (*File, error) { doc, err := tomledit.Parse(r) @@ -80,6 +89,8 @@ func (f *File) refuseUnsupportedShapes() error { } name := s.Name.String() if headings[name] { + // Also refused by the decoder, and kept for the same reason as a duplicate key: this says + // which heading and what an edit would reach. return fmt.Errorf("[%s] appears more than once, and an edit reaches only the first, so a "+ "value written into this file would not be the one read back", name) } @@ -100,9 +111,9 @@ func (f *File) refuseUnsupportedShapes() error { bad = err return false } - // One value per key, checked here rather than left to the decoder. A duplicate is the one shape - // the editing parser accepts and a conforming decoder rejects, so without this the file parses - // and then every read of it fails. + // The decoder below refuses this too. It stays because it names the key and says what an edit + // would do to it, where the decoder names a line, and a duplicate key is the mistake an operator + // is most likely to make by hand. key := full.String() if written[key] { bad = fmt.Errorf("%s is written more than once, and an edit reaches only the first, so a "+ @@ -291,6 +302,11 @@ func (f *File) Save(path string) error { if err != nil { return err } + // The one function every write to disk passes through, so the check belongs here rather than at each + // verb that edits. A verb added later cannot forget an invariant it does not have to remember. + if _, err := decodeBytes(raw); err != nil { + return err + } mode, err := modeToWrite(path) if err != nil { @@ -331,6 +347,14 @@ func (f *File) Save(path string) error { // wrong about what happened. var ErrNotDurable = errors.New("the file is installed and its directory entry is not yet flushed") +// Landed reports whether a Save put the values on disk, which is what a caller acting on the outcome +// wants to know. +// +// Save reports one outcome that is not a failure through the error it returns, so the plain err != nil +// check reads a landed save as a failed one and tells an operator their change did not apply when it +// did. This is that check written correctly, in one call, next to the sentinel it accounts for. +func Landed(err error) bool { return err == nil || errors.Is(err, ErrNotDurable) } + // modeToWrite returns the permission a save should use, and refuses a destination it must not replace. // // An existing file keeps its own mode, so a save never widens what an operator narrowed. A symbolic diff --git a/config/seitoml/seitoml_test.go b/config/seitoml/seitoml_test.go index cbd0c949d6..faac47e0ef 100644 --- a/config/seitoml/seitoml_test.go +++ b/config/seitoml/seitoml_test.go @@ -1034,7 +1034,7 @@ func TestThePreambleGoesAboveEverythingInAFileThatStartsWithATable(t *testing.T) f.SetPreamble([]string{" a header"}) out := render(t, f) - if !strings.HasPrefix(strings.TrimSpace(out), "# a header") { + if !strings.HasPrefix(strings.TrimSpace(out), "# ---- generated by seid ----\n# a header") { t.Errorf("the preamble is not the first thing in the file:\n%s", out) } } @@ -1680,3 +1680,221 @@ func TestAPreambleOnlyReplacesOneItWrote(t *testing.T) { t.Errorf("the operator's comment was deleted by the second preamble:\n%s", second) } } + +// TestAPreambleOwnsItsLinesRatherThanTheBlockTheySitIn holds what regenerating may touch. +// +// The mark ends the generated lines, and an operator writes wherever they like around them: above the +// header, or on a line the parser groups into the same comment item. Anchored at the first item, a +// header above which anything was written is never found and grows on every run; anchored at a block, an +// operator's line inside that block is deleted with it. Both are the failures the mark exists to prevent. +func TestAPreambleOwnsItsLinesRatherThanTheBlockTheySitIn(t *testing.T) { + const note = "ops: we pin flatkv, see INC-4412" + tail := "\nschema_version = 1\nnode_mode = \"validator\"\n" + + for _, tc := range []struct{ name, before string }{ + { + "an operator block above the generated region", + "# " + note + "\n\n" + generated("run one") + tail, + }, + { + "an operator line below the region, no blank line", + generated("run one") + "# " + note + "\n" + tail, + }, + { + "an operator line above the region, no blank line", + "# " + note + "\n" + generated("run one") + tail, + }, + { + "an operator line between two generated regions", + generated("run one") + "# " + note + "\n" + generated("stale run") + tail, + }, + } { + t.Run(tc.name, func(t *testing.T) { + f := parse(t, tc.before) + f.SetPreamble([]string{" written by run two"}) + out := render(t, f) + + if strings.Contains(out, "run one") { + t.Errorf("the previous header survived, so it grows on every regenerate:\n%s", out) + } + if !strings.Contains(out, note) { + t.Errorf("the operator's comment was deleted:\n%s", out) + } + if n := strings.Count(out, "generated by seid"); n != 1 { + t.Errorf("the file carries %d generated regions, want one:\n%s", n, out) + } + + // And again through a parse, since a regenerate reads what the last one wrote. + third := parse(t, out) + third.SetPreamble([]string{" written by run three"}) + last := render(t, third) + if strings.Contains(last, "run two") { + t.Errorf("the second header survived the third:\n%s", last) + } + if !strings.Contains(last, note) { + t.Errorf("the operator's comment was deleted on the third run:\n%s", last) + } + }) + } +} + +// TestATableKeepsOneSpellingWhateverOrderItsKeysWereWritten holds insert's choice between the two forms. +// +// A table nothing has created is new and gets a heading, which is the form an operator reads. A table an +// ancestor's dotted key already created has no heading and cannot be given one, so its keys join that +// dotted name. Deciding by whether an ancestor section merely exists would spell the same table either +// way depending on which key was set first. +func TestATableKeepsOneSpellingWhateverOrderItsKeysWereWritten(t *testing.T) { + t.Run("a new table under an existing section gets a heading", func(t *testing.T) { + f := parse(t, "schema_version = 1\nnode_mode = \"validator\"\n\n[state-commit]\nbuffer = 100\n") + if err := f.Set("state-commit.flatkv.dir", "/data"); err != nil { + t.Fatalf("Set: %v", err) + } + out := render(t, f) + if !strings.Contains(out, "[state-commit.flatkv]") { + t.Errorf("a table nothing had created did not get a heading:\n%s", out) + } + requireStillReadable(t, f) + }) + + t.Run("a table a dotted key created joins that name", func(t *testing.T) { + f := parse(t, "schema_version = 1\nnode_mode = \"validator\"\n\n[state-commit]\nflatkv.enable = true\n") + if err := f.Set("state-commit.flatkv.dir", "/data"); err != nil { + t.Fatalf("Set: %v", err) + } + out := render(t, f) + if strings.Contains(out, "[state-commit.flatkv]") { + t.Errorf("a table the document already created was given a second definition:\n%s", out) + } + requireStillReadable(t, f) + }) +} + +// TestLandedSeparatesAnInstalledFileFromAFailedWrite gives the outcome check a correct spelling. +// +// Save reports one outcome through its error that is not a failure, so the plain err != nil check reads a +// landed save as a failed one. Landed is that check written once, so a caller cannot get it wrong by +// writing the idiom every other Go call wants. +func TestLandedSeparatesAnInstalledFileFromAFailedWrite(t *testing.T) { + for _, tc := range []struct { + name string + err error + landed bool + }{ + {"a save that completed", nil, true}, + {"a save whose directory entry is not flushed", fmt.Errorf("x: %w", seitoml.ErrNotDurable), true}, + {"a save that could not write", errors.New("permission denied"), false}, + } { + if got := seitoml.Landed(tc.err); got != tc.landed { + t.Errorf("%s: Landed = %v, want %v", tc.name, got, tc.landed) + } + } +} + +// TestReadingTwiceDoesNotDecodeTwice holds the cache an edit invalidates. +// +// A read renders the document and decodes it, so a caller walking every declared key would pay that per +// key, and building a file would be quadratic in its size because every edit checks the result. The +// values a read returns still have to follow the document, which is what makes the invalidation the part +// worth testing rather than the caching. +func TestReadingTwiceDoesNotDecodeTwice(t *testing.T) { + f := parse(t, "schema_version = 1\nnode_mode = \"validator\"\n\n[probe]\nn = 1\n") + + first, err := f.Values() + if err != nil { + t.Fatalf("Values: %v", err) + } + if first["probe.n"] != int64(1) { + t.Fatalf("probe.n = %#v, want 1", first["probe.n"]) + } + + // Every edit has to be visible to the next read, or a cache is a correctness bug rather than a saving. + for _, step := range []struct { + name string + edit func() error + key string + want any + }{ + {"Set replacing a value", func() error { return f.Set("probe.n", 2) }, "probe.n", int64(2)}, + {"Set adding a key", func() error { return f.Set("probe.m", 3) }, "probe.m", int64(3)}, + {"Unset removing one", func() error { _, err := f.Unset("probe.m"); return err }, "probe.m", nil}, + } { + if err := step.edit(); err != nil { + t.Fatalf("%s: %v", step.name, err) + } + got, err := f.Values() + if err != nil { + t.Fatalf("%s: Values: %v", step.name, err) + } + if got[step.key] != step.want { + t.Errorf("after %s, %s = %#v, want %#v", step.name, step.key, got[step.key], step.want) + } + } + + // A preamble changes the document too, and must not strand a decode of the version before it. + f.SetPreamble([]string{" header"}) + if _, err := f.Values(); err != nil { + t.Errorf("Values after a preamble: %v", err) + } +} + +// generated renders a preamble region the way SetPreamble writes one, so a fixture can start from a file +// a previous run produced. +func generated(header string) string { + return "# ---- generated by seid ----\n# " + header + + "\n# ---- end generated; your notes are safe below ----\n" +} + +// TestAnOperatorWritingInsideTheGeneratedRegionIsTold pins the one line the region does claim. +// +// The delimiters say where the generated lines start and stop, so a note written between them is inside +// the part a regenerate replaces. That is the contract the file states in words, and it is worth a test +// because the alternative reading, that nothing a person typed may ever be replaced, would make the +// region unreplaceable. +func TestAnOperatorWritingInsideTheGeneratedRegionIsTold(t *testing.T) { + inside := "# ---- generated by seid ----\n# written by run one\n# a note typed inside the region\n" + + "# ---- end generated; your notes are safe below ----\nschema_version = 1\nnode_mode = \"v\"\n" + + f := parse(t, inside) + f.SetPreamble([]string{" written by run two"}) + out := render(t, f) + + if strings.Contains(out, "typed inside the region") { + t.Errorf("a line inside the generated region survived a regenerate, so the region cannot be "+ + "replaced at all:\n%s", out) + } + if !strings.Contains(out, "run two") { + t.Errorf("the new header is missing:\n%s", out) + } +} + +// TestAnUnpairedDelimiterLeavesTheLinesAlone covers a file hand-edited into a shape with one delimiter. +// +// A region is the text between two delimiters, so one on its own bounds nothing. Guessing where it ends +// would delete an operator's lines on the strength of a marker they may have typed themselves, and +// leaving them is the answer that cannot lose their writing. +func TestAnUnpairedDelimiterLeavesTheLinesAlone(t *testing.T) { + for _, tc := range []struct{ name, before string }{ + { + "a begin with no end", + "# ---- generated by seid ----\n# a note under a stray delimiter\n\nschema_version = 1\nnode_mode = \"v\"\n", + }, + { + "an end with no begin", + "# a note above a stray delimiter\n# ---- end generated; your notes are safe below ----\n\nschema_version = 1\nnode_mode = \"v\"\n", + }, + } { + t.Run(tc.name, func(t *testing.T) { + f := parse(t, tc.before) + f.SetPreamble([]string{" written by this run"}) + + out := render(t, f) + if !strings.Contains(out, "stray delimiter") { + t.Errorf("the operator's line was removed on the strength of one delimiter:\n%s", out) + } + if !strings.Contains(out, "written by this run") { + t.Errorf("the new header is missing:\n%s", out) + } + }) + } +} diff --git a/config/seitoml/values.go b/config/seitoml/values.go index e561157549..9d7f827df2 100644 --- a/config/seitoml/values.go +++ b/config/seitoml/values.go @@ -51,10 +51,26 @@ func (f *File) Get(key string) (any, bool, error) { // path a later process would use, so a value this package cannot express fails here rather than on a // node. func (f *File) decoded() (map[string]any, error) { + if f.values != nil { + return f.values, nil + } raw, err := f.Bytes() if err != nil { return nil, err } + out, err := decodeBytes(raw) + if err != nil { + return nil, err + } + f.values = out + return out, nil +} + +// decodeBytes reads a rendered document as dotted paths to Go values. +// +// Separate from decoded so that a caller holding the rendering already, such as Save, does not render it +// a second time to check it. +func decodeBytes(raw []byte) (map[string]any, error) { var nested map[string]any if err := toml.Unmarshal(raw, &nested); err != nil { return nil, fmt.Errorf("read sei.toml: %w", err) From 5fa839d821e0433a0cd74651843b8607556f326a Mon Sep 17 00:00:00 2001 From: bdchatham Date: Tue, 18 Aug 2026 17:47:40 -0700 Subject: [PATCH 12/24] config/seitoml: stop a read from changing what the next read answers Values filtered the two keys describing the file out of the map decoded returns, and that map is now the cache. So one Values call left the cache without them, and every later Version, Mode and Get read a file that holds them as missing them. Reproduced: Version returns "sei.toml has no schema_version" on a file whose first line is schema_version = 1. The same aliasing runs outward and was not reported: Values handed the cache to its caller, so a caller writing into or deleting from the map they were given changed what the next read of the file answered. Values now builds its own map. decoded's contract says the map is the cache and a caller needing to hand one outward or change it builds its own, which is the rule Values broke. The cache was mine, added in the commit before this one, and the test that drove it only checked that an edit invalidates. It now also reads every way twice in an order that exposes a read leaking, and writes into a returned map to check the next read is unaffected. Restoring the in-place delete fails it. Co-Authored-By: Claude Opus 5 (1M context) --- config/seitoml/seitoml_test.go | 61 ++++++++++++++++++++++++++++++++++ config/seitoml/values.go | 17 ++++++++-- 2 files changed, 75 insertions(+), 3 deletions(-) diff --git a/config/seitoml/seitoml_test.go b/config/seitoml/seitoml_test.go index faac47e0ef..bf51c71bd3 100644 --- a/config/seitoml/seitoml_test.go +++ b/config/seitoml/seitoml_test.go @@ -1898,3 +1898,64 @@ func TestAnUnpairedDelimiterLeavesTheLinesAlone(t *testing.T) { }) } } + +// TestReadingOneWayDoesNotChangeWhatAnotherAnswers holds the reads against each other. +// +// A read renders and decodes the document, and the result is cached so that walking every key does not +// pay that per key. The cache is one map, so a read that filtered or handed it out would change what the +// next read of any other kind answers: Values omits the two keys describing the file, and deleting them +// from the shared map made Version, Mode and Get report a file that holds them as missing them. +func TestReadingOneWayDoesNotChangeWhatAnotherAnswers(t *testing.T) { + f := parse(t, "schema_version = 1\nnode_mode = \"validator\"\n\n[probe]\nn = 1\n") + + // Read every way, twice, in an order that would expose a read leaking into the cache. + for round := 1; round <= 2; round++ { + values, err := f.Values() + if err != nil { + t.Fatalf("round %d: Values: %v", round, err) + } + for _, describing := range []string{seitoml.VersionKey, seitoml.ModeKey} { + if _, present := values[describing]; present { + t.Errorf("round %d: Values reports %s, which describes the file rather than the node", + round, describing) + } + } + if values["probe.n"] != int64(1) { + t.Errorf("round %d: probe.n = %#v, want 1", round, values["probe.n"]) + } + + version, err := f.Version() + if err != nil || version != 1 { + t.Errorf("round %d: Version = (%d, %v) after Values, want 1", round, version, err) + } + mode, err := f.Mode() + if err != nil || mode != "validator" { + t.Errorf("round %d: Mode = (%q, %v) after Values, want validator", round, mode, err) + } + for _, key := range []string{seitoml.VersionKey, seitoml.ModeKey, "probe.n"} { + if _, present, err := f.Get(key); err != nil || !present { + t.Errorf("round %d: Get(%q) = (present %v, %v) after Values", round, key, present, err) + } + } + } + + // What a caller does with the map they were handed cannot reach a later read either. + mine, err := f.Values() + if err != nil { + t.Fatalf("Values: %v", err) + } + mine["probe.n"] = "written by the caller" + delete(mine, "probe.n") + mine["invented"] = true + + again, err := f.Values() + if err != nil { + t.Fatalf("Values: %v", err) + } + if again["probe.n"] != int64(1) { + t.Errorf("a caller's write reached the next read: probe.n = %#v, want 1", again["probe.n"]) + } + if _, present := again["invented"]; present { + t.Error("a key the caller invented appeared in the next read") + } +} diff --git a/config/seitoml/values.go b/config/seitoml/values.go index 9d7f827df2..0764631505 100644 --- a/config/seitoml/values.go +++ b/config/seitoml/values.go @@ -17,9 +17,17 @@ func (f *File) Values() (map[string]any, error) { if err != nil { return nil, err } - delete(all, VersionKey) - delete(all, ModeKey) - return all, nil + // Built rather than filtered in place, because decoded hands back the cache itself. Deleting the two + // describing keys from it made every later Version, Mode and Get read them as absent, and handing the + // cache to a caller let their own writes into it. + out := make(map[string]any, len(all)) + for key, v := range all { + if key == VersionKey || key == ModeKey { + continue + } + out[key] = v + } + return out, nil } // Get returns one key's written value. @@ -50,6 +58,9 @@ func (f *File) Get(key string) (any, bool, error) { // Rendering first rather than holding the source means an unsaved edit is read back through the same // path a later process would use, so a value this package cannot express fails here rather than on a // node. +// +// The map returned is the cache. Every caller here reads it, and one that needs to hand a map outward or +// change it has to build its own; writing into this one would change what a later read answers. func (f *File) decoded() (map[string]any, error) { if f.values != nil { return f.values, nil From 1dcef6fed22c50a189b11cf8fa97cd9a8f4c8c79 Mon Sep 17 00:00:00 2001 From: bdchatham Date: Tue, 18 Aug 2026 17:53:57 -0700 Subject: [PATCH 13/24] config/seitoml: remove SetPreamble, and make the cache and ordering tests able to fail Four rounds of review found a hole in SetPreamble each time, and each fix moved the boundary rather than closing it: items[0], then the leading run of comment items plus items[0].Block, and each time the shape one step outside failed. The last round reproduced a stale generated region left in a file forever when a key sits above it. The cause is the design, not the fixtures. SetPreamble located its region by walking the parser's item and block structure, and an operator chooses that structure by where they put blank lines. Delimiting both ends made the region's extent well defined and did nothing for finding it. It has no caller in this PR; the two that exist are in the CLI slice. So it goes, and comes back with them, written to search every comment surface a document has rather than a position. That removes four findings outright. Three findings from the same round stand on their own and are fixed: - The longest-prefix comparison in ancestorOf was dead. At most one section can qualify, because two would name one table twice and the decoder refuses that at the door, so the code claimed to resolve an ambiguity that cannot arise. - appendItem returned a no-op undo when the key was already present, so a write that inserted nothing returned success. insert now reports whether it inserted and Set says so. Still unreachable; no longer silent if it ever is not. - createsTable's empty-key guard was unreachable, since insert only asks about a table FindTable missed. Two claims in the previous commit message were not true of the suite, which is the part worth recording: - It said the test drove the invalidation after an undone edit. Removing that call killed nothing, and the reason is that it was redundant: a decode that fails leaves no cache behind, so there is nothing to drop after the undo. The call is gone and the comment says why none is needed. - It said the quadratic build was fixed. It is not. Every insert still renders and decodes the whole document to check it, measured at 700us per insert at 1600 keys, so building a file stays quadratic in its size. The cache helps consecutive reads, which is the half that was true. The cache test could not fail: deleting the cache passed it. The properties it names are only visible inside the package, so they moved there, and they are the two that matter rather than the one I asserted. A read with no edit before it reuses the last decode, driven by writing a sentinel into the first result. And no edit leaves a decode describing another document, driven by comparing what is held against a fresh decode after each kind of edit, including a refused one. Both mutations now fail. The external test keeps the half that belongs outside: every edit is visible to the next read. The ordering test never varied the order it was named for. It now sets four keys spanning both branches in all 24 orders and compares the resulting shape, and making the spelling depend on a section merely existing fails it. Coverage is 94.2% of statements over 131 cases. Co-Authored-By: Claude Opus 5 (1M context) --- config/seitoml/doc.go | 6 +- config/seitoml/edit.go | 177 +++------------- config/seitoml/guards_test.go | 100 ++++++++- config/seitoml/seitoml_test.go | 376 ++++++++------------------------- 4 files changed, 200 insertions(+), 459 deletions(-) diff --git a/config/seitoml/doc.go b/config/seitoml/doc.go index 373ed7a78e..503c03f16c 100644 --- a/config/seitoml/doc.go +++ b/config/seitoml/doc.go @@ -24,9 +24,9 @@ // because a release migrates the file on the node's own disk and rolling the binary back does not roll // the file back with it. Read anyway, the older binary would apply only the keys it still recognises. // -// Editing preserves the document. An operator may hand-edit the file, and comments are how they -// explain a choice to whoever reads it next, so set and unset change the one line they name and leave -// every other line of content untouched. Vertical spacing is normalised once, on the first save of a +// Editing preserves the document. An operator may hand-edit the file, and comments are how they explain +// a choice to whoever reads it next, so set and unset change the one line they name and leave every other +// line of content untouched. Vertical spacing is normalised once, on the first save of a // file nothing has saved before, and holds steady after that. This is why the package edits a parsed // document rather than re-rendering a decoded map, which would drop every comment in the file. // diff --git a/config/seitoml/edit.go b/config/seitoml/edit.go index 95185efd5a..3933e37ec0 100644 --- a/config/seitoml/edit.go +++ b/config/seitoml/edit.go @@ -40,10 +40,17 @@ func (f *File) Set(key string, v any) error { // table, or use a table's name for it. Rather than enumerate the shapes that collide, insert and then // ask the decoder, which is the same one the node reads with: if the document no longer decodes, the // insert is undone and the key is named. Enumerating them by hand missed three. - undo := f.insert(path, value) + undo, inserted := f.insert(path, value) + if !inserted { + // Unreachable: the lookup above established the key is absent, and the dotted name insert builds + // addresses the same place. Reported rather than returned as success, because a caller told a + // write landed when nothing was written has no way to find out. + return fmt.Errorf("%s: the document already holds this key", key) + } if err := f.decodable(); err != nil { + // Nothing to drop: the decode that just failed left no cache behind, which is why the undo needs + // no invalidation of its own. undo() - f.changed() return fmt.Errorf("%s: %w", key, err) } return nil @@ -64,7 +71,7 @@ func (f *File) decodable() error { // A key with no dots belongs at the top level. Otherwise it goes in the table its prefix names, // which is created when it is absent so writing the first key of a section works without the // operator having to add the heading by hand. -func (f *File) insert(path parser.Key, value parser.Value) func() { +func (f *File) insert(path parser.Key, value parser.Value) (func(), bool) { leaf := parser.Key{path[len(path)-1]} kv := &parser.KeyValue{Name: leaf, Value: value} @@ -88,7 +95,7 @@ func (f *File) insert(path parser.Key, value parser.Value) func() { Heading: &parser.Heading{Name: copyKey(table)}, Items: []parser.Item{kv}, }) - return func() { f.doc.Sections = f.doc.Sections[:before] } + return func() { f.doc.Sections = f.doc.Sections[:before] }, true } // ancestorOf returns the section a table's keys belong in when the table has no heading of its own, and @@ -99,23 +106,17 @@ func (f *File) insert(path parser.Key, value parser.Value) func() { // has no heading, so no prefix can find it, and a heading written for the table would be the second // definition the decoder refuses. func (f *File) ancestorOf(table parser.Key) (*tomledit.Section, parser.Key) { - var best *tomledit.Section - var under parser.Key + // At most one section can qualify, so the first match is the only match. Two would need a section + // [a] holding a dotted key beginning b. alongside a section [a.b], and that names the table b twice, + // which the decoder refuses at the door. for _, s := range f.doc.Sections { if s.Heading == nil || !s.Name.IsPrefixOf(table) { continue } - below := table[len(s.Name):] - if !createsTable(s, below) { - continue - } - if best == nil || len(s.Name) > len(best.Name) { - best, under = s, copyKey(below) + if below := table[len(s.Name):]; createsTable(s, below) { + return s, copyKey(below) } } - if best != nil { - return best, under - } if f.doc.Global != nil && createsTable(f.doc.Global, table) { return f.doc.Global, copyKey(table) } @@ -128,9 +129,6 @@ func (f *File) ancestorOf(table parser.Key) (*tomledit.Section, parser.Key) { // heading. Only such a table is joined by extending a dotted name; one nothing has created gets a // heading of its own, so a table is spelled the same way whatever order its keys were written in. func createsTable(s *tomledit.Section, table parser.Key) bool { - if len(table) == 0 { - return false - } for _, item := range s.Items { kv, ok := item.(*parser.KeyValue) if !ok { @@ -156,14 +154,14 @@ func dottedName(under parser.Key, leaf parser.Key) parser.Key { return append(copyKey(under), leaf...) } -// appendItem adds an item to a section and reports how to remove it again. +// appendItem adds an item to a section, and reports how to remove it again and whether it went in. // -// Told not to replace, so a key already present is reported rather than overwritten. Set rules that out -// by looking the key up first, and the distinction matters for the undo: replacing would leave it -// deleting an entry that was there before the edit. -func appendItem(s *tomledit.Section, kv *parser.KeyValue) func() { +// Told not to replace, so a key already present is reported rather than overwritten. The distinction +// matters twice: replacing would leave the undo deleting an entry that predated the edit, and a caller +// needs to know a write did not happen rather than being told it did. +func appendItem(s *tomledit.Section, kv *parser.KeyValue) (func(), bool) { if !transform.InsertMapping(s, kv, false) { - return func() {} // nothing was inserted, so nothing needs undoing + return nil, false } return func() { for i, item := range s.Items { @@ -172,144 +170,17 @@ func appendItem(s *tomledit.Section, kv *parser.KeyValue) func() { return } } - } + }, true } // insertGlobal adds a top-level key, creating the global section when the document has none. -func (f *File) insertGlobal(kv *parser.KeyValue) func() { +func (f *File) insertGlobal(kv *parser.KeyValue) (func(), bool) { if f.doc.Global == nil { f.doc.Global = &tomledit.Section{} } return appendItem(f.doc.Global, kv) } -// The lines delimiting the region SetPreamble owns. -// -// A comment at the top of a file may be one an operator wrote, and this method has to replace its own -// without touching theirs. One mark cannot tell a line before the region from a line inside it, so the -// region is delimited at both ends: everything between these two lines belongs to this method, and -// everything outside them belongs to whoever wrote it. -const ( - preambleBegin = " ---- generated by seid ----" - preambleEnd = " ---- end generated; your notes are safe below ----" -) - -// SetPreamble puts a comment block at the top of the document, above everything else. -// -// Comments rather than keys, so nothing a reader needs in order to understand the file becomes -// configuration the node has to recognize. This replaces a region it wrote before, so regenerating does -// not stack one preamble on the last, and leaves every other comment alone: an operator's explanation at -// the top of the file is exactly what this package exists to preserve. -func (f *File) SetPreamble(lines []string) { - if f.doc.Global == nil { - f.doc.Global = &tomledit.Section{} - } - f.changed() - - kept := withoutGeneratedLines(f.takeLeadingComments()) - var leading []parser.Item - if len(lines) > 0 { - region := append([]string{preambleBegin}, lines...) - leading = append(leading, parser.Comments(append(region, preambleEnd))) - } - if len(kept) > 0 { - leading = append(leading, parser.Comments(kept)) - } - f.doc.Global.Items = append(leading, f.doc.Global.Items...) -} - -// takeLeadingComments removes and returns every comment line standing above the document's first value. -// -// A comment block is an item of its own when a blank line follows it, and part of the item below it -// otherwise, and an operator decides which by how they type. Taking both means a region is found wherever -// the parser put it, and returning what is kept as one block puts it somewhere a later run still looks. -func (f *File) takeLeadingComments() []string { - var lines []string - - cut := 0 - for _, item := range f.doc.Global.Items { - block, comments := item.(parser.Comments) - if !comments { - break - } - lines = append(lines, block...) - cut++ - } - f.doc.Global.Items = f.doc.Global.Items[cut:] - - if attached := f.firstBlock(); attached != nil { - lines = append(lines, *attached...) - *attached = nil - } - return lines -} - -// firstBlock returns the comment block carried by whatever the document holds first, or nil. -func (f *File) firstBlock() *parser.Comments { - if len(f.doc.Global.Items) > 0 { - switch first := f.doc.Global.Items[0].(type) { - case *parser.KeyValue: - return &first.Block - case *parser.Heading: - return &first.Block - } - return nil - } - // Nothing at the top level, so the first thing in the file is a section heading. - for _, s := range f.doc.Sections { - if s.Heading != nil { - return &s.Block - } - } - return nil -} - -// withoutGeneratedLines drops every generated region from a set of comment lines. -// -// Every region rather than the first, so the result does not depend on how many a previous run left -// behind: dropping one and keeping another would leave a stale header in the file forever. -func withoutGeneratedLines(lines []string) []string { - var out []string - for i := 0; i < len(lines); { - from, to := generatedRegion(lines[i:]) - if from < 0 { - return append(out, lines[i:]...) - } - out = append(out, lines[i:i+from]...) - i += to + 1 - } - return out -} - -// generatedRegion returns the first and last line of the earliest generated region, or -1 for none. -// -// Both delimiters have to be present. A file holding only one of them was hand-edited into a shape this -// cannot reason about, and leaving those lines alone keeps an operator's writing over its own tidiness. -// -// Compared on each line's content, because a block this package inserts holds the text alone while the -// same block read back from a file carries the comment character the renderer added. -func generatedRegion(lines []string) (int, int) { - from := -1 - for i, line := range lines { - switch content(line) { - case content(preambleBegin): - if from < 0 { - from = i - } - case content(preambleEnd): - if from >= 0 { - return from, i - } - } - } - return -1, -1 -} - -// content is a comment line without its marker or surrounding space. -func content(line string) string { - return strings.TrimSpace(strings.TrimPrefix(strings.TrimSpace(line), "#")) -} - // Unset removes a key and reports whether the file carried one. // // This removes the key rather than writing a zero, because an absent key resolves to the running diff --git a/config/seitoml/guards_test.go b/config/seitoml/guards_test.go index 963d48414f..7b308ba66e 100644 --- a/config/seitoml/guards_test.go +++ b/config/seitoml/guards_test.go @@ -1,6 +1,7 @@ package seitoml import ( + "reflect" "strings" "testing" @@ -26,16 +27,95 @@ func TestATopLevelKeyReachesADocumentWithNoGlobalSection(t *testing.T) { } } -// TestAPreambleReachesADocumentWithNoGlobalSection is the preamble's half of the same shape. -func TestAPreambleReachesADocumentWithNoGlobalSection(t *testing.T) { - f := &File{doc: &tomledit.Document{}} - f.SetPreamble([]string{" a header"}) - - raw, err := f.Bytes() - if err != nil { - t.Fatalf("Bytes: %v", err) +// TestAReadReusesItsDecodeAndNeverAStaleOne drives the cache itself, which only this package can see. +// +// A read renders the document and decodes it, so a caller walking every key would pay that per key. Two +// properties make the saving safe, and neither is visible from outside: a read with no edit before it +// reuses the last decode, and no edit ever leaves a decode behind that describes the document as it was. +// The second is the one that would be a correctness bug rather than a lost saving. +func TestAReadReusesItsDecodeAndNeverAStaleOne(t *testing.T) { + newFile := func(t *testing.T) *File { + t.Helper() + f, err := Parse(strings.NewReader("schema_version = 1\nnode_mode = \"validator\"\n\n[probe]\nn = 1\n")) + if err != nil { + t.Fatalf("Parse: %v", err) + } + return f } - if !strings.Contains(string(raw), "# a header") { - t.Errorf("the preamble is not in the rendered document: %q", raw) + + t.Run("a second read reuses the first", func(t *testing.T) { + f := newFile(t) + first, err := f.decoded() + if err != nil { + t.Fatalf("decoded: %v", err) + } + // Written into the map the first read returned. A second read that decoded again would hand back + // a map without it. + first["probe.sentinel"] = true + second, err := f.decoded() + if err != nil { + t.Fatalf("decoded: %v", err) + } + if _, reused := second["probe.sentinel"]; !reused { + t.Error("a read with no edit before it decoded the document again") + } + }) + + for _, tc := range []struct { + name string + edit func(*testing.T, *File) + }{ + {"Set replacing a value", func(t *testing.T, f *File) { + if err := f.Set("probe.n", 2); err != nil { + t.Fatalf("Set: %v", err) + } + }}, + {"Set adding a key", func(t *testing.T, f *File) { + if err := f.Set("probe.m", 3); err != nil { + t.Fatalf("Set: %v", err) + } + }}, + {"Set adding a section", func(t *testing.T, f *File) { + if err := f.Set("p2p.laddr", "x"); err != nil { + t.Fatalf("Set: %v", err) + } + }}, + {"Unset", func(t *testing.T, f *File) { + if _, err := f.Unset("probe.n"); err != nil { + t.Fatalf("Unset: %v", err) + } + }}, + {"a Set the decoder refused", func(t *testing.T, f *File) { + // Refused after the write, so the document changed and changed back. A decode of either state + // in between describes neither. + if err := f.Set("probe.n.deeper", 4); err == nil { + t.Fatal("writing a table over a value was accepted") + } + }}, + } { + t.Run("after "+tc.name, func(t *testing.T) { + f := newFile(t) + if _, err := f.decoded(); err != nil { + t.Fatalf("decoded: %v", err) + } + tc.edit(t, f) + + held := f.values + if held == nil { + return // nothing cached, so nothing can be stale + } + raw, err := f.Bytes() + if err != nil { + t.Fatalf("Bytes: %v", err) + } + fresh, err := decodeBytes(raw) + if err != nil { + t.Fatalf("decodeBytes: %v", err) + } + if !reflect.DeepEqual(held, fresh) { + t.Errorf("%s left a decode describing another document:\n held %v\nfresh %v", + tc.name, held, fresh) + } + }) } } diff --git a/config/seitoml/seitoml_test.go b/config/seitoml/seitoml_test.go index bf51c71bd3..49fe97cd78 100644 --- a/config/seitoml/seitoml_test.go +++ b/config/seitoml/seitoml_test.go @@ -7,6 +7,7 @@ import ( "os" "path/filepath" "reflect" + "sort" "strings" "testing" "time" @@ -994,51 +995,6 @@ func TestSetWritesATopLevelKeyIntoAFileThatStartsWithATable(t *testing.T) { } } -// TestThePreambleIsReplacedRatherThanStacked holds what regenerating a file does to its header. -// -// The preamble explains the file to whoever opens it, and a generate or adopt run writes one. Stacking -// a new block on the last would grow the header on every run until the explanation is buried in copies -// of itself. An empty list removes it, which is how a caller drops a header it no longer wants. -func TestThePreambleIsReplacedRatherThanStacked(t *testing.T) { - f := parse(t, "schema_version = 1\nnode_mode = \"validator\"\n\n[probe]\nfirst = 1\n") - - f.SetPreamble([]string{" written by the first run"}) - first := render(t, f) - if !strings.Contains(first, "# written by the first run") { - t.Fatalf("the preamble is not in the file:\n%s", first) - } - - f.SetPreamble([]string{" written by the second run"}) - second := render(t, f) - if strings.Contains(second, "first run") { - t.Errorf("the second preamble stacked on the first, so a header grows on every run:\n%s", second) - } - if !strings.Contains(second, "# written by the second run") { - t.Errorf("the second preamble is not in the file:\n%s", second) - } - - // The keys are untouched throughout, since a header is not configuration. - if got, ok, err := parse(t, second).Get("probe.first"); err != nil || !ok || got != int64(1) { - t.Errorf("probe.first = (%#v, %v, %v) after two preambles, want 1", got, ok, err) - } - - f.SetPreamble(nil) - if bare := render(t, f); strings.Contains(bare, "second run") { - t.Errorf("an empty preamble left the old one in place:\n%s", bare) - } -} - -// TestThePreambleGoesAboveEverythingInAFileThatStartsWithATable covers the no-global-section case. -func TestThePreambleGoesAboveEverythingInAFileThatStartsWithATable(t *testing.T) { - f := parse(t, "[probe]\nfirst = 1\n") - f.SetPreamble([]string{" a header"}) - - out := render(t, f) - if !strings.HasPrefix(strings.TrimSpace(out), "# ---- generated by seid ----\n# a header") { - t.Errorf("the preamble is not the first thing in the file:\n%s", out) - } -} - // TestAMalformedKeyIsRefusedByEveryVerbThatTakesOne holds the four entry points to one answer. // // Set, Unset and Get each take a dotted key from a caller, and a key TOML cannot express has to be @@ -1500,79 +1456,6 @@ func TestEveryVerbTakingAKeyAppliesOneRule(t *testing.T) { } } -// TestThePreambleIsReplacedAcrossASaveAndReload holds the property over the flow that actually runs. -// -// Regenerating is Load, SetPreamble, Save, and the same again on the next release, so the block this -// method must recognise is one that has been through the parser rather than one it just inserted. Held -// in memory only, the test cannot see a parser that reattaches a leading comment to whatever follows it, -// and the header would grow on every run. -func TestThePreambleIsReplacedAcrossASaveAndReload(t *testing.T) { - dir := t.TempDir() - path := filepath.Join(dir, "sei.toml") - - f, err := seitoml.New("validator") - if err != nil { - t.Fatalf("New: %v", err) - } - if err := f.Set("probe.n", 1); err != nil { - t.Fatalf("Set: %v", err) - } - f.SetPreamble([]string{" written by run one"}) - if err := f.Save(path); err != nil { - t.Fatalf("Save: %v", err) - } - - reread, err := seitoml.Load(path) - if err != nil { - t.Fatalf("Load: %v", err) - } - reread.SetPreamble([]string{" written by run two"}) - if err := reread.Save(path); err != nil { - t.Fatalf("Save: %v", err) - } - - raw, err := os.ReadFile(path) //nolint:gosec // a path this test created under t.TempDir - if err != nil { - t.Fatalf("ReadFile: %v", err) - } - if strings.Contains(string(raw), "run one") { - t.Errorf("the first preamble survived the second, so a header grows on every regenerate:\n%s", raw) - } - if !strings.Contains(string(raw), "# written by run two") { - t.Errorf("the second preamble is not in the file:\n%s", raw) - } - // The values are untouched by either header, since a preamble is not configuration. - final, err := seitoml.Load(path) - if err != nil { - t.Fatalf("Load after two preambles: %v", err) - } - if got, ok, err := final.Get("probe.n"); err != nil || !ok || got != int64(1) { - t.Errorf("probe.n = (%#v, %v, %v) after two preambles, want 1", got, ok, err) - } -} - -// TestAPreambleLeavesAnOperatorsOwnTopCommentAlone covers the comment this method must not claim. -// -// SetPreamble replaces the block it put there before, and an operator may have written their own -// explanation at the top of the file. Removing that would be the comment loss this package exists to -// prevent, so it has to survive a header being written above it. -func TestAPreambleLeavesAnOperatorsOwnTopCommentAlone(t *testing.T) { - // The blank line matters: the parser keeps a comment block standalone only when one follows it, and a - // block attached to the next key is one SetPreamble never looks at. Without it this test passes - // whatever the code does. - f := parse(t, "# I wrote this and it explains the file\n# do not delete it\n\nschema_version = 1\n"+ - "node_mode = \"validator\"\n\n[probe]\nn = 1\n") - - f.SetPreamble([]string{" generated header"}) - - out := render(t, f) - for _, want := range []string{"I wrote this", "do not delete it", "# generated header"} { - if !strings.Contains(out, want) { - t.Errorf("%q is not in the file after a preamble was written:\n%s", want, out) - } - } -} - // TestAnUnsignedValueTooLargeToReadBackIsRefused holds the writer to what a reader can return. // // A TOML integer is signed and decodes into an int64, so a larger unsigned value renders as a line that @@ -1650,94 +1533,6 @@ func TestAValueThatIsNotTextIsRefused(t *testing.T) { } } -// TestAPreambleOnlyReplacesOneItWrote holds the block this method may claim. -// -// A comment block at the top of a file is an operator's explanation unless this method put it there, and -// it has no way to tell the two apart but a mark it writes and looks for. Deleting theirs would be the -// comment loss the package exists to prevent; leaving its own would grow a header on every regenerate. -func TestAPreambleOnlyReplacesOneItWrote(t *testing.T) { - const operator = "# ops: do not raise flatkv without asking" - f := parse(t, operator+"\n\nschema_version = 1\nnode_mode = \"validator\"\n\n[probe]\nn = 1\n") - - f.SetPreamble([]string{" run one"}) - first := render(t, f) - if !strings.Contains(first, "do not raise flatkv") { - t.Fatalf("the operator's comment was deleted by the first preamble:\n%s", first) - } - - // Through a parse, which is the state the next release's run sees. - again := parse(t, first) - again.SetPreamble([]string{" run two"}) - second := render(t, again) - - if strings.Contains(second, "run one") { - t.Errorf("the first preamble survived the second, so a header grows on every run:\n%s", second) - } - if !strings.Contains(second, "run two") { - t.Errorf("the second preamble is not in the file:\n%s", second) - } - if !strings.Contains(second, "do not raise flatkv") { - t.Errorf("the operator's comment was deleted by the second preamble:\n%s", second) - } -} - -// TestAPreambleOwnsItsLinesRatherThanTheBlockTheySitIn holds what regenerating may touch. -// -// The mark ends the generated lines, and an operator writes wherever they like around them: above the -// header, or on a line the parser groups into the same comment item. Anchored at the first item, a -// header above which anything was written is never found and grows on every run; anchored at a block, an -// operator's line inside that block is deleted with it. Both are the failures the mark exists to prevent. -func TestAPreambleOwnsItsLinesRatherThanTheBlockTheySitIn(t *testing.T) { - const note = "ops: we pin flatkv, see INC-4412" - tail := "\nschema_version = 1\nnode_mode = \"validator\"\n" - - for _, tc := range []struct{ name, before string }{ - { - "an operator block above the generated region", - "# " + note + "\n\n" + generated("run one") + tail, - }, - { - "an operator line below the region, no blank line", - generated("run one") + "# " + note + "\n" + tail, - }, - { - "an operator line above the region, no blank line", - "# " + note + "\n" + generated("run one") + tail, - }, - { - "an operator line between two generated regions", - generated("run one") + "# " + note + "\n" + generated("stale run") + tail, - }, - } { - t.Run(tc.name, func(t *testing.T) { - f := parse(t, tc.before) - f.SetPreamble([]string{" written by run two"}) - out := render(t, f) - - if strings.Contains(out, "run one") { - t.Errorf("the previous header survived, so it grows on every regenerate:\n%s", out) - } - if !strings.Contains(out, note) { - t.Errorf("the operator's comment was deleted:\n%s", out) - } - if n := strings.Count(out, "generated by seid"); n != 1 { - t.Errorf("the file carries %d generated regions, want one:\n%s", n, out) - } - - // And again through a parse, since a regenerate reads what the last one wrote. - third := parse(t, out) - third.SetPreamble([]string{" written by run three"}) - last := render(t, third) - if strings.Contains(last, "run two") { - t.Errorf("the second header survived the third:\n%s", last) - } - if !strings.Contains(last, note) { - t.Errorf("the operator's comment was deleted on the third run:\n%s", last) - } - }) - } -} - // TestATableKeepsOneSpellingWhateverOrderItsKeysWereWritten holds insert's choice between the two forms. // // A table nothing has created is new and gets a heading, which is the form an operator reads. A table an @@ -1791,112 +1586,39 @@ func TestLandedSeparatesAnInstalledFileFromAFailedWrite(t *testing.T) { } } -// TestReadingTwiceDoesNotDecodeTwice holds the cache an edit invalidates. +// TestEveryEditIsVisibleToTheNextRead holds the values a read returns against the document. // -// A read renders the document and decodes it, so a caller walking every declared key would pay that per -// key, and building a file would be quadratic in its size because every edit checks the result. The -// values a read returns still have to follow the document, which is what makes the invalidation the part -// worth testing rather than the caching. -func TestReadingTwiceDoesNotDecodeTwice(t *testing.T) { +// Reads are cached, so this is the half that makes the cache safe rather than merely fast: an edit the +// cache outlived would have a read answering for the document as it used to be. What is cached, and when +// it is dropped, is driven in the package's own tests where the field is visible. +func TestEveryEditIsVisibleToTheNextRead(t *testing.T) { f := parse(t, "schema_version = 1\nnode_mode = \"validator\"\n\n[probe]\nn = 1\n") - first, err := f.Values() - if err != nil { - t.Fatalf("Values: %v", err) - } - if first["probe.n"] != int64(1) { - t.Fatalf("probe.n = %#v, want 1", first["probe.n"]) + if values, err := f.Values(); err != nil || values["probe.n"] != int64(1) { + t.Fatalf("probe.n = (%#v, %v), want 1", values["probe.n"], err) } - - // Every edit has to be visible to the next read, or a cache is a correctness bug rather than a saving. for _, step := range []struct { name string edit func() error key string want any }{ - {"Set replacing a value", func() error { return f.Set("probe.n", 2) }, "probe.n", int64(2)}, - {"Set adding a key", func() error { return f.Set("probe.m", 3) }, "probe.m", int64(3)}, - {"Unset removing one", func() error { _, err := f.Unset("probe.m"); return err }, "probe.m", nil}, + {"replacing a value", func() error { return f.Set("probe.n", 2) }, "probe.n", int64(2)}, + {"adding a key", func() error { return f.Set("probe.m", 3) }, "probe.m", int64(3)}, + {"removing one", func() error { _, err := f.Unset("probe.m"); return err }, "probe.m", nil}, + {"adding a section", func() error { return f.Set("p2p.laddr", "x") }, "p2p.laddr", "x"}, } { if err := step.edit(); err != nil { t.Fatalf("%s: %v", step.name, err) } - got, err := f.Values() + values, err := f.Values() if err != nil { t.Fatalf("%s: Values: %v", step.name, err) } - if got[step.key] != step.want { - t.Errorf("after %s, %s = %#v, want %#v", step.name, step.key, got[step.key], step.want) + if values[step.key] != step.want { + t.Errorf("after %s, %s = %#v, want %#v", step.name, step.key, values[step.key], step.want) } } - - // A preamble changes the document too, and must not strand a decode of the version before it. - f.SetPreamble([]string{" header"}) - if _, err := f.Values(); err != nil { - t.Errorf("Values after a preamble: %v", err) - } -} - -// generated renders a preamble region the way SetPreamble writes one, so a fixture can start from a file -// a previous run produced. -func generated(header string) string { - return "# ---- generated by seid ----\n# " + header + - "\n# ---- end generated; your notes are safe below ----\n" -} - -// TestAnOperatorWritingInsideTheGeneratedRegionIsTold pins the one line the region does claim. -// -// The delimiters say where the generated lines start and stop, so a note written between them is inside -// the part a regenerate replaces. That is the contract the file states in words, and it is worth a test -// because the alternative reading, that nothing a person typed may ever be replaced, would make the -// region unreplaceable. -func TestAnOperatorWritingInsideTheGeneratedRegionIsTold(t *testing.T) { - inside := "# ---- generated by seid ----\n# written by run one\n# a note typed inside the region\n" + - "# ---- end generated; your notes are safe below ----\nschema_version = 1\nnode_mode = \"v\"\n" - - f := parse(t, inside) - f.SetPreamble([]string{" written by run two"}) - out := render(t, f) - - if strings.Contains(out, "typed inside the region") { - t.Errorf("a line inside the generated region survived a regenerate, so the region cannot be "+ - "replaced at all:\n%s", out) - } - if !strings.Contains(out, "run two") { - t.Errorf("the new header is missing:\n%s", out) - } -} - -// TestAnUnpairedDelimiterLeavesTheLinesAlone covers a file hand-edited into a shape with one delimiter. -// -// A region is the text between two delimiters, so one on its own bounds nothing. Guessing where it ends -// would delete an operator's lines on the strength of a marker they may have typed themselves, and -// leaving them is the answer that cannot lose their writing. -func TestAnUnpairedDelimiterLeavesTheLinesAlone(t *testing.T) { - for _, tc := range []struct{ name, before string }{ - { - "a begin with no end", - "# ---- generated by seid ----\n# a note under a stray delimiter\n\nschema_version = 1\nnode_mode = \"v\"\n", - }, - { - "an end with no begin", - "# a note above a stray delimiter\n# ---- end generated; your notes are safe below ----\n\nschema_version = 1\nnode_mode = \"v\"\n", - }, - } { - t.Run(tc.name, func(t *testing.T) { - f := parse(t, tc.before) - f.SetPreamble([]string{" written by this run"}) - - out := render(t, f) - if !strings.Contains(out, "stray delimiter") { - t.Errorf("the operator's line was removed on the strength of one delimiter:\n%s", out) - } - if !strings.Contains(out, "written by this run") { - t.Errorf("the new header is missing:\n%s", out) - } - }) - } } // TestReadingOneWayDoesNotChangeWhatAnotherAnswers holds the reads against each other. @@ -1959,3 +1681,71 @@ func TestReadingOneWayDoesNotChangeWhatAnotherAnswers(t *testing.T) { t.Error("a key the caller invented appeared in the next read") } } + +// TestATableIsSpelledTheSameWhateverOrderItsKeysArrive drives the property by varying the order. +// +// insert chooses between a heading and a dotted name, and the choice has to follow the document rather +// than the sequence of calls. Set the same keys in every order and the file has to come out the same, or a +// table is a heading or a dotted name depending on which key an operator happened to write first. +func TestATableIsSpelledTheSameWhateverOrderItsKeysArrive(t *testing.T) { + keys := []string{ + "state-commit.buffer", + "state-commit.flatkv.enable", + "state-commit.flatkv.dir", + "p2p.laddr", + } + + var want string + for _, order := range permutations(keys) { + f, err := seitoml.New("validator") + if err != nil { + t.Fatalf("New: %v", err) + } + for _, key := range order { + if err := f.Set(key, "v"); err != nil { + t.Fatalf("Set(%q) in order %v: %v", key, order, err) + } + } + got := shape(t, f) + if want == "" { + want = got + continue + } + if got != want { + t.Fatalf("order %v produced\n %s\nand another order produced\n %s", order, got, want) + } + } +} + +// shape is a file's headings and keys, sorted, so two files can be compared without regard to the order +// their lines happen to sit in. +func shape(t *testing.T, f *seitoml.File) string { + t.Helper() + var out []string + for _, line := range strings.Split(render(t, f), "\n") { + line = strings.TrimSpace(line) + switch { + case strings.HasPrefix(line, "["): + out = append(out, "heading "+line) + case strings.Contains(line, "="): + out = append(out, "key "+strings.TrimSpace(strings.SplitN(line, "=", 2)[0])) + } + } + sort.Strings(out) + return strings.Join(out, " | ") +} + +// permutations returns every ordering of the given keys. +func permutations(keys []string) [][]string { + if len(keys) <= 1 { + return [][]string{append([]string(nil), keys...)} + } + var out [][]string + for i := range keys { + rest := append(append([]string(nil), keys[:i]...), keys[i+1:]...) + for _, tail := range permutations(rest) { + out = append(out, append([]string{keys[i]}, tail...)) + } + } + return out +} From 9b80106dee817bddb1354217ac5dd8cab97e35b0 Mon Sep 17 00:00:00 2001 From: bdchatham Date: Tue, 18 Aug 2026 18:08:06 -0700 Subject: [PATCH 14/24] config/seitoml: stop handing out the cache's lists A read caches its decode, and Values built its own map so a caller's writes could not reach it. The map was shallow, so a list stayed shared: a caller sorting the slice they were handed sorted the cache, and every later read reported the new order. Get returned the cached value with no copy at all. Both now hand values out through one function that copies a list, recursively for a list of lists. The map already had a comment saying it is the cache; the comment now says the lists are too, and where a caller-facing copy happens. The test that holds the reads against each other carried no list, which is why it agreed. Its fixture now has one flat list and one nested, and it mutates what each read handed back. Reverting either call site, or the recursion, fails a named assertion. Two claims elsewhere were stronger than the code. The package doc said every edit is asked the decoder's question; two of the three paths change no namespace and are covered by the check in Save instead, which the paragraph did not mention. Save's own doc did not mention that check either. Both now say what happens. Unset returned "the file carried no such key" when a removal found the key and did nothing. That is the shape corrected in Set, and it is corrected here for the same reason: a caller told nothing was there cannot find out otherwise. Unreachable, so uncovered, and it is the shape the next verb's author copies. 131 cases, 94.1% of statements, race clean, 0 lint issues. --- config/seitoml/doc.go | 9 +++---- config/seitoml/edit.go | 7 +++++- config/seitoml/file.go | 2 ++ config/seitoml/seitoml_test.go | 43 +++++++++++++++++++++++++++++----- config/seitoml/values.go | 32 ++++++++++++++++++++----- 5 files changed, 76 insertions(+), 17 deletions(-) diff --git a/config/seitoml/doc.go b/config/seitoml/doc.go index 503c03f16c..7759cf7693 100644 --- a/config/seitoml/doc.go +++ b/config/seitoml/doc.go @@ -56,10 +56,11 @@ // integer or a multi-line string means is a second implementation of the specification and a // hand-written one went wrong in four places. // -// Every edit is asked the same question. Set writes the key, renders, and offers the result to the -// decoder; if the document no longer reads, the write is undone and the key is named. So a shape nobody -// anticipated is refused as surely as one somebody did, which is the property enumerating shapes by hand -// could not give. +// A new key is asked the same question where it is written. Set writes the key, renders, and offers the +// result to the decoder; if the document no longer reads, the write is undone and the key is named. Save +// asks again over the whole rendering, which is what covers the verbs that change no namespace, so no +// file reaches disk unread. Together that refuses a shape nobody anticipated as surely as one somebody +// did, which is the property enumerating shapes by hand could not give. // // Four things that decoder allows are refused anyway, because this package has to write back what it // reads: diff --git a/config/seitoml/edit.go b/config/seitoml/edit.go index 3933e37ec0..7d5b46d2e8 100644 --- a/config/seitoml/edit.go +++ b/config/seitoml/edit.go @@ -196,7 +196,12 @@ func (f *File) Unset(key string) (bool, error) { return false, nil } f.changed() - return e.Remove(), nil + if !e.Remove() { + // Reported rather than returned as an absent key, which is what the file carrying one and the + // removal doing nothing would otherwise look like to a caller. + return false, fmt.Errorf("%s: the file carries this key and it could not be removed", key) + } + return true, nil } // tomlValue renders a Go value as the TOML literal that parses back to it. diff --git a/config/seitoml/file.go b/config/seitoml/file.go index e28a95f656..0eabb35ad9 100644 --- a/config/seitoml/file.go +++ b/config/seitoml/file.go @@ -294,6 +294,8 @@ func (f *File) Bytes() ([]byte, error) { // Save writes the document to path, atomically. // +// The document is offered to the decoder first, so no file reaches disk that a node cannot read. +// // The rename makes it atomic, and the temporary file sits in the destination's own directory so the // rename stays within one filesystem. A crash at any point leaves either the previous file or the // new one, never a truncated file a node cannot parse. diff --git a/config/seitoml/seitoml_test.go b/config/seitoml/seitoml_test.go index 49fe97cd78..4cd0fec6d0 100644 --- a/config/seitoml/seitoml_test.go +++ b/config/seitoml/seitoml_test.go @@ -1390,7 +1390,8 @@ func TestANameCannotBeAValueAndATableAtOnce(t *testing.T) { t.Run("a value named like a section holding nothing", func(t *testing.T) { // An empty section contributes no value, so a check over written values cannot see it. - f := parse(t, "schema_version = 1\nnode_mode = \"validator\"\n\n[probe]\nn = 1\n") + f := parse(t, "schema_version = 1\nnode_mode = \"validator\"\n\n[probe]\nn = 1\n"+ + "seeds = [\"first\", \"second\"]\npairs = [[\"inner\"]]\n") if _, err := f.Unset("probe.n"); err != nil { t.Fatalf("Unset: %v", err) } @@ -1592,7 +1593,8 @@ func TestLandedSeparatesAnInstalledFileFromAFailedWrite(t *testing.T) { // cache outlived would have a read answering for the document as it used to be. What is cached, and when // it is dropped, is driven in the package's own tests where the field is visible. func TestEveryEditIsVisibleToTheNextRead(t *testing.T) { - f := parse(t, "schema_version = 1\nnode_mode = \"validator\"\n\n[probe]\nn = 1\n") + f := parse(t, "schema_version = 1\nnode_mode = \"validator\"\n\n[probe]\nn = 1\n"+ + "seeds = [\"first\", \"second\"]\npairs = [[\"inner\"]]\n") if values, err := f.Values(); err != nil || values["probe.n"] != int64(1) { t.Fatalf("probe.n = (%#v, %v), want 1", values["probe.n"], err) @@ -1624,11 +1626,14 @@ func TestEveryEditIsVisibleToTheNextRead(t *testing.T) { // TestReadingOneWayDoesNotChangeWhatAnotherAnswers holds the reads against each other. // // A read renders and decodes the document, and the result is cached so that walking every key does not -// pay that per key. The cache is one map, so a read that filtered or handed it out would change what the -// next read of any other kind answers: Values omits the two keys describing the file, and deleting them -// from the shared map made Version, Mode and Get report a file that holds them as missing them. +// pay that per key. The cache is one map holding one slice per list, so a read that filtered it, or handed +// out either, would change what the next read of any other kind answers: Values omits the two keys +// describing the file, and deleting them from the shared map made Version, Mode and Get report a file that +// holds them as missing them. A list reaches the same end through its elements, since a caller sorting the +// slice they were handed sorts the cache. func TestReadingOneWayDoesNotChangeWhatAnotherAnswers(t *testing.T) { - f := parse(t, "schema_version = 1\nnode_mode = \"validator\"\n\n[probe]\nn = 1\n") + f := parse(t, "schema_version = 1\nnode_mode = \"validator\"\n\n[probe]\nn = 1\n"+ + "seeds = [\"first\", \"second\"]\npairs = [[\"inner\"]]\n") // Read every way, twice, in an order that would expose a read leaking into the cache. for round := 1; round <= 2; round++ { @@ -1680,6 +1685,32 @@ func TestReadingOneWayDoesNotChangeWhatAnotherAnswers(t *testing.T) { if _, present := again["invented"]; present { t.Error("a key the caller invented appeared in the next read") } + + // A list the caller was handed is theirs to change, whichever read handed it over. + mine["probe.seeds"].([]any)[0] = "written by the caller" + mine["probe.pairs"].([]any)[0].([]any)[0] = "written by the caller" + + got, _, err := f.Get("probe.seeds") + if err != nil { + t.Fatalf("Get: %v", err) + } + if first := got.([]any)[0]; first != "first" { + t.Errorf("a caller's write into a list reached the next read: probe.seeds[0] = %#v, want first", first) + } + got.([]any)[1] = "written by the caller" + + last, err := f.Values() + if err != nil { + t.Fatalf("Values: %v", err) + } + if second := last["probe.seeds"].([]any)[1]; second != "second" { + t.Errorf("a caller's write into a list reached the next read: probe.seeds[1] = %#v, want second", + second) + } + if inner := last["probe.pairs"].([]any)[0].([]any)[0]; inner != "inner" { + t.Errorf("a caller's write into a nested list reached the next read: probe.pairs[0][0] = %#v, "+ + "want inner", inner) + } } // TestATableIsSpelledTheSameWhateverOrderItsKeysArrive drives the property by varying the order. diff --git a/config/seitoml/values.go b/config/seitoml/values.go index 0764631505..11dfedf848 100644 --- a/config/seitoml/values.go +++ b/config/seitoml/values.go @@ -18,14 +18,13 @@ func (f *File) Values() (map[string]any, error) { return nil, err } // Built rather than filtered in place, because decoded hands back the cache itself. Deleting the two - // describing keys from it made every later Version, Mode and Get read them as absent, and handing the - // cache to a caller let their own writes into it. + // describing keys from it made every later Version, Mode and Get read them as absent. out := make(map[string]any, len(all)) for key, v := range all { if key == VersionKey || key == ModeKey { continue } - out[key] = v + out[key] = handedOut(v) } return out, nil } @@ -41,7 +40,27 @@ func (f *File) Get(key string) (any, bool, error) { return nil, false, err } v, ok := all[path.String()] - return v, ok, nil + return handedOut(v), ok, nil +} + +// handedOut returns a value a caller can change without changing what a later read answers. +// +// A list decodes to a slice the cache holds, so returning that slice shares its backing array, and a +// caller sorting or index-assigning what it was given rewrites the cache. Copied on the way out rather +// than once at decode, because a caller changes what it holds at any point after it holds it. +// +// Only a list needs copying. A scalar is copied by the assignment, and a leaf is never a table: an +// inline table is refused when the file is read and cannot be written, so nothing reaches here as a map. +func handedOut(v any) any { + list, ok := v.([]any) + if !ok { + return v + } + out := make([]any, len(list)) + for i, element := range list { + out[i] = handedOut(element) + } + return out } // decoded renders the document and reads it back as Go values, keyed by dotted path. @@ -59,8 +78,9 @@ func (f *File) Get(key string) (any, bool, error) { // path a later process would use, so a value this package cannot express fails here rather than on a // node. // -// The map returned is the cache. Every caller here reads it, and one that needs to hand a map outward or -// change it has to build its own; writing into this one would change what a later read answers. +// The map returned is the cache, and so is every list in it. Every caller here reads them, and one that +// hands either outward passes it through handedOut; writing into this map or a list it holds would +// change what a later read answers. func (f *File) decoded() (map[string]any, error) { if f.values != nil { return f.values, nil From e88c4bbfef6218361870e563b16bf1af746522b5 Mon Sep 17 00:00:00 2001 From: bdchatham Date: Tue, 18 Aug 2026 18:29:39 -0700 Subject: [PATCH 15/24] config/seitoml: keep the comment beside a value across an edit Set replaced the whole parsed value on the existing line. A comment above the key hangs off the key and survived; a comment beside the value hangs off the value and was dropped. So an operator who wrote their reason at the end of the line lost it the first time anything changed that key, which is the loss this package exists to prevent. The fixture said it held "a reason beside a value" and both its comments were on their own lines, so the property was stated and never driven. The comment on one key moves to the end of its line, both keys are now edited, and reverting the one-line carry fails the test by name. The fixture keeps its two keys rather than gaining a third, so the tests that count them are untouched. The two value assertions were anchored to a line start: "enabled = " is a suffix of "occ_enabled = ", so the unanchored form found the other key's line and stopped discriminating. Three documentation gaps, all found by reading a claim against the code: - File did not say it is for one goroutine at a time. Reading is not pure since every read decodes and holds the result, so two concurrent reads of a shared File race. - Save did not say a non-nil error can mean the values are on disk. It now names ErrNotDurable and Landed at the call a caller is looking at. - A test comment listed a date and an inline table among the shapes its fixture drives. Both are refused when the file is read, so the fixture cannot hold either. 131 cases, 94.1% of statements, race clean, 0 lint issues. --- config/seitoml/edit.go | 3 +++ config/seitoml/file.go | 7 +++++++ config/seitoml/seitoml_test.go | 36 +++++++++++++++++++++++----------- 3 files changed, 35 insertions(+), 11 deletions(-) diff --git a/config/seitoml/edit.go b/config/seitoml/edit.go index 7d5b46d2e8..6da9e17082 100644 --- a/config/seitoml/edit.go +++ b/config/seitoml/edit.go @@ -31,6 +31,9 @@ func (f *File) Set(key string, v any) error { if e := f.doc.First(path...); e != nil && e.KeyValue != nil { f.changed() + // The comment beside a value hangs off the value, so replacing the value drops it unless it is + // carried across. The block above the key hangs off the key instead and survives on its own. + value.Trailer = e.Value.Trailer e.Value = value return nil } diff --git a/config/seitoml/file.go b/config/seitoml/file.go index 0eabb35ad9..b9a6f10268 100644 --- a/config/seitoml/file.go +++ b/config/seitoml/file.go @@ -42,6 +42,9 @@ const ModeKey = "node_mode" const newFileMode os.FileMode = 0o600 // File is a parsed sei.toml that survives editing with its comments and layout intact. +// +// A File is for one goroutine at a time. Reading is not a pure operation: every read decodes the +// document and holds the result, so two concurrent reads of a shared File race. type File struct { doc *tomledit.Document // values caches the last decode, and is nil whenever the document has changed since. @@ -296,6 +299,10 @@ func (f *File) Bytes() ([]byte, error) { // // The document is offered to the decoder first, so no file reaches disk that a node cannot read. // +// A non-nil error does not always mean the values are absent from disk: ErrNotDurable reports a file +// that is installed with its directory entry not yet flushed. Call Landed on the error rather than +// comparing it against nil, which reads that outcome as a failed write. +// // The rename makes it atomic, and the temporary file sits in the destination's own directory so the // rename stays within one filesystem. A crash at any point leaves either the previous file or the // new one, never a truncated file a node cannot parse. diff --git a/config/seitoml/seitoml_test.go b/config/seitoml/seitoml_test.go index 4cd0fec6d0..97ad48759a 100644 --- a/config/seitoml/seitoml_test.go +++ b/config/seitoml/seitoml_test.go @@ -15,14 +15,17 @@ import ( "github.com/sei-protocol/sei-chain/config/seitoml" ) -// commented is a file written the way an operator writes one: a heading comment, a reason beside a -// value, and a blank line for legibility. +// commented is a file written the way an operator writes one: a heading comment, a reason above a value, +// a reason beside another, and a blank line for legibility. +// +// Both comment positions are here because they are preserved by different means. A block above the key +// hangs off the key, and a comment beside the value hangs off the value an edit replaces. const commented = `schema_version = 1 node_mode = "validator" # The giga executor. Turned on after the load test in March. [giga_executor] -enabled = true +enabled = true # Left on through the upgrade; the load test covered this path. # Off deliberately: this node serves historical queries and OCC cost us more than it saved. occ_enabled = false ` @@ -50,18 +53,23 @@ func render(t *testing.T, f *seitoml.File) string { // // An operator's comments are how they explain a choice to whoever reads the file next. Rewriting // the file from a decoded map would drop all of them, and the operator would have no way to get -// that reasoning back. Held by editing a value that has a comment explaining it. +// that reasoning back. Held by editing both values that carry a comment explaining them, because a +// comment above a key and a comment beside a value survive an edit by different means. func TestEditingPreservesAnOperatorsComments(t *testing.T) { f := parse(t, commented) if err := f.Set("giga_executor.occ_enabled", true); err != nil { t.Fatalf("Set: %v", err) } + if err := f.Set("giga_executor.enabled", false); err != nil { + t.Fatalf("Set: %v", err) + } got := render(t, f) for _, comment := range []string{ "# The giga executor. Turned on after the load test in March.", "# Off deliberately: this node serves historical queries and OCC cost us more than it saved.", + "# Left on through the upgrade; the load test covered this path.", } { if !strings.Contains(got, comment) { t.Errorf("editing one value dropped a comment:\n %s\n\nThe file now reads:\n%s\n\n"+ @@ -69,11 +77,17 @@ func TestEditingPreservesAnOperatorsComments(t *testing.T) { comment, got) } } - if !strings.Contains(got, "occ_enabled = true") { - t.Errorf("the value was not written. The file reads:\n%s", got) + // Anchored to the start of a line: "enabled = " is a suffix of "occ_enabled = ", so an unanchored + // search for one key's value finds the other's and the assertion stops discriminating. + for _, written := range []string{"\nocc_enabled = true", "\nenabled = false"} { + if !strings.Contains(got, written) { + t.Errorf("%q was not written. The file reads:\n%s", written, got) + } } - if strings.Contains(got, "occ_enabled = false") { - t.Errorf("the old value is still present, so the key is written twice:\n%s", got) + for _, stale := range []string{"\nocc_enabled = false", "\nenabled = true"} { + if strings.Contains(got, stale) { + t.Errorf("%q is still present, so the key is written twice:\n%s", stale, got) + } } } @@ -794,9 +808,9 @@ func TestAMigrationCarriesTheNodeModeForward(t *testing.T) { // // The file is hand-written, and TOML gives an operator more ways to write a value than a generated // file would ever use: two string quotings and their multi-line forms, an integer in hex or with -// separators, a date, an array with a comment inside it, an inline table. Each has to come back as the -// Go value a reader compares against a default, because a shape that decodes wrongly is a value an -// operator wrote and the node silently disagrees about. +// separators, a float, an array with a comment inside it. Each has to come back as the Go value a +// reader compares against a default, because a shape that decodes wrongly is a value an operator +// wrote and the node silently disagrees about. func TestEveryValueShapeTomlAllowsReadsBack(t *testing.T) { f := parse(t, `schema_version = 1 node_mode = "validator" From 169f9615cd54c9fc2a540c43943a538d3a9e6f73 Mon Sep 17 00:00:00 2001 From: bdchatham Date: Tue, 18 Aug 2026 18:47:20 -0700 Subject: [PATCH 16/24] config/seitoml: Save's error means one thing Save returned an error that sometimes meant the values were on disk. A caller writing the check every other Go call wants read a landed save as a failed one, and the package answered that with an exported sentinel plus an exported helper to test for it. That put the invariant in every caller's memory rather than in the code, and a caller who forgets writes something that looks correct. The distinction is gone rather than moved. Past the rename the new file is what a node reads, so whether the directory entry has been flushed does not change whether the save landed, and no caller has an action either way. Retrying is actively wrong: Linux reports a writeback error once per descriptor and does not write the pages again, so a second flush can succeed over data that never reached the device. syncDir returns nothing, and there is no value left to get wrong. Two exported names go with it. The test that pinned the old behavior drove it with a directory mode of 0300, which fails opening the directory rather than flushing it. So the sentinel's text named a state the code had not established, in the only case the suite produced. The test now asserts the save succeeds and is renamed for what it proves. modeToWrite mapped every inspect failure onto "no file there yet" and accepted any destination that was not a symbolic link: - A path whose parent is a file is neither present nor absent. Reading it as a first save chose the default mode on a guess, then failed further down on the temporary file and named that instead of the path given. - A pipe, a socket or a device node is replaced by the rename, not written through. The destination was destroyed and the configuration took whatever permission it carried, which for a device node is world-writable on a file naming key paths. Both are refused where the mode is decided, before anything is written. The refused save leaves the destination in place. 132 cases, 94.0% of statements, race clean, 0 lint issues. Reverting any of the three guards fails a named test. --- config/seitoml/file.go | 69 ++++++++++------------ config/seitoml/seitoml_test.go | 101 +++++++++++++++++++++++---------- 2 files changed, 102 insertions(+), 68 deletions(-) diff --git a/config/seitoml/file.go b/config/seitoml/file.go index b9a6f10268..c91c3d6e4f 100644 --- a/config/seitoml/file.go +++ b/config/seitoml/file.go @@ -5,6 +5,7 @@ import ( "errors" "fmt" "io" + "io/fs" "os" "path/filepath" "strconv" @@ -297,11 +298,8 @@ func (f *File) Bytes() ([]byte, error) { // Save writes the document to path, atomically. // -// The document is offered to the decoder first, so no file reaches disk that a node cannot read. -// -// A non-nil error does not always mean the values are absent from disk: ErrNotDurable reports a file -// that is installed with its directory entry not yet flushed. Call Landed on the error rather than -// comparing it against nil, which reads that outcome as a failed write. +// The document is offered to the decoder first, so no file reaches disk that a node cannot read. A +// non-nil error means the values are not on disk. // // The rename makes it atomic, and the temporary file sits in the destination's own directory so the // rename stays within one filesystem. A crash at any point leaves either the previous file or the @@ -340,40 +338,27 @@ func (f *File) Save(path string) error { if err := os.Rename(tmpName, path); err != nil { return fmt.Errorf("install %s: %w", path, err) } - // Past this point the new file is what the node will read, so a failure to flush the directory - // entry is not a failure of the save. Returning one would tell a caller their change did not land - // when it did, and the next thing they do is write it again or open an incident. - if err := syncDir(dir); err != nil { - return fmt.Errorf("%w: %w", ErrNotDurable, err) - } + syncDir(dir) return nil } -// ErrNotDurable reports that a save installed the file and could not flush the directory entry. -// -// The values are in place and a reader sees them. Only their survival of a machine losing power before -// the filesystem flushes on its own is unproven, so a caller that treats this as a failed write is -// wrong about what happened. -var ErrNotDurable = errors.New("the file is installed and its directory entry is not yet flushed") - -// Landed reports whether a Save put the values on disk, which is what a caller acting on the outcome -// wants to know. -// -// Save reports one outcome that is not a failure through the error it returns, so the plain err != nil -// check reads a landed save as a failed one and tells an operator their change did not apply when it -// did. This is that check written correctly, in one call, next to the sentinel it accounts for. -func Landed(err error) bool { return err == nil || errors.Is(err, ErrNotDurable) } - // modeToWrite returns the permission a save should use, and refuses a destination it must not replace. // -// An existing file keeps its own mode, so a save never widens what an operator narrowed. A symbolic -// link is refused: renaming onto one replaces the link with a regular file, leaving whatever it pointed -// at holding the old values, and nothing about the result says the link is gone. +// An existing file keeps its own mode, so a save never widens what an operator narrowed. Two +// destinations are refused instead, because a rename replaces either one rather than writing through +// it. A symbolic link would leave whatever it pointed at holding the old values, with nothing about the +// result saying the link is gone. Anything else that is not a regular file is a device node, a socket +// or a pipe, and replacing one destroys it and hands the configuration whatever permission it carried, +// which for a device node is world-writable. func modeToWrite(path string) (os.FileMode, error) { info, err := os.Lstat(path) switch { - case err != nil: + case errors.Is(err, fs.ErrNotExist): return newFileMode, nil // no file there yet, which is the ordinary first save + case err != nil: + // Separated from the absent case, which used to absorb it. A path this process cannot inspect + // is not a first save, and calling it one writes at the default mode on a guess. + return 0, fmt.Errorf("inspect %s: %w", path, err) case info.Mode()&os.ModeSymlink != 0: target, err := os.Readlink(path) if err != nil { @@ -382,6 +367,10 @@ func modeToWrite(path string) (os.FileMode, error) { return 0, fmt.Errorf("%s is a symbolic link to %s. Writing here would replace the link with a "+ "regular file and leave %s holding the old values; edit the target directly", path, target, target) + case !info.Mode().IsRegular(): + return 0, fmt.Errorf("%s is a %s, not a regular file. A save renames over it, which would "+ + "destroy it and write the configuration at its permission (%#o)", path, + info.Mode().Type(), info.Mode().Perm()) default: return info.Mode().Perm(), nil } @@ -412,17 +401,21 @@ func writeAndSync(tmp *os.File, raw []byte, mode os.FileMode) error { return tmp.Close() } -// syncDir flushes the directory entry so the rename itself survives a crash. -func syncDir(dir string) error { +// syncDir asks the filesystem to flush dir's entries, so a rename into it survives a power loss. +// +// It reports nothing, because nothing a caller does with the answer is right. Past the rename the new +// file is what a node reads, so a flush that did not complete is not a failed save. Retrying is worse +// than doing nothing: Linux reports a writeback error once per descriptor and does not write the pages +// again, so a second flush can succeed over data that never reached the device. +func syncDir(dir string) { d, err := os.Open(dir) //nolint:gosec // the destination's own directory if err != nil { - return fmt.Errorf("open %s: %w", dir, err) + // A directory this process cannot open for reading is not a save that failed. The rename has + // already happened and a reader sees the new values. + return } - defer func() { _ = d.Close() }() - if err := d.Sync(); err != nil { - return fmt.Errorf("sync %s: %w", dir, err) - } - return nil + _ = d.Sync() + _ = d.Close() } // keyOf splits a dotted key into its parser path. diff --git a/config/seitoml/seitoml_test.go b/config/seitoml/seitoml_test.go index 97ad48759a..063f1d4466 100644 --- a/config/seitoml/seitoml_test.go +++ b/config/seitoml/seitoml_test.go @@ -1,7 +1,6 @@ package seitoml_test import ( - "errors" "fmt" "math" "os" @@ -9,6 +8,7 @@ import ( "reflect" "sort" "strings" + "syscall" "testing" "time" @@ -1221,13 +1221,75 @@ func TestAFileFromANewerReleaseIsRefused(t *testing.T) { } } -// TestAnUnflushedDirectoryEntryIsNotAFailedSave separates two outcomes a caller must not confuse. +// TestSaveNamesAPathItCannotInspect covers a destination that is neither present nor absent. // -// After the rename the new values are what the node reads. A directory entry that has not been flushed -// only leaves their survival of a power loss unproven, so reporting it the same way as a failed write -// tells an operator their change did not land when it did. The next thing they do is write it again or -// open an incident. -func TestAnUnflushedDirectoryEntryIsNotAFailedSave(t *testing.T) { +// A path whose parent is a file rather than a directory cannot be inspected at all, which is not the +// same as nothing being there yet. Reading it as a first save picks the default mode on a guess and then +// fails further down on the temporary file, naming that instead of the path the operator gave. +func TestSaveNamesAPathItCannotInspect(t *testing.T) { + dir := t.TempDir() + parent := filepath.Join(dir, "not-a-directory") + if err := os.WriteFile(parent, []byte("x"), 0o600); err != nil { + t.Fatalf("write: %v", err) + } + path := filepath.Join(parent, "sei.toml") + + f, err := seitoml.New("validator") + if err != nil { + t.Fatalf("New: %v", err) + } + err = f.Save(path) + if err == nil { + t.Fatal("a save under a path that is a file was accepted") + } + if !strings.Contains(err.Error(), path) { + t.Errorf("Save refused with %q, which does not name the path the operator gave", err) + } +} + +// TestSaveRefusesADestinationARenameWouldDestroy covers the destinations that are not files. +// +// A save installs by renaming over the path, and a rename replaces what is there rather than writing +// through it. For a pipe, a socket or a device node that means the destination is gone, and the +// configuration lands carrying whatever permission it had, which for a device node is world-writable on +// a file naming key paths. Refused where the mode is decided, before anything is written. +func TestSaveRefusesADestinationARenameWouldDestroy(t *testing.T) { + dir := t.TempDir() + path := filepath.Join(dir, "sei.toml") + if err := syscall.Mkfifo(path, 0o600); err != nil { + t.Skipf("this platform will not make a pipe to save onto: %v", err) + } + + f, err := seitoml.New("validator") + if err != nil { + t.Fatalf("New: %v", err) + } + err = f.Save(path) + if err == nil { + t.Fatal("saving onto a pipe was accepted, so the pipe is gone and the configuration carries " + + "whatever permission it had") + } + if !strings.Contains(err.Error(), "not a regular file") { + t.Errorf("Save refused with %q, which does not say what the destination is", err) + } + + // Nothing was written, so the destination an operator pointed at is still there to look at. + info, statErr := os.Lstat(path) + if statErr != nil { + t.Fatalf("the refused save removed the destination: %v", statErr) + } + if info.Mode()&os.ModeNamedPipe == 0 { + t.Errorf("the destination is now %v, want the pipe it was", info.Mode()) + } +} + +// TestADirectoryThisProcessCannotReadDoesNotFailTheSave holds Save's error to one meaning. +// +// Flushing the directory entry needs the directory open for reading, and a save does not. After the +// rename the new values are already what the node reads, so nothing about the flush changes whether the +// save landed. Reporting it would tell an operator their change did not apply when it did, and the next +// thing they do is write it again or open an incident. +func TestADirectoryThisProcessCannotReadDoesNotFailTheSave(t *testing.T) { if os.Geteuid() == 0 { t.Skip("this drives failure through directory permissions, which do not apply to uid 0") } @@ -1254,7 +1316,7 @@ func TestAnUnflushedDirectoryEntryIsNotAFailedSave(t *testing.T) { t.Fatalf("restore: %v", err) } - if saveErr != nil && !errors.Is(saveErr, seitoml.ErrNotDurable) { + if saveErr != nil { t.Fatalf("Save reported %v, which a caller reads as the file not being written", saveErr) } reread, err := seitoml.Load(path) @@ -1264,7 +1326,7 @@ func TestAnUnflushedDirectoryEntryIsNotAFailedSave(t *testing.T) { got, present, err := reread.Get("probe.value") if err != nil || !present || got != int64(7) { t.Errorf("the value on disk is (%#v, %v, %v), want 7. The rename completed, so the new "+ - "configuration is what the node reads whatever the sync reported", got, present, err) + "configuration is what the node reads whether or not the directory was flushed", got, present, err) } } @@ -1580,27 +1642,6 @@ func TestATableKeepsOneSpellingWhateverOrderItsKeysWereWritten(t *testing.T) { }) } -// TestLandedSeparatesAnInstalledFileFromAFailedWrite gives the outcome check a correct spelling. -// -// Save reports one outcome through its error that is not a failure, so the plain err != nil check reads a -// landed save as a failed one. Landed is that check written once, so a caller cannot get it wrong by -// writing the idiom every other Go call wants. -func TestLandedSeparatesAnInstalledFileFromAFailedWrite(t *testing.T) { - for _, tc := range []struct { - name string - err error - landed bool - }{ - {"a save that completed", nil, true}, - {"a save whose directory entry is not flushed", fmt.Errorf("x: %w", seitoml.ErrNotDurable), true}, - {"a save that could not write", errors.New("permission denied"), false}, - } { - if got := seitoml.Landed(tc.err); got != tc.landed { - t.Errorf("%s: Landed = %v, want %v", tc.name, got, tc.landed) - } - } -} - // TestEveryEditIsVisibleToTheNextRead holds the values a read returns against the document. // // Reads are cached, so this is the half that makes the cache safe rather than merely fast: an edit the From 37bde479794b57951e0b689d56a32510f2e6d535 Mon Sep 17 00:00:00 2001 From: bdchatham Date: Wed, 19 Aug 2026 07:23:08 -0700 Subject: [PATCH 17/24] config/seitoml: refuse an unreadable schema version at the door Version refused a counter this binary cannot read, and nothing else did. Parse accepted a file from a newer release, Values answered from it, Set edited it and Save wrote it back to an operator's disk. The guard sat on a read a caller may never make, so the package's own claim that such a file is refused held only for callers who happened to ask. Parse now asks. One line, calling the same Version, so there is one implementation of the rule and no verb below can answer from a file whose shape is not established. The four refusals move with it: absent, not an integer, below the first schema, ahead of this binary. Removing the call fails all four by name. Two consequences worth recording. A valid file carries its counter at the top level, above any heading, so a parsed document always has a global section and the case where one has to be created is now reachable only from inside the package, where it was already driven. And the external test for a file starting with a table could no longer build its fixture, so it goes; the in-package test that actually exercises that branch stays, and reverting the branch still fails it. Version compared after narrowing int64 to int. Where int is 32 bits a counter above 2^32 wrapped into the accepted range and the file read as a version it does not hold. It compares first now and narrows only past both bounds. No test can fail on a 64-bit target, so none was added. Two comments claimed more than the code does. keyOf said it applies the rule Parse applies, and it folds case first, which Parse refuses; the fold is deliberate and pinned, so the comment was the error. handedOut gave one of the two reasons a leaf is never a map. doc.go is rewritten rather than patched. It had been edited across the workstream and carried three false claims, four imprecise ones, three counts pointing at lists that had changed, and about 40% of its text as a second copy of a why already sitting at the line that needs it. It also recorded what went wrong while the package was built, which a godoc does not. The schema-counter argument moves onto the constant it governs rather than being deleted, because it is what stops a release version being written into an on-disk field. 930 words and one heading become 671 and three. 131 cases, 94.1% of statements, race clean, 0 lint issues. --- config/seitoml/doc.go | 92 +++++++++++++++------------------- config/seitoml/file.go | 24 +++++++-- config/seitoml/seitoml_test.go | 48 +++++------------- config/seitoml/values.go | 5 +- 4 files changed, 77 insertions(+), 92 deletions(-) diff --git a/config/seitoml/doc.go b/config/seitoml/doc.go index 7759cf7693..b6787b49c2 100644 --- a/config/seitoml/doc.go +++ b/config/seitoml/doc.go @@ -1,68 +1,57 @@ // Package seitoml reads, edits and writes the node's sei.toml. // +// A File is a mutable in-memory document for one goroutine at a time. +// // The file holds only what an operator decided. A key present in it is authoritative; a key absent // from it resolves to the running binary's default for the node's mode. Nothing here writes a // default into the file, because a value the binary put there reads exactly like one an operator // chose. // // Two keys at the top level describe the file rather than configure the node, and Values leaves both -// out so a check comparing written keys against the declared set never reports them as keys no section -// owns. +// out. // // schema_version which migration the file has reached // node_mode which mode's defaults its values were chosen against // -// schema_version is a counter that rises by exactly one per migration, and a migration chain reads it -// to decide which steps a file still needs. It is deliberately not a release version. Most releases -// change no schema, so a release version could not answer whether the schema moved between two of -// them without a release-to-schema table, which is this counter reintroduced as an indirection. -// Releases also do not form the total order a chain needs: a hotfix can ship after a later minor, so -// ordering steps by release would run them in an order nobody intended. -// -// Nothing here migrates a file. This package reads the counter and writes it; the chain that acts on it -// arrives with the migrations themselves. A file whose counter is ahead of this binary's is refused, -// because a release migrates the file on the node's own disk and rolling the binary back does not roll -// the file back with it. Read anyway, the older binary would apply only the keys it still recognises. -// -// Editing preserves the document. An operator may hand-edit the file, and comments are how they explain -// a choice to whoever reads it next, so set and unset change the one line they name and leave every other -// line of content untouched. Vertical spacing is normalised once, on the first save of a -// file nothing has saved before, and holds steady after that. This is why the package edits a parsed -// document rather than re-rendering a decoded map, which would drop every comment in the file. -// -// Every write is atomic. A node cannot boot from a configuration file a crash truncated mid-write, -// so a save lands in full or not at all. -// -// A value has to survive a round trip as its own type. TOML tells a float from an integer by the -// fractional part, so an integral float needs one written explicitly or it reads back as an integer, -// and a key declared as a float would resolve as one type from a node's own files and another from its -// sei.toml. Infinities and NaN have no form here at all, and are refused in both directions rather than -// written as a line no reader can load. +// schema_version counts migrations, one per migration, and is deliberately not a release version. Parse +// refuses a file whose counter is absent, below the first schema, or ahead of the one this binary +// understands, so every verb below answers from a file whose shape is established. Nothing here migrates +// a file; this package reads and writes the counter, and the chain that acts on it arrives with the +// migrations. Nothing here resolves a value or knows what keys exist. // -// # What This File May Hold +// # Editing Preserves The Document +// +// An operator hand-edits this file, and the comments in it are how they explain a choice to whoever +// reads it next. So Set and Unset change the value they name and add no line the caller did not ask +// for, leaving every other line of content as it was. A comment above a key and a comment beside a +// value both survive an edit, and a key's own comment leaves with it when the key is unset. +// +// Vertical spacing normalises once, on the first save of a file nothing has saved before, and holds +// from then on. // -// Values come from the decoder a node reads its configuration with. viper decodes TOML with -// pelletier/go-toml/v2, so this package does too, which makes "this file parses here" and "the node can -// boot from it" the same statement rather than two that can drift apart. +// Every write is atomic. A save lands in full or not at all, and a failed save leaves no temporary file +// behind. // -// That decoder is therefore the authority on shape, and Parse asks it rather than keeping a list. A -// name used for both a value and a table, a table defined twice, a key written twice, a table given a -// heading after an ancestor's dotted key already created it: all of it is one question with one answer. -// A list kept here instead missed three of those, and every one of them produced a file this package -// would save and no node could read. +// A value read from the file is written back as the same type. // -// The editing parser is a second library and does a different job: it locates lines and preserves -// comments, and stops short of interpreting a literal, because deciding what an underscore-separated -// integer or a multi-line string means is a second implementation of the specification and a -// hand-written one went wrong in four places. +// # One Decoder Decides What Parses // -// A new key is asked the same question where it is written. Set writes the key, renders, and offers the -// result to the decoder; if the document no longer reads, the write is undone and the key is named. Save -// asks again over the whole rendering, which is what covers the verbs that change no namespace, so no -// file reaches disk unread. Together that refuses a shape nobody anticipated as surely as one somebody -// did, which is the property enumerating shapes by hand could not give. +// The decoder is the one a node reads its configuration with. viper decodes TOML with +// pelletier/go-toml/v2, so this package decodes with it too, which makes "this file parses" and "this +// node can boot from it" one statement. +// +// So the question a shape has to answer is put to that decoder rather than to a list kept here. Parse +// asks it, and Set renders the document and asks it again, undoing the write and naming the key when the +// answer is no. Two of the decoder's answers are also checked here ahead of it, a repeated heading and a +// repeated key, because those name the key and say what an edit would reach where the decoder names a +// line. +// +// The editing parser is a second library doing a different job: it locates lines and preserves comments, +// and stops short of interpreting a literal. +// +// # What This File May Hold // -// Four things that decoder allows are refused anyway, because this package has to write back what it +// These are refused although the decoder accepts them, because this package has to write back what it // reads: // // - an infinity or a NaN, which have no form to write @@ -70,10 +59,11 @@ // type it was read as // - an inline table, whose keys flatten into the same space a table's do, so an edit to one of them // has no line of its own to change -// - an array of tables, where every entry but the last disappears from the flattened key space +// - an array of tables, which flattens to one key holding a list of tables, so no entry has a line of +// its own to edit // -// A key is a bare TOML key: lower-case letters, digits, underscores and hyphens. Anything else has to be +// A key is a lower-case bare TOML key: letters, digits, underscores and hyphens. Anything else has to be // quoted where it is written, and a quoted key is spelled one way by the decoder and another by a -// lookup, so Values would report a key Get answers absent for. Set, Unset and Get apply that same rule -// to the key a caller hands them, so a key one of them writes is a key the file reads back. +// lookup, so Values would report a key Get answers absent for. Set, Unset and Get fold a caller's key to +// lower case and then hold it to that rule, so a key one of them writes is a key the file reads back. package seitoml diff --git a/config/seitoml/file.go b/config/seitoml/file.go index c91c3d6e4f..30d05e2469 100644 --- a/config/seitoml/file.go +++ b/config/seitoml/file.go @@ -17,6 +17,12 @@ import ( ) // SchemaVersion is the schema this binary writes and reads. +// +// A counter rising by one per migration, and not a release version. Most releases change no schema, so a +// release version could not answer whether the schema moved between two of them without a +// release-to-schema table, which is this counter reintroduced as an indirection. Releases also do not +// form the total order a chain needs: a hotfix can ship after a later minor, so ordering steps by +// release would run them in an order nobody intended. const SchemaVersion = 1 // VersionKey records which schema the file follows. @@ -69,6 +75,13 @@ func Parse(r io.Reader) (*File, error) { if err := f.refuseUnsupportedShapes(); err != nil { return nil, err } + // The counter is read here rather than left to whoever calls Version, so no verb can answer from a + // file whose schema this binary does not understand. Asked at the door, every read below is reading a + // file whose shape is established; asked only by Version, a caller that never calls it resolves + // values from a file a newer release wrote and boots on a configuration neither release produced. + if _, err := f.Version(); err != nil { + return nil, err + } return f, nil } @@ -275,7 +288,7 @@ func (f *File) Version() (int, error) { return 0, fmt.Errorf("sei.toml is at %s %d, and the first schema this format had is 1. Its shape "+ "cannot be established, so no migration can safely run against it", VersionKey, n) } - if int(n) > SchemaVersion { + if n > int64(SchemaVersion) { // The rollback case, and the reason the counter exists. A release migrates the file forward on // the node's own disk, so rolling the binary back does not roll the file back with it. Read // anyway, this binary would silently ignore every key the newer schema added or renamed and boot @@ -284,6 +297,9 @@ func (f *File) Version() (int, error) { "newer release, so reading it would apply only the keys this binary still recognises", VersionKey, n, SchemaVersion) } + // Narrowed only past both bounds, so the counter fits whatever width int has here. Comparing after + // the cast let a counter too wide for int wrap into the accepted range, and the file then read as a + // version it does not hold. return int(n), nil } @@ -430,8 +446,10 @@ func keyOf(key string) (parser.Key, error) { } } out := parser.Key(parts) - // The same rule Parse applies to a key it reads. Stated once, so a key a caller writes is a key - // the file reads back; checked only here, Set could write a key the next Parse refuses. + // Folded to lower case above and then held to the rule Parse applies, which is not quite that rule: + // Parse refuses an upper-case segment because a file is read lower-cased and the written name would + // not be the one read, where a caller naming a key has no written spelling to disagree with. Held + // here rather than at each verb, so Set cannot write a key the next Parse refuses. if err := keyIsAddressable(out); err != nil { return nil, fmt.Errorf("key %q: %w", key, err) } diff --git a/config/seitoml/seitoml_test.go b/config/seitoml/seitoml_test.go index 063f1d4466..fe34b685c7 100644 --- a/config/seitoml/seitoml_test.go +++ b/config/seitoml/seitoml_test.go @@ -159,16 +159,16 @@ func TestFormattingNormalizesOnceAndThenHoldsSteady(t *testing.T) { // A migration chain reads the version to decide which steps to run. Defaulting an absent one to // zero would run every step in history against a file nobody established the shape of, and the // result would look like a successful upgrade. -func TestAnAbsentSchemaVersionIsAnError(t *testing.T) { +func TestAnAbsentSchemaVersionIsRefusedAtTheDoor(t *testing.T) { for _, tc := range []struct{ name, body string }{ {"absent", "[giga_executor]\nenabled = true\n"}, {"not an integer", "schema_version = \"1\"\n"}, {"a float", "schema_version = 1.0\n"}, } { t.Run(tc.name, func(t *testing.T) { - if _, err := parse(t, tc.body).Version(); err == nil { - t.Errorf("a %s schema version was accepted. A migration would then run against a file "+ - "whose shape nobody established, and report success", tc.name) + if _, err := seitoml.Parse(strings.NewReader(tc.body)); err == nil { + t.Errorf("a %s schema version was accepted. Every read below would then answer from a "+ + "file whose shape nobody established, and a migration would report success", tc.name) } }) } @@ -985,30 +985,6 @@ func TestSetWritesIntoATableTheFileAlreadyHas(t *testing.T) { } } -// TestSetWritesATopLevelKeyIntoAFileThatStartsWithATable covers a document with no global section. -// -// A file whose first line is a heading has nothing above it, so writing the schema version or the node -// mode into one has to create that space rather than fail or land inside the first table. Landing -// inside it would make the key read as section.schema_version, which no reader asks for. -func TestSetWritesATopLevelKeyIntoAFileThatStartsWithATable(t *testing.T) { - f := parse(t, "[probe]\nfirst = 1\n") - if err := f.Set(seitoml.ModeKey, "archive"); err != nil { - t.Fatalf("Set: %v", err) - } - - reread := parse(t, render(t, f)) - mode, err := reread.Mode() - if err != nil { - t.Fatalf("Mode after writing it into a file that had no top level: %v\n%s", err, render(t, f)) - } - if mode != "archive" { - t.Errorf("mode read back as %q, want archive", mode) - } - if _, ok, _ := reread.Get("probe." + seitoml.ModeKey); ok { - t.Error("the key landed inside the first table, so it reads as one that section owns") - } -} - // TestAMalformedKeyIsRefusedByEveryVerbThatTakesOne holds the four entry points to one answer. // // Set, Unset and Get each take a dotted key from a caller, and a key TOML cannot express has to be @@ -1094,16 +1070,16 @@ func TestAFileDescribingItselfWithANonValueIsRefused(t *testing.T) { }) t.Run("a node mode that is not a string", func(t *testing.T) { - _, err := parse(t, "node_mode = 3\n").Mode() + _, err := parse(t, "schema_version = 1\nnode_mode = 3\n").Mode() if err == nil || !strings.Contains(err.Error(), "want a mode name") { t.Errorf("Mode on a numeric mode returned %v, want a refusal naming what it wanted", err) } }) t.Run("a schema version that is not an integer", func(t *testing.T) { - _, err := parse(t, "schema_version = \"one\"\n").Version() + _, err := seitoml.Parse(strings.NewReader("schema_version = \"one\"\n")) if err == nil || !strings.Contains(err.Error(), "want an integer") { - t.Errorf("Version on a string version returned %v, want a refusal naming what it wanted", err) + t.Errorf("Parse on a string version returned %v, want a refusal naming what it wanted", err) } }) } @@ -1196,9 +1172,9 @@ func TestAListCarryingAValueThatCannotBeWrittenNamesTheElement(t *testing.T) { func TestAFileFromANewerReleaseIsRefused(t *testing.T) { ahead := fmt.Sprintf("schema_version = %d\nnode_mode = \"validator\"\n", seitoml.SchemaVersion+1) - _, err := parse(t, ahead).Version() + _, err := seitoml.Parse(strings.NewReader(ahead)) if err == nil { - t.Fatal("a file from a newer release was read, so this binary would apply only the keys it " + + t.Fatal("a file from a newer release was accepted, so this binary would apply only the keys it " + "still recognises and boot on a configuration neither release produced") } for _, want := range []string{ @@ -1569,10 +1545,10 @@ func TestAnUnsignedValueTooLargeToReadBackIsRefused(t *testing.T) { // one it gets alongside an error. func TestASchemaVersionBelowTheFirstOneIsRefused(t *testing.T) { for _, body := range []string{"schema_version = 0\n", "schema_version = -5\n"} { - got, err := parse(t, body).Version() + _, err := seitoml.Parse(strings.NewReader(body)) if err == nil { - t.Errorf("%q read as version %d; a counter below the first schema names no shape", - strings.TrimSpace(body), got) + t.Errorf("%q was accepted; a counter below the first schema names no shape", + strings.TrimSpace(body)) } else if !strings.Contains(err.Error(), "first schema") { t.Errorf("the refusal reads %q and does not say what the floor is", err) } diff --git a/config/seitoml/values.go b/config/seitoml/values.go index 11dfedf848..ff6de1aebc 100644 --- a/config/seitoml/values.go +++ b/config/seitoml/values.go @@ -49,8 +49,9 @@ func (f *File) Get(key string) (any, bool, error) { // caller sorting or index-assigning what it was given rewrites the cache. Copied on the way out rather // than once at decode, because a caller changes what it holds at any point after it holds it. // -// Only a list needs copying. A scalar is copied by the assignment, and a leaf is never a table: an -// inline table is refused when the file is read and cannot be written, so nothing reaches here as a map. +// Only a list needs copying. A scalar is copied by the assignment, and a leaf is never a table: the two +// shapes that would put one here, an inline table and an array of tables, are both refused when the file +// is read and neither can be written, so nothing reaches here as a map. func handedOut(v any) any { list, ok := v.([]any) if !ok { From 100489931a892da3e06fffa48ca2c2c9580bb83b Mon Sep 17 00:00:00 2001 From: bdchatham Date: Wed, 19 Aug 2026 07:49:45 -0700 Subject: [PATCH 18/24] config/seitoml: one spelling per table, so nothing has to choose between two TOML names a table two ways: a heading, and the segments before the last in a dotted key. This file accepted both, and every insert then had to work out which one applied. A table an existing dotted key had created has no heading, so writing one would define it twice and the document would stop parsing; the key had to join the dotted name instead. Working that out took a walk over the sections, a scan of every key's prefixes for an implicitly created table, and a proof that at most one section can qualify. That is a hand-written analysis of a shape, in a package whose answer to a shape is to write it, render, and ask the decoder. It read as the exception it was, and it behaved like one: a longest-prefix comparison in it was dead, and a document whose table came from a top-level dotted key was mishandled until it was fixed here. So a dotted key is refused where the file is read, beside the two shapes refused for the same reason. An inline table and an array of tables are already turned away because an edit to what they name has no line of its own to change, and every segment before the last in a dotted key names a table with exactly that problem. No expressiveness goes with it: a.b = 1 is [a] and b = 1, and one spelling per table is the rule the file already applies to keys and headings. Insert now joins the section that carries the table or brings a heading with it. ancestorOf, createsTable and dottedName are gone, and so are the fixtures whose only subject was the choice between two spellings. The property those fixtures held, that a table is spelled the same whatever order its keys arrive, is now true by construction; the assertion that a new table gets a heading stays. Reverting the refusal fails the two cases that name it. 129 cases, 94.1% of statements, race clean, 0 lint issues. edit.go loses 57 lines. --- config/seitoml/doc.go | 5 +++ config/seitoml/edit.go | 59 +------------------------ config/seitoml/file.go | 17 +++++--- config/seitoml/seitoml_test.go | 79 +++++----------------------------- 4 files changed, 29 insertions(+), 131 deletions(-) diff --git a/config/seitoml/doc.go b/config/seitoml/doc.go index b6787b49c2..998feebb80 100644 --- a/config/seitoml/doc.go +++ b/config/seitoml/doc.go @@ -26,6 +26,9 @@ // for, leaving every other line of content as it was. A comment above a key and a comment beside a // value both survive an edit, and a key's own comment leaves with it when the key is unset. // +// A table is named one way, by a heading. So a new key either joins the section that already carries its +// table or brings a heading with it, and there is no second spelling for an insert to choose between. +// // Vertical spacing normalises once, on the first save of a file nothing has saved before, and holds // from then on. // @@ -59,6 +62,8 @@ // type it was read as // - an inline table, whose keys flatten into the same space a table's do, so an edit to one of them // has no line of its own to change +// - a dotted key, whose segments before the last name tables with no line of their own, so a key +// added to one of those tables has nowhere to go // - an array of tables, which flattens to one key holding a list of tables, so no entry has a line of // its own to edit // diff --git a/config/seitoml/edit.go b/config/seitoml/edit.go index 6da9e17082..f3a42cb508 100644 --- a/config/seitoml/edit.go +++ b/config/seitoml/edit.go @@ -86,13 +86,8 @@ func (f *File) insert(path parser.Key, value parser.Value) (func(), bool) { if e := transform.FindTable(f.doc, table...); e != nil { return appendItem(e.Section, kv) } - // No section carries this name. It may still be a table the document created by writing a dotted key, - // and a heading for one of those defines it twice, so the leaf joins that dotted name instead. Where - // nothing has created it, the heading is new and correct, which is the form an operator expects to - // read. - if owner, under := f.ancestorOf(table); owner != nil { - return appendItem(owner, &parser.KeyValue{Name: dottedName(under, leaf), Value: value}) - } + // No section carries this name, so the table is new and gets a heading. There is no second spelling to + // choose between: a dotted key is the other way to name a table and the document cannot hold one. before := len(f.doc.Sections) f.doc.Sections = append(f.doc.Sections, &tomledit.Section{ Heading: &parser.Heading{Name: copyKey(table)}, @@ -101,62 +96,12 @@ func (f *File) insert(path parser.Key, value parser.Value) (func(), bool) { return func() { f.doc.Sections = f.doc.Sections[:before] }, true } -// ancestorOf returns the section a table's keys belong in when the table has no heading of its own, and -// the path from that section down to the table. -// -// A section whose heading is a prefix of the table owns it, and the longest such heading is the nearest -// ancestor. The global section owns it when a top-level dotted key has already created it: that section -// has no heading, so no prefix can find it, and a heading written for the table would be the second -// definition the decoder refuses. -func (f *File) ancestorOf(table parser.Key) (*tomledit.Section, parser.Key) { - // At most one section can qualify, so the first match is the only match. Two would need a section - // [a] holding a dotted key beginning b. alongside a section [a.b], and that names the table b twice, - // which the decoder refuses at the door. - for _, s := range f.doc.Sections { - if s.Heading == nil || !s.Name.IsPrefixOf(table) { - continue - } - if below := table[len(s.Name):]; createsTable(s, below) { - return s, copyKey(below) - } - } - if f.doc.Global != nil && createsTable(f.doc.Global, table) { - return f.doc.Global, copyKey(table) - } - return nil, nil -} - -// createsTable reports whether a dotted key in this section already names the given table. -// -// Every proper prefix of a dotted key names a table, so flatkv.enable creates flatkv without giving it a -// heading. Only such a table is joined by extending a dotted name; one nothing has created gets a -// heading of its own, so a table is spelled the same way whatever order its keys were written in. -func createsTable(s *tomledit.Section, table parser.Key) bool { - for _, item := range s.Items { - kv, ok := item.(*parser.KeyValue) - if !ok { - continue - } - for i := 1; i < len(kv.Name); i++ { - if kv.Name[:i].Equals(table) { - return true - } - } - } - return false -} - // copyKey returns a key that shares no storage with its argument. // // The paths here are slices of one another, so appending to a shorter one would write into the longer // one's storage. func copyKey(k parser.Key) parser.Key { return append(parser.Key(nil), k...) } -// dottedName joins the path down to a table with the key inside it. -func dottedName(under parser.Key, leaf parser.Key) parser.Key { - return append(copyKey(under), leaf...) -} - // appendItem adds an item to a section, and reports how to remove it again and whether it went in. // // Told not to replace, so a key already present is reported rather than overwritten. The distinction diff --git a/config/seitoml/file.go b/config/seitoml/file.go index 30d05e2469..49d470d8ce 100644 --- a/config/seitoml/file.go +++ b/config/seitoml/file.go @@ -87,11 +87,11 @@ func Parse(r io.Reader) (*File, error) { // refuseUnsupportedShapes rejects TOML this format does not carry. // -// TOML permits more shapes than a node's configuration uses, and each of these was previously accepted -// and then lost or corrupted somewhere downstream: a mixed-case key read back under a different name, -// an inline table that Set split into a second definition of the same table, an array of tables whose -// earlier entries vanished from Values. Refusing at the door is what keeps one answer per key, and it -// is only free while no operator has written a file that uses them. +// TOML permits more shapes than a node's configuration uses, and each of these reaches an edit that has +// nowhere to land: a mixed-case key is read back under a different name, an inline table and a dotted key +// each name a table with no line of its own, and an array of tables gives no entry a line of its own. +// Refusing at the door is what keeps one spelling per table and one answer per key, and it leaves every +// verb below with a document it can round-trip. func (f *File) refuseUnsupportedShapes() error { headings := map[string]bool{} // Every entry in Sections is a named table, so each carries a heading; the global section is a field @@ -128,6 +128,13 @@ func (f *File) refuseUnsupportedShapes() error { bad = err return false } + if len(e.Name) > 1 { + bad = fmt.Errorf("%s is written as a dotted key, which this file does not carry. Every "+ + "segment before the last names a table with no line of its own, so a key added to one "+ + "of those tables has nowhere to go; write [%s] as a section instead, and put %s in it", + full, full[:len(full)-1], full[len(full)-1]) + return false + } // The decoder below refuses this too. It stays because it names the key and says what an edit // would do to it, where the decoder names a line, and a duplicate key is the mistake an operator // is most likely to make by hand. diff --git a/config/seitoml/seitoml_test.go b/config/seitoml/seitoml_test.go index fe34b685c7..a7d1e33188 100644 --- a/config/seitoml/seitoml_test.go +++ b/config/seitoml/seitoml_test.go @@ -400,6 +400,16 @@ func TestAShapeThisFileDoesNotCarryIsRefusedAtTheDoor(t *testing.T) { "[[peer]]\nhost = \"a\"\n\n[[peer]]\nhost = \"b\"\n", "is an array of tables", }, + { + "a dotted key inside a table", + "[state-commit]\nflatkv.enable = true\n", + "is written as a dotted key", + }, + { + "a dotted key at the top level", + "giga.enabled = true\n", + "is written as a dotted key", + }, { "a key written twice in one table", "[probe]\nn = 1\nn = 2\n", @@ -484,9 +494,6 @@ sc-async-commit-buffer = 100 enable = true dir = "/data" -[pruning] -memiavl.snapshot-interval = 100 - [p2p] persistent-peers = ["a", "b"] `) @@ -499,9 +506,6 @@ persistent-peers = ["a", "b"] "state-commit.sc-async-commit-buffer": int64(100), "state-commit.flatkv.enable": true, "state-commit.flatkv.dir": "/data", - // A dotted key inside a table, which is a different shape from a nested heading and reads to the - // same flattened key. - "pruning.memiavl.snapshot-interval": int64(100), } { if values[key] != want { t.Errorf("%s read back as %#v, want %#v", key, values[key], want) @@ -1368,58 +1372,6 @@ func TestANameCannotBeAValueAndATableAtOnce(t *testing.T) { } }) - t.Run("a table an ancestor's dotted key created", func(t *testing.T) { - // The table exists without a heading of its own, so a heading for it would define it twice. The - // key has to join the dotted name instead, and the result has to satisfy the node's decoder. - f := parse(t, "schema_version = 1\nnode_mode = \"validator\"\n\n[state-commit]\nflatkv.enable = true\n") - if err := f.Set("state-commit.flatkv.dir", "/data"); err != nil { - t.Fatalf("writing a sibling into an implicitly created table was refused: %v", err) - } - requireStillReadable(t, f) - values, err := f.Values() - if err != nil { - t.Fatalf("Values: %v", err) - } - for key, want := range map[string]any{ - "state-commit.flatkv.enable": true, - "state-commit.flatkv.dir": "/data", - } { - if values[key] != want { - t.Errorf("%s = %#v, want %#v", key, values[key], want) - } - } - }) - - t.Run("a table a top-level dotted key created", func(t *testing.T) { - // The same shape as the headed case below it, except the table's ancestor is the file itself. The - // global section carries no heading, so no prefix can find it, and a heading written for the table - // would be the second definition the decoder refuses. - f := parse(t, "schema_version = 1\nnode_mode = \"validator\"\ngiga.enabled = true\n") - if err := f.Set("giga.workers", 4); err != nil { - t.Fatalf("writing a sibling under a top-level dotted table was refused: %v", err) - } - requireStillReadable(t, f) - values, err := f.Values() - if err != nil { - t.Fatalf("Values: %v", err) - } - for key, want := range map[string]any{"giga.enabled": true, "giga.workers": int64(4)} { - if values[key] != want { - t.Errorf("%s = %#v, want %#v", key, values[key], want) - } - } - }) - - t.Run("a table two levels below a top-level dotted key", func(t *testing.T) { - f := parse(t, "schema_version = 1\nnode_mode = \"validator\"\na.b.c = 1\n") - for _, key := range []string{"a.b.d", "a.e"} { - if err := f.Set(key, 2); err != nil { - t.Errorf("Set(%q) was refused: %v", key, err) - } - } - requireStillReadable(t, f) - }) - t.Run("a section nothing has created still gets a heading", func(t *testing.T) { // The other half: a table no key has named is new, and a heading is the form an operator expects // to read. Treating the global section as everything's ancestor would write dotted keys instead. @@ -1605,17 +1557,6 @@ func TestATableKeepsOneSpellingWhateverOrderItsKeysWereWritten(t *testing.T) { requireStillReadable(t, f) }) - t.Run("a table a dotted key created joins that name", func(t *testing.T) { - f := parse(t, "schema_version = 1\nnode_mode = \"validator\"\n\n[state-commit]\nflatkv.enable = true\n") - if err := f.Set("state-commit.flatkv.dir", "/data"); err != nil { - t.Fatalf("Set: %v", err) - } - out := render(t, f) - if strings.Contains(out, "[state-commit.flatkv]") { - t.Errorf("a table the document already created was given a second definition:\n%s", out) - } - requireStillReadable(t, f) - }) } // TestEveryEditIsVisibleToTheNextRead holds the values a read returns against the document. From a5565a3c9a0dfa92b9de682e16bfc642d817aa98 Mon Sep 17 00:00:00 2001 From: bdchatham Date: Wed, 19 Aug 2026 08:11:07 -0700 Subject: [PATCH 19/24] config/seitoml: say why two shapes are checked twice The package doc named the decoder as the authority on what parses and then mentioned in passing that two of its answers are also checked here. Read as an aside, that invites the wrong deletion: a reader who takes the stronger claim literally removes the repeated-key and repeated-heading checks as redundant. It now says what the arrangement is and what each half buys. Whether the file loads rests on the decoder alone, so those two checks are not load-bearing for safety; what they add is the diagnosis, naming the dotted key an operator typed and saying that an edit reaches only the first, where the decoder names a line and reports that the name already exists. Both are the mistake hand-editing produces most. It also says that the refusal list below runs the other way, shapes the decoder accepts and this file does not carry, so the two directions are not read as one. --- config/seitoml/doc.go | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/config/seitoml/doc.go b/config/seitoml/doc.go index 998feebb80..5cba5c91a2 100644 --- a/config/seitoml/doc.go +++ b/config/seitoml/doc.go @@ -45,9 +45,16 @@ // // So the question a shape has to answer is put to that decoder rather than to a list kept here. Parse // asks it, and Set renders the document and asks it again, undoing the write and naming the key when the -// answer is no. Two of the decoder's answers are also checked here ahead of it, a repeated heading and a -// repeated key, because those name the key and say what an edit would reach where the decoder names a -// line. +// answer is no. A shape nobody anticipated is refused as surely as one somebody did. +// +// Two shapes are checked here as well, and deliberately: a repeated key and a repeated heading. The +// decoder refuses both, so nothing about whether the file loads rests on these; what they add is the +// diagnosis. They name the dotted key an operator typed and say that an edit reaches only the first, +// where the decoder names a line and reports that the name already exists. Both are the mistake +// hand-editing produces most. +// +// The refusals in the next section are the other direction: shapes the decoder accepts and this file +// does not carry. // // The editing parser is a second library doing a different job: it locates lines and preserves comments, // and stops short of interpreting a literal. From a85dae69e402a9235bb8a12ae5cf12ff3885c967 Mon Sep 17 00:00:00 2001 From: bdchatham Date: Wed, 19 Aug 2026 08:31:46 -0700 Subject: [PATCH 20/24] config/seitoml: Save asks whether the file it writes can be loaded Parse refuses a schema counter this binary cannot read. Save asked only whether the bytes decode as TOML, and the describing keys are written through Set, which is how New puts them there. So a caller could set the counter ahead of this binary, below the first schema, or to a string, or unset it, and Save wrote the result to disk. All four produced a file this package then refused to load, and the refusal landed at the next boot rather than at the write that caused it. Save now asks Parse. What has to hold of a file on disk is that this package can load it, and Parse is where that is decided, so asking anything narrower leaves the two answers free to drift. This also covers the refusals Parse already made and every one added to it later, without the verb that writes having to remember any of them. Four cases pin it, one per way the counter can go wrong, and each checks the refused save left nothing behind. Putting the bare decode back fails all four. 134 cases, 94.4% of statements, race clean, 0 lint issues. --- config/seitoml/file.go | 6 ++-- config/seitoml/seitoml_test.go | 52 ++++++++++++++++++++++++++++++++++ 2 files changed, 56 insertions(+), 2 deletions(-) diff --git a/config/seitoml/file.go b/config/seitoml/file.go index 49d470d8ce..bddc0d9a0b 100644 --- a/config/seitoml/file.go +++ b/config/seitoml/file.go @@ -333,8 +333,10 @@ func (f *File) Save(path string) error { return err } // The one function every write to disk passes through, so the check belongs here rather than at each - // verb that edits. A verb added later cannot forget an invariant it does not have to remember. - if _, err := decodeBytes(raw); err != nil { + // verb that edits. Asked as Parse rather than as a decode, because what has to hold of a file on disk + // is that this package can load it, and Parse is where that is decided. A refusal added there is then + // enforced on the way out too, without the verb that writes having to remember it. + if _, err := Parse(bytes.NewReader(raw)); err != nil { return err } diff --git a/config/seitoml/seitoml_test.go b/config/seitoml/seitoml_test.go index a7d1e33188..f16c5557dc 100644 --- a/config/seitoml/seitoml_test.go +++ b/config/seitoml/seitoml_test.go @@ -1088,6 +1088,58 @@ func TestAFileDescribingItselfWithANonValueIsRefused(t *testing.T) { }) } +// TestSaveRefusesADocumentParseWouldRefuse holds the writer to what the reader accepts. +// +// The describing keys are written through Set, which is how New puts them there, so a caller can also +// change one to something Parse will not take. Nothing about TOML is wrong with the result, so a check +// that only asked whether the bytes decode let it through, and the file on disk was one this package +// could not load. The failure appears at the next boot rather than at the write that caused it. +func TestSaveRefusesADocumentParseWouldRefuse(t *testing.T) { + for _, tc := range []struct { + name string + edit func(*seitoml.File) error + want string + }{ + {"a version ahead of this binary", func(f *seitoml.File) error { + return f.Set(seitoml.VersionKey, seitoml.SchemaVersion+1) + }, "newer release"}, + {"a version below the first schema", func(f *seitoml.File) error { + return f.Set(seitoml.VersionKey, 0) + }, "first schema"}, + {"a version that is not an integer", func(f *seitoml.File) error { + return f.Set(seitoml.VersionKey, "one") + }, "want an integer"}, + {"no version at all", func(f *seitoml.File) error { + _, err := f.Unset(seitoml.VersionKey) + return err + }, "has no schema_version"}, + } { + t.Run(tc.name, func(t *testing.T) { + f, err := seitoml.New("validator") + if err != nil { + t.Fatalf("New: %v", err) + } + if err := tc.edit(f); err != nil { + t.Fatalf("the edit itself failed, so this case drives nothing: %v", err) + } + + path := filepath.Join(t.TempDir(), "sei.toml") + saveErr := f.Save(path) + if saveErr == nil { + _, loadErr := seitoml.Load(path) + t.Fatalf("the save was accepted, and loading what it wrote reports %v. A node reads this "+ + "file at boot, so the failure lands there rather than at the write", loadErr) + } + if !strings.Contains(saveErr.Error(), tc.want) { + t.Errorf("the refusal reads %q and does not mention %q", saveErr, tc.want) + } + if _, err := os.Lstat(path); !os.IsNotExist(err) { + t.Errorf("the refused save left something at %s", path) + } + }) + } +} + // TestSaveNamesThePathWhenItCannotWriteThere holds the failure an operator is most likely to hit. // // A configured directory that does not exist is an ordinary mistake, and the error has to name the path From 0fc32b5f6c59a6f097cbbfc75e2247bb5ff1e657 Mon Sep 17 00:00:00 2001 From: bdchatham Date: Wed, 19 Aug 2026 09:43:13 -0700 Subject: [PATCH 21/24] config/seitoml: Parse reads the node mode too, not only the counter Parse asked for the schema counter and never for the node mode. The mode's only guard sat inside New, a constructor rather than a function every path reaches, so Load then Set then Save carried no guard at all. That is the path a config command and a migration take. Three edits therefore reached disk and failed at the next boot: unsetting the mode, setting it to a number, and setting it to an empty string. Unsetting it left a file holding nothing but schema_version = 1. The mode matters more than the counter it sat beside. It is the only durable record of an archive node, because seid init writes config.toml's mode as full for one, so a file that loses it is compared against a validator's defaults and nothing about the file says so. Parse now reads both describing keys, and Save inherits both by asking Parse. The write side of the mode key had no test at all, which is how the gap survived a suite that covers the counter's four cases; the same table now carries three for the mode. Removing the guard fails seven cases by name. One claim in values.go said both describing keys are read before anything else. That was the intent and not the code. It is true now, and says which function does it. 137 cases, 94.4% of statements, race clean, 0 lint issues. --- config/seitoml/file.go | 12 ++++++++---- config/seitoml/seitoml_test.go | 31 ++++++++++++++++++++++--------- config/seitoml/values.go | 4 ++-- 3 files changed, 32 insertions(+), 15 deletions(-) diff --git a/config/seitoml/file.go b/config/seitoml/file.go index bddc0d9a0b..f216160e80 100644 --- a/config/seitoml/file.go +++ b/config/seitoml/file.go @@ -75,13 +75,17 @@ func Parse(r io.Reader) (*File, error) { if err := f.refuseUnsupportedShapes(); err != nil { return nil, err } - // The counter is read here rather than left to whoever calls Version, so no verb can answer from a - // file whose schema this binary does not understand. Asked at the door, every read below is reading a - // file whose shape is established; asked only by Version, a caller that never calls it resolves - // values from a file a newer release wrote and boots on a configuration neither release produced. + // Both keys that describe the file are read here rather than left to whoever calls Version or Mode. + // Asked at the door, every verb below answers for a file whose schema and mode are established; + // asked only by the verb that returns one, a caller that never calls it resolves values from a file + // a newer release wrote, or against the wrong mode's defaults, and boots on a configuration nobody + // intended. if _, err := f.Version(); err != nil { return nil, err } + if _, err := f.Mode(); err != nil { + return nil, err + } return f, nil } diff --git a/config/seitoml/seitoml_test.go b/config/seitoml/seitoml_test.go index f16c5557dc..0f19cd69ea 100644 --- a/config/seitoml/seitoml_test.go +++ b/config/seitoml/seitoml_test.go @@ -285,7 +285,8 @@ func equal(a, b any) bool { // Decoding a literal string as though it had escapes turns a Windows path's separators into // control characters, and the value the node runs is not the one in the file. func TestALiteralStringIsTakenAsWritten(t *testing.T) { - f := parse(t, "schema_version = 1\n[probe]\nliteral = 'C:\\sei\\data'\nbasic = \"a\\tb\"\n") + f := parse(t, "schema_version = 1\nnode_mode = \"validator\"\n\n[probe]\n"+ + "literal = 'C:\\sei\\data'\nbasic = \"a\\tb\"\n") values, err := f.Values() if err != nil { @@ -755,16 +756,16 @@ func TestANewFileRecordsItsNodeMode(t *testing.T) { // // Guessing picks one binary's idea of a default and silently measures an archive node's file against // a validator's defaults, which is the mistake this key exists to make impossible. -func TestAnAbsentOrUnreadableNodeModeIsAnError(t *testing.T) { +func TestAnAbsentOrUnreadableNodeModeIsRefusedAtTheDoor(t *testing.T) { for _, tc := range []struct{ name, body string }{ {"absent", "schema_version = 1\n"}, {"not text", "schema_version = 1\nnode_mode = 3\n"}, {"empty", "schema_version = 1\nnode_mode = \"\"\n"}, } { t.Run(tc.name, func(t *testing.T) { - if _, err := parse(t, tc.body).Mode(); err == nil { - t.Errorf("a %s node mode was accepted, so a reader would compare the file against "+ - "whichever defaults it happened to pick", tc.name) + if _, err := seitoml.Parse(strings.NewReader(tc.body)); err == nil { + t.Errorf("a %s node mode was accepted, so every verb below would answer for a file the "+ + "reader compares against whichever defaults it happened to pick", tc.name) } }) } @@ -1074,9 +1075,9 @@ func TestAFileDescribingItselfWithANonValueIsRefused(t *testing.T) { }) t.Run("a node mode that is not a string", func(t *testing.T) { - _, err := parse(t, "schema_version = 1\nnode_mode = 3\n").Mode() + _, err := seitoml.Parse(strings.NewReader("schema_version = 1\nnode_mode = 3\n")) if err == nil || !strings.Contains(err.Error(), "want a mode name") { - t.Errorf("Mode on a numeric mode returned %v, want a refusal naming what it wanted", err) + t.Errorf("Parse on a numeric mode returned %v, want a refusal naming what it wanted", err) } }) @@ -1090,8 +1091,10 @@ func TestAFileDescribingItselfWithANonValueIsRefused(t *testing.T) { // TestSaveRefusesADocumentParseWouldRefuse holds the writer to what the reader accepts. // -// The describing keys are written through Set, which is how New puts them there, so a caller can also -// change one to something Parse will not take. Nothing about TOML is wrong with the result, so a check +// Both describing keys are written through Set, which is how New puts them there, so a caller can also +// change either to something Parse will not take. The mode matters more than the counter: it is the only +// durable record of an archive node, so a file that loses it is compared against a validator's defaults +// and nothing says so. Nothing about TOML is wrong with the result, so a check // that only asked whether the bytes decode let it through, and the file on disk was one this package // could not load. The failure appears at the next boot rather than at the write that caused it. func TestSaveRefusesADocumentParseWouldRefuse(t *testing.T) { @@ -1113,6 +1116,16 @@ func TestSaveRefusesADocumentParseWouldRefuse(t *testing.T) { _, err := f.Unset(seitoml.VersionKey) return err }, "has no schema_version"}, + {"a mode that is not text", func(f *seitoml.File) error { + return f.Set(seitoml.ModeKey, 5) + }, "want a mode name"}, + {"an empty mode", func(f *seitoml.File) error { + return f.Set(seitoml.ModeKey, "") + }, "is empty"}, + {"no mode at all", func(f *seitoml.File) error { + _, err := f.Unset(seitoml.ModeKey) + return err + }, "has no node_mode"}, } { t.Run(tc.name, func(t *testing.T) { f, err := seitoml.New("validator") diff --git a/config/seitoml/values.go b/config/seitoml/values.go index ff6de1aebc..1f9186e345 100644 --- a/config/seitoml/values.go +++ b/config/seitoml/values.go @@ -166,8 +166,8 @@ func flatten(prefix string, in, out map[string]any) { // stringValue reads one of the keys that describe the file. // -// Both are read before anything else, and neither has a sensible reading when it is absent or holds -// something other than a string, so each caller states its own consequence rather than sharing one. +// Parse reads both before it returns a file, and neither has a sensible reading when it is absent or +// holds something other than a string, so each caller states its own consequence rather than sharing one. func (f *File) stringValue(key string) (string, bool, error) { all, err := f.decoded() if err != nil { From b64a935b324b63e7a852b200cae8142bf7120535 Mon Sep 17 00:00:00 2001 From: bdchatham Date: Wed, 19 Aug 2026 09:59:20 -0700 Subject: [PATCH 22/24] config/seitoml: hold the documentation to what the code does Eight claims the code does not support, or supports only when narrowed. The package doc said Set renders the document and asks the decoder again. Only an insert does. Replacing a value on an existing line changes no shape and does not ask, and neither does Unset; Save asks over the whole document instead. This claim had been corrected once already and the rewrite reintroduced it, so the same overstatement in decodable's own godoc is corrected with it rather than left to drift from its pair again. It said parsing and booting are one statement, which its own scope paragraph contradicts four lines later. A file this package accepts is a file the node's decoder accepts; whether the values make a bootable node is answered elsewhere, and that is the one inference this package must not license. It said the file holds only what an operator decided, and New writes both describing keys itself. It said Set and Unset add no line the caller did not ask for, then conceded four lines later that a new table brings a heading. It read as a closed list of the counter's refusals and omitted a counter that is not a whole number, which two cases pin. It said a key is a lower-case bare TOML key, where the rule is per segment, so read literally it forbade the dotted key every verb here takes. Load reports fs.ErrNotExist for a path with no file, which is the one outcome a caller acts on rather than reports, since a node with no sei.toml needs New. The godoc did not say so and the test asserted only that some error came back, so wrapping that line would have broken callers silently. Both are closed, and wrapping it now fails the test by name. The round-trip table compared elements with !=, which panics on a list element rather than failing. So the table could not carry a list of lists, which is a shape this package supports and recurses for. It compares with reflect.DeepEqual now, as the rest of this workstream does, and carries that row. The scope boundaries moved out of the version paragraph into a section of their own, matching the sibling package, so a reader scanning for what this does not do finds it where they look. 138 cases, 94.4% of statements, race clean, 0 lint issues. --- config/seitoml/doc.go | 54 +++++++++++++++++++++------------- config/seitoml/edit.go | 7 +++-- config/seitoml/file.go | 3 ++ config/seitoml/seitoml_test.go | 34 +++++++++------------ 4 files changed, 54 insertions(+), 44 deletions(-) diff --git a/config/seitoml/doc.go b/config/seitoml/doc.go index 5cba5c91a2..cc29bff162 100644 --- a/config/seitoml/doc.go +++ b/config/seitoml/doc.go @@ -2,10 +2,10 @@ // // A File is a mutable in-memory document for one goroutine at a time. // -// The file holds only what an operator decided. A key present in it is authoritative; a key absent -// from it resolves to the running binary's default for the node's mode. Nothing here writes a -// default into the file, because a value the binary put there reads exactly like one an operator -// chose. +// Apart from the two keys below, the file holds only what an operator decided. A key present in it is +// authoritative; a key absent from it resolves to the running binary's default for the node's mode. +// Nothing here writes a default into the file, because a value the binary put there reads exactly like +// one an operator chose. // // Two keys at the top level describe the file rather than configure the node, and Values leaves both // out. @@ -14,20 +14,29 @@ // node_mode which mode's defaults its values were chosen against // // schema_version counts migrations, one per migration, and is deliberately not a release version. Parse -// refuses a file whose counter is absent, below the first schema, or ahead of the one this binary -// understands, so every verb below answers from a file whose shape is established. Nothing here migrates -// a file; this package reads and writes the counter, and the chain that acts on it arrives with the -// migrations. Nothing here resolves a value or knows what keys exist. +// reads both keys before it returns a file, so every verb below answers for one whose schema and mode are +// established. It refuses a counter that is absent, not a whole number, below the first schema, or ahead +// of the one this binary understands, and a mode that is absent, not text, or empty. // -// # Editing Preserves The Document +// # What This Package Is Not // -// An operator hand-edits this file, and the comments in it are how they explain a choice to whoever -// reads it next. So Set and Unset change the value they name and add no line the caller did not ask -// for, leaving every other line of content as it was. A comment above a key and a comment beside a -// value both survive an edit, and a key's own comment leaves with it when the key is unset. +// Nothing here migrates a file. This package reads and writes the counter; whatever runs the steps +// arrives with them, and no step exists here. // -// A table is named one way, by a heading. So a new key either joins the section that already carries its -// table or brings a heading with it, and there is no second spelling for an insert to choose between. +// Nothing here resolves a value or knows what keys exist. A key this file carries may be one no section +// declares, and this package does not say so. +// +// Nothing here writes a default, and nothing here decides whether the values make a bootable node. +// +// # Editing Preserves the Document +// +// An operator hand-edits this file, and the comments in it are how they explain a choice to whoever reads +// it next. So Set and Unset write the key they name and, when its table is new, that table's heading. +// They add nothing else, and every other line of content stays as it was. A comment above a key and a +// comment beside a value both survive an edit, and a key's own comment leaves with it when the key is +// unset. +// +// A table is named one way, by a heading, so there is no second spelling for an insert to choose between. // // Vertical spacing normalises once, on the first save of a file nothing has saved before, and holds // from then on. @@ -40,12 +49,15 @@ // # One Decoder Decides What Parses // // The decoder is the one a node reads its configuration with. viper decodes TOML with -// pelletier/go-toml/v2, so this package decodes with it too, which makes "this file parses" and "this -// node can boot from it" one statement. +// pelletier/go-toml/v2, so this package decodes with it too, and a file this package accepts is a file +// the node's own decoder accepts. Whether its values make a bootable node is answered elsewhere. // -// So the question a shape has to answer is put to that decoder rather than to a list kept here. Parse -// asks it, and Set renders the document and asks it again, undoing the write and naming the key when the -// answer is no. A shape nobody anticipated is refused as surely as one somebody did. +// So the question a shape has to answer is put to that decoder rather than to a list kept here. Parse asks +// it. So does a Set that adds a key: it inserts, renders, asks again, and undoes the write and names the +// key when the answer is no. A Set that replaces a value on an existing line changes no shape and does +// not ask, nor does Unset, and Save asks once more over the whole document before anything reaches disk. +// So a shape nobody anticipated is refused as surely as one somebody did, and nothing reaches a node's +// disk unread. // // Two shapes are checked here as well, and deliberately: a repeated key and a repeated heading. The // decoder refuses both, so nothing about whether the file loads rests on these; what they add is the @@ -74,7 +86,7 @@ // - an array of tables, which flattens to one key holding a list of tables, so no entry has a line of // its own to edit // -// A key is a lower-case bare TOML key: letters, digits, underscores and hyphens. Anything else has to be +// Every segment of a key is a lower-case bare TOML key: letters, digits, underscores and hyphens. Anything else has to be // quoted where it is written, and a quoted key is spelled one way by the decoder and another by a // lookup, so Values would report a key Get answers absent for. Set, Unset and Get fold a caller's key to // lower case and then hold it to that rule, so a key one of them writes is a key the file reads back. diff --git a/config/seitoml/edit.go b/config/seitoml/edit.go index f3a42cb508..55c43eec30 100644 --- a/config/seitoml/edit.go +++ b/config/seitoml/edit.go @@ -61,9 +61,10 @@ func (f *File) Set(key string, v any) error { // decodable reports whether the document still renders to something the node's decoder can read. // -// The one check every edit passes through, so a shape no caller anticipated is refused as surely as one -// somebody did. Rendering is what a later process reads, so this asks the question in the form the -// answer matters in. +// The check an insert passes through, because only an insert can name a place the document already uses. +// Replacing a value on an existing line changes no shape, and neither does removing a name, so neither +// asks; Save asks over the whole document instead, which is the gate every file reaching disk crosses. +// Rendering is what a later process reads, so this asks the question in the form the answer matters in. func (f *File) decodable() error { _, err := f.decoded() return err diff --git a/config/seitoml/file.go b/config/seitoml/file.go index f216160e80..9f200c11c6 100644 --- a/config/seitoml/file.go +++ b/config/seitoml/file.go @@ -230,6 +230,9 @@ func valueIsAddressable(key parser.Key, v parser.Value) error { } // Load reads the document at path. +// +// A path with no file there reports fs.ErrNotExist, which errors.Is matches. That is the one outcome a +// caller acts on rather than reports, since a node with no sei.toml yet needs New instead. func Load(path string) (*File, error) { raw, err := os.ReadFile(path) //nolint:gosec // the caller's configured path is the subject if err != nil { diff --git a/config/seitoml/seitoml_test.go b/config/seitoml/seitoml_test.go index 0f19cd69ea..fc7b84cd33 100644 --- a/config/seitoml/seitoml_test.go +++ b/config/seitoml/seitoml_test.go @@ -1,7 +1,9 @@ package seitoml_test import ( + "errors" "fmt" + "io/fs" "math" "os" "path/filepath" @@ -234,6 +236,10 @@ func TestSetRoundTripsEveryTypeItAccepts(t *testing.T) { {"empty string", "", ""}, {"duration", 90 * time.Second, "1m30s"}, {"string list", []string{"a", "b"}, []any{"a", "b"}}, + // A list of lists, which is the shape handedOut recurses for. The comparison here has to be + // reflect.DeepEqual for this row to exist: comparing elements with != panics on a list element, + // so a helper doing that could not fail this case, it could only abort it. + {"list of lists", []any{[]any{"a"}, []any{"b", "c"}}, []any{[]any{"a"}, []any{"b", "c"}}}, } { t.Run(tc.name, func(t *testing.T) { f, err := seitoml.New("validator") @@ -252,7 +258,7 @@ func TestSetRoundTripsEveryTypeItAccepts(t *testing.T) { t.Fatalf("Get after a round trip: (%#v, %v, %v)\nfile:\n%s", got, ok, err, render(t, f)) } - if !equal(got, tc.want) { + if !reflect.DeepEqual(got, tc.want) { t.Errorf("wrote %#v and read back %#v, want %#v.\nfile:\n%s\n\nA value that does not "+ "survive a round trip means the file looks correct while the node runs something "+ "else", tc.set, got, tc.want, render(t, f)) @@ -261,24 +267,6 @@ func TestSetRoundTripsEveryTypeItAccepts(t *testing.T) { } } -// equal compares two read values, including lists. -func equal(a, b any) bool { - as, aok := a.([]any) - bs, bok := b.([]any) - if aok || bok { - if !aok || !bok || len(as) != len(bs) { - return false - } - for i := range as { - if as[i] != bs[i] { - return false - } - } - return true - } - return a == b -} - // TestALiteralStringIsTakenAsWritten holds the difference between TOML's two string forms. // // A basic string carries escapes and a literal string does not, which is why TOML has both. @@ -949,8 +937,14 @@ func TestParseRefusesAFileTomlCannotRead(t *testing.T) { if _, err := seitoml.Parse(strings.NewReader("[unterminated\nkey = 1\n")); err == nil { t.Error("a malformed document parsed, and an empty one reads as a node that chose nothing") } - if _, err := seitoml.Load(filepath.Join(t.TempDir(), "absent.toml")); err == nil { + // Classified, not merely non-nil: a caller chooses New over Load on this one outcome, so wrapping it + // into something errors.Is cannot match would break them with nothing here failing. + _, absent := seitoml.Load(filepath.Join(t.TempDir(), "absent.toml")) + if absent == nil { t.Error("loading a file that does not exist succeeded") + } else if !errors.Is(absent, fs.ErrNotExist) { + t.Errorf("a missing file reports %v, which errors.Is(fs.ErrNotExist) does not match, so a caller "+ + "cannot tell it from a file that is present and unreadable", absent) } bad := filepath.Join(t.TempDir(), "sei.toml") if err := os.WriteFile(bad, []byte("[unterminated\n"), 0o600); err != nil { From ea05e6d576d9ef0a5987e35cae8244d72f4d57e0 Mon Sep 17 00:00:00 2001 From: bdchatham Date: Wed, 19 Aug 2026 10:24:53 -0700 Subject: [PATCH 23/24] config/seitoml: state each boundary where it applies, not in a section of its own A section listing what this package is not restated two things the doc already said. Nothing here writes a default was in the opening paragraph, qualifying the sentence about what the file holds. Nothing here decides whether the values make a bootable node was already the closing clause of the decoder paragraph, which is the sentence it corrects. It also gave the least durable statement in the file the most navigational weight. Nothing here migrates a file stops being true when the migration chain lands, and a heading on pkg.go.dev is where a reader goes first. A negative reads weakest standing alone and strongest attached to the thing it bounds, and here every boundary has an obvious host. What acts on the counter is a clause on the counter paragraph. What Values does not answer is a clause on Values, which is what tells a reader why it does not check a key against the declared set. The other two were already where they belong. The sibling package earns its own such section because a registration is spread across a struct tag, an init and a test, so no single paragraph can host the boundary. This package has one type and eight verbs. 85 lines and three headings, from 93 and four. 138 cases, 94.4% of statements, race clean, 0 lint issues. --- config/seitoml/doc.go | 16 ++++------------ 1 file changed, 4 insertions(+), 12 deletions(-) diff --git a/config/seitoml/doc.go b/config/seitoml/doc.go index cc29bff162..a366364cca 100644 --- a/config/seitoml/doc.go +++ b/config/seitoml/doc.go @@ -8,7 +8,8 @@ // one an operator chose. // // Two keys at the top level describe the file rather than configure the node, and Values leaves both -// out. +// out. Values reports the keys the file writes and nothing about them: which section owns a key, and +// whether any section does, is answered elsewhere. // // schema_version which migration the file has reached // node_mode which mode's defaults its values were chosen against @@ -16,17 +17,8 @@ // schema_version counts migrations, one per migration, and is deliberately not a release version. Parse // reads both keys before it returns a file, so every verb below answers for one whose schema and mode are // established. It refuses a counter that is absent, not a whole number, below the first schema, or ahead -// of the one this binary understands, and a mode that is absent, not text, or empty. -// -// # What This Package Is Not -// -// Nothing here migrates a file. This package reads and writes the counter; whatever runs the steps -// arrives with them, and no step exists here. -// -// Nothing here resolves a value or knows what keys exist. A key this file carries may be one no section -// declares, and this package does not say so. -// -// Nothing here writes a default, and nothing here decides whether the values make a bootable node. +// of the one this binary understands, and a mode that is absent, not text, or empty. This package reads +// and writes the counter; what acts on it arrives with the migrations. // // # Editing Preserves the Document // From c27dd3844d416887b374ad58091e433030e84ecc Mon Sep 17 00:00:00 2001 From: bdchatham Date: Wed, 19 Aug 2026 10:29:46 -0700 Subject: [PATCH 24/24] config/seitoml: name the same key on every read, and clear four residues A file can hold more than one value this format cannot write, and the refusal names the first one found. It walked a map to find it, so the first one differed between reads: measured across 50 reads of one file, two different keys were named 39 and 11 times. An operator fixes the key they were told about and the next read names the other one. The walk is sorted now, and the test drives 50 reads and refuses more than one spelling. An unsorted walk fails it, and so does sorting the other way. The remaining four are things earlier commits here left behind. A comment recorded that a branch was separated from the one that used to absorb it. That is a former version of the same function, which a godoc does not carry; the present-state sentence stands without it. A test comment still opened with the name its function had before it was renamed, the only such mismatch in the file. A test named for an order property never varied one. Its comment described a table whose keys join an ancestor's dotted name, which no file can hold since dotted keys are refused at the door, and a blank line marked where its second case had been. It is now named and written for the one property it holds, that a new table brings a heading. Save's godoc stopped at atomicity and said nothing about the destinations it refuses. A symlink and any non-regular file are both refused, because a rename replaces either rather than writing through it, and that is an operator-facing failure a caller reads about here or discovers at runtime. It also now says an existing file keeps its permission and a new one is owner-only. 138 cases, 94.5% of statements, race clean, 0 lint issues. --- config/seitoml/file.go | 8 +++-- config/seitoml/seitoml_test.go | 61 +++++++++++++++++++++++++--------- config/seitoml/values.go | 13 ++++++-- 3 files changed, 62 insertions(+), 20 deletions(-) diff --git a/config/seitoml/file.go b/config/seitoml/file.go index 9f200c11c6..9868738654 100644 --- a/config/seitoml/file.go +++ b/config/seitoml/file.go @@ -334,6 +334,10 @@ func (f *File) Bytes() ([]byte, error) { // The rename makes it atomic, and the temporary file sits in the destination's own directory so the // rename stays within one filesystem. A crash at any point leaves either the previous file or the // new one, never a truncated file a node cannot parse. +// +// A destination that is a symbolic link, or that is not a regular file, is refused: a rename replaces +// either one rather than writing through it. An existing file keeps its own permission, and a new one is +// created readable and writable by its owner alone. func (f *File) Save(path string) error { raw, err := f.Bytes() if err != nil { @@ -388,8 +392,8 @@ func modeToWrite(path string) (os.FileMode, error) { case errors.Is(err, fs.ErrNotExist): return newFileMode, nil // no file there yet, which is the ordinary first save case err != nil: - // Separated from the absent case, which used to absorb it. A path this process cannot inspect - // is not a first save, and calling it one writes at the default mode on a guess. + // A path this process cannot inspect is not a first save, and calling it one writes at the + // default mode on a guess. return 0, fmt.Errorf("inspect %s: %w", path, err) case info.Mode()&os.ModeSymlink != 0: target, err := os.Readlink(path) diff --git a/config/seitoml/seitoml_test.go b/config/seitoml/seitoml_test.go index fc7b84cd33..3d9eacc306 100644 --- a/config/seitoml/seitoml_test.go +++ b/config/seitoml/seitoml_test.go @@ -156,7 +156,7 @@ func TestFormattingNormalizesOnceAndThenHoldsSteady(t *testing.T) { } } -// TestAnAbsentSchemaVersionIsAnError holds that the file's shape is never guessed. +// TestAnAbsentSchemaVersionIsRefusedAtTheDoor holds that the file's shape is never guessed. // // A migration chain reads the version to decide which steps to run. Defaulting an absent one to // zero would run every step in history against a file nobody established the shape of, and the @@ -1597,25 +1597,54 @@ func TestAValueThatIsNotTextIsRefused(t *testing.T) { } } -// TestATableKeepsOneSpellingWhateverOrderItsKeysWereWritten holds insert's choice between the two forms. +// TestARefusalNamesTheSameKeyEveryTime holds the diagnosis steady across reads. // -// A table nothing has created is new and gets a heading, which is the form an operator reads. A table an -// ancestor's dotted key already created has no heading and cannot be given one, so its keys join that -// dotted name. Deciding by whether an ancestor section merely exists would spell the same table either -// way depending on which key was set first. -func TestATableKeepsOneSpellingWhateverOrderItsKeysWereWritten(t *testing.T) { - t.Run("a new table under an existing section gets a heading", func(t *testing.T) { - f := parse(t, "schema_version = 1\nnode_mode = \"validator\"\n\n[state-commit]\nbuffer = 100\n") - if err := f.Set("state-commit.flatkv.dir", "/data"); err != nil { - t.Fatalf("Set: %v", err) +// A file can hold more than one value this format cannot write, and the refusal names the first one +// found. Found by walking a map, the first one differs between reads, so an operator fixes the key they +// were told about and the next read names the other. Two of them here, so an unsorted walk reports both +// spellings over enough reads and a sorted one reports the one that sorts first. +func TestARefusalNamesTheSameKeyEveryTime(t *testing.T) { + body := "schema_version = 1\nnode_mode = \"validator\"\n\n[probe]\naaa = nan\nzzz = inf\n" + + named := map[string]int{} + for i := 0; i < 50; i++ { + _, err := seitoml.Parse(strings.NewReader(body)) + if err == nil { + t.Fatal("a file holding a NaN and an infinity parsed") } - out := render(t, f) - if !strings.Contains(out, "[state-commit.flatkv]") { - t.Errorf("a table nothing had created did not get a heading:\n%s", out) + switch { + case strings.Contains(err.Error(), "probe.aaa"): + named["probe.aaa"]++ + case strings.Contains(err.Error(), "probe.zzz"): + named["probe.zzz"]++ + default: + t.Fatalf("the refusal names neither key: %v", err) } - requireStillReadable(t, f) - }) + } + if len(named) != 1 { + t.Errorf("50 reads of one file named %v; an operator cannot fix a key that changes between "+ + "reads, and a test asserting one of them would flake", named) + } + if named["probe.aaa"] == 0 { + t.Errorf("the refusal named %v rather than the key that sorts first", named) + } +} +// TestANewTableUnderAnExistingSectionGetsAHeading covers the form an operator reads. +// +// A key whose table nothing has created brings that table's heading with it, rather than joining the +// section above it as a dotted name. Both spell the same key to a reader, and only one of them gives the +// table a line an operator can edit and comment on. +func TestANewTableUnderAnExistingSectionGetsAHeading(t *testing.T) { + f := parse(t, "schema_version = 1\nnode_mode = \"validator\"\n\n[state-commit]\nbuffer = 100\n") + if err := f.Set("state-commit.flatkv.dir", "/data"); err != nil { + t.Fatalf("Set: %v", err) + } + out := render(t, f) + if !strings.Contains(out, "[state-commit.flatkv]") { + t.Errorf("a table nothing had created did not get a heading:\n%s", out) + } + requireStillReadable(t, f) } // TestEveryEditIsVisibleToTheNextRead holds the values a read returns against the document. diff --git a/config/seitoml/values.go b/config/seitoml/values.go index 1f9186e345..e5796fb870 100644 --- a/config/seitoml/values.go +++ b/config/seitoml/values.go @@ -3,6 +3,7 @@ package seitoml import ( "fmt" "math" + "sort" toml "github.com/pelletier/go-toml/v2" ) @@ -121,8 +122,16 @@ func decodeBytes(raw []byte) (map[string]any, error) { // cannot write one back, because rendering it produces a line no reader loads, so accepting one here // would mean any later edit of any other key failed on a value this package had handed out. func refuseNonFiniteNumbers(values map[string]any) error { - for key, v := range values { - if err := finite(key, v); err != nil { + // Sorted, because this returns the first refusal it finds and a map hands its keys back in a + // different order each time. Unsorted, a file holding two of these names one of them on one read and + // the other on the next, so an operator fixes what they were told and meets the same refusal again. + keys := make([]string, 0, len(values)) + for key := range values { + keys = append(keys, key) + } + sort.Strings(keys) + for _, key := range keys { + if err := finite(key, values[key]); err != nil { return err } }