From 87ea627976cc59765f70609f917956a2e022af48 Mon Sep 17 00:00:00 2001 From: Alessandro De Blasis Date: Thu, 1 Oct 2026 09:05:16 +0300 Subject: [PATCH 1/3] fix(engine): preserve response header case for x-amz-meta --- internal/engine/adapter_dispatch.go | 2 +- internal/engine/engine.go | 2 +- internal/engine/header_case_test.go | 99 +++++++++++++++++++++++++++++ 3 files changed, 101 insertions(+), 2 deletions(-) create mode 100644 internal/engine/header_case_test.go diff --git a/internal/engine/adapter_dispatch.go b/internal/engine/adapter_dispatch.go index dd01d81a..efd4b0d7 100644 --- a/internal/engine/adapter_dispatch.go +++ b/internal/engine/adapter_dispatch.go @@ -160,7 +160,7 @@ func (e *Engine) runHandler( // Write headers. for k, v := range resp.Headers { - w.Header().Set(k, v) + w.Header()[k] = []string{v} } status := resp.Status diff --git a/internal/engine/engine.go b/internal/engine/engine.go index 5ae49da9..e1b36d32 100644 --- a/internal/engine/engine.go +++ b/internal/engine/engine.go @@ -585,7 +585,7 @@ func applyDecision(w http.ResponseWriter, r *http.Request, d rules.Decision) { time.Sleep(time.Duration(d.LatencyMS) * time.Millisecond) } for k, v := range d.Headers { - w.Header().Set(k, v) + w.Header()[k] = []string{v} } if len(d.BodyBytes) > 0 && w.Header().Get("Content-Type") == "" { w.Header().Set("Content-Type", "application/json") diff --git a/internal/engine/header_case_test.go b/internal/engine/header_case_test.go new file mode 100644 index 00000000..2959acd8 --- /dev/null +++ b/internal/engine/header_case_test.go @@ -0,0 +1,99 @@ +package engine + +import ( + "net/http/httptest" + "os" + "path/filepath" + "testing" + + "stuntapi.com/stunt/internal/manifest" + "stuntapi.com/stunt/internal/rules" +) + +// Real S3 sends user-metadata headers lowercase on the wire +// (x-amz-meta-). The AWS SDK for .NET preserves the +// suffix case after stripping the prefix, so a canonicalized wire form +// (X-Amz-Meta-Kind) surfaces as "Kind" instead of "kind". +func TestHeaderCaseApplyDecisionPreservesLowercase(t *testing.T) { + d := rules.Decision{ + Matched: true, + Status: 200, + Headers: map[string]string{"x-amz-meta-kind": "sample"}, + BodyBytes: []byte(`{}`), + } + rec := httptest.NewRecorder() + req := httptest.NewRequest("GET", "/", nil) + applyDecision(rec, req, d) + + h := rec.Header() + vals, ok := h["x-amz-meta-kind"] + if !ok { + t.Fatalf("expected exact lowercase key %q in header map, got %v", "x-amz-meta-kind", h) + } + if len(vals) == 0 || vals[0] != "sample" { + t.Fatalf("x-amz-meta-kind = %v, want %q", vals, "sample") + } + if _, ok := h["X-Amz-Meta-Kind"]; ok { + t.Fatalf("header map must not contain canonicalized key %q, got %v", "X-Amz-Meta-Kind", h) + } +} + +func TestHeaderCaseRunHandlerPreservesLowercase(t *testing.T) { + adapterDir := t.TempDir() + scriptsDir := filepath.Join(adapterDir, "scripts") + if err := os.MkdirAll(scriptsDir, 0o755); err != nil { + t.Fatal(err) + } + adapterYAML := "id: test-header-case\n" + + "name: header case test\n" + + "version: \"0.1.0\"\n" + + "endpoints:\n" + + " - route: /obj\n" + + " method: GET\n" + + " handler: scripts/h.star#on_get\n" + if err := os.WriteFile(filepath.Join(adapterDir, "adapter.yaml"), []byte(adapterYAML), 0o644); err != nil { + t.Fatal(err) + } + handlerSrc := "def on_get(req):\n" + + " return respond(200, {\"ok\": True}, {\"x-amz-meta-kind\": \"sample\"})\n" + if err := os.WriteFile(filepath.Join(scriptsDir, "h.star"), []byte(handlerSrc), 0o644); err != nil { + t.Fatal(err) + } + + stateDir := t.TempDir() + m := &manifest.Manifest{ + Path: filepath.Join(stateDir, "stunt.yaml"), + Version: 1, + Network: manifest.Network{Mode: "port", BasePort: 0}, + Services: map[string]manifest.Service{ + "svc": {Adapter: adapterDir}, + }, + } + e, err := newEngine(m, t.TempDir()) + if err != nil { + t.Fatalf("newEngine: %v", err) + } + defer e.Close() + + st, ok := e.states["svc"] + if !ok { + t.Fatalf("service state missing: loadErrors=%v", e.loadErrors) + } + ep := st.adapter.Endpoints[0] + + rec := httptest.NewRecorder() + req := httptest.NewRequest("GET", "/obj", nil) + e.runHandler(rec, req, st, ep, nil, map[string]string{}) + + h := rec.Header() + vals, ok := h["x-amz-meta-kind"] + if !ok { + t.Fatalf("expected exact lowercase key %q in header map, got %v", "x-amz-meta-kind", h) + } + if len(vals) == 0 || vals[0] != "sample" { + t.Fatalf("x-amz-meta-kind = %v, want %q", vals, "sample") + } + if _, ok := h["X-Amz-Meta-Kind"]; ok { + t.Fatalf("header map must not contain canonicalized key %q, got %v", "X-Amz-Meta-Kind", h) + } +} From c3e6baa39f22b0874feb33c17eab72637c78efc6 Mon Sep 17 00:00:00 2001 From: Alessandro De Blasis Date: Thu, 1 Oct 2026 07:17:01 +0300 Subject: [PATCH 2/3] fix(s3): x-amz-meta persist/echo/validate 2KB # Conflicts: # adapters/aws-s3-style/scripts/lib.star # adapters/aws-s3-style/scripts/objects.star # internal/engine/aws_s3_style_test.go --- adapters/aws-s3-style/scripts/lib.star | 100 ++++++++- adapters/aws-s3-style/scripts/objects.star | 14 +- internal/engine/aws_s3_style_test.go | 227 +++++++++++++++++++++ 3 files changed, 333 insertions(+), 8 deletions(-) diff --git a/adapters/aws-s3-style/scripts/lib.star b/adapters/aws-s3-style/scripts/lib.star index 9648d64b..b489655f 100644 --- a/adapters/aws-s3-style/scripts/lib.star +++ b/adapters/aws-s3-style/scripts/lib.star @@ -739,8 +739,10 @@ def _find_object(bucket, key): # _upsert_object writes an object's content bytes (reusing the existing # blob id when overwriting, so the blob store has one file per object) and # refreshes its metadata doc. Returns nothing; the ETag is derived by the -# caller (it differs for simple vs multipart uploads). -def _upsert_object(bucket, key, raw, ct, etag): +# caller (it differs for simple vs multipart uploads). meta is the object's +# x-amz-meta-* map; {} when the request carried none, which clears any +# previously stored value. +def _upsert_object(bucket, key, raw, ct, etag, meta): oc = store_collection("objects") bid = "" obj_id = "" @@ -761,12 +763,94 @@ def _upsert_object(bucket, key, raw, ct, etag): "lastModified": _unix_to_iso8601(now_unix), "lastModifiedUnix": now_unix, "size": len(raw), + "metadata": meta, } if obj_id != None and obj_id != "": oc.update(obj_id, doc) else: oc.insert(doc) +# ==================================================================== +# User metadata (x-amz-meta-*) +# ==================================================================== +# S3 user metadata: request headers with the x-amz-meta- prefix are stored +# with the object and echoed back on GET/HEAD 200 only (never on 304/412, +# List, or error responses). The first occurrence wins: the engine already +# collapses duplicate wire headers to v[0] (headerMap), and the collect +# loop below keeps the first suffix on post-lower collision as +# defense-in-depth. Suffixes are lowercased ASCII-only (hand-rolled, not +# str.lower(), so non-ASCII bytes pass through unfolded and are preserved +# for size/echo). The byte sum len(suffix)+len(value) over all collected +# entries must fit in 2048 (the AWS 2KB user-metadata total; the prefix is +# excluded, measured post-dedup with byte len, so 2048 passes / 2049 fails). + +# _ascii_lower lowercases ASCII A-Z only, preserving every other byte +# (unlike str.lower(), which would fold non-ASCII too). +def _ascii_lower(s): + out = "" + for i in range(len(s)): + ch = s[i] + if ch >= "A" and ch <= "Z": + out = out + chr(ord(ch) + 32) + else: + out = out + ch + return out + +# _meta_bad_value returns True when s holds a byte real S3 rejects in user +# metadata: CR, LF, NUL, any other C0 control except TAB, or DEL. (CR/LF +# cannot arrive over HTTP — Go rejects them at the transport — so this is +# defense-in-depth covered by inspection, not e2e.) +def _meta_bad_value(s): + for i in range(len(s)): + o = ord(s[i]) + if o == 9: + continue + if o < 32 or o == 127: + return True + return False + +# _collect_metadata gathers x-amz-meta-* request headers (iteration keys are +# already the lowercase canonical form). Empty suffix or rejected bytes -> +# 400 InvalidArgument; post-dedup byte total over 2048 -> 400 +# MetadataTooLarge. Empty values are allowed; spaces are preserved verbatim. +# Returns (meta, None) on success or (None, error_response). +def _collect_metadata(req): + headers = req.get("headers") + if headers == None: + return {}, None + meta = {} + total = 0 + for k in headers.keys(): + if not _has_prefix(k, "x-amz-meta-"): + continue + suffix = _ascii_lower(k[len("x-amz-meta-"):]) + v = headers.get(k, "") + if v == None: + v = "" + v = str(v) + if suffix == "": + return None, _invalid_argument(k, v, "Metadata name must not be empty.") + if _meta_bad_value(suffix) or _meta_bad_value(v): + return None, _invalid_argument(k, v, "Metadata contains invalid characters.") + if suffix in meta: + continue + meta[suffix] = v + total = total + len(suffix) + len(v) + if total > 2048: + return None, _xml_error("MetadataTooLarge", "Your metadata headers exceed the maximum allowed metadata size.", "", 400) + return meta, None + +# _meta_response_headers merges stored user metadata into a GET/HEAD 200 +# response-header dict (suffixes were lowercased at collect time). Legacy +# docs stored without a metadata field fall back to {}. +def _meta_response_headers(obj, base): + meta = obj.get("metadata", {}) + if meta == None: + meta = {} + for k in meta.keys(): + base["x-amz-meta-" + k] = meta[k] + return base + # ==================================================================== # Multipart upload core # ==================================================================== @@ -853,11 +937,15 @@ def _mpu_create(req, bucket, key): ct = "application/octet-stream" upload_id = "mpu_" + str(store_kv_incr("s3", "mpu_seq")) + meta, merr = _collect_metadata(req) + if merr != None: + return merr store_collection("mpu_uploads").insert({ "id": upload_id, "bucket": bucket, "key": key, "contentType": ct, + "metadata": meta, "initiatedUnix": clock.now_unix(), }) @@ -1062,7 +1150,13 @@ def _mpu_complete(req, bucket, key): ct = upload.get("contentType", "application/octet-stream") if ct == None or ct == "": ct = "application/octet-stream" - _upsert_object(bucket, key, full, ct, etag) + # The object's user metadata is the Create request's (stored on the + # upload row); Complete-request and UploadPart meta are ignored. Legacy + # uploads stored without a metadata field fall back to {}. + meta = upload.get("metadata", {}) + if meta == None: + meta = {} + _upsert_object(bucket, key, full, ct, etag, meta) _mpu_discard(upload_id) store_collection("mpu_uploads").delete(upload_id) diff --git a/adapters/aws-s3-style/scripts/objects.star b/adapters/aws-s3-style/scripts/objects.star index b509f6d5..0005380a 100644 --- a/adapters/aws-s3-style/scripts/objects.star +++ b/adapters/aws-s3-style/scripts/objects.star @@ -86,8 +86,12 @@ def on_put_object(req): # Content-derived ETag (SHA-256 of the verbatim bytes); the write path # (blob + metadata doc) is shared with CompleteMultipartUpload. + # User metadata replaces any previous value (PUT without meta clears). etag = _etag(raw) - _upsert_object(bucket, key, raw, ct, etag) + meta, merr = _collect_metadata(req) + if merr != None: + return merr + _upsert_object(bucket, key, raw, ct, etag, meta) return respond(200, "", { "ETag": '"' + etag + '"', @@ -122,13 +126,13 @@ def on_get_object(req): if etag == None: etag = "" - return respond(200, content, { + return respond(200, content, _meta_response_headers(obj, { "Content-Type": ct, "ETag": '"' + etag + '"', "Last-Modified": _obj_last_modified_rfc1123(obj), "Content-Length": str(len(content)), "x-amz-request-id": _req_id(), - }) + })) # on_head_object returns metadata headers only (no body). def on_head_object(req): @@ -153,13 +157,13 @@ def on_head_object(req): if size == None: size = 0 - return respond(200, "", { + return respond(200, "", _meta_response_headers(obj, { "Content-Type": ct, "ETag": '"' + etag + '"', "Last-Modified": _obj_last_modified_rfc1123(obj), "Content-Length": _to_int_str(size), "x-amz-request-id": _req_id(), - }) + })) # on_delete_object removes an object, or aborts an in-progress multipart # upload when the request carries an uploadId. Returns 204. diff --git a/internal/engine/aws_s3_style_test.go b/internal/engine/aws_s3_style_test.go index bc72b33b..69c78e81 100644 --- a/internal/engine/aws_s3_style_test.go +++ b/internal/engine/aws_s3_style_test.go @@ -876,3 +876,230 @@ func TestAWSS3StyleBinaryRoundTrip(t *testing.T) { t.Fatalf("HEAD Content-Length = %q, want %d", hdr.Header.Get("Content-Length"), len(bin)) } } + +func TestAWSS3Metadata(t *testing.T) { + adapterDir, err := filepath.Abs(filepath.Join("..", "..", "adapters", "aws-s3-style")) + if err != nil { + t.Fatal(err) + } + stateDir := t.TempDir() + m := &manifest.Manifest{ + Path: filepath.Join(stateDir, "stunt.yaml"), + Version: 1, + Network: manifest.Network{Mode: "port", BasePort: 0}, + Services: map[string]manifest.Service{ + "s3": {Adapter: adapterDir}, + }, + } + e, err := New(m) + if err != nil { + t.Fatalf("engine.New: %v", err) + } + defer e.Close() + addrs, cancel, err := e.ServeForTest(context.Background()) + if err != nil { + t.Fatalf("ServeForTest: %v", err) + } + defer cancel() + time.Sleep(50 * time.Millisecond) + base := addrs["s3"] + now := time.Now() + + doReq := func(method, rawurl string, body []byte, hdrs map[string]string) (string, int, http.Header) { + t.Helper() + req := s3SignedReq(t, method, rawurl, body, now, awsStyleAccessKey, awsStyleSecretKey) + for k, v := range hdrs { + req.Header.Set(k, v) + } + resp, err := http.DefaultClient.Do(req) + if err != nil { + t.Fatal(err) + } + defer resp.Body.Close() + b, _ := io.ReadAll(resp.Body) + return string(b), resp.StatusCode, resp.Header + } + hasMeta := func(hdr http.Header) bool { + for k := range hdr { + if strings.HasPrefix(strings.ToLower(k), "x-amz-meta-") { + return true + } + } + return false + } + + if _, status := s3Put(t, base+"/metabucket", nil, now); status != 200 { + t.Fatalf("create bucket -> %d", status) + } + + // PUT with user meta (mixed-case suffix, interior spaces preserved). + meta := map[string]string{ + "X-Amz-Meta-Kind": "sample", + "X-Amz-Meta-Mixed": "VaLue", + "X-Amz-Meta-Sp": "a b", + } + if body, st, _ := doReq("PUT", base+"/metabucket/file.txt", []byte("hello"), meta); st != 200 { + t.Fatalf("put with meta -> status %d, want 200; body %s", st, body) + } + + // GET echoes on 200 (suffix lowercased, values verbatim). + body, st, hdr := doReq("GET", base+"/metabucket/file.txt", nil, nil) + if st != 200 || body != "hello" { + t.Fatalf("get -> status %d body %q, want 200 hello", st, body) + } + if got := hdr.Get("x-amz-meta-kind"); got != "sample" { + t.Fatalf("get x-amz-meta-kind = %q, want %q (customMetadataPreserved:false)", got, "sample") + } + if got := hdr.Get("x-amz-meta-mixed"); got != "VaLue" { + t.Fatalf("get x-amz-meta-mixed = %q, want %q", got, "VaLue") + } + if got := hdr.Get("x-amz-meta-sp"); got != "a b" { + t.Fatalf("get x-amz-meta-sp = %q, want interior spaces preserved", got) + } + + // HEAD echoes too. + _, st, hdr = doReq("HEAD", base+"/metabucket/file.txt", nil, nil) + if st != 200 { + t.Fatalf("head -> status %d, want 200", st) + } + if got := hdr.Get("x-amz-meta-kind"); got != "sample" { + t.Fatalf("head x-amz-meta-kind = %q, want %q", got, "sample") + } + + // First-wins: duplicate wire headers collapse to the first value + // (engine headerMap v[0]; Starlark keeps the first suffix on collision). + req := s3SignedReq(t, "PUT", base+"/metabucket/dup.txt", []byte("hello"), now, awsStyleAccessKey, awsStyleSecretKey) + req.Header.Add("x-amz-meta-kind", "first") + req.Header.Add("x-amz-meta-kind", "second") + resp, err := http.DefaultClient.Do(req) + if err != nil { + t.Fatal(err) + } + io.Copy(io.Discard, resp.Body) + resp.Body.Close() + if resp.StatusCode != 200 { + t.Fatalf("put dup meta -> status %d, want 200", resp.StatusCode) + } + _, _, hdr = doReq("GET", base+"/metabucket/dup.txt", nil, nil) + if got := hdr.Get("x-amz-meta-kind"); got != "first" { + t.Fatalf("get dup x-amz-meta-kind = %q, want first-wins %q", got, "first") + } + + // Empty value is allowed (header present, value empty). + if body, st, _ := doReq("PUT", base+"/metabucket/empty.txt", []byte("hello"), map[string]string{"x-amz-meta-empty": ""}); st != 200 { + t.Fatalf("put empty-value meta -> status %d, want 200; body %s", st, body) + } + _, st, hdr = doReq("GET", base+"/metabucket/empty.txt", nil, nil) + if st != 200 { + t.Fatalf("get empty-value -> status %d, want 200", st) + } + if _, ok := hdr["X-Amz-Meta-Empty"]; !ok { + t.Fatal("get empty-value: x-amz-meta-empty header missing, want present-but-empty") + } + if got := hdr.Get("x-amz-meta-empty"); got != "" { + t.Fatalf("get empty-value = %q, want %q", got, "") + } + + // Empty suffix (wire name "x-amz-meta-") -> 400 InvalidArgument. + if body, st, _ := doReq("PUT", base+"/metabucket/bad.txt", []byte("hello"), map[string]string{"x-amz-meta-": "v"}); st != 400 || !strings.Contains(body, "InvalidArgument") { + t.Fatalf("put empty-suffix meta -> %d %q, want 400 InvalidArgument", st, body) + } + + // Size cap: sum(len(suffix)+len(value)) over 2048 -> 400 MetadataTooLarge + // (prefix excluded, post-dedup byte length; 2048 passes, 2049 fails). + if body, st, _ := doReq("PUT", base+"/metabucket/ok2048.txt", []byte("hello"), map[string]string{"x-amz-meta-k": strings.Repeat("a", 2047)}); st != 200 { + t.Fatalf("put 2048-total meta -> status %d, want 200; body %s", st, body) + } + if body, st, _ := doReq("PUT", base+"/metabucket/big.txt", []byte("hello"), map[string]string{"x-amz-meta-k": strings.Repeat("a", 2048)}); st != 400 || !strings.Contains(body, "MetadataTooLarge") { + t.Fatalf("put 2049-total meta -> %d %q, want 400 MetadataTooLarge", st, body) + } + multi := map[string]string{"x-amz-meta-a": strings.Repeat("a", 1024), "x-amz-meta-b": strings.Repeat("b", 1024)} + if body, st, _ := doReq("PUT", base+"/metabucket/multi-big.txt", []byte("hello"), multi); st != 400 || !strings.Contains(body, "MetadataTooLarge") { + t.Fatalf("put multi-header oversize meta -> %d %q, want 400 MetadataTooLarge", st, body) + } + + // Overwrite without meta clears (PUT replaces, not merges). + if _, st, _ := doReq("PUT", base+"/metabucket/file.txt", []byte("hello"), nil); st != 200 { + t.Fatalf("overwrite no-meta -> status %d, want 200", st) + } + _, st, hdr = doReq("GET", base+"/metabucket/file.txt", nil, nil) + if st != 200 { + t.Fatalf("get after overwrite -> status %d, want 200", st) + } + if hasMeta(hdr) { + t.Fatalf("get after overwrite carries meta headers, want cleared: %v", hdr) + } + // Objects stored without meta carry none (legacy {} fallback: no crash). + _, st, hdr = doReq("HEAD", base+"/metabucket/file.txt", nil, nil) + if st != 200 || hasMeta(hdr) { + t.Fatalf("head after overwrite -> status %d meta %v, want 200 no-meta", st, hdr) + } + + // Re-arm meta for the no-meta-on-error paths below. + if body, st, _ := doReq("PUT", base+"/metabucket/file.txt", []byte("hello"), map[string]string{"x-amz-meta-kind": "sample"}); st != 200 { + t.Fatalf("re-put with meta -> status %d; body %s", st, body) + } + _, st, hdr = doReq("GET", base+"/metabucket/file.txt", nil, nil) + if hdr.Get("x-amz-meta-kind") != "sample" { + t.Fatalf("re-put meta echo = %q, want sample", hdr.Get("x-amz-meta-kind")) + } + + // List carries no user meta in response headers. + _, st, hdr = doReq("GET", base+"/metabucket?list-type=2", nil, nil) + if st != 200 { + t.Fatalf("list -> status %d, want 200", st) + } + if hasMeta(hdr) { + t.Fatalf("list carries meta headers, want none: %v", hdr) + } + + // MPU: Create meta propagates through Complete; UploadPart meta and + // Complete-request meta are ignored (no 400, not stored). + body, st, _ = doReq("POST", base+"/metabucket/mpu.bin?uploads", nil, map[string]string{"x-amz-meta-kind": "mpuval"}) + if st != 200 { + t.Fatalf("mpu create with meta -> status %d; body %s", st, body) + } + uploadID := s3XMLTag(t, body, "UploadId") + if uploadID == "" { + t.Fatalf("mpu create: no UploadId in %s", body) + } + partURL := base + "/metabucket/mpu.bin?uploadId=" + uploadID + if body, st, _ := doReq("PUT", partURL+"&partNumber=1", []byte("hello"), map[string]string{"x-amz-meta-evil": "yes"}); st != 200 { + t.Fatalf("upload part with meta -> status %d, want 200 ignored; body %s", st, body) + } + // The ETag digest algorithm is covered separately; CompleteMultipartUpload + // matches on whatever the UploadPart response returned. + partETag := awsSHA256Hex([]byte("hello")) + completeBody := s3CompleteBody([][2]string{{"1", partETag}}) + if body, st, _ := doReq("POST", partURL, []byte(completeBody), map[string]string{"x-amz-meta-other": "yes"}); st != 200 { + t.Fatalf("complete with meta -> status %d, want 200 ignored; body %s", st, body) + } + _, st, hdr = doReq("GET", base+"/metabucket/mpu.bin", nil, nil) + if st != 200 { + t.Fatalf("get mpu object -> status %d, want 200", st) + } + if got := hdr.Get("x-amz-meta-kind"); got != "mpuval" { + t.Fatalf("get mpu x-amz-meta-kind = %q, want %q (Create→Complete propagation)", got, "mpuval") + } + if hasMetaKey(hdr, "x-amz-meta-evil") || hasMetaKey(hdr, "x-amz-meta-other") { + t.Fatalf("get mpu carries ignored part/complete meta, want only create meta: %v", hdr) + } + + // MPU Create validates too: oversize -> 400 MetadataTooLarge, empty + // suffix -> 400 InvalidArgument; no upload is created. + if body, st, _ := doReq("POST", base+"/metabucket/mpu-big.bin?uploads", nil, map[string]string{"x-amz-meta-k": strings.Repeat("a", 2048)}); st != 400 || !strings.Contains(body, "MetadataTooLarge") { + t.Fatalf("mpu create oversize meta -> %d %q, want 400 MetadataTooLarge", st, body) + } + if body, st, _ := doReq("POST", base+"/metabucket/mpu-bad.bin?uploads", nil, map[string]string{"x-amz-meta-": "v"}); st != 400 || !strings.Contains(body, "InvalidArgument") { + t.Fatalf("mpu create empty-suffix meta -> %d %q, want 400 InvalidArgument", st, body) + } +} + +func hasMetaKey(hdr http.Header, key string) bool { + for k := range hdr { + if strings.ToLower(k) == key { + return true + } + } + return false +} From f051397a6874543f6f8a8d0a146833a8903d84a1 Mon Sep 17 00:00:00 2001 From: Alessandro De Blasis Date: Thu, 1 Oct 2026 10:22:03 +0300 Subject: [PATCH 3/3] docs: S3 user metadata behavior and validation --- CHANGELOG.md | 13 +++++++++++++ CONFORMANCE.md | 5 +++-- adapters/aws-s3-style/README.md | 20 +++++++++++++++++++- conformance/matrix.json | 1 + conformance/matrix.yaml | 1 + 5 files changed, 37 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 542a6f0d..a754b238 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,19 @@ All notable changes to **stunt** are documented here. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). +## [Unreleased] + +### Adapters + +- **aws-s3-style: user metadata.** `x-amz-meta-*` request headers are stored + with the object and echoed on GET/HEAD, with control-byte and 2 KB total + validation. + +### Engine + +- **Response headers keep the adapter's casing.** `x-amz-meta-*` is emitted + lowercase, as real S3 does, instead of Go's canonical form. + ## [0.52.0] — 2026-08-24 The conformance campaign: every real API adapter now carries a real test diff --git a/CONFORMANCE.md b/CONFORMANCE.md index b43255d3..86101faf 100644 --- a/CONFORMANCE.md +++ b/CONFORMANCE.md @@ -40,7 +40,7 @@ Behavior columns come in two kinds: **verified** (an official SDK was driven aga | [avalara-style](adapters/avalara-style/) | Avalara AvaTax REST API `2` | 8 | VM | — | — | [5](#avalara-style) | [3](#avalara-style) | | [aws-cognito-style](adapters/aws-cognito-style/) | Amazon Cognito Identity Provider API `2016-04-18` | 7 | VM | — | — | [6](#aws-cognito-style) | [3](#aws-cognito-style) | | [aws-iam-sts-style](adapters/aws-iam-sts-style/) | AWS STS + IAM API `2011-06-15` | 2 | SDK | aws-sdk-go-v2 @ v1.43.7 | 2 | [3](#aws-iam-sts-style) | [3](#aws-iam-sts-style) | -| [aws-s3-style](adapters/aws-s3-style/) | Amazon S3 API `2006-03-01` | 8 | SDK | aws-sdk-go-v2 @ v1.43.7 | 6 | [5](#aws-s3-style) | [5](#aws-s3-style) | +| [aws-s3-style](adapters/aws-s3-style/) | Amazon S3 API `2006-03-01` | 8 | SDK | aws-sdk-go-v2 @ v1.43.7 | 6 | [5](#aws-s3-style) | [6](#aws-s3-style) | | [azure-devops-style](adapters/azure-devops-style/) | Azure DevOps REST API `7.1` | 17 | VM | — | — | [8](#azure-devops-style) | [6](#azure-devops-style) | | [azure-servicebus-style](adapters/azure-servicebus-style/) | Azure Service Bus + Storage `2024-01-01` | 18 | VM | — | — | [6](#azure-servicebus-style) | [3](#azure-servicebus-style) | | [azure-storage-style](adapters/azure-storage-style/) | Azure Storage Blob REST API `2024-08-04` | 9 | VM | — | — | [6](#azure-storage-style) | [3](#azure-storage-style) | @@ -2598,11 +2598,12 @@ behavior notes live in each adapter's README. - No browser form POST uploads (POST policy) - No ListMultipartUploads (GET /{bucket}?uploads) -**Deviations** (5) +**Deviations** (6) - ETags are SHA-256-based (real S3 uses MD5); multipart ETag is sha256(etags)-N - Multipart 5 MiB minimum part size not enforced (small parts allowed) - DELETE of a missing bucket is an idempotent 204 (real S3: 404 NoSuchBucket) +- x-amz-meta-* suffixes are lowercased and the first occurrence wins - SigV4 canonical URI/query rebuilt from decoded values — duplicates indistinguishable - No RFC 1123 Date fallback; canonical header whitespace collapsing not applied diff --git a/adapters/aws-s3-style/README.md b/adapters/aws-s3-style/README.md index a25c51c9..355b0736 100644 --- a/adapters/aws-s3-style/README.md +++ b/adapters/aws-s3-style/README.md @@ -109,6 +109,23 @@ all list filters are applied before pagination, as in real S3. All XML responses use the correct S3 namespace: `http://s3.amazonaws.com/doc/2006-03-01/`. +### User metadata + +Request headers with the `x-amz-meta-` prefix are stored with the object +and echoed on `GET` and `HEAD` with status `200`. They are not returned on +error responses or on `ListObjectsV2`, matching real S3. + +For multipart uploads, metadata is captured at `CreateMultipartUpload` and +propagated at `CompleteMultipartUpload`; `UploadPart` metadata is ignored. A +`PUT` without metadata clears whatever was stored before. + +Validation rejects `\r`, `\n`, NUL, and other C0 bytes except TAB with +`400 InvalidArgument`, and caps total user metadata at 2048 bytes with +`400 MetadataTooLarge`. Suffixes are lowercased and the first occurrence +wins. Response headers are emitted with the adapter's own casing rather +than Go's canonical form, because real S3 sends them lowercase and SDKs +preserve the suffix case after stripping the prefix. + ## Auth — AWS Signature Version 4 (SigV4), verified for real Amazon S3 uses **AWS Signature Version 4** (SigV4) for authentication. This @@ -256,10 +273,11 @@ All errors use S3-shaped XML: | `InvalidAccessKeyId` | 403 | Access key is not the documented synthetic AKID | | `RequestTimeTooSkewed` | 403 | `x-amz-date` outside the ±15-minute window | | `XAmzContentSHA256Mismatch` | 400 | `x-amz-content-sha256` header does not match the body bytes | -| `InvalidArgument` | 400 | `encoding-type` other than `url` on a list request; invalid `x-amz-content-sha256`; `partNumber` outside `1..10000` | +| `InvalidArgument` | 400 | `encoding-type` other than `url` on a list request; invalid `x-amz-content-sha256`; `partNumber` outside `1..10000`; an `x-amz-meta-*` name is empty or its value holds a rejected control byte | | `InvalidPart` | 400 | CompleteMultipartUpload lists a part that was never uploaded, or whose ETag does not match | | `InvalidPartOrder` | 400 | CompleteMultipartUpload part list is not in ascending order | | `MalformedXML` | 400 | CompleteMultipartUpload body is not valid `CompleteMultipartUpload` XML | +| `MetadataTooLarge` | 400 | `x-amz-meta-*` exceeds 2048 bytes total | | `MethodNotAllowed` | 405 | POST to an object without `?uploads`/`?uploadId` | | `NoSuchBucket` | 404 | Bucket doesn't exist | | `NoSuchKey` | 404 | Object key doesn't exist | diff --git a/conformance/matrix.json b/conformance/matrix.json index af35391f..db33b4c2 100644 --- a/conformance/matrix.json +++ b/conformance/matrix.json @@ -9383,6 +9383,7 @@ "ETags are SHA-256-based (real S3 uses MD5); multipart ETag is sha256(etags)-N", "Multipart 5 MiB minimum part size not enforced (small parts allowed)", "DELETE of a missing bucket is an idempotent 204 (real S3: 404 NoSuchBucket)", + "x-amz-meta-* suffixes are lowercased and the first occurrence wins", "SigV4 canonical URI/query rebuilt from decoded values — duplicates indistinguishable", "No RFC 1123 Date fallback; canonical header whitespace collapsing not applied" ], diff --git a/conformance/matrix.yaml b/conformance/matrix.yaml index 9245a04a..cb41ff95 100644 --- a/conformance/matrix.yaml +++ b/conformance/matrix.yaml @@ -152,6 +152,7 @@ adapters: - "ETags are SHA-256-based (real S3 uses MD5); multipart ETag is sha256(etags)-N" - "Multipart 5 MiB minimum part size not enforced (small parts allowed)" - "DELETE of a missing bucket is an idempotent 204 (real S3: 404 NoSuchBucket)" + - "x-amz-meta-* suffixes are lowercased and the first occurrence wins" - "SigV4 canonical URI/query rebuilt from decoded values — duplicates indistinguishable" - "No RFC 1123 Date fallback; canonical header whitespace collapsing not applied" missing: