From f2ef8da19053801a005a421df3f6b383b76fed86 Mon Sep 17 00:00:00 2001 From: MoeMahhouk Date: Wed, 19 Aug 2026 14:25:31 +0000 Subject: [PATCH 1/2] feat: add optional Prometheus metrics endpoint New -metrics-addr flag (env METRICS_ADDR, empty = disabled) serves /metrics on a separate listener. Records syncproxy_forwards_total {method,backend,result} where success means the backend answered 2xx (i.e. the EL demonstrably received and accepted the forwarded call, including in mirror mode where the caller only gets an empty ACK), and a forward duration histogram. --- go.mod | 14 ++++++++++++-- go.sum | 32 ++++++++++++++++++++++++++++---- main.go | 21 +++++++++++++++------ metrics.go | 44 ++++++++++++++++++++++++++++++++++++++++++++ metrics_test.go | 47 +++++++++++++++++++++++++++++++++++++++++++++++ proxy.go | 3 +++ 6 files changed, 149 insertions(+), 12 deletions(-) create mode 100644 metrics.go create mode 100644 metrics_test.go diff --git a/go.mod b/go.mod index 74fb566..0811def 100644 --- a/go.mod +++ b/go.mod @@ -7,7 +7,9 @@ toolchain go1.24.2 require ( github.com/Microsoft/go-winio v0.6.2 // indirect github.com/ProjectZKM/Ziren/crates/go-runtime/zkvm_runtime v0.0.0-20251001021608-1fe7b43fc4d6 // indirect + github.com/beorn7/perks v1.0.1 // indirect github.com/bits-and-blooms/bitset v1.20.0 // indirect + github.com/cespare/xxhash/v2 v2.3.0 // indirect github.com/consensys/gnark-crypto v0.18.0 // indirect github.com/crate-crypto/go-eth-kzg v1.4.0 // indirect github.com/crate-crypto/go-ipa v0.0.0-20240724233137-53bbb0ceb27a // indirect @@ -24,20 +26,27 @@ require ( github.com/gorilla/websocket v1.5.0 // indirect github.com/holiman/uint256 v1.3.2 // indirect github.com/klauspost/cpuid/v2 v2.0.9 // indirect + github.com/kylelemons/godebug v1.1.0 // indirect github.com/mattn/go-runewidth v0.0.14 // indirect github.com/minio/sha256-simd v1.0.0 // indirect github.com/mitchellh/mapstructure v1.4.1 // indirect + github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect github.com/olekukonko/tablewriter v0.0.5 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect + github.com/prometheus/client_model v0.6.2 // indirect + github.com/prometheus/common v0.66.1 // indirect + github.com/prometheus/procfs v0.16.1 // indirect github.com/rivo/uniseg v0.4.4 // indirect github.com/shirou/gopsutil v3.21.11+incompatible // indirect github.com/supranational/blst v0.3.16-0.20250831170142-f48500c1fdbe // indirect github.com/tklauser/go-sysconf v0.3.12 // indirect github.com/tklauser/numcpus v0.6.1 // indirect github.com/yusufpapurcu/wmi v1.2.2 // indirect + go.yaml.in/yaml/v2 v2.4.2 // indirect golang.org/x/crypto v0.36.0 // indirect - golang.org/x/sync v0.12.0 // indirect + golang.org/x/sync v0.13.0 // indirect golang.org/x/sys v0.36.0 // indirect + google.golang.org/protobuf v1.36.8 // indirect gopkg.in/yaml.v2 v2.4.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect ) @@ -46,6 +55,7 @@ require ( github.com/ethereum/go-ethereum v1.16.7 github.com/golang-jwt/jwt/v5 v5.2.2 github.com/gorilla/mux v1.8.0 + github.com/prometheus/client_golang v1.23.2 github.com/sirupsen/logrus v1.9.3 - github.com/stretchr/testify v1.10.0 + github.com/stretchr/testify v1.11.1 ) diff --git a/go.sum b/go.sum index 6f08875..8a975da 100644 --- a/go.sum +++ b/go.sum @@ -2,8 +2,12 @@ github.com/Microsoft/go-winio v0.6.2 h1:F2VQgta7ecxGYO8k3ZZz3RS8fVIXVxONVUPlNERo github.com/Microsoft/go-winio v0.6.2/go.mod h1:yd8OoFMLzJbo9gZq8j5qaps8bJ9aShtEA8Ipt1oGCvU= github.com/ProjectZKM/Ziren/crates/go-runtime/zkvm_runtime v0.0.0-20251001021608-1fe7b43fc4d6 h1:1zYrtlhrZ6/b6SAjLSfKzWtdgqK0U+HtH/VcBWh1BaU= github.com/ProjectZKM/Ziren/crates/go-runtime/zkvm_runtime v0.0.0-20251001021608-1fe7b43fc4d6/go.mod h1:ioLG6R+5bUSO1oeGSDxOV3FADARuMoytZCSX6MEMQkI= +github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= +github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= github.com/bits-and-blooms/bitset v1.20.0 h1:2F+rfL86jE2d/bmw7OhqUg2Sj/1rURkBn3MdfoPyRVU= github.com/bits-and-blooms/bitset v1.20.0/go.mod h1:7hO7Gc7Pp1vODcmWvKMRA9BNmbv6a/7QIWpPxHddWR8= +github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= +github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/consensys/gnark-crypto v0.18.0 h1:vIye/FqI50VeAr0B3dx+YjeIvmc3LWz4yEfbWBpTUf0= github.com/consensys/gnark-crypto v0.18.0/go.mod h1:L3mXGFTe1ZN+RSJ+CLjUt9x7PNdx8ubaYfDROyp2Z8c= github.com/crate-crypto/go-eth-kzg v1.4.0 h1:WzDGjHk4gFg6YzV0rJOAsTK4z3Qkz5jd4RE3DAvPFkg= @@ -38,12 +42,16 @@ github.com/golang-jwt/jwt/v5 v5.2.2 h1:Rl4B7itRWVtYIHFrSNd7vhTiz9UpLdi6gZhZ3wEeD github.com/golang-jwt/jwt/v5 v5.2.2/go.mod h1:pqrtFR0X4osieyHYxtmOUWsAWrfe1Q5UVIyoH402zdk= github.com/golang/snappy v1.0.0 h1:Oy607GVXHs7RtbggtPBnr2RmDArIsAefDwvrdWvRhGs= github.com/golang/snappy v1.0.0/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= +github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8= +github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU= github.com/gorilla/mux v1.8.0 h1:i40aqfkR1h2SlN9hojwV5ZA91wcXFOvkdNIeFDP5koI= github.com/gorilla/mux v1.8.0/go.mod h1:DVbg23sWSpFRCP0SfiEN6jmj59UnW/n46BH5rLB71So= github.com/gorilla/websocket v1.5.0 h1:PPwGk2jz7EePpoHN/+ClbZu8SPxiqlu12wZP/3sWmnc= github.com/gorilla/websocket v1.5.0/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= github.com/holiman/uint256 v1.3.2 h1:a9EgMPSC1AAaj1SZL5zIQD3WbwTuHrMGOerLjGmM/TA= github.com/holiman/uint256 v1.3.2/go.mod h1:EOMSn4q6Nyt9P6efbI3bueV4e1b3dGlUCXeiRV4ng7E= +github.com/klauspost/compress v1.18.0 h1:c/Cqfb0r+Yi+JtIEq73FWXVkRonBlf0CRNYc8Zttxdo= +github.com/klauspost/compress v1.18.0/go.mod h1:2Pp+KzxcywXVXMr50+X0Q/Lsb43OQHYWRCY2AiWywWQ= github.com/klauspost/cpuid/v2 v2.0.4/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg= github.com/klauspost/cpuid/v2 v2.0.9 h1:lgaqFMSdTdQYdZ04uHyN2d/eKdOMyi2YLSvlQIBFYa4= github.com/klauspost/cpuid/v2 v2.0.9/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg= @@ -62,10 +70,20 @@ github.com/minio/sha256-simd v1.0.0 h1:v1ta+49hkWZyvaKwrQB8elexRqm6Y0aMLjCNsrYxo github.com/minio/sha256-simd v1.0.0/go.mod h1:OuYzVNI5vcoYIAmbIvHPl3N3jUzVedXbKy5RFepssQM= github.com/mitchellh/mapstructure v1.4.1 h1:CpVNEelQCZBooIPDn+AR3NpivK/TIKU8bDxdASFVQag= github.com/mitchellh/mapstructure v1.4.1/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= +github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA= +github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= github.com/olekukonko/tablewriter v0.0.5 h1:P2Ga83D34wi1o9J6Wh1mRuqd4mF/x/lgBS7N7AbDhec= github.com/olekukonko/tablewriter v0.0.5/go.mod h1:hPp6KlRPjbx+hW8ykQs1w3UBbZlj6HuIJcUGPhkA7kY= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/prometheus/client_golang v1.23.2 h1:Je96obch5RDVy3FDMndoUsjAhG5Edi49h0RJWRi/o0o= +github.com/prometheus/client_golang v1.23.2/go.mod h1:Tb1a6LWHB3/SPIzCoaDXI4I8UHKeFTEQ1YCr+0Gyqmg= +github.com/prometheus/client_model v0.6.2 h1:oBsgwpGs7iVziMvrGhE53c/GrLUsZdHnqNwqPLxwZyk= +github.com/prometheus/client_model v0.6.2/go.mod h1:y3m2F6Gdpfy6Ut/GBsUqTWZqCUvMVzSfMLjcu6wAwpE= +github.com/prometheus/common v0.66.1 h1:h5E0h5/Y8niHc5DlaLlWLArTQI7tMrsfQjHV+d9ZoGs= +github.com/prometheus/common v0.66.1/go.mod h1:gcaUsgf3KfRSwHY4dIMXLPV0K/Wg1oZ8+SbZk/HH/dA= +github.com/prometheus/procfs v0.16.1 h1:hZ15bTNuirocR6u0JZ6BAHHmwS1p8B4P6MRqxtzMyRg= +github.com/prometheus/procfs v0.16.1/go.mod h1:teAbpZRB1iIAJYREa1LsoWUXykVXA1KlTmWl8x/U+Is= github.com/prysmaticlabs/gohashtree v0.0.4-beta h1:H/EbCuXPeTV3lpKeXGPpEV9gsUpkqOOVnWapUyeWro4= github.com/prysmaticlabs/gohashtree v0.0.4-beta/go.mod h1:BFdtALS+Ffhg3lGQIHv9HDWuHS8cTvHZzrHWxwOtGOs= github.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc= @@ -79,8 +97,8 @@ github.com/sirupsen/logrus v1.9.3 h1:dueUQJ1C2q9oE3F7wvmSGAaVtTmUizReu6fjN8uqzbQ github.com/sirupsen/logrus v1.9.3/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= -github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOfJA= -github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= +github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U= +github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U= github.com/supranational/blst v0.3.16-0.20250831170142-f48500c1fdbe h1:nbdqkIGOGfUAD54q1s2YBcBz/WcsxCO9HUQ4aGV5hUw= github.com/supranational/blst v0.3.16-0.20250831170142-f48500c1fdbe/go.mod h1:jZJtfjgudtNl4en1tzwPIV3KjUnQUvG3/j+w+fVonLw= github.com/tklauser/go-sysconf v0.3.12 h1:0QaGUFOdQaIVdPgfITYzaTegZvdCjmYO52cSFAEVmqU= @@ -89,10 +107,14 @@ github.com/tklauser/numcpus v0.6.1 h1:ng9scYS7az0Bk4OZLvrNXNSAO2Pxr1XXRAPyjhIx+F github.com/tklauser/numcpus v0.6.1/go.mod h1:1XfjsgE2zo8GVw7POkMbHENHzVg3GzmoZ9fESEdAacY= github.com/yusufpapurcu/wmi v1.2.2 h1:KBNDSne4vP5mbSWnJbO+51IMOXJB67QiYCSBrubbPRg= github.com/yusufpapurcu/wmi v1.2.2/go.mod h1:SBZ9tNy3G9/m5Oi98Zks0QjeHVDvuK0qfxQmPyzfmi0= +go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= +go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE= +go.yaml.in/yaml/v2 v2.4.2 h1:DzmwEr2rDGHl7lsFgAHxmNz/1NlQ7xLIrlN2h5d1eGI= +go.yaml.in/yaml/v2 v2.4.2/go.mod h1:081UH+NErpNdqlCXm3TtEran0rJZGxAYx9hb/ELlsPU= golang.org/x/crypto v0.36.0 h1:AnAEvhDddvBdpY+uR+MyHmuZzzNqXSe/GvuDeob5L34= golang.org/x/crypto v0.36.0/go.mod h1:Y4J0ReaxCR1IMaabaSMugxJES1EpwhBHhv2bDHklZvc= -golang.org/x/sync v0.12.0 h1:MHc5BpPuC30uJk597Ri8TV3CNZcTLu6B6z4lJy+g6Jw= -golang.org/x/sync v0.12.0/go.mod h1:1dzgHSNfp02xaA81J2MS99Qcpr2w7fw1gpm99rleRqA= +golang.org/x/sync v0.13.0 h1:AauUjRAJ9OSnvULf/ARrrVywoJDy0YS2AwQ98I37610= +golang.org/x/sync v0.13.0/go.mod h1:1dzgHSNfp02xaA81J2MS99Qcpr2w7fw1gpm99rleRqA= golang.org/x/sys v0.0.0-20190916202348-b4ddaad3f8a3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= @@ -100,6 +122,8 @@ golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.11.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.36.0 h1:KVRy2GtZBrk1cBYA7MKu5bEZFxQk4NIDV6RLVcC8o0k= golang.org/x/sys v0.36.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks= +google.golang.org/protobuf v1.36.8 h1:xHScyCOEuuwZEc6UtSOvPbAT4zRh0xcNRYekJwfqyMc= +google.golang.org/protobuf v1.36.8/go.mod h1:fuxRtAxBytpl4zzqUh6/eyUujkJdNiuEkXntxiD/uRU= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= diff --git a/main.go b/main.go index c69e2b8..7efc57c 100644 --- a/main.go +++ b/main.go @@ -15,11 +15,12 @@ var ( version = "dev" // is set during build process // Default values - defaultLogLevel = getEnv("LOG_LEVEL", "info") - defaultLogJSON = os.Getenv("LOG_JSON") != "" - defaultListenAddr = getEnv("PROXY_LISTEN_ADDR", "localhost:25590") - defaultTimeoutMs = getEnvInt("BUILDER_TIMEOUT_MS", 2000) // timeout for all the requests to the builders - defaultMirrorMode = os.Getenv("MIRROR_MODE") != "" + defaultLogLevel = getEnv("LOG_LEVEL", "info") + defaultLogJSON = os.Getenv("LOG_JSON") != "" + defaultListenAddr = getEnv("PROXY_LISTEN_ADDR", "localhost:25590") + defaultTimeoutMs = getEnvInt("BUILDER_TIMEOUT_MS", 2000) // timeout for all the requests to the builders + defaultMirrorMode = os.Getenv("MIRROR_MODE") != "" + defaultMetricsAddr = getEnv("METRICS_ADDR", "") // Flags logJSON = flag.Bool("json", defaultLogJSON, "log in JSON format instead of text") @@ -30,6 +31,7 @@ var ( proxyURLs = flag.String("proxies", "", "proxy urls - other proxies to forward BN requests to (scheme://host)") proxyTimeoutMs = flag.Int("proxy-request-timeout", defaultTimeoutMs, "timeout for redundant beacon node requests to another proxy [ms]") mirrorMode = flag.Bool("mirror-mode", defaultMirrorMode, "acknowledge requests immediately with an empty 200 and forward to builders asynchronously (for use behind an nginx mirror)") + metricsAddr = flag.String("metrics-addr", defaultMetricsAddr, "listen address for the Prometheus /metrics endpoint (empty = disabled)") ) var log = logrus.WithField("module", "sync-proxy") @@ -70,11 +72,11 @@ func main() { proxyTimeout := time.Duration(*proxyTimeoutMs) * time.Millisecond - // Create a new proxy service. if *mirrorMode { log.Info("mirror mode enabled: acknowledging requests with an empty 200 and forwarding asynchronously") } + // Create a new proxy service. opts := ProxyServiceOpts{ ListenAddr: *listenAddr, Builders: builders, @@ -90,6 +92,13 @@ func main() { log.WithError(err).Fatal("failed creating the server") } + if *metricsAddr != "" { + go func() { + log.Infof("metrics listening on %s", *metricsAddr) + log.WithError(StartMetricsServer(*metricsAddr)).Fatal("metrics server exited") + }() + } + log.Println("listening on", *listenAddr) log.Fatal(proxyService.StartHTTPServer()) } diff --git a/metrics.go b/metrics.go new file mode 100644 index 0000000..510925e --- /dev/null +++ b/metrics.go @@ -0,0 +1,44 @@ +package main + +import ( + "net/http" + "time" + + "github.com/prometheus/client_golang/prometheus" + "github.com/prometheus/client_golang/prometheus/promauto" + "github.com/prometheus/client_golang/prometheus/promhttp" +) + +var ( + forwardsTotal = promauto.NewCounterVec(prometheus.CounterOpts{ + Name: "syncproxy_forwards_total", + Help: "Engine API requests forwarded to backends, by method, backend and result (success = 2xx response, http_error = non-2xx response, error = transport failure/timeout).", + }, []string{"method", "backend", "result"}) + + forwardDuration = promauto.NewHistogramVec(prometheus.HistogramOpts{ + Name: "syncproxy_forward_duration_seconds", + Help: "Duration of forwarded requests to backends.", + Buckets: prometheus.DefBuckets, + }, []string{"method", "backend"}) +) + +// StartMetricsServer serves /metrics on addr. Blocking. +func StartMetricsServer(addr string) error { + mux := http.NewServeMux() + mux.Handle("/metrics", promhttp.Handler()) + srv := &http.Server{ + Addr: addr, + Handler: mux, + ReadHeaderTimeout: 5 * time.Second, + } + return srv.ListenAndServe() +} + +func observeForward(method, backend string, statusCode int, start time.Time) { + result := "http_error" + if statusCode >= 200 && statusCode < 300 { + result = "success" + } + forwardsTotal.WithLabelValues(method, backend, result).Inc() + forwardDuration.WithLabelValues(method, backend).Observe(time.Since(start).Seconds()) +} diff --git a/metrics_test.go b/metrics_test.go new file mode 100644 index 0000000..c9993c3 --- /dev/null +++ b/metrics_test.go @@ -0,0 +1,47 @@ +package main + +import ( + "testing" + "time" + + "github.com/prometheus/client_golang/prometheus/testutil" + "github.com/stretchr/testify/require" +) + +func TestForwardMetrics(t *testing.T) { + t.Run("successful forward increments success counter", func(t *testing.T) { + backend := newTestBackend(t, 1, 0, time.Second, time.Second) + backendURL := backend.builders[0].Server.URL + + backend.request(t, []byte(mockNewPayloadRequest), from) + + require.Equal(t, float64(1), testutil.ToFloat64( + forwardsTotal.WithLabelValues(newPayloadPath, backendURL, "success"))) + require.Equal(t, float64(0), testutil.ToFloat64( + forwardsTotal.WithLabelValues(newPayloadPath, backendURL, "error"))) + }) + + t.Run("unreachable backend increments error counter", func(t *testing.T) { + backend := newTestBackend(t, 1, 0, time.Second, time.Second) + backendURL := backend.builders[0].Server.URL + backend.builders[0].Server.Close() + + backend.request(t, []byte(mockNewPayloadRequest), from) + + require.Equal(t, float64(1), testutil.ToFloat64( + forwardsTotal.WithLabelValues(newPayloadPath, backendURL, "error"))) + }) + + t.Run("mirror mode records metrics asynchronously", func(t *testing.T) { + backend := newTestBackend(t, 1, 0, time.Second, time.Second) + backend.proxyService.mirrorMode = true + backendURL := backend.builders[0].Server.URL + + backend.request(t, []byte(mockNewPayloadRequest), from) + + require.Eventually(t, func() bool { + return testutil.ToFloat64( + forwardsTotal.WithLabelValues(newPayloadPath, backendURL, "success")) == 1 + }, 2*time.Second, 10*time.Millisecond) + }) +} diff --git a/proxy.go b/proxy.go index fe411d1..fa1c154 100644 --- a/proxy.go +++ b/proxy.go @@ -235,13 +235,16 @@ func (p *ProxyService) callBuilders(req *http.Request, requestJSON JSONRPCReques go func(entry *ProxyEntry) { defer wg.Done() url := entry.URL + start := time.Now() resp, cancel, err := SendProxyRequest(req, entry, bodyBytes) if err != nil { + forwardsTotal.WithLabelValues(requestJSON.Method, url.String(), "error").Inc() log.WithError(err).WithField("url", url.String()).Error("error sending request to builder") return } defer cancel() defer resp.Body.Close() + observeForward(requestJSON.Method, url.String(), resp.StatusCode, start) responseBytes, err := io.ReadAll(resp.Body) if err != nil { From 870715b80836997b9a872843836ace21afd511c7 Mon Sep 17 00:00:00 2001 From: MoeMahhouk Date: Wed, 19 Aug 2026 14:30:27 +0000 Subject: [PATCH 2/2] docs: document mirror mode, metrics endpoint and timeout semantics --- README.md | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/README.md b/README.md index 19a8056..d7bf6f3 100644 --- a/README.md +++ b/README.md @@ -28,6 +28,14 @@ To run with multiple EL endpoins: ./sync-proxy -builders="localhost:8551,localhost:8552" ``` +### Notable flags + +- `-request-timeout` / `-proxy-request-timeout` (ms): bound each forwarded request end to end (connection, response headers and body). A backend that does not answer within the timeout is treated as failed and the proxy falls back to the next builder. +- `-mirror-mode` (env `MIRROR_MODE`): acknowledge every request immediately with an empty `200` and forward to the builders asynchronously. Intended for deployments behind an nginx `mirror`, where the caller discards responses anyway — this frees the caller instead of blocking on the slowest backend. Requests are still parsed, filtered and (bounded by `-request-timeout`) awaited internally for logging and response-divergence detection. +- `-metrics-addr` (env `METRICS_ADDR`): listen address for a Prometheus `/metrics` endpoint on a separate listener (empty = disabled). Exposes: + - `syncproxy_forwards_total{method, backend, result}` — forwarded requests; `result` is `success` (backend answered 2xx — i.e. the EL demonstrably received and accepted the call, also in mirror mode), `http_error` (non-2xx answer) or `error` (transport failure or timeout). + - `syncproxy_forward_duration_seconds{method, backend}` — forward latency histogram. + ### Nginx The sync proxy can also be used with nginx, with requests proxied from the beacon node to a local execution client and mirrored to multiple sync proxies.