From 93acc072d96d656e5859b1e0edbb34066d7aee62 Mon Sep 17 00:00:00 2001 From: Saumya Shah Date: Sun, 20 Sep 2026 10:06:26 +0530 Subject: [PATCH] feat(db): introduce new seaweedfs object store usage for e2e Signed-off-by: Saumya Shah refactor a bit make a generalized constant for s3 secret and access key Signed-off-by: Saumya Shah defer close idle conn in ready() check and add -master.telemetry=false so non-related to e2e stuff doesnt start Signed-off-by: Saumya Shah adress pr review changes, add optional iceberg port expose option Signed-off-by: Saumya Shah update deps in examples/thanos and lint changes Signed-off-by: Saumya Shah --- README.md | 2 +- db/db.go | 163 ++++++++++++++++++++++++++++++++++++++++- db/db_test.go | 28 ++++++- examples/thanos/go.mod | 9 +++ examples/thanos/go.sum | 6 ++ 5 files changed, 201 insertions(+), 7 deletions(-) diff --git a/README.md b/README.md index 34367ba..454684d 100644 --- a/README.md +++ b/README.md @@ -48,7 +48,7 @@ Let's go through an example leveraging the `go test` flow: 4. Use `e2emon.AsInstrumented` if you want to be able to query your service for metrics, which is a great way to assess it's internal state in tests! For example see following Etcd definition: - ```go mdox-exec="sed -n '462,474p' db/db.go" + ```go mdox-exec="sed -n '485,497p' db/db.go" return e2emon.AsInstrumented(env.Runnable(name).WithPorts(map[string]int{AccessPortName: 2379, "metrics": 9000}).Init( e2e.StartOptions{ Image: o.image, diff --git a/db/db.go b/db/db.go index cc140da..0016019 100644 --- a/db/db.go +++ b/db/db.go @@ -6,13 +6,16 @@ package e2edb import ( + "context" "crypto/rand" "crypto/rsa" + "crypto/tls" "crypto/x509" "encoding/pem" "fmt" "math/big" "net" + "net/http" "os" "path/filepath" "strconv" @@ -23,11 +26,13 @@ import ( "github.com/efficientgo/e2e" e2emon "github.com/efficientgo/e2e/monitoring" e2eprof "github.com/efficientgo/e2e/profiling" + "github.com/minio/minio-go/v7" + "github.com/minio/minio-go/v7/pkg/credentials" ) const ( - MinioAccessKey = "Cheescake" - MinioSecretKey = "supersecret" + S3AccessKey = "Cheescake" + S3SecretKey = "supersecret" ) type Option func(*options) @@ -36,6 +41,7 @@ type options struct { image string flagOverride map[string]string minioOptions minioOptions + seaweedOptions seaweedFSOptions azuriteOptions azuriteOptions } @@ -50,6 +56,11 @@ type minioOptions struct { enableTLS bool } +type seaweedFSOptions struct { + enableTLS bool + enableIceberg bool +} + func WithImage(image string) Option { return func(o *options) { o.image = image @@ -74,6 +85,18 @@ func WithMinioTLS() Option { } } +func WithSeaweedFSTLS() Option { + return func(o *options) { + o.seaweedOptions.enableTLS = true + } +} + +func WithSeaweedFSIceberg() Option { + return func(o *options) { + o.seaweedOptions.enableIceberg = true + } +} + func WithAzuriteStorageAccounts(acc string) Option { return func(o *options) { o.azuriteOptions.customStorageAccounts = acc @@ -116,8 +139,8 @@ func NewMinio(env e2e.Environment, name, bktName string, opts ...Option) *e2emon userID := strconv.Itoa(os.Getuid()) ports := map[string]int{AccessPortName: 8090} envVars := []string{ - "MINIO_ROOT_USER=" + MinioAccessKey, - "MINIO_ROOT_PASSWORD=" + MinioSecretKey, + "MINIO_ROOT_USER=" + S3AccessKey, + "MINIO_ROOT_PASSWORD=" + S3SecretKey, "MINIO_BROWSER=" + "off", } @@ -473,3 +496,135 @@ func NewETCD(env e2e.Environment, name string, opts ...Option) *e2emon.Instrumen }, ), "metrics") } + +func NewSeaweedFS(env e2e.Environment, name, bucket string, opts ...Option) *e2emon.InstrumentedRunnable { + o := options{image: "chrislusf/seaweedfs:4.47"} + for _, opt := range opts { + opt(&o) + } + + const ( + adminPortName = "admin" + icebergPortName = "iceberg" + seaweedFSIcebergPort = 8181 + ) + ports := map[string]int{ + AccessPortName: 8333, + adminPortName: 23646, + } + icebergPort := 0 + if o.seaweedOptions.enableIceberg { + icebergPort = seaweedFSIcebergPort + ports[icebergPortName] = seaweedFSIcebergPort + } + f := env.Runnable(name).WithPorts(ports).Future() + dataDir := filepath.Join(f.Dir(), "data") + if err := os.MkdirAll(dataDir, 0750); err != nil { + return &e2emon.InstrumentedRunnable{Runnable: e2e.NewFailedRunnable(name, errors.Wrap(err, "create SeaweedFS data directory"))} + } + + args := []string{ + "mini", + "-dir=/data", + fmt.Sprintf("-s3.port=%d", ports[AccessPortName]), + "-master.telemetry=false", + "-webdav=false", + fmt.Sprintf("-s3.port.iceberg=%d", icebergPort), + "-s3.port.lance=0", + "-s3.autoCreateBucket=false", + } + readiness := e2e.ReadinessProbe(e2e.NewHTTPReadinessProbe(AccessPortName, "/readyz", http.StatusOK, http.StatusOK)) + instrumentedOpts := []e2emon.InstrumentedOption{} + caFile := "" + + if o.seaweedOptions.enableTLS { + certDir := filepath.Join(f.Dir(), "certs") + caDir := filepath.Join(certDir, "CAs") + if err := os.MkdirAll(caDir, 0750); err != nil { + return &e2emon.InstrumentedRunnable{Runnable: e2e.NewFailedRunnable(name, errors.Wrap(err, "create SeaweedFS certificate directory"))} + } + + certFile := filepath.Join(certDir, "public.crt") + keyFile := filepath.Join(certDir, "private.key") + caFile = filepath.Join(caDir, "ca.crt") + if err := genCerts(certFile, keyFile, caFile, fmt.Sprintf("%s-%s", env.Name(), name)); err != nil { + return &e2emon.InstrumentedRunnable{Runnable: e2e.NewFailedRunnable(name, errors.Wrap(err, "generate SeaweedFS certificates"))} + } + + args = append(args, "-s3.cert.file="+certFile, "-s3.key.file="+keyFile) + readiness = e2e.NewHTTPSReadinessProbe(AccessPortName, "/readyz", http.StatusOK, http.StatusOK) + instrumentedOpts = append(instrumentedOpts, e2emon.WithInstrumentedScheme("https")) + } + + envVars := map[string]string{ + "AWS_ACCESS_KEY_ID": S3AccessKey, + "AWS_SECRET_ACCESS_KEY": S3SecretKey, + } + if bucket != "" { + envVars["S3_BUCKET"] = bucket + readiness = &seaweedFSBucketReadinessProbe{ + readiness: readiness, + bucket: bucket, + enableTLS: o.seaweedOptions.enableTLS, + caFile: caFile, + } + } + + r := f.Init(e2e.StartOptions{ + Image: o.image, + User: strconv.Itoa(os.Getuid()), + Command: e2e.NewCommand(args[0], args[1:]...), + EnvVars: envVars, + Readiness: readiness, + Volumes: []string{dataDir + ":/data:z"}, + }) + return e2emon.AsInstrumented(r, AccessPortName, instrumentedOpts...) +} + +type seaweedFSBucketReadinessProbe struct { + readiness e2e.ReadinessProbe + bucket string + enableTLS bool + caFile string +} + +// This explicitly checks if bucket exists or not (seaweedfs s3 gateway server might report healthy while bucket creation is still pending, which happens in later stage). +func (p *seaweedFSBucketReadinessProbe) Ready(runnable e2e.Runnable) error { + if err := p.readiness.Ready(runnable); err != nil { + return err + } + + transport := http.DefaultTransport.(*http.Transport).Clone() + defer transport.CloseIdleConnections() + if p.enableTLS { + caPEM, err := os.ReadFile(p.caFile) + if err != nil { + return errors.Wrap(err, "read SeaweedFS CA certificate") + } + rootCAs := x509.NewCertPool() + if !rootCAs.AppendCertsFromPEM(caPEM) { + return errors.New("parse SeaweedFS CA certificate") + } + transport.TLSClientConfig = &tls.Config{RootCAs: rootCAs} + } + + client, err := minio.New(runnable.Endpoint(AccessPortName), &minio.Options{ + Creds: credentials.NewStaticV4(S3AccessKey, S3SecretKey, ""), + Secure: p.enableTLS, + Transport: transport, + }) + if err != nil { + return errors.Wrap(err, "create SeaweedFS readiness client") + } + + ctx, cancel := context.WithTimeout(context.Background(), time.Second) + defer cancel() + exists, err := client.BucketExists(ctx, p.bucket) + if err != nil { + return errors.Wrap(err, "check SeaweedFS bucket") + } + if !exists { + return errors.Newf("SeaweedFS bucket %q does not exist", p.bucket) + } + return nil +} diff --git a/db/db_test.go b/db/db_test.go index 37d0639..8b8e8c0 100644 --- a/db/db_test.go +++ b/db/db_test.go @@ -25,8 +25,8 @@ func TestMinio(t *testing.T) { testutil.Ok(t, e2e.StartAndWaitReady(minioContainer)) endpoint := minioContainer.Endpoint("http") - accessKeyID := MinioAccessKey - secretAccessKey := MinioSecretKey + accessKeyID := S3AccessKey + secretAccessKey := S3SecretKey minioClient, err := minio.New(endpoint, &minio.Options{ Creds: credentials.NewStaticV4(accessKeyID, secretAccessKey, ""), Secure: false, @@ -38,3 +38,27 @@ func TestMinio(t *testing.T) { minioClient.MakeBucket(context.Background(), "test-bucket", minio.MakeBucketOptions{}), ) } + +func TestSeaweedFS(t *testing.T) { + for _, tc := range []struct { + name string + secure bool + }{ + {name: "HTTP"}, + {name: "HTTPS", secure: true}, + } { + t.Run(tc.name, func(t *testing.T) { + e, err := e2e.New() + testutil.Ok(t, err) + t.Cleanup(e.Close) + + opts := []Option{} + if tc.secure { + opts = append(opts, WithSeaweedFSTLS()) + } + const bucket = "test-bucket" + seaweedFS := NewSeaweedFS(e, "seaweedfs", bucket, opts...) + testutil.Ok(t, e2e.StartAndWaitReady(seaweedFS)) + }) + } +} diff --git a/examples/thanos/go.mod b/examples/thanos/go.mod index c76b29e..8ce7cd7 100644 --- a/examples/thanos/go.mod +++ b/examples/thanos/go.mod @@ -14,23 +14,32 @@ require ( github.com/beorn7/perks v1.0.1 // indirect github.com/cespare/xxhash/v2 v2.3.0 // indirect github.com/davecgh/go-spew v1.1.1 // indirect + github.com/dustin/go-humanize v1.0.0 // indirect github.com/golang-jwt/jwt/v5 v5.3.0 // indirect github.com/google/uuid v1.6.0 // indirect github.com/jpillora/backoff v1.0.0 // indirect github.com/json-iterator/go v1.1.12 // indirect github.com/klauspost/compress v1.17.9 // indirect + github.com/klauspost/cpuid/v2 v2.1.0 // indirect + github.com/minio/md5-simd v1.1.2 // indirect + github.com/minio/minio-go/v7 v7.0.45 // indirect + github.com/minio/sha256-simd v1.0.0 // indirect github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect github.com/modern-go/reflect2 v1.0.2 // indirect github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f // indirect github.com/prometheus/client_model v0.6.2 // indirect github.com/prometheus/procfs v0.15.1 // indirect + github.com/rs/xid v1.4.0 // indirect + github.com/sirupsen/logrus v1.9.0 // indirect go.yaml.in/yaml/v2 v2.4.3 // indirect + golang.org/x/crypto v0.46.0 // indirect golang.org/x/net v0.48.0 // indirect golang.org/x/oauth2 v0.34.0 // indirect golang.org/x/sys v0.39.0 // indirect golang.org/x/text v0.32.0 // indirect google.golang.org/protobuf v1.36.11 // indirect + gopkg.in/ini.v1 v1.66.6 // indirect gopkg.in/yaml.v2 v2.4.0 // indirect ) diff --git a/examples/thanos/go.sum b/examples/thanos/go.sum index 10f6172..d749acb 100644 --- a/examples/thanos/go.sum +++ b/examples/thanos/go.sum @@ -22,6 +22,8 @@ github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnr github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= github.com/klauspost/compress v1.17.9 h1:6KIumPrER1LHsvBVuDa0r5xaG0Es51mhhB9BQB2qeMA= github.com/klauspost/compress v1.17.9/go.mod h1:Di0epgTjJY877eYKx5yC51cX2A2Vl2ibi7bDH9ttBbw= +github.com/klauspost/cpuid/v2 v2.0.1/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg= +github.com/klauspost/cpuid/v2 v2.0.4/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg= github.com/klauspost/cpuid/v2 v2.1.0 h1:eyi1Ad2aNJMW95zcSbmGg7Cg6cq3ADwLpMAP96d8rF0= github.com/klauspost/cpuid/v2 v2.1.0/go.mod h1:RVVoqg1df56z8g3pUjL/3lE5UfnlrJX8tyFgg4nqhuY= github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= @@ -65,6 +67,7 @@ github.com/sirupsen/logrus v1.9.0 h1:trlNQbNUG3OdDrDil03MCb1H2o9nJ1x4/5LYw7byDE0 github.com/sirupsen/logrus v1.9.0/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= +github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U= github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U= go.yaml.in/yaml/v2 v2.4.3 h1:6gvOSjQoTB3vt1l+CU+tSyi/HOjfOjRLJ4YwYZGwRO0= @@ -75,6 +78,8 @@ golang.org/x/net v0.48.0 h1:zyQRTTrjc33Lhh0fBgT/H3oZq9WuvRR5gPC70xpDiQU= golang.org/x/net v0.48.0/go.mod h1:+ndRgGjkh8FGtu1w1FGbEC31if4VrNVMuKTgcAAnQRY= golang.org/x/oauth2 v0.34.0 h1:hqK/t4AKgbqWkdkcAeI8XLmbK+4m4G5YeQRrmiotGlw= golang.org/x/oauth2 v0.34.0/go.mod h1:lzm5WQJQwKZ3nwavOZ3IS5Aulzxi68dUSgRHujetwEA= +golang.org/x/sys v0.0.0-20220704084225-05e143d24a9e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.39.0 h1:CvCKL8MeisomCi6qNZ+wbb0DN9E5AATixKsvNtMoMFk= golang.org/x/sys v0.39.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks= golang.org/x/text v0.32.0 h1:ZD01bjUt1FQ9WJ0ClOL5vxgxOI/sVCNgX1YtKwcY0mU= @@ -88,5 +93,6 @@ gopkg.in/ini.v1 v1.66.6 h1:LATuAqN/shcYAOkv3wl2L4rkaKqkcgTBQjOyYDvcPKI= gopkg.in/ini.v1 v1.66.6/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=