feature(requests): run requests concurrently

.cz.toml

@@ -2,6 +2,6 @@
 name = "cz_conventional_commits"
 tag_format = "$version"
 version_scheme = "semver"
-version = "0.15.4"
+version = "0.16.0"
 update_changelog_on_bump = false
 major_version_zero = true

.gitignore

@@ -21,6 +21,8 @@ manpages
 
 # Output of the go coverage tool, specifically when used with LiteIDE
 *.out
+*.prof
+profiles/
 
 # Dependency directories (remove the comment below to include it)
 # vendor/

CHANGELOG.md

@@ -2,6 +2,44 @@
 
 # https-wrench - changelog
 
+## 0.16.0 (2026-09-17)
+
+### Dependencies
+
+    Deps: upgrade golang.org/x/sync to v0.23.0 and promote to direct dependency.
+
+### Feat
+
+    Requests: support concurrent HTTP requests execution via bounded worker pool (`--concurrency` / `-c` flag, default: 10).
+
+    Requests: add `concurrency` property to configuration schema and examples, defaulting to 10 if omitted or <= 0.
+
+    Requests: extend concurrency boundaries down to individual single requests, enabling multiple hosts and URIs within a single `RequestConfig` to execute concurrently.
+
+    Requests: bound active host goroutines in `processHostsConcurrently` using `errgroup.Group.SetLimit`, keeping the HTTP request limiter exclusively for URI requests.
+
+    Requests: acquire limiter slot before launching goroutines in `processURIsConcurrently` to eliminate unbounded goroutine allocation.
+
+    Requests: serialize debug output writing (`PrintRequestDebug`, `PrintResponseDebug`) via mutex to eliminate log tearing and data races across concurrent requests.
+
+    Requests: preserve deterministic slice ordering for hosts and URIs, aggregating response data by request name.
+
+### Fix
+
+    Devenv: update profiling cleanup traps to remove generated `requests.test` binary from the invocation directory instead of `internal/requests/requests.test`.
+
+    Devenv: add `EXIT` trap to aggregate profiling script to guarantee test binary cleanup on command failures, and remove redundant test binary cleanup from trace script.
+
+    Schema: remove `minimum: 0` constraint on `concurrency` in both `https-wrench.schema.json` and `internal/mcp/assets/schema.json` so negative values are accepted and fall back cleanly to 10.
+
+### Tests
+
+    Requests: add `TestRequests_ExecuteWithWriter_PeakInFlight` regression test measuring peak in-flight requests with barrier synchronization to eliminate timing jitter under race detection and coverage instrumentation.
+
+    Requests: add `BenchmarkExecuteWithWriter` benchmarking performance across concurrency tiers (`1`, `2`, `5`, `10`, `20`).
+
+    Devenv: add isolated profiling scripts (`profile-requests-concurrency-cpu`, `profile-requests-concurrency-mem`, `profile-requests-concurrency-block`, `profile-requests-concurrency-mutex`, `profile-requests-concurrency-all`, and `trace-requests-concurrency-goroutines`) to avoid measurement cross-talk.
+
 ## 0.15.4 (2026-09-15)
 
 ### Feat

devenv.nix

@@ -1027,6 +1027,90 @@ in
     go tool trace -http=:3111 /tmp/BenchmarkProbeCiphersConcurrently.trace.out
   '';
 
+  scripts.bench-requests-concurrency-tiers.exec = ''
+    set -e
+    gum format "## BenchmarkExecuteWithWriter concurrency tiers (1, 2, 5, 10, 20)"
+
+    go test ./internal/requests/ -run '^$' \
+      -bench BenchmarkExecuteWithWriter -benchmem -count=3
+  '';
+
+  scripts.profile-requests-concurrency-cpu.exec = ''
+    set -e
+    trap 'rm -f /tmp/BenchmarkExecuteWithWriter.cpu.out requests.test' EXIT
+    gum format "## BenchmarkExecuteWithWriter CPU profile (pprof :3112)"
+
+    go test ./internal/requests/ -run '^$' \
+      -bench BenchmarkExecuteWithWriter/concurrency-10 -benchtime 2s -benchmem \
+      -cpuprofile BenchmarkExecuteWithWriter.cpu.out \
+      -outputdir /tmp
+    go tool pprof -http=:3112 /tmp/BenchmarkExecuteWithWriter.cpu.out
+  '';
+
+  scripts.profile-requests-concurrency-mem.exec = ''
+    set -e
+    trap 'rm -f /tmp/BenchmarkExecuteWithWriter.mem.out requests.test' EXIT
+    gum format "## BenchmarkExecuteWithWriter heap profile (pprof :3112, -alloc_objects)"
+
+    go test ./internal/requests/ -run '^$' \
+      -bench BenchmarkExecuteWithWriter/concurrency-10 -benchtime 2s -benchmem \
+      -memprofile BenchmarkExecuteWithWriter.mem.out \
+      -outputdir /tmp
+    go tool pprof -http=:3112 -alloc_objects /tmp/BenchmarkExecuteWithWriter.mem.out
+  '';
+
+  scripts.profile-requests-concurrency-block.exec = ''
+    set -e
+    trap 'rm -f /tmp/BenchmarkExecuteWithWriter.block.out requests.test' EXIT
+    gum format "## BenchmarkExecuteWithWriter block profile (pprof :3112)"
+
+    go test ./internal/requests/ -run '^$' \
+      -bench BenchmarkExecuteWithWriter/concurrency-10 -benchtime 2s \
+      -blockprofile BenchmarkExecuteWithWriter.block.out \
+      -outputdir /tmp
+    go tool pprof -http=:3112 /tmp/BenchmarkExecuteWithWriter.block.out
+  '';
+
+  scripts.profile-requests-concurrency-mutex.exec = ''
+    set -e
+    trap 'rm -f /tmp/BenchmarkExecuteWithWriter.mutex.out requests.test' EXIT
+    gum format "## BenchmarkExecuteWithWriter mutex profile (pprof :3112)"
+
+    go test ./internal/requests/ -run '^$' \
+      -bench BenchmarkExecuteWithWriter/concurrency-10 -benchtime 2s \
+      -mutexprofile BenchmarkExecuteWithWriter.mutex.out \
+      -outputdir /tmp
+    go tool pprof -http=:3112 /tmp/BenchmarkExecuteWithWriter.mutex.out
+  '';
+
+  scripts.trace-requests-concurrency-goroutines.exec = ''
+    set -e
+    trap 'rm -f /tmp/BenchmarkExecuteWithWriter.trace.out' EXIT
+    gum format "## BenchmarkExecuteWithWriter execution trace (:3112)"
+
+    go test ./internal/requests/ -run '^$' \
+      -bench BenchmarkExecuteWithWriter/concurrency-10 -benchtime 1s \
+      -trace /tmp/BenchmarkExecuteWithWriter.trace.out \
+      -outputdir /tmp
+    go tool trace -http=:3112 /tmp/BenchmarkExecuteWithWriter.trace.out
+  '';
+
+  scripts.profile-requests-concurrency-all.exec = ''
+    set -e
+    trap 'rm -f requests.test' EXIT
+    gum format "## Running isolated profiles sequentially to avoid measurement cross-talk"
+    mkdir -p profiles
+    echo "1/4 Collecting CPU profile..."
+    go test ./internal/requests/ -run '^$' -bench BenchmarkExecuteWithWriter/concurrency-10 -benchtime 2s -cpuprofile profiles/requests-cpu.prof
+    echo "2/4 Collecting Memory profile..."
+    go test ./internal/requests/ -run '^$' -bench BenchmarkExecuteWithWriter/concurrency-10 -benchtime 2s -memprofile profiles/requests-mem.prof
+    echo "3/4 Collecting Block profile..."
+    go test ./internal/requests/ -run '^$' -bench BenchmarkExecuteWithWriter/concurrency-10 -benchtime 2s -blockprofile profiles/requests-block.prof
+    echo "4/4 Collecting Mutex profile..."
+    go test ./internal/requests/ -run '^$' -bench BenchmarkExecuteWithWriter/concurrency-10 -benchtime 2s -mutexprofile profiles/requests-mutex.prof
+    gum format "### All profiles saved in profiles/. Inspect with: go tool pprof -http=:3112 profiles/<name>.prof"
+  '';
+
   enterShell = ''
     echo "https-wrench devenv ready"
     go version

go.mod

@@ -21,6 +21,7 @@ require (
 	github.com/spf13/viper v1.21.0
 	github.com/stretchr/testify v1.11.1
 	github.com/youmark/pkcs8 v0.0.0-20240726163527-a2c0da244d78
+	golang.org/x/sync v0.23.0
 	golang.org/x/term v0.45.0
 )
 
@@ -60,7 +61,6 @@ require (
 	go.yaml.in/yaml/v3 v3.0.5 // indirect
 	golang.org/x/crypto v0.55.0 // indirect
 	golang.org/x/oauth2 v0.36.0 // indirect
-	golang.org/x/sync v0.22.0 // indirect
 	golang.org/x/sys v0.47.0 // indirect
 	golang.org/x/text v0.41.0 // indirect
 	golang.org/x/time v0.15.0 // indirect

go.sum

@@ -121,8 +121,8 @@ golang.org/x/exp v0.0.0-20231006140011-7918f672742d h1:jtJma62tbqLibJ5sFQz8bKtEM
 golang.org/x/exp v0.0.0-20231006140011-7918f672742d/go.mod h1:ldy0pHrwJyGW56pPQzzkH36rKxoZW1tw7ZJpeKx+hdo=
 golang.org/x/oauth2 v0.36.0 h1:peZ/1z27fi9hUOFCAZaHyrpWG5lwe0RJEEEeH0ThlIs=
 golang.org/x/oauth2 v0.36.0/go.mod h1:YDBUJMTkDnJS+A4BP4eZBjCqtokkg1hODuPjwiGPO7Q=
-golang.org/x/sync v0.22.0 h1:SZjpbeLmrCk4xhRSZFNZW5gFUeCeFgjekvI/+gfScek=
-golang.org/x/sync v0.22.0/go.mod h1:9xrNwdLfx4jkKbNva9FpL6vEN7evnE43NNNJQ2LF3+0=
+golang.org/x/sync v0.23.0 h1:KameEIfc1IkluZyXWLn39Wd4tURc6GbCiISGiZm2bQk=
+golang.org/x/sync v0.23.0/go.mod h1:sUUOizhqBxiL6pEWpqNLUiaJn1ShEbZ6BBqskPbjZm0=
 golang.org/x/sys v0.47.0 h1:o7XGOvZQCADBQQ4Y7VNq2dRWQR7JmOUW8Kxx4ZsNgWs=
 golang.org/x/sys v0.47.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw=
 golang.org/x/term v0.45.0 h1:NwWyBmoJCbfTHpxrWoZ9C6/VxOf7ic219I8xZZFdrf0=

https-wrench.schema.json

@@ -19,6 +19,11 @@
           "type": "string",
           "description": "PEM-encoded CA certificate bundle used to verify server certificates. When omitted, the system trust store is used."
         },
+        "concurrency": {
+          "type": "integer",
+          "default": 10,
+          "description": "Maximum number of concurrent HTTP requests (1 for sequential; defaults to 10 if omitted or <= 0)."
+        },
         "baseRequest": {
           "$ref": "#/definitions/RequestDefaults",
           "description": "YAML-only shared request template. Define an anchor (e.g. baseRequest: &base) and merge into requests with <<: *base. Ignored by https-wrench at runtime."

internal/cmd/embedded/config-example.yaml

@@ -10,6 +10,9 @@ debug: false
 ## verbose - Enables verbose output, showing more details during execution. Required option.
 verbose: true
 
+## concurrency - Maximum number of concurrent HTTP requests (1 for sequential, defaults to 10 if omitted or <= 0).
+concurrency: 10
+
 ## caBundle - A PEM-encoded CA certificate bundle as a multiline string to be used for verifying server certificates.
 ## When testing inside the devenv environment, the 'devenv up' command will create new self-signed certificates and
 ## start a local, HTTPS-enabled Nginx server.

internal/cmd/requests.go

@@ -20,9 +20,10 @@ import (
 
 var (
 	//go:embed  embedded/config-example.yaml
-	sampleYamlConfig string
-	showSampleConfig bool
-	requestsFmt      string
+	sampleYamlConfig    string
+	showSampleConfig    bool
+	requestsFmt         string
+	requestsConcurrency int
 )
 
 var requestsCmd = &cobra.Command{
@@ -95,8 +96,14 @@ Examples:
 			return
 		}
 
+		concurrency := requestsConcurrency
+		if !cmd.Flags().Changed("concurrency") && cfg.Concurrency > 0 {
+			concurrency = cfg.Concurrency
+		}
+
 		requestsCfg.SetVerbose(cfg.Verbose).
 			SetDebug(cfg.Debug).
+			SetConcurrency(concurrency).
 			SetRequests(cfg.Requests)
 
 		if err := requestsCfg.SetCaPoolFromYAML(cfg.CaBundle); err != nil {
@@ -166,5 +173,12 @@ func init() {
 		"text",
 		"Output format: text (human-readable) or json (machine-readable, no ANSI)",
 	)
+	requestsCmd.Flags().IntVarP(
+		&requestsConcurrency,
+		"concurrency",
+		"c",
+		requests.DefaultRequestsConcurrency,
+		"Maximum number of concurrent HTTP requests (1 for sequential)",
+	)
 	rootCmd.AddCommand(requestsCmd)
 }

internal/cmd/requests_test.go

@@ -33,6 +33,7 @@ func TestRequestsCmd(t *testing.T) {
 				"Global Flags:",
 				"--config",
 				"--ca-bundle",
+				"--concurrency",
 				"--format",
 				"--show-sample-config",
 				"--version",
@@ -45,6 +46,7 @@ func TestRequestsCmd(t *testing.T) {
 			expectError: false,
 			expected: []string{
 				"https-wrench.schema.json",
+				"concurrency:",
 				"requests:",
 				"transportOverrideUrl:",
 				"requestHeaders:",

internal/cmd/root_test.go

@@ -84,6 +84,7 @@ func TestRootCmd_LoadConfig(t *testing.T) {
 		require.IsType(t, expectedRequestsConfigs, config.Requests)
 
 		// testing against the current values of the embedded config
+		require.Equal(t, 10, config.Concurrency)
 		require.Equal(t, "SampleRequestAgainstLocalWebserver", config.Requests[0].Name)
 		require.Equal(t, "https://127.0.0.1:9443", config.Requests[0].TransportOverrideURL)
 	})

internal/mcp/assets/sample-config.yaml

@@ -10,6 +10,9 @@ debug: false
 ## verbose - Enables verbose output, showing more details during execution. Required option.
 verbose: true
 
+## concurrency - Maximum number of concurrent HTTP requests (1 for sequential, defaults to 10 if omitted or <= 0).
+concurrency: 10
+
 ## caBundle - A PEM-encoded CA certificate bundle as a multiline string to be used for verifying server certificates.
 ## When testing inside the devenv environment, the 'devenv up' command will create new self-signed certificates and
 ## start a local, HTTPS-enabled Nginx server.

internal/mcp/assets/schema.json

@@ -19,6 +19,11 @@
           "type": "string",
           "description": "PEM-encoded CA certificate bundle used to verify server certificates. When omitted, the system trust store is used."
         },
+        "concurrency": {
+          "type": "integer",
+          "default": 10,
+          "description": "Maximum number of concurrent HTTP requests (1 for sequential; defaults to 10 if omitted or <= 0)."
+        },
         "baseRequest": {
           "$ref": "#/definitions/RequestDefaults",
           "description": "YAML-only shared request template. Define an anchor (e.g. baseRequest: &base) and merge into requests with <<: *base. Ignored by https-wrench at runtime."