diff --git a/bot/stats.go b/bot/stats.go index 6cf914f..bf66a10 100644 --- a/bot/stats.go +++ b/bot/stats.go @@ -24,6 +24,8 @@ type Stats struct { persistOnce atomic.Bool networkMu sync.RWMutex networks map[string]*networkStats + persistedPluginCommands map[string]map[string]uint64 + persistWG sync.WaitGroup } type networkStats struct { @@ -43,11 +45,12 @@ type networkStats struct { } type persistedStats struct { - Received uint64 `json:"messages_received"` - Sent uint64 `json:"messages_sent"` - Commands uint64 `json:"commands_handled"` - Reconnects uint64 `json:"reconnects"` - Dropped uint64 `json:"messages_dropped"` + Received uint64 `json:"messages_received"` + Sent uint64 `json:"messages_sent"` + Commands uint64 `json:"commands_handled"` + Reconnects uint64 `json:"reconnects"` + Dropped uint64 `json:"messages_dropped"` + PluginCommands map[string]map[string]uint64 `json:"plugin_commands,omitempty"` } type metricLabel struct { @@ -61,7 +64,7 @@ type prometheusWriter struct { } func NewStats(dbs ...*storage.DB) *Stats { - s := &Stats{started: time.Now(), networks: make(map[string]*networkStats)} + s := &Stats{started: time.Now(), networks: make(map[string]*networkStats), persistedPluginCommands: make(map[string]map[string]uint64)} if len(dbs) > 0 && dbs[0] != nil { s.db = dbs[0] if raw, err := s.db.Get("stats", "global"); err == nil { @@ -72,10 +75,17 @@ func NewStats(dbs ...*storage.DB) *Stats { s.commands.Store(saved.Commands) s.reconnects.Store(saved.Reconnects) s.dropped.Store(saved.Dropped) + if saved.PluginCommands != nil { + s.persistedPluginCommands = saved.PluginCommands + } } } s.persistDone = make(chan struct{}) - go s.persistLoop() + s.persistWG.Add(1) + go func() { + defer s.persistWG.Done() + s.persistLoop() + }() } return s } @@ -94,6 +104,9 @@ func (s *Stats) registerNetwork(name string, configuredChannels int, queue *Queu return existing } network := &networkStats{name: name, configuredChannels: configuredChannels, queue: queue, joinedChannels: make(map[string]string)} + for plugin, count := range s.persistedPluginCommands[name] { + atomicCounter(&network.pluginCommands, plugin).Store(count) + } s.networks[name] = network return network } @@ -213,8 +226,15 @@ func (s *Stats) Persist() { if s.db == nil { return } + pluginCommands := make(map[string]map[string]uint64) + for _, network := range s.sortedNetworks() { + counters := snapshotCounters(&network.pluginCommands) + if len(counters) > 0 { + pluginCommands[network.name] = counters + } + } _ = s.db.Set("stats", "global", persistedStats{ - Received: s.received.Load(), Sent: s.sent.Load(), Commands: s.commands.Load(), Reconnects: s.reconnects.Load(), Dropped: s.dropped.Load(), + Received: s.received.Load(), Sent: s.sent.Load(), Commands: s.commands.Load(), Reconnects: s.reconnects.Load(), Dropped: s.dropped.Load(), PluginCommands: pluginCommands, }) } @@ -223,6 +243,7 @@ func (s *Stats) Close() { return } close(s.persistDone) + s.persistWG.Wait() s.Persist() } @@ -327,17 +348,11 @@ func (s *Stats) PrometheusSnapshot() string { } func (w *prometheusWriter) syncMapCounters(name, help, network string, counters *sync.Map, includeHandler bool) { - values := make(map[string]uint64) - keys := make([]string, 0) - counters.Range(func(key, value interface{}) bool { - text, ok := key.(string) - counter, counterOK := value.(*atomic.Uint64) - if ok && counterOK { - keys = append(keys, text) - values[text] = counter.Load() - } - return true - }) + values := snapshotCounters(counters) + keys := make([]string, 0, len(values)) + for key := range values { + keys = append(keys, key) + } sort.Strings(keys) for _, key := range keys { plugin, handler := key, "" @@ -352,6 +367,19 @@ func (w *prometheusWriter) syncMapCounters(name, help, network string, counters } } +func snapshotCounters(counters *sync.Map) map[string]uint64 { + values := make(map[string]uint64) + counters.Range(func(key, value interface{}) bool { + text, ok := key.(string) + counter, counterOK := value.(*atomic.Uint64) + if ok && counterOK { + values[text] = counter.Load() + } + return true + }) + return values +} + func (w *prometheusWriter) metric(name, help, metricType string, labels []metricLabel, value interface{}) { if _, ok := w.described[name]; !ok { fmt.Fprintf(&w.output, "# HELP %s %s\n# TYPE %s %s\n", name, help, name, metricType) diff --git a/bot/stats_test.go b/bot/stats_test.go index 12dac6d..6923346 100644 --- a/bot/stats_test.go +++ b/bot/stats_test.go @@ -2,6 +2,7 @@ package bot import ( "context" + "path/filepath" "strings" "testing" @@ -95,6 +96,36 @@ func TestExpandedPrometheusMetricsRemainBackwardCompatible(t *testing.T) { } } +func TestPluginCommandMetricsPersistAcrossRestart(t *testing.T) { + dbPath := filepath.Join(t.TempDir(), "stats.db") + db, err := storage.Open(dbPath) + if err != nil { + t.Fatal(err) + } + first := NewStats(db) + network := first.registerNetwork("ouch", 0, nil) + first.recordCommand(network, "help") + first.Close() + if err := db.Close(); err != nil { + t.Fatal(err) + } + + db, err = storage.Open(dbPath) + if err != nil { + t.Fatal(err) + } + second := NewStats(db) + second.registerNetwork("ouch", 0, nil) + metrics := second.PrometheusSnapshot() + if !strings.Contains(metrics, `bot_plugin_commands_handled_total{network="ouch",plugin="help"} 1`) { + t.Fatalf("persisted plugin command metric missing:\n%s", metrics) + } + second.Close() + if err := db.Close(); err != nil { + t.Fatal(err) + } +} + func TestOwnChannelMembershipTracksJoinPartKickAndDisconnect(t *testing.T) { stats := NewStats() instance := NewWithStats(Config{NetworkName: "libera", Identity: IdentityConfig{Nick: "GoBot"}}, nil, nil, zap.NewNop(), stats) diff --git a/docs/monitoring.md b/docs/monitoring.md index ede370d..1543c29 100644 --- a/docs/monitoring.md +++ b/docs/monitoring.md @@ -50,15 +50,17 @@ GoBot also exposes operational metrics for richer dashboards: | `bot_network_channel_joined{network,channel}` | gauge | Current channel membership; one series with value 1 per joined channel | | `bot_outgoing_queue_depth{network}` | gauge | Messages currently waiting to be sent | | `bot_outgoing_queue_capacity{network}` | gauge | Maximum outbound queue size | -| `bot_plugin_commands_handled_total{network,plugin}` | counter | Handled commands grouped by plugin | +| `bot_plugin_commands_handled_total{network,plugin}` | counter | Persistent handled-command totals grouped by plugin | | `bot_plugin_panics_total{network,plugin,handler}` | counter | Recovered message/event handler panics | -Per-network and per-plugin counters reset when the GoBot process restarts; -Prometheus `rate()` and `increase()` account for counter resets. Labels are -limited to network names, the bot's current joined channels, built-in plugin -names, and the bounded handler type. Nicknames, accounts, and message contents -are not exported. Because joined channel names are exposed as metric labels, -keep `/metrics` on a private monitoring network. +Per-network traffic, reconnect, dropped-message, and panic counters reset when +the GoBot process restarts; Prometheus `rate()` and `increase()` account for +those counter resets. Plugin command totals are persisted in BoltDB and remain +available after a restart, so the Grafana most-used-plugins panel does not lose +its ranking. Labels are limited to network names, the bot's current joined +channels, built-in plugin names, and the bounded handler type. Nicknames, +accounts, and message contents are not exported. Because joined channel names +are exposed as metric labels, keep `/metrics` on a private monitoring network. The `/stats` JSON response includes a `networks` object with connection, traffic, command, reconnect, queue, configured-channel, and current joined- diff --git a/grafana/README.md b/grafana/README.md index c60b486..658e697 100644 --- a/grafana/README.md +++ b/grafana/README.md @@ -9,7 +9,7 @@ identifier. It covers: - process uptime and reliability events - per-network incoming and outgoing message rates - handled-command rates grouped by plugin -- the ten most-used plugins during the current GoBot process lifetime +- the ten most-used plugins across the persisted GoBot command history - current networks and channels GoBot has actually joined - outbound queue depth and capacity by network - filtering by Prometheus job, environment, hostname, instance, and IRC network @@ -86,10 +86,11 @@ The dashboard refreshes every 30 seconds, matching the Prometheus scrape interval. A newly started bot may need two scrapes (about one minute) before rate panels have enough samples to draw a line. -The **Most-used plugins** panel counts handled commands since the current -GoBot process started, so its ranking resets after a service restart. The -**Joined networks and channels** panel reflects live JOIN/PART/KICK state and -clears a network's channels when its IRC connection ends. +The **Most-used plugins** panel uses persistent per-network command totals from +BoltDB, so its ranking survives service restarts. A new database has no plugin +series until the first command is handled. The **Joined networks and channels** +panel reflects live JOIN/PART/KICK state and clears a network's channels when +its IRC connection ends. ## Security note diff --git a/grafana/gobot-dashboard.json b/grafana/gobot-dashboard.json index b836e5c..580f070 100644 --- a/grafana/gobot-dashboard.json +++ b/grafana/gobot-dashboard.json @@ -215,7 +215,7 @@ "uid": "${DS_PROMETHEUS}" }, "editorMode": "code", - "expr": "min(up{job=~\"$job\", environment=~\"$environment\", hostname=~\"$hostname\", instance=~\"$instance\"})", + "expr": "min(up{job=~\"$job\", environment=~\"$environment\", hostname=~\"$hostname\", instance=~\"$instance\"} and on (job, environment, hostname, instance) bot_process_start_time_seconds{job=~\"$job\", environment=~\"$environment\", hostname=~\"$hostname\", instance=~\"$instance\"})", "legendFormat": "__auto", "range": true, "refId": "A" @@ -702,7 +702,7 @@ "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, - "description": "Top plugins by commands handled during the current GoBot process lifetime. Counts reset when GoBot restarts.", + "description": "Top plugins by persistent per-network command totals. Counts survive GoBot restarts while the BoltDB database is retained.", "fieldConfig": { "defaults": { "unit": "short",