Columns in search

README.md

@@ -1464,6 +1464,8 @@ seqcli search -f "@Exception like '%TimeoutException%'" -c 30
 | ------ | ----------- |
 | `-f`, `--filter=VALUE` | A filter to apply to the search, for example `Host = 'xmpweb-01.example.com'` |
 | `-c`, `--count=VALUE` | The maximum number of events to retrieve; the default is 1 |
+|       `--column=VALUE` | A column to display preceding each event's message; any Seq expression can be supplied, for example `OrderId`, `@SpanKind`, or `@Resource['service.name']`; this argument can be used multiple times, adding columns in order; applies to plain-text output only |
+|       `--no-signal-columns` | Do not show columns associated with the specified signal expression |
 |       `--start=VALUE` | ISO 8601 date/time to query from |
 |       `--end=VALUE` | ISO 8601 date/time to query to |
 |       `--json` | Print output in newline-delimited JSON (the default is plain text) |
@@ -1636,6 +1638,8 @@ Stream log events matching a filter.
 | Option | Description |
 | ------ | ----------- |
 | `-f`, `--filter=VALUE` | An optional server-side filter to apply to the stream, for example `@Level = 'Error'` |
+|       `--column=VALUE` | A column to display preceding each event's message; any Seq expression can be supplied, for example `OrderId`, `@SpanKind`, or `@Resource['service.name']`; this argument can be used multiple times, adding columns in order; applies to plain-text output only |
+|       `--no-signal-columns` | Do not show columns associated with the specified signal expression |
 |       `--json` | Print output in newline-delimited JSON (the default is plain text) |
 |       `--no-color` | Don't colorize text output |
 |       `--force-color` | Force redirected output to have ANSI color (unless `--no-color` is also specified) |

src/SeqCli/Cli/Commands/SearchCommand.cs

@@ -18,6 +18,7 @@
 using SeqCli.Api;
 using SeqCli.Cli.Features;
 using SeqCli.Config;
+using SeqCli.Output;
 using Serilog;
 
 // ReSharper disable UnusedType.Global
@@ -33,6 +34,7 @@ class SearchCommand : Command
     readonly DateRangeFeature _range;
     readonly SignalExpressionFeature _signal;
     readonly StoragePathFeature _storagePath;
+    readonly EventColumnsFeature _eventColumns;
     string? _filter;
     int _count = 1;
     int _httpClientTimeout = 100000;
@@ -44,11 +46,13 @@ public SearchCommand()
             "f=|filter=",
             "A filter to apply to the search, for example `Host = 'xmpweb-01.example.com'`",
             v => _filter = v);
+
         Options.Add(
             "c=|count=",
             $"The maximum number of events to retrieve; the default is {_count}",
             v => _count = int.Parse(v, CultureInfo.InvariantCulture));
 
+        _eventColumns = Enable<EventColumnsFeature>();
         _range = Enable<DateRangeFeature>();
         _output = Enable(new OutputFormatFeature(supportNative: true, supportJson: true));
         _storagePath = Enable<StoragePathFeature>();
@@ -71,10 +75,13 @@ protected override async Task<int> Run()
         try
         {
             var config = RuntimeConfigurationLoader.Load(_storagePath);
-            var output = _output.GetOutputFormat(config);
+
             var connection = SeqConnectionFactory.Connect(_connection, config);
             connection.Client.HttpClient.Timeout = TimeSpan.FromMilliseconds(_httpClientTimeout);
 
+            var columns = await _eventColumns.GetColumns(connection, _signal.Signal);
+            var output = _output.GetOutputFormat(config, TextFormatters.PlainOutputTemplate(columns));
+
             string? filter = null;
             if (!string.IsNullOrWhiteSpace(_filter))
                 filter = (await connection.Expressions.ToStrictAsync(_filter)).StrictExpression;

src/SeqCli/Cli/Commands/TailCommand.cs

@@ -20,6 +20,7 @@
 using SeqCli.Api;
 using SeqCli.Cli.Features;
 using SeqCli.Config;
+using SeqCli.Output;
 
 namespace SeqCli.Cli.Commands;
 
@@ -31,6 +32,7 @@ class TailCommand : Command
     readonly OutputFormatFeature _output;
     readonly SignalExpressionFeature _signal;
     readonly StoragePathFeature _storagePath;
+    readonly EventColumnsFeature _eventColumns;
     string? _filter;
 
     public TailCommand()
@@ -40,6 +42,7 @@ public TailCommand()
             "An optional server-side filter to apply to the stream, for example `@Level = 'Error'`",
             v => _filter = v);
 
+        _eventColumns = Enable<EventColumnsFeature>();
         _output = Enable(new OutputFormatFeature(supportNative: true, supportJson: true));
         _storagePath = Enable<StoragePathFeature>();
         _signal = Enable<SignalExpressionFeature>();
@@ -61,7 +64,8 @@ protected override async Task<int> Run()
             strict = converted.StrictExpression;
         }
 
-        var output = _output.GetOutputFormat(config);
+        var columns = await _eventColumns.GetColumns(connection, _signal.Signal);
+        var output = _output.GetOutputFormat(config, TextFormatters.PlainOutputTemplate(columns));
 
         try
         {

src/SeqCli/Cli/Features/EventColumnsFeature.cs

@@ -0,0 +1,72 @@
+// Copyright © Datalust and contributors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+using System;
+using System.Collections.Generic;
+using System.Threading.Tasks;
+using Seq.Api;
+using Seq.Api.Model.Signals;
+using SeqCli.Signals;
+using SeqCli.Syntax;
+using SeqCli.Util;
+
+namespace SeqCli.Cli.Features;
+
+class EventColumnsFeature : CommandFeature
+{
+    readonly List<string> _columns = [];
+    bool _noSignalColumns;
+
+    public override void Enable(OptionSet options)
+    {
+        options.Add(
+            "column=",
+            "A column to display preceding each event's message; any Seq expression can be supplied, for " +
+            "example `OrderId`, `@SpanKind`, or `@Resource.service.name`; this argument can be used multiple " +
+            "times, adding columns in order; applies to plain-text output only",
+            c => _columns.Add(ArgumentString.Normalize(c) ?? throw new ArgumentException("Columns require a value.")));
+
+        options.Add(
+            "no-signal-columns",
+            "Do not show columns associated with the specified signal expression",
+            _ => _noSignalColumns = true);
+    }
+
+    public async Task<IReadOnlyList<string>> GetColumns(SeqConnection connection, SignalExpressionPart? signal)
+    {
+        var columns = new List<string>();
+        if (!_noSignalColumns && signal is { } signalExpression)
+        {
+            foreach (var signalId in signalExpression.ReferencedSignalIds())
+            {
+                var signalEntity = await connection.Signals.FindAsync(signalId);
+                foreach (var column in signalEntity.Columns)
+                {
+                    columns.Add(column.Expression);
+                }
+            }
+        }
+
+        columns.AddRange(_columns);
+
+        foreach (var column in columns)
+        {
+            // A better error than a failed output template parse.
+            if (!SeqSyntax.TryCompileExpression(column, out _, out var error))
+                throw new ArgumentException($"The column expression `{column}` could not be compiled: {error}");
+        }
+
+        return columns;
+    }
+}

src/SeqCli/Cli/Features/SignalExpressionFeature.cs

@@ -13,6 +13,7 @@
 // limitations under the License.
 
 using Seq.Api.Model.Signals;
+using SeqCli.Signals;
 
 namespace SeqCli.Cli.Features;
 
@@ -27,9 +28,7 @@ public SignalExpressionPart? Signal
             if (string.IsNullOrWhiteSpace(_signalExpression))
                 return null;
 
-            // This is a hack that just happens to work because of the way
-            // signal ids are passed through ToString() as literals
-            return SignalExpressionPart.Signal(_signalExpression.Trim());
+            return SignalExpressionParser.ParseExpression(_signalExpression);
         }
     }
 

src/SeqCli/Output/TextFormatters.cs

@@ -13,6 +13,8 @@
 // limitations under the License.
 
 using System;
+using System.Collections.Generic;
+using System.Text;
 using Seq.Syntax.Templates;
 using Seq.Syntax.Templates.Encoding;
 using Seq.Syntax.Templates.Themes;
@@ -30,14 +32,28 @@ static class TextFormatters
         "{@Data}" + Environment.NewLine,
         encoder: Encoder(theme));
 
-    // Guarding on `@Elapsed` rather than the built-in `IsSpan()` shows elapsed time for any
-    // event carrying a span start timestamp, whether or not trace and span ids accompany it.
-    static readonly string DefaultPlainTextOutputTemplate =
-        "[{@Timestamp:o} {@Level:u3}] {@Message}{#if @Elapsed is not null} ({TotalMilliseconds(@Elapsed):0.###} ms){#end}" +
+    /// <summary>
+    /// The default plain-text template, showing <paramref name="columns"/> ahead of each
+    /// event's message.
+    /// </summary>
+    /// <param name="columns">Column expressions, evaluated against each event; any Seq expression can be
+    /// supplied.</param>
+    internal static string PlainOutputTemplate(IEnumerable<string>? columns = null) =>
+        "[{@Timestamp:o} {@Level:u3}] " + ColumnsFragment(columns ?? []) +
+        "{@Message}{#if @Elapsed is not null} ({TotalMilliseconds(@Elapsed):0.###} ms){#end}" +
         Environment.NewLine + "{@Exception}";
 
+    internal static string ColumnsFragment(IEnumerable<string> columns)
+    {
+        var fragment = new StringBuilder();
+        foreach (var column in columns)
+            fragment.Append($"{{#if ({column}) <> ''}}{{({column})}} {{#end}}");
+
+        return fragment.ToString();
+    }
+
     public static ExpressionTemplate Plain(TemplateTheme? theme, string? outputTemplate) =>
-        SeqSyntax.ParseTemplate(outputTemplate ?? DefaultPlainTextOutputTemplate, Encoder(theme));
+        SeqSyntax.ParseTemplate(outputTemplate ?? PlainOutputTemplate(), Encoder(theme));
 
     static TemplateOutputEncoder? Encoder(TemplateTheme? theme) =>
         theme != null ? TemplateOutputEncoder.Ansi(theme) : null;

src/SeqCli/Output/TraceFormatter.cs

@@ -15,6 +15,7 @@
 using System;
 using System.Collections.Generic;
 using System.Globalization;
+using System.Linq;
 using System.Text;
 using System.Text.Json.Nodes;
 using SeqCli.Api;
@@ -32,20 +33,14 @@ static class TraceFormatter
     const string SpanConnector = "├─ ", LastSpanConnector = "└─ ", LogConnector = "┊  ",
         Continuation = "│  ", Gap = "   ";
 
-    static string ColumnPropertyName(int index) => $"{ColumnPrefixProperty}_{index}";
+    // The trace query evaluates column expressions server-side, so their results are carried in
+    // surrogate properties rather than being recomputed by the output template.
+    static string ColumnProperty(int index) => $"{ColumnPrefixProperty}_{index}";
 
     public static string OutputTemplate(int columnCount)
     {
         var template = new StringBuilder($"[{{@Timestamp:o}} {{@Level:u3}}] {{{TreePrefixProperty}}}");
-
-        // `<> ''` is undefined, and hence falsy, when the property is missing; the guard thus
-        // drops the column, and its trailing space, for both missing and empty values.
-        for (var i = 0; i < columnCount; ++i)
-        {
-            var column = ColumnPropertyName(i);
-            template.Append($"{{#if {column} <> ''}}{{{column}}} {{#end}}");
-        }
-
+        template.Append(TextFormatters.ColumnsFragment(Enumerable.Range(0, columnCount).Select(ColumnProperty)));
         template.Append($"{{@Message}}{{#if {ElapsedProperty} is not null}} ({{TotalMilliseconds({ElapsedProperty}):0.###}} ms){{#end}}");
         template.Append(Environment.NewLine).Append("{@Exception}");
         return template.ToString();
@@ -106,7 +101,7 @@ static JsonObject ToEventJson(TraceTreeNode treeNode, string treePrefix)
         for (var i = 0; i < evt.Columns.Count; ++i)
         {
             if (evt.Columns[i] is { } value)
-                eventJson[ColumnPropertyName(i)] = ToSystemTextJson.FromApiValue(value);
+                eventJson[ColumnProperty(i)] = ToSystemTextJson.FromApiValue(value);
         }
 
         return eventJson;

src/SeqCli/SeqCli.csproj

@@ -42,7 +42,7 @@
     <PackageReference Include="System.ServiceProcess.ServiceController" Version="10.0.10" />
     <PackageReference Include="Serilog.Sinks.Seq" Version="9.1.0" />
     <PackageReference Include="Seq.Apps" Version="2023.4.0" />
-    <PackageReference Include="Seq.Syntax" Version="2.0.0-dev-00100" />
+    <PackageReference Include="Seq.Syntax" Version="2.0.0-dev-00103" />
     <PackageReference Include="Tavis.UriTemplates" Version="2.0.0" />
   </ItemGroup>
   <ItemGroup>

src/SeqCli/Signals/SignalExpressionPartExtensions.cs

@@ -0,0 +1,34 @@
+// Copyright © Datalust and contributors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using Seq.Api.Model.Signals;
+
+namespace SeqCli.Signals;
+
+static class SignalExpressionPartExtensions
+{
+    public static IEnumerable<string> ReferencedSignalIds(this SignalExpressionPart expr)
+    {
+        return expr.Kind switch
+        {
+            SignalExpressionKind.Signal => [expr.SignalId],
+            SignalExpressionKind.Intersection or SignalExpressionKind.Union => expr.Left.ReferencedSignalIds()
+                .Concat(expr.Right.ReferencedSignalIds()),
+            _ => throw new ArgumentOutOfRangeException(nameof(expr))
+        };
+    }
+}

test/SeqCli.EndToEnd/Events/SearchColumnsTestCase.cs

@@ -0,0 +1,45 @@
+using System.IO;
+using System.Threading.Tasks;
+using Seq.Api;
+using SeqCli.EndToEnd.Support;
+using Serilog;
+using Xunit;
+
+#nullable enable
+
+namespace SeqCli.EndToEnd.Events;
+
+public class SearchColumnsTestCase : ICliTestCase
+{
+    const string TraceId = "7d4dedcc73b18e449e0e4ea08cbe346d";
+
+    public Task ExecuteAsync(
+        SeqConnection connection,
+        ILogger logger,
+        CliCommandRunner runner)
+    {
+        var inputFile = Path.Combine("Data", "trace-tree.clef");
+        Assert.True(File.Exists(inputFile));
+
+        var exit = runner.Exec("ingest", $"--json -i {inputFile}");
+        Assert.Equal(0, exit);
+
+        var filter = $"--filter=\"@TraceId = '{TraceId}' and Customer is not null\"";
+
+        exit = runner.Exec("search", $"{filter} -c 10 --column Customer --column RowCount");
+        Assert.Equal(0, exit);
+        Assert.Contains("] scott GET /orders", runner.LastRunProcess!.Output);
+
+        // Columns apply to plain-text output only.
+        exit = runner.Exec("search", $"{filter} -c 10 --column Customer --json");
+        Assert.Equal(0, exit);
+        Assert.Contains("GET {Route}", runner.LastRunProcess!.Output);
+        Assert.DoesNotContain("_SeqcliColumn", runner.LastRunProcess!.Output);
+
+        exit = runner.Exec("search", $"{filter} -c 10 --column \"not a valid (\"");
+        Assert.Equal(1, exit);
+        Assert.Contains("could not be compiled", runner.LastRunProcess!.Output);
+
+        return Task.CompletedTask;
+    }
+}