From 7d368fd0fe0e9b60dfef8a2e6124c57a054f7f6f Mon Sep 17 00:00:00 2001 From: Kelechi Oliver Azorji Date: Sun, 27 Sep 2026 10:12:39 +0000 Subject: [PATCH 1/5] feature: Add more query type support to Opensearch plugin Signed-off-by: Kelechi Oliver Azorji --- opensearch/README.md | 9 +- .../queries/opensearch-log-query/query.cue | 37 ++- .../tests/invalid/dsl-without-index.json | 4 + .../tests/invalid/lucene-without-index.json | 4 + .../tests/invalid/negative-limit.json | 4 + .../tests/invalid/unknown-language.json | 4 + .../tests/invalid/zero-limit.json | 4 + .../opensearch-log-query/tests/valid/dsl.json | 4 + .../tests/valid/lucene.json | 4 + .../tests/valid/no-language.json | 4 + .../opensearch-log-query/tests/valid/ppl.json | 12 + .../opensearch-log-query/tests/valid/sql.json | 4 + opensearch/sdk/go/query/log/log.go | 22 ++ opensearch/sdk/go/query/log/log_test.go | 94 +++++++ opensearch/sdk/go/query/log/options.go | 25 ++ .../OpenSearchDatasource.tsx | 4 +- .../OpenSearchDatasourceEditor.tsx | 10 + .../src/model/opensearch-client-types.ts | 24 +- .../src/model/opensearch-client.test.ts | 136 +++++++-- opensearch/src/model/opensearch-client.ts | 151 ++++++++-- opensearch/src/queries/constants.ts | 72 ++++- .../OpenSearchLogQuery.tsx | 2 +- .../OpenSearchLogQueryEditor.test.tsx | 107 ++++++- .../OpenSearchLogQueryEditor.tsx | 106 ++++++- .../opensearch-log-query/build/dsl.test.ts | 91 ++++++ .../queries/opensearch-log-query/build/dsl.ts | 69 +++++ .../opensearch-log-query/build/index.ts | 18 ++ .../opensearch-log-query/build/lucene.test.ts | 71 +++++ .../opensearch-log-query/build/lucene.ts | 44 +++ .../queries/opensearch-log-query/build/ppl.ts | 92 ++++++ .../opensearch-log-query/build/search.ts | 52 ++++ .../opensearch-log-query/build/sql.test.ts | 44 +++ .../queries/opensearch-log-query/build/sql.ts | 44 +++ .../convert/datarows-to-logs.ts | 77 +++++ .../convert/hits-to-logs.test.ts | 122 ++++++++ .../convert/hits-to-logs.ts | 105 +++++++ .../opensearch-log-query/convert/index.ts | 16 ++ .../convert/parse-timestamp.ts | 51 ++++ .../get-opensearch-log-data.test.ts | 136 ++++++++- .../get-opensearch-log-data.ts | 262 ++++-------------- .../src/queries/opensearch-log-query/index.ts | 2 + .../opensearch-log-query-types.test.ts | 31 +++ .../opensearch-log-query-types.ts | 33 ++- 43 files changed, 1919 insertions(+), 288 deletions(-) create mode 100644 opensearch/schemas/queries/opensearch-log-query/tests/invalid/dsl-without-index.json create mode 100644 opensearch/schemas/queries/opensearch-log-query/tests/invalid/lucene-without-index.json create mode 100644 opensearch/schemas/queries/opensearch-log-query/tests/invalid/negative-limit.json create mode 100644 opensearch/schemas/queries/opensearch-log-query/tests/invalid/unknown-language.json create mode 100644 opensearch/schemas/queries/opensearch-log-query/tests/invalid/zero-limit.json create mode 100644 opensearch/schemas/queries/opensearch-log-query/tests/valid/dsl.json create mode 100644 opensearch/schemas/queries/opensearch-log-query/tests/valid/lucene.json create mode 100644 opensearch/schemas/queries/opensearch-log-query/tests/valid/no-language.json create mode 100644 opensearch/schemas/queries/opensearch-log-query/tests/valid/ppl.json create mode 100644 opensearch/schemas/queries/opensearch-log-query/tests/valid/sql.json create mode 100644 opensearch/src/queries/opensearch-log-query/build/dsl.test.ts create mode 100644 opensearch/src/queries/opensearch-log-query/build/dsl.ts create mode 100644 opensearch/src/queries/opensearch-log-query/build/index.ts create mode 100644 opensearch/src/queries/opensearch-log-query/build/lucene.test.ts create mode 100644 opensearch/src/queries/opensearch-log-query/build/lucene.ts create mode 100644 opensearch/src/queries/opensearch-log-query/build/ppl.ts create mode 100644 opensearch/src/queries/opensearch-log-query/build/search.ts create mode 100644 opensearch/src/queries/opensearch-log-query/build/sql.test.ts create mode 100644 opensearch/src/queries/opensearch-log-query/build/sql.ts create mode 100644 opensearch/src/queries/opensearch-log-query/convert/datarows-to-logs.ts create mode 100644 opensearch/src/queries/opensearch-log-query/convert/hits-to-logs.test.ts create mode 100644 opensearch/src/queries/opensearch-log-query/convert/hits-to-logs.ts create mode 100644 opensearch/src/queries/opensearch-log-query/convert/index.ts create mode 100644 opensearch/src/queries/opensearch-log-query/convert/parse-timestamp.ts create mode 100644 opensearch/src/queries/opensearch-log-query/opensearch-log-query-types.test.ts diff --git a/opensearch/README.md b/opensearch/README.md index a1c10be30..d3370ca33 100644 --- a/opensearch/README.md +++ b/opensearch/README.md @@ -1,7 +1,12 @@ # Plugin Module: opensearch -Datasource plugin for [OpenSearch](https://opensearch.org/) in Perses. Supports log queries using -[PPL (Piped Processing Language)](https://opensearch.org/docs/latest/search-plugins/sql/ppl/index/). +Datasource plugin for [OpenSearch](https://opensearch.org/) in Perses. Log queries can +be written in [PPL](https://docs.opensearch.org/latest/sql-and-ppl/ppl/index/), +[SQL](https://docs.opensearch.org/latest/sql-and-ppl/sql/index/), +[Lucene / query string](https://docs.opensearch.org/latest/query-dsl/full-text/query-string/), +or raw [Query DSL](https://docs.opensearch.org/latest/query-dsl/), selected per query. +PPL and SQL need the SQL/PPL plugin enabled on the cluster; Lucene and Query DSL use +the core `_search` API. This plugin is intended to display logs alongside the traces surfaced by the Tempo plugin, so a user can pivot from a trace to the related logs in OpenSearch. diff --git a/opensearch/schemas/queries/opensearch-log-query/query.cue b/opensearch/schemas/queries/opensearch-log-query/query.cue index 8dcd6005e..47fc1beb2 100644 --- a/opensearch/schemas/queries/opensearch-log-query/query.cue +++ b/opensearch/schemas/queries/opensearch-log-query/query.cue @@ -19,11 +19,46 @@ import ( ) kind: "OpenSearchLogQuery" + +// `index` is required only for the languages that put it in the request URL +// (`lucene`, `dsl`, which hit `_search`); it's optional for `ppl`/`sql`, where +// the index lives in the query text itself. +// +// A disjunction of two closed structs (one branch requiring `index` for +// lucene/dsl, one leaving it optional for ppl/sql) does NOT work: a missing +// *required* field in CUE produces an *incomplete* value, not a conflicting +// one, so the lucene/dsl branch is never eliminated just because +// `queryLanguage` is absent from the input. That leaves the whole disjunction +// unresolved, and every spec that omits `queryLanguage` (which must be legal — +// see tests/valid/no-language.json) fails as "incomplete value" instead of +// validating against the ppl/sql branch. Do not retry this approach. +// +// The `if` comprehension below works instead, but only because of two things +// together: `queryLanguage` is given a default (`*"ppl" | ...`) so it is +// always concrete, even when the input omits it entirely; and every field +// referenced by the comprehension is declared directly in this one struct +// literal. A CUE comprehension condition can only see fields declared +// lexically in the same struct literal — it cannot see fields that arrive via +// unification (`someBase & {...}`) or embedding a separately-declared struct +// (`{someBase, if ...}`). A prior version of this schema factored the shared +// fields out into a `_base` field and unified/embedded it here; that broke +// the comprehension, because `queryLanguage` (declared in `_base`) wasn't +// visible to the `if` conditions declared alongside the embedding. So all +// fields are inlined directly into this single `spec: close({...})` struct +// literal instead of being factored out — do not re-extract a shared base +// without re-testing the comprehension against it. spec: close({ ds.#selector query: strings.MinRunes(1) - index?: strings.MinRunes(1) timestampField?: strings.MinRunes(1) messageField?: strings.MinRunes(1) disableTimeFilter?: bool + limit?: int & >0 + queryLanguage: *"ppl" | "sql" | "lucene" | "dsl" + if queryLanguage == "lucene" || queryLanguage == "dsl" { + index: strings.MinRunes(1) + } + if queryLanguage != "lucene" && queryLanguage != "dsl" { + index?: strings.MinRunes(1) + } }) diff --git a/opensearch/schemas/queries/opensearch-log-query/tests/invalid/dsl-without-index.json b/opensearch/schemas/queries/opensearch-log-query/tests/invalid/dsl-without-index.json new file mode 100644 index 000000000..23e0184dc --- /dev/null +++ b/opensearch/schemas/queries/opensearch-log-query/tests/invalid/dsl-without-index.json @@ -0,0 +1,4 @@ +{ + "kind": "OpenSearchLogQuery", + "spec": { "queryLanguage": "dsl", "query": "{\"query\":{\"match\":{\"level\":\"error\"}}}" } +} diff --git a/opensearch/schemas/queries/opensearch-log-query/tests/invalid/lucene-without-index.json b/opensearch/schemas/queries/opensearch-log-query/tests/invalid/lucene-without-index.json new file mode 100644 index 000000000..ec7bf41c1 --- /dev/null +++ b/opensearch/schemas/queries/opensearch-log-query/tests/invalid/lucene-without-index.json @@ -0,0 +1,4 @@ +{ + "kind": "OpenSearchLogQuery", + "spec": { "queryLanguage": "lucene", "query": "level:error" } +} diff --git a/opensearch/schemas/queries/opensearch-log-query/tests/invalid/negative-limit.json b/opensearch/schemas/queries/opensearch-log-query/tests/invalid/negative-limit.json new file mode 100644 index 000000000..b83bdfb89 --- /dev/null +++ b/opensearch/schemas/queries/opensearch-log-query/tests/invalid/negative-limit.json @@ -0,0 +1,4 @@ +{ + "kind": "OpenSearchLogQuery", + "spec": { "queryLanguage": "lucene", "query": "level:error", "index": "logs-*", "limit": -3 } +} diff --git a/opensearch/schemas/queries/opensearch-log-query/tests/invalid/unknown-language.json b/opensearch/schemas/queries/opensearch-log-query/tests/invalid/unknown-language.json new file mode 100644 index 000000000..172a82651 --- /dev/null +++ b/opensearch/schemas/queries/opensearch-log-query/tests/invalid/unknown-language.json @@ -0,0 +1,4 @@ +{ + "kind": "OpenSearchLogQuery", + "spec": { "queryLanguage": "kql", "query": "level:error", "index": "logs-*" } +} diff --git a/opensearch/schemas/queries/opensearch-log-query/tests/invalid/zero-limit.json b/opensearch/schemas/queries/opensearch-log-query/tests/invalid/zero-limit.json new file mode 100644 index 000000000..1db868137 --- /dev/null +++ b/opensearch/schemas/queries/opensearch-log-query/tests/invalid/zero-limit.json @@ -0,0 +1,4 @@ +{ + "kind": "OpenSearchLogQuery", + "spec": { "queryLanguage": "lucene", "query": "level:error", "index": "logs-*", "limit": 0 } +} diff --git a/opensearch/schemas/queries/opensearch-log-query/tests/valid/dsl.json b/opensearch/schemas/queries/opensearch-log-query/tests/valid/dsl.json new file mode 100644 index 000000000..70c3b3096 --- /dev/null +++ b/opensearch/schemas/queries/opensearch-log-query/tests/valid/dsl.json @@ -0,0 +1,4 @@ +{ + "kind": "OpenSearchLogQuery", + "spec": { "queryLanguage": "dsl", "query": "{\"query\":{\"match\":{\"level\":\"error\"}}}", "index": "logs-*" } +} diff --git a/opensearch/schemas/queries/opensearch-log-query/tests/valid/lucene.json b/opensearch/schemas/queries/opensearch-log-query/tests/valid/lucene.json new file mode 100644 index 000000000..df3a8a334 --- /dev/null +++ b/opensearch/schemas/queries/opensearch-log-query/tests/valid/lucene.json @@ -0,0 +1,4 @@ +{ + "kind": "OpenSearchLogQuery", + "spec": { "queryLanguage": "lucene", "query": "level:error AND service:api", "index": "logs-*", "limit": 250 } +} diff --git a/opensearch/schemas/queries/opensearch-log-query/tests/valid/no-language.json b/opensearch/schemas/queries/opensearch-log-query/tests/valid/no-language.json new file mode 100644 index 000000000..deff77e09 --- /dev/null +++ b/opensearch/schemas/queries/opensearch-log-query/tests/valid/no-language.json @@ -0,0 +1,4 @@ +{ + "kind": "OpenSearchLogQuery", + "spec": { "query": "source=logs-* | head 10" } +} diff --git a/opensearch/schemas/queries/opensearch-log-query/tests/valid/ppl.json b/opensearch/schemas/queries/opensearch-log-query/tests/valid/ppl.json new file mode 100644 index 000000000..927e76f89 --- /dev/null +++ b/opensearch/schemas/queries/opensearch-log-query/tests/valid/ppl.json @@ -0,0 +1,12 @@ +{ + "kind": "OpenSearchLogQuery", + "spec": { + "datasource": { "kind": "OpenSearchDatasource", "name": "opensearch-demo" }, + "queryLanguage": "ppl", + "query": "source=logs-* | where level='error'", + "index": "logs-*", + "timestampField": "@timestamp", + "messageField": "message", + "disableTimeFilter": false + } +} diff --git a/opensearch/schemas/queries/opensearch-log-query/tests/valid/sql.json b/opensearch/schemas/queries/opensearch-log-query/tests/valid/sql.json new file mode 100644 index 000000000..4cae7fea2 --- /dev/null +++ b/opensearch/schemas/queries/opensearch-log-query/tests/valid/sql.json @@ -0,0 +1,4 @@ +{ + "kind": "OpenSearchLogQuery", + "spec": { "queryLanguage": "sql", "query": "SELECT @timestamp, message FROM `logs-2025`" } +} diff --git a/opensearch/sdk/go/query/log/log.go b/opensearch/sdk/go/query/log/log.go index 20b653675..c5afac5c9 100644 --- a/opensearch/sdk/go/query/log/log.go +++ b/opensearch/sdk/go/query/log/log.go @@ -27,6 +27,8 @@ const PluginKind = "OpenSearchLogQuery" type PluginSpec struct { Datasource *datasource.Selector `json:"datasource,omitempty" yaml:"datasource,omitempty"` Query string `json:"query" yaml:"query"` + QueryLanguage string `json:"queryLanguage,omitempty" yaml:"queryLanguage,omitempty"` + Limit int `json:"limit,omitempty" yaml:"limit,omitempty"` Index string `json:"index,omitempty" yaml:"index,omitempty"` TimestampField string `json:"timestampField,omitempty" yaml:"timestampField,omitempty"` MessageField string `json:"messageField,omitempty" yaml:"messageField,omitempty"` @@ -59,10 +61,26 @@ func (s *PluginSpec) UnmarshalYAML(unmarshal func(interface{}) error) error { return nil } +// validLanguages mirrors the CUE schema. An empty value means PPL. +var validLanguages = map[string]bool{"": true, "ppl": true, "sql": true, "lucene": true, "dsl": true} + func (s *PluginSpec) validate() error { if len(s.Query) == 0 { return fmt.Errorf("query cannot be empty") } + if !validLanguages[s.QueryLanguage] { + return fmt.Errorf("unknown query language %q: want one of ppl, sql, lucene, dsl", s.QueryLanguage) + } + if (s.QueryLanguage == "lucene" || s.QueryLanguage == "dsl") && len(s.Index) == 0 { + return fmt.Errorf("index is required when queryLanguage is %q", s.QueryLanguage) + } + // Only negative values are rejected here: with a bare int + omitempty, an omitted + // `limit` also unmarshals to 0, so rejecting 0 would reject every spec that simply + // omits the field. An explicit `"limit": 0` in hand-written YAML/JSON therefore + // passes here and is caught by the CUE schema instead. + if s.Limit < 0 { + return fmt.Errorf("limit cannot be negative") + } return nil } @@ -83,6 +101,10 @@ func create(query string, options ...Option) (Builder, error) { } } + if err := builder.PluginSpec.validate(); err != nil { + return *builder, err + } + return *builder, nil } diff --git a/opensearch/sdk/go/query/log/log_test.go b/opensearch/sdk/go/query/log/log_test.go index ef761d76e..278530c16 100644 --- a/opensearch/sdk/go/query/log/log_test.go +++ b/opensearch/sdk/go/query/log/log_test.go @@ -15,6 +15,7 @@ package log import ( "encoding/json" + "strings" "testing" ) @@ -103,3 +104,96 @@ func TestOpenSearchLogQueryDisableTimeFilter(t *testing.T) { t.Errorf("disableTimeFilter mismatch: %v", out["disableTimeFilter"]) } } + +func TestQueryLanguageAndLimit(t *testing.T) { + q := OpenSearchLogQuery("level:error", + QueryLanguage("lucene"), + Index("logs-*"), + Limit(250), + ) + if q.Error != nil { + t.Fatalf("unexpected error: %v", q.Error) + } + + raw, err := json.Marshal(q.Plugin.Spec) + if err != nil { + t.Fatalf("marshal: %v", err) + } + var out map[string]any + if err := json.Unmarshal(raw, &out); err != nil { + t.Fatalf("unmarshal: %v", err) + } + + if out["queryLanguage"] != "lucene" { + t.Errorf("queryLanguage mismatch: %v", out["queryLanguage"]) + } + // JSON numbers decode as float64 into map[string]any. + if out["limit"] != float64(250) { + t.Errorf("limit mismatch: %v", out["limit"]) + } +} + +func TestDefaultLanguageIsOmitted(t *testing.T) { + q := OpenSearchLogQuery("source=logs-*") + if q.Error != nil { + t.Fatalf("unexpected error: %v", q.Error) + } + + raw, err := json.Marshal(q.Plugin.Spec) + if err != nil { + t.Fatalf("marshal: %v", err) + } + var out map[string]any + if err := json.Unmarshal(raw, &out); err != nil { + t.Fatalf("unmarshal: %v", err) + } + + // No queryLanguage or limit key at all: the plugin treats absence as PPL. + if _, ok := out["queryLanguage"]; ok { + t.Errorf("queryLanguage should be omitted, got %v", out["queryLanguage"]) + } + if _, ok := out["limit"]; ok { + t.Errorf("limit should be omitted, got %v", out["limit"]) + } +} + +func TestRejectsUnknownLanguage(t *testing.T) { + q := OpenSearchLogQuery("level:error", QueryLanguage("kql"), Index("logs-*")) + if q.Error == nil { + t.Fatal("expected an error for an unknown query language") + } + if !strings.Contains(q.Error.Error(), "unknown query language") { + t.Errorf("unexpected error text: %v", q.Error) + } +} + +func TestRequiresIndexForLucene(t *testing.T) { + var spec PluginSpec + err := json.Unmarshal([]byte(`{"query":"level:error","queryLanguage":"lucene"}`), &spec) + if err == nil { + t.Fatal("expected an error when index is missing for lucene") + } + if !strings.Contains(err.Error(), "index is required") { + t.Errorf("unexpected error text: %v", err) + } +} + +func TestLimitRejectsZero(t *testing.T) { + q := OpenSearchLogQuery("level:error", QueryLanguage("lucene"), Index("logs-*"), Limit(0)) + if q.Error == nil { + t.Fatal("expected an error for Limit(0)") + } + if !strings.Contains(q.Error.Error(), "limit must be greater than 0") { + t.Errorf("unexpected error text: %v", q.Error) + } +} + +func TestLimitRejectsNegative(t *testing.T) { + q := OpenSearchLogQuery("level:error", QueryLanguage("lucene"), Index("logs-*"), Limit(-3)) + if q.Error == nil { + t.Fatal("expected an error for Limit(-3)") + } + if !strings.Contains(q.Error.Error(), "limit must be greater than 0") { + t.Errorf("unexpected error text: %v", q.Error) + } +} diff --git a/opensearch/sdk/go/query/log/options.go b/opensearch/sdk/go/query/log/options.go index e76c0da1c..e4fe1c341 100644 --- a/opensearch/sdk/go/query/log/options.go +++ b/opensearch/sdk/go/query/log/options.go @@ -14,6 +14,8 @@ package log import ( + "fmt" + opensearchDatasource "github.com/perses/plugins/opensearch/sdk/go/datasource" ) @@ -58,3 +60,26 @@ func DisableTimeFilter(disable bool) Option { return nil } } + +// QueryLanguage sets the language the query is written in: "ppl" (the default when +// unset), "sql", "lucene", or "dsl". "lucene" and "dsl" also require Index. +func QueryLanguage(lang string) Option { + return func(builder *Builder) error { + builder.QueryLanguage = lang + return nil + } +} + +// Limit caps the number of log documents fetched. Applies to the "lucene" and "dsl" +// languages, which map it to the _search request's `size`. Must be greater than zero: +// the CUE schema constrains `limit` to `int & >0`, and because the field is a bare int +// with omitempty, a zero would be indistinguishable from an omitted value. +func Limit(n int) Option { + return func(builder *Builder) error { + if n <= 0 { + return fmt.Errorf("limit must be greater than 0, got %d", n) + } + builder.Limit = n + return nil + } +} diff --git a/opensearch/src/datasources/opensearch-datasource/OpenSearchDatasource.tsx b/opensearch/src/datasources/opensearch-datasource/OpenSearchDatasource.tsx index 5de98b136..bf6e566e9 100644 --- a/opensearch/src/datasources/opensearch-datasource/OpenSearchDatasource.tsx +++ b/opensearch/src/datasources/opensearch-datasource/OpenSearchDatasource.tsx @@ -14,7 +14,7 @@ import type { DatasourcePlugin } from '@perses-dev/plugin-system'; import type { OpenSearchClient } from '../../model/opensearch-client'; -import { ppl } from '../../model/opensearch-client'; +import { ppl, search, sql } from '../../model/opensearch-client'; import type { OpenSearchDatasourceSpec } from './opensearch-datasource-types'; import { OpenSearchDatasourceEditor } from './OpenSearchDatasourceEditor'; @@ -34,6 +34,8 @@ const createClient: DatasourcePlugin datasourceUrl, }, ppl: (params, headers, signal) => ppl(params, { datasourceUrl, headers: headers ?? specHeaders }, signal), + sql: (params, headers, signal) => sql(params, { datasourceUrl, headers: headers ?? specHeaders }, signal), + search: (params, headers, signal) => search(params, { datasourceUrl, headers: headers ?? specHeaders }, signal), }; }; diff --git a/opensearch/src/datasources/opensearch-datasource/OpenSearchDatasourceEditor.tsx b/opensearch/src/datasources/opensearch-datasource/OpenSearchDatasourceEditor.tsx index 0f59b1326..91450c597 100644 --- a/opensearch/src/datasources/opensearch-datasource/OpenSearchDatasourceEditor.tsx +++ b/opensearch/src/datasources/opensearch-datasource/OpenSearchDatasourceEditor.tsx @@ -35,6 +35,16 @@ export function OpenSearchDatasourceEditor(props: OpenSearchDatasourceEditorProp endpointPattern: '/_plugins/_ppl', method: 'POST', }, + { + endpointPattern: '/_plugins/_sql', + method: 'POST', + }, + { + // Lucene and Query DSL queries POST to /{index}/_search, so the index + // segment has to be matched as a pattern. + endpointPattern: '/.*/_search', + method: 'POST', + }, ], url: '', }, diff --git a/opensearch/src/model/opensearch-client-types.ts b/opensearch/src/model/opensearch-client-types.ts index 81fd2de25..c95111404 100644 --- a/opensearch/src/model/opensearch-client-types.ts +++ b/opensearch/src/model/opensearch-client-types.ts @@ -13,13 +13,14 @@ export type OpenSearchRequestHeaders = Record; -export interface OpenSearchPPLColumn { +export interface OpenSearchDatarowsColumn { name: string; type: string; } -export interface OpenSearchPPLResponse { - schema: OpenSearchPPLColumn[]; +/** The schema + datarows shape returned by _ppl and _sql. OpenSearch calls this its `jdbc` response format (?format=jdbc), which is the default. */ +export interface OpenSearchDatarowsResponse { + schema: OpenSearchDatarowsColumn[]; datarows: Array>; total?: number; size?: number; @@ -33,3 +34,20 @@ export interface OpenSearchErrorResponse { }; status?: number; } + +export interface OpenSearchSearchHit { + _index?: string; + _id?: string; + _source?: Record; +} + +export interface OpenSearchSearchResponse { + hits: { + /** Absent when the request sets `track_total_hits: false`. */ + total?: { value: number; relation: 'eq' | 'gte' }; + hits: OpenSearchSearchHit[]; + }; +} + +/** A `_search` request body. Kept loose so a user-supplied Query DSL body passes through unchanged. */ +export type OpenSearchSearchRequestBody = Record; diff --git a/opensearch/src/model/opensearch-client.test.ts b/opensearch/src/model/opensearch-client.test.ts index 9d60af651..bd5c01438 100644 --- a/opensearch/src/model/opensearch-client.test.ts +++ b/opensearch/src/model/opensearch-client.test.ts @@ -13,7 +13,7 @@ import type { Mock } from 'vitest'; -import { ppl, OpenSearchPPLError } from './opensearch-client'; +import { ppl, sql, search, OpenSearchApiError } from './opensearch-client'; describe('opensearch-client', () => { const originalFetch = global.fetch; @@ -62,15 +62,22 @@ describe('opensearch-client', () => { expect((init.headers as Record).Authorization).toBe('Basic xyz'); }); - it('throws OpenSearchPPLError carrying status and body on non-200', async () => { + it('throws OpenSearchApiError carrying status and body on non-200', async () => { mockFetch({ ok: false, status: 400, body: '{"error":{"reason":"bad PPL"}}' }); await expect(ppl({ query: 'broken' }, { datasourceUrl: 'http://localhost:9200' })).rejects.toMatchObject({ - name: 'OpenSearchPPLError', + name: 'OpenSearchApiError', status: 400, body: '{"error":{"reason":"bad PPL"}}', }); await expect(ppl({ query: 'broken' }, { datasourceUrl: 'http://localhost:9200' })).rejects.toBeInstanceOf( - OpenSearchPPLError, + OpenSearchApiError, + ); + }); + + it('labels the error message PPL', async () => { + mockFetch({ ok: false, status: 400, body: '{"error":{"reason":"bad PPL"}}' }); + await expect(ppl({ query: 'broken' }, { datasourceUrl: 'http://localhost:9200' })).rejects.toThrow( + 'OpenSearch PPL request failed (400): bad PPL', ); }); @@ -81,55 +88,142 @@ describe('opensearch-client', () => { expect(url).toContain('/api/datasources/proxy/1/_plugins/_ppl'); }); - describe('OpenSearchPPLError.message', () => { + describe('OpenSearchApiError.message', () => { it('uses error.reason from a JSON body when present', () => { - const err = new OpenSearchPPLError(400, '{"error":{"reason":"bad PPL","type":"SyntaxCheckException"}}'); - expect(err.message).toBe('OpenSearch PPL request failed (400): bad PPL'); + const err = new OpenSearchApiError(400, '{"error":{"reason":"bad PPL","type":"SyntaxCheckException"}}'); + expect(err.message).toBe('OpenSearch API request failed (400): bad PPL'); expect(err.body).toBe('{"error":{"reason":"bad PPL","type":"SyntaxCheckException"}}'); }); it('falls back to error.details when reason is absent', () => { - const err = new OpenSearchPPLError(400, '{"error":{"details":"timestamp:2026-... in unsupported format"}}'); - expect(err.message).toBe('OpenSearch PPL request failed (400): timestamp:2026-... in unsupported format'); + const err = new OpenSearchApiError(400, '{"error":{"details":"timestamp:2026-... in unsupported format"}}'); + expect(err.message).toBe('OpenSearch API request failed (400): timestamp:2026-... in unsupported format'); }); it('omits the body entirely when the body is not JSON', () => { - const err = new OpenSearchPPLError(502, 'Bad Gateway'); - expect(err.message).toBe('OpenSearch PPL request failed (502)'); + const err = new OpenSearchApiError(502, 'Bad Gateway'); + expect(err.message).toBe('OpenSearch API request failed (502)'); expect(err.body).toBe('Bad Gateway'); }); it('omits the body when JSON has no error field', () => { - const err = new OpenSearchPPLError(500, '{"status":"weird"}'); - expect(err.message).toBe('OpenSearch PPL request failed (500)'); + const err = new OpenSearchApiError(500, '{"status":"weird"}'); + expect(err.message).toBe('OpenSearch API request failed (500)'); }); it('combines reason and details when both are present and differ', () => { - const err = new OpenSearchPPLError( + const err = new OpenSearchApiError( 400, '{"error":{"reason":"Invalid Query","details":"can\'t resolve Symbol(name=@timestamp)"}}', ); expect(err.message).toBe( - "OpenSearch PPL request failed (400): Invalid Query — can't resolve Symbol(name=@timestamp)", + "OpenSearch API request failed (400): Invalid Query — can't resolve Symbol(name=@timestamp)", ); }); it('does not duplicate text when reason equals details', () => { - const err = new OpenSearchPPLError(400, '{"error":{"reason":"same text","details":"same text"}}'); - expect(err.message).toBe('OpenSearch PPL request failed (400): same text'); + const err = new OpenSearchApiError(400, '{"error":{"reason":"same text","details":"same text"}}'); + expect(err.message).toBe('OpenSearch API request failed (400): same text'); }); it('uses reason alone when details is an empty string', () => { - const err = new OpenSearchPPLError(400, '{"error":{"reason":"Invalid Query","details":""}}'); - expect(err.message).toBe('OpenSearch PPL request failed (400): Invalid Query'); + const err = new OpenSearchApiError(400, '{"error":{"reason":"Invalid Query","details":""}}'); + expect(err.message).toBe('OpenSearch API request failed (400): Invalid Query'); }); it('falls through to details when reason is an empty string', () => { - const err = new OpenSearchPPLError( + const err = new OpenSearchApiError( 400, '{"error":{"reason":"","details":"can\'t resolve Symbol(name=@timestamp)"}}', ); - expect(err.message).toBe("OpenSearch PPL request failed (400): can't resolve Symbol(name=@timestamp)"); + expect(err.message).toBe("OpenSearch API request failed (400): can't resolve Symbol(name=@timestamp)"); + }); + }); + + describe('sql', () => { + it('POSTs to /_plugins/_sql with query and filter', async () => { + const mock = mockFetch({ ok: true, status: 200, body: { schema: [], datarows: [] } }); + const filter = { range: { '@timestamp': { gte: 1, lte: 2, format: 'epoch_millis' } } }; + await sql({ query: 'SELECT * FROM logs', filter }, { datasourceUrl: 'http://localhost:9200' }); + + const [url, init] = mock.mock.calls[0] as [string, RequestInit]; + expect(url).toBe('http://localhost:9200/_plugins/_sql'); + expect(init.method).toBe('POST'); + expect(JSON.parse(init.body as string)).toEqual({ query: 'SELECT * FROM logs', filter }); + }); + + it('omits filter from the body when not provided', async () => { + const mock = mockFetch({ ok: true, status: 200, body: { schema: [], datarows: [] } }); + await sql({ query: 'SELECT 1' }, { datasourceUrl: 'http://localhost:9200' }); + const [, init] = mock.mock.calls[0] as [string, RequestInit]; + expect(JSON.parse(init.body as string)).toEqual({ query: 'SELECT 1' }); + }); + + it('labels the error message SQL', async () => { + mockFetch({ ok: false, status: 400, body: '{"error":{"reason":"bad SQL"}}' }); + await expect(sql({ query: 'x' }, { datasourceUrl: 'http://localhost:9200' })).rejects.toThrow( + 'OpenSearch SQL request failed (400): bad SQL' + ); + }); + }); + + describe('search', () => { + it('POSTs the body to /{index}/_search', async () => { + const mock = mockFetch({ ok: true, status: 200, body: { hits: { hits: [] } } }); + const body = { size: 500, query: { match_all: {} } }; + await search({ index: 'logs-*', body }, { datasourceUrl: 'http://localhost:9200' }); + + const [url, init] = mock.mock.calls[0] as [string, RequestInit]; + expect(url).toBe('http://localhost:9200/logs-*/_search'); + expect(JSON.parse(init.body as string)).toEqual(body); + }); + + it('preserves commas for multi-index patterns', async () => { + const mock = mockFetch({ ok: true, status: 200, body: { hits: { hits: [] } } }); + await search({ index: 'a-*,b-*', body: {} }, { datasourceUrl: 'http://localhost:9200' }); + const [url] = mock.mock.calls[0] as [string]; + expect(url).toBe('http://localhost:9200/a-*,b-*/_search'); + }); + + it('allows dots within an index name (date-suffixed indices)', async () => { + const mock = mockFetch({ ok: true, status: 200, body: { hits: { hits: [] } } }); + await search({ index: 'logs-2025.01.01', body: {} }, { datasourceUrl: 'http://localhost:9200' }); + const [url] = mock.mock.calls[0] as [string]; + expect(url).toBe('http://localhost:9200/logs-2025.01.01/_search'); + }); + + it.each([ + '../_cluster/settings', + 'logs/../..', + 'a?b', + 'a#b', + 'logs *', + '', + '%2e%2e', + '%2E%2E', + '%2e', + '.', + '..', + '\\', + 'logs-*,.', + 'a%20b', + ])('rejects unsafe index %p without issuing a request', async (index) => { + const mock = mockFetch({ ok: true, status: 200, body: { hits: { hits: [] } } }); + await expect(search({ index, body: {} }, { datasourceUrl: 'http://localhost:9200' })).rejects.toThrow( + /invalid OpenSearch index/i + ); + expect(mock).not.toHaveBeenCalled(); + }); + + it('surfaces root_cause reason from a _search error body', async () => { + mockFetch({ + ok: false, + status: 400, + body: '{"error":{"type":"search_phase_execution_exception","root_cause":[{"reason":"failed to parse date field"}]},"status":400}', + }); + await expect(search({ index: 'logs-*', body: {} }, { datasourceUrl: 'http://localhost:9200' })).rejects.toThrow( + 'OpenSearch Search request failed (400): failed to parse date field' + ); }); }); }); diff --git a/opensearch/src/model/opensearch-client.ts b/opensearch/src/model/opensearch-client.ts index 6eee66fee..3d6ba6b20 100644 --- a/opensearch/src/model/opensearch-client.ts +++ b/opensearch/src/model/opensearch-client.ts @@ -11,7 +11,12 @@ // See the License for the specific language governing permissions and // limitations under the License. -import type { OpenSearchPPLResponse, OpenSearchRequestHeaders } from './opensearch-client-types'; +import type { + OpenSearchDatarowsResponse, + OpenSearchRequestHeaders, + OpenSearchSearchRequestBody, + OpenSearchSearchResponse, +} from './opensearch-client-types'; export interface OpenSearchPPLParams { query: string; @@ -22,6 +27,16 @@ export interface OpenSearchApiOptions { headers?: OpenSearchRequestHeaders; } +export interface OpenSearchSQLParams { + query: string; + filter?: unknown; +} + +export interface OpenSearchSearchParams { + index: string; + body: OpenSearchSearchRequestBody; +} + export interface OpenSearchClient { options: { datasourceUrl: string; @@ -30,36 +45,59 @@ export interface OpenSearchClient { params: OpenSearchPPLParams, headers?: OpenSearchRequestHeaders, signal?: AbortSignal, - ) => Promise; + ) => Promise; + sql: ( + params: OpenSearchSQLParams, + headers?: OpenSearchRequestHeaders, + signal?: AbortSignal, + ) => Promise; + search: ( + params: OpenSearchSearchParams, + headers?: OpenSearchRequestHeaders, + signal?: AbortSignal, + ) => Promise; } -export class OpenSearchPPLError extends Error { +export class OpenSearchApiError extends Error { constructor( public readonly status: number, public readonly body: string, message?: string, ) { - super(message ?? buildShortMessage(status, body)); - this.name = 'OpenSearchPPLError'; + super(message ?? buildShortMessage('API', status, body)); + this.name = 'OpenSearchApiError'; } } -function buildShortMessage(status: number, body: string): string { +/** + * Label used in error messages so the user knows which API rejected the request. + * `'API'` is a neutral fallback for a caller that constructs `OpenSearchApiError` + * without an explicit message — unreachable today since all three transports pass + * one, but a future caller doing so should get a generic label rather than a + * misleading `'PPL'`. + */ +type ApiLabel = 'PPL' | 'SQL' | 'Search' | 'API'; + +function buildShortMessage(label: ApiLabel, status: number, body: string): string { try { - const parsed = JSON.parse(body) as { error?: { reason?: string; details?: string } }; + const parsed = JSON.parse(body) as { + error?: { reason?: string; details?: string; root_cause?: Array<{ reason?: string }> }; + }; const reason = parsed?.error?.reason; const details = parsed?.error?.details; // OpenSearch often puts a generic phrase in `reason` ("Invalid Query") and the // actionable text in `details` ("can't resolve Symbol(name=@timestamp)"). Surface - // both so the user can diagnose the failure. - const suffix = reason && details && reason !== details ? `${reason} — ${details}` : reason || details; + // both so the user can diagnose the failure. `_search` errors carry neither and + // put the useful text in root_cause[0].reason instead. + const rootCause = parsed?.error?.root_cause?.[0]?.reason; + const suffix = reason && details && reason !== details ? `${reason} — ${details}` : reason || details || rootCause; if (suffix) { - return `OpenSearch PPL request failed (${status}): ${suffix}`; + return `OpenSearch ${label} request failed (${status}): ${suffix}`; } } catch { // body wasn't JSON — fall through to the bare-status form } - return `OpenSearch PPL request failed (${status})`; + return `OpenSearch ${label} request failed (${status})`; } function buildUrl(path: string, datasourceUrl: string): URL { @@ -79,27 +117,96 @@ function buildUrl(path: string, datasourceUrl: string): URL { return new URL(fullPath, window.location.origin); } -export async function ppl( - params: OpenSearchPPLParams, +async function postJson( + label: ApiLabel, + path: string, + body: unknown, options: OpenSearchApiOptions, signal?: AbortSignal, -): Promise { - const url = buildUrl('/_plugins/_ppl', options.datasourceUrl); +): Promise { + const url = buildUrl(path, options.datasourceUrl); const response = await fetch(url.toString(), { method: 'POST', - headers: { - 'Content-Type': 'application/json', - ...options.headers, - }, - body: JSON.stringify({ query: params.query }), + headers: { 'Content-Type': 'application/json', ...options.headers }, + body: JSON.stringify(body), signal, }); if (!response.ok) { - const body = await response.text(); - throw new OpenSearchPPLError(response.status, body); + const text = await response.text(); + throw new OpenSearchApiError(response.status, text, buildShortMessage(label, response.status, text)); } return response.json(); } + +/** + * `_search` puts the index in the URL *path*, so an index like `../_cluster/settings` + * would reach a different endpoint entirely. `sanitizeIndex` in build/ppl.ts is the + * wrong filter here (it targets PPL metacharacters), so guard the path separately. + * + * This MUST be an allowlist, not a denylist of specific characters: the value is + * interpolated into a URL path and then re-parsed by `new URL` (via `buildUrl`), and + * that WHATWG parser normalises the path *after* this guard would run — it collapses + * percent-encoded and bare dot segments (`%2e%2e`, `.`, `..`) and treats a backslash + * as a path separator, so a denylist that only checks for literal `/`, `?`, `#`, `..` + * misses `%2e%2e` (decodes to `..` and climbs out of the index segment to the + * cluster-wide root) and `\` (reparsed as an authority separator, escaping the host + * entirely). Only an allowlist of characters that OpenSearch itself permits in an + * index name closes this off in one rule, because it rejects `%` and `\` outright + * regardless of what they'd decode or reparse to. + */ +export function assertSafeIndexPath(index: string): void { + const invalid = (): never => { + throw new Error( + `invalid OpenSearch index for a _search request: ${JSON.stringify(index)}. ` + + 'An index (or comma-separated list of indices) must be non-empty and may only contain ' + + 'letters, digits, "_", ".", "+", "-", and "*" as path segments, with no empty or ' + + 'dots-only segment.' + ); + }; + + if (!index || /[^A-Za-z0-9_.+*,-]/.test(index)) { + invalid(); + } + + // Reject a comma-separated segment that is empty or consists only of dots — this + // catches the bare `.` / `..` traversal forms (and a dot segment hidden inside a + // multi-index list like `logs-*,.`) that the character allowlist above lets through, + // since `.` is itself a legitimate character in an index name like `logs-2025.01.01`. + for (const segment of index.split(',')) { + if (segment === '' || /^\.+$/.test(segment)) { + invalid(); + } + } +} + +export async function ppl( + params: OpenSearchPPLParams, + options: OpenSearchApiOptions, + signal?: AbortSignal +): Promise { + return postJson('PPL', '/_plugins/_ppl', { query: params.query }, options, signal); +} + +export async function sql( + params: OpenSearchSQLParams, + options: OpenSearchApiOptions, + signal?: AbortSignal +): Promise { + const body: Record = { query: params.query }; + if (params.filter !== undefined) { + body.filter = params.filter; + } + return postJson('SQL', '/_plugins/_sql', body, options, signal); +} + +export async function search( + params: OpenSearchSearchParams, + options: OpenSearchApiOptions, + signal?: AbortSignal +): Promise { + assertSafeIndexPath(params.index); + return postJson('Search', `/${params.index}/_search`, params.body, options, signal); +} diff --git a/opensearch/src/queries/constants.ts b/opensearch/src/queries/constants.ts index be59e5dbe..eda6f43e3 100644 --- a/opensearch/src/queries/constants.ts +++ b/opensearch/src/queries/constants.ts @@ -11,6 +11,8 @@ // See the License for the specific language governing permissions and // limitations under the License. +import type { OpenSearchQueryLanguage } from './opensearch-log-query/opensearch-log-query-types'; + export const DATASOURCE_KIND = 'OpenSearchDatasource'; export const DEFAULT_DATASOURCE = { kind: DATASOURCE_KIND }; @@ -19,11 +21,71 @@ export const DEFAULT_DATASOURCE = { kind: DATASOURCE_KIND }; export const DEFAULT_TIMESTAMP_FIELDS = ['@timestamp', 'timestamp', 'time']; export const DEFAULT_MESSAGE_FIELDS = ['message', 'log', 'body']; -// OpenSearch PPL reference documentation. -export const PPL_DOCS_URL = 'https://opensearch.org/docs/latest/search-plugins/sql/ppl/index/'; +export interface LanguageMeta { + label: string; + docsUrl: string; + docsLabel: string; + placeholder: string; + examples: string; + indexHelperText: string; + usageNote: string; +} -// A handful of copy-pasteable PPL examples shown in the editor's "Query Examples" disclosure. -export const PPL_QUERY_EXAMPLES = `source=logs-* | where level="error" | head 20 +/** + * Single source of truth for the editor's language dropdown and every piece of + * language-specific copy. Adding a language is one entry here plus a `case` in the + * dispatcher. + */ +export const QUERY_LANGUAGES: Record = { + ppl: { + label: 'PPL', + docsUrl: 'https://docs.opensearch.org/latest/sql-and-ppl/ppl/index/', + docsLabel: 'PPL', + placeholder: 'e.g. source=logs-* | where level="error"', + examples: `source=logs-* | where level="error" | head 20 source=logs-* | fields @timestamp, service, message | head 50 source=logs-* | stats count() by service -source=logs-* | where traceId='$traceId'`; +source=logs-* | where traceId='$traceId'`, + indexHelperText: 'Ignored when the PPL query starts with source=.', + usageNote: 'Requires the SQL/PPL plugin enabled on the OpenSearch cluster.', + }, + sql: { + label: 'SQL', + docsUrl: 'https://docs.opensearch.org/latest/sql-and-ppl/sql/index/', + docsLabel: 'SQL', + placeholder: "e.g. SELECT @timestamp, message FROM `logs-*` WHERE level = 'error'", + examples: `SELECT @timestamp, message, level FROM \`logs-*\` WHERE level = 'error' LIMIT 20 +SELECT service, COUNT(*) FROM \`logs-*\` GROUP BY service +SELECT * FROM \`otel-logs-*\` WHERE traceId = '$traceId' LIMIT 50`, + indexHelperText: 'Ignored — the index comes from the FROM clause of your SQL.', + usageNote: 'Requires the SQL/PPL plugin enabled on the OpenSearch cluster.', + }, + lucene: { + label: 'Lucene', + docsUrl: 'https://docs.opensearch.org/latest/query-dsl/full-text/query-string/', + docsLabel: 'query string', + placeholder: 'e.g. level:error AND service:api', + examples: `level:error AND service:api +message:"connection refused" +status:[500 TO 599] +traceId:$traceId +NOT level:debug`, + indexHelperText: 'Required. Sent as the index in POST /{index}/_search.', + usageNote: 'The dashboard time range is added to your query automatically.', + }, + dsl: { + label: 'Query DSL', + docsUrl: 'https://docs.opensearch.org/latest/query-dsl/', + docsLabel: 'Query DSL', + placeholder: 'e.g. {"query": {"match": {"level": "error"}}}', + examples: `{"query": {"match": {"level": "error"}}} +{"query": {"bool": {"must_not": [{"term": {"level": "debug"}}]}}} +{"size": 100, "query": {"term": {"traceId": "$traceId"}}}`, + indexHelperText: 'Required. Sent as the index in POST /{index}/_search.', + usageNote: 'The dashboard time range is added to your query automatically.', + }, +}; + +// `_search` returns only 10 hits by default, which would silently truncate a log panel +// to ten lines. 500 matches the log default used by Grafana's OpenSearch datasource. +export const DEFAULT_SEARCH_SIZE = 500; diff --git a/opensearch/src/queries/opensearch-log-query/OpenSearchLogQuery.tsx b/opensearch/src/queries/opensearch-log-query/OpenSearchLogQuery.tsx index 94963ab9a..f54bd4681 100644 --- a/opensearch/src/queries/opensearch-log-query/OpenSearchLogQuery.tsx +++ b/opensearch/src/queries/opensearch-log-query/OpenSearchLogQuery.tsx @@ -21,7 +21,7 @@ import { OpenSearchLogQueryEditor } from './OpenSearchLogQueryEditor'; export const OpenSearchLogQuery: LogQueryPlugin = { getLogData: getOpenSearchLogData, OptionsEditorComponent: OpenSearchLogQueryEditor, - createInitialOptions: () => ({ query: '' }), + createInitialOptions: () => ({ query: '', queryLanguage: 'ppl' }), dependsOn: (spec) => { const queryVariables = parseVariables(spec.query); const indexVariables = spec.index ? parseVariables(spec.index) : []; diff --git a/opensearch/src/queries/opensearch-log-query/OpenSearchLogQueryEditor.test.tsx b/opensearch/src/queries/opensearch-log-query/OpenSearchLogQueryEditor.test.tsx index 4b924feb3..d71ede9e7 100644 --- a/opensearch/src/queries/opensearch-log-query/OpenSearchLogQueryEditor.test.tsx +++ b/opensearch/src/queries/opensearch-log-query/OpenSearchLogQueryEditor.test.tsx @@ -15,9 +15,9 @@ import type * as PluginSystemModule from '@perses-dev/plugin-system'; import { render, screen, fireEvent } from '@testing-library/react'; import type { Mock } from 'vitest'; -import { PPL_DOCS_URL } from '../constants'; +import { QUERY_LANGUAGES } from '../constants'; import type { OpenSearchLogQuerySpec } from './opensearch-log-query-types'; -import { OpenSearchLogQueryEditor } from './OpenSearchLogQueryEditor'; +import { OpenSearchLogQueryEditor, parseLimitInput } from './OpenSearchLogQueryEditor'; vi.mock('@perses-dev/plugin-system', async (importOriginal) => ({ ...(await importOriginal()), @@ -80,7 +80,7 @@ describe('OpenSearchLogQueryEditor', () => { it('links to the OpenSearch PPL documentation', () => { setup(); const link = screen.getByRole('link', { name: /PPL/i }); - expect(link).toHaveAttribute('href', PPL_DOCS_URL); + expect(link).toHaveAttribute('href', QUERY_LANGUAGES.ppl.docsUrl); }); it('renders a Query Examples disclosure', () => { @@ -94,3 +94,104 @@ describe('OpenSearchLogQueryEditor', () => { expect(onChange).toHaveBeenCalledWith(expect.objectContaining({ query: 'source=logs-*' })); }); }); + +describe('OpenSearchLogQueryEditor language selector', () => { + it('shows PPL as the selected language when the spec has none', () => { + setup({ query: 'source=logs-*' }); + expect(screen.getByLabelText('Query language')).toHaveTextContent('PPL'); + }); + + it('writes queryLanguage to the spec without clearing the query', () => { + const { onChange } = setup({ query: 'source=logs-*' }); + + fireEvent.mouseDown(screen.getByLabelText('Query language')); + fireEvent.click(screen.getByRole('option', { name: 'Lucene' })); + + expect(onChange).toHaveBeenCalledWith(expect.objectContaining({ queryLanguage: 'lucene', query: 'source=logs-*' })); + }); + + it('marks the index field required for Lucene when it is empty', () => { + setup({ query: 'level:error', queryLanguage: 'lucene' }); + expect(screen.getByText('Required for Lucene and Query DSL')).toBeInTheDocument(); + }); + + it('does not mark the index field required for PPL', () => { + setup({ query: 'source=logs-*', queryLanguage: 'ppl' }); + expect(screen.queryByText('Required for Lucene and Query DSL')).not.toBeInTheDocument(); + }); + + it('shows the limit field only for the _search languages', () => { + setup({ query: 'source=logs-*', queryLanguage: 'ppl' }); + expect(screen.queryByLabelText(/Limit/)).not.toBeInTheDocument(); + }); + + it('shows the limit field for lucene and dsl', () => { + setup({ query: 'level:error', queryLanguage: 'lucene', index: 'logs-*' }); + expect(screen.getByLabelText(/Limit/)).toBeInTheDocument(); + }); + + it('switches the query field label and placeholder with the language', () => { + setup({ query: '', queryLanguage: 'sql' }); + expect(screen.getByText('SQL Query')).toBeInTheDocument(); + expect(screen.getByPlaceholderText(/SELECT/)).toBeInTheDocument(); + }); + + it('renders without throwing and falls back to PPL for an out-of-enum queryLanguage', () => { + // Cast through `as` since 'kql' is not a valid OpenSearchQueryLanguage — this simulates a + // hand-edited or hand-crafted spec reaching the editor at runtime, bypassing the type system. + expect(() => + setup({ query: 'source=logs-*', queryLanguage: 'kql' as OpenSearchLogQuerySpec['queryLanguage'] }) + ).not.toThrow(); + expect(screen.getByLabelText('Query language')).toHaveTextContent('PPL'); + }); + + it('carries forward an uncommitted (unblurred) query edit when the language is switched', () => { + const { onChange } = setup({ query: 'source=logs-*' }); + + // Edit the query field without blurring it — useQueryState only commits to the spec on blur, + // so at this point value.query is still the old text and only local state has the new text. + fireEvent.change(screen.getByPlaceholderText(/source=logs-/), { + target: { value: 'source=logs-* | where level="warn"' }, + }); + + fireEvent.mouseDown(screen.getByLabelText('Query language')); + fireEvent.click(screen.getByRole('option', { name: 'Lucene' })); + + expect(onChange).toHaveBeenCalledWith( + expect.objectContaining({ queryLanguage: 'lucene', query: 'source=logs-* | where level="warn"' }) + ); + }); +}); + +describe('OpenSearchLogQueryEditor limit field', () => { + // A native sanitizes an invalid string ("abc", " 5" — leading whitespace + // is not a valid floating-point-number character) to "" before React ever sees a change event, + // so those two inputs cannot be produced by firing a DOM change on the rendered field itself. + // We cover them with a direct unit test of parseLimitInput below instead, and cover the + // DOM-reachable values (empty, zero, negative, decimal) end-to-end through the rendered input. + it.each([ + ['', undefined], + ['0', undefined], + ['-5', undefined], + ['3.7', 3], + ])('sets limit to %p -> %p via the rendered field', (input, expected) => { + // Start from a non-empty value so every target value below is a genuine DOM change — + // starting from "" would make the "" case a same-value no-op that never fires onChange. + const { onChange } = setup({ query: 'level:error', queryLanguage: 'lucene', index: 'logs-*', limit: 999 }); + + fireEvent.change(screen.getByLabelText(/Limit/), { target: { value: input } }); + + expect(onChange).toHaveBeenCalledWith(expect.objectContaining({ limit: expected })); + }); + + it.each([ + ['', undefined], + ['0', undefined], + ['-5', undefined], + ['abc', undefined], + ['3.7', 3], + [' 5', 5], + ])('parseLimitInput(%p) -> %p', (input, expected) => { + expect(parseLimitInput(input)).toBe(expected); + }); +}); diff --git a/opensearch/src/queries/opensearch-log-query/OpenSearchLogQueryEditor.tsx b/opensearch/src/queries/opensearch-log-query/OpenSearchLogQueryEditor.tsx index b048bcc43..c7441d02c 100644 --- a/opensearch/src/queries/opensearch-log-query/OpenSearchLogQueryEditor.tsx +++ b/opensearch/src/queries/opensearch-log-query/OpenSearchLogQueryEditor.tsx @@ -11,7 +11,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -import { Box, Checkbox, FormControlLabel, InputLabel, Link, Stack, TextField, Typography } from '@mui/material'; +import { Box, Checkbox, FormControlLabel, InputLabel, Link, MenuItem, Select, Stack, TextField, Typography } from '@mui/material'; import { createModEnterHandler } from '@perses-dev/dashboards'; import type { DatasourceSelectProps, OptionsEditorProps } from '@perses-dev/plugin-system'; import { DatasourceSelect, isVariableDatasource, useDatasourceSelectValueToSelector } from '@perses-dev/plugin-system'; @@ -20,12 +20,32 @@ import type { ReactElement } from 'react'; import type { OpenSearchDatasourceSelector } from '../../model'; import { isDefaultOpenSearchSelector, OPENSEARCH_DATASOURCE_KIND } from '../../model'; -import { DATASOURCE_KIND, DEFAULT_DATASOURCE, PPL_DOCS_URL, PPL_QUERY_EXAMPLES } from '../constants'; +import { DATASOURCE_KIND, DEFAULT_DATASOURCE, QUERY_LANGUAGES } from '../constants'; import { useQueryState } from '../query-editor-model'; -import type { OpenSearchLogQuerySpec } from './opensearch-log-query-types'; +import type { OpenSearchLogQuerySpec, OpenSearchQueryLanguage } from './opensearch-log-query-types'; +import { OPENSEARCH_QUERY_LANGUAGES, requiresIndex, resolveQueryLanguage } from './opensearch-log-query-types'; type OpenSearchQueryEditorProps = OptionsEditorProps; +// Constant across renders — derived once from the language registry so a fifth +// `_search`-based language only needs a `QUERY_LANGUAGES` entry, not an editor edit. +const REQUIRED_INDEX_MESSAGE = `Required for ${OPENSEARCH_QUERY_LANGUAGES.filter(requiresIndex) + .map((l) => QUERY_LANGUAGES[l].label) + .join(' and ')}`; + +/** + * Parses the Limit field's raw text into a spec-safe value: `undefined` for anything empty, + * non-numeric, or not a positive integer, since the CUE schema constrains `limit` to `int & >0`. + * Exported (rather than inlined in the change handler) so it can be unit tested directly — + * a native `` sanitizes invalid strings like `"abc"` or `" 5"` to `""` + * before a change event ever reaches React, so those inputs can't be exercised through the + * rendered field itself. + */ +export function parseLimitInput(raw: string): number | undefined { + const parsed = Number.parseInt(raw, 10); + return Number.isFinite(parsed) && parsed > 0 ? parsed : undefined; +} + const examplesSx = { fontSize: '11px', color: 'text.secondary', @@ -48,6 +68,30 @@ export function OpenSearchLogQueryEditor(props: OpenSearchQueryEditorProps): Rea const { query, handleQueryChange, handleQueryBlur } = useQueryState(props); + const language = resolveQueryLanguage(value); + const meta = QUERY_LANGUAGES[language]; + const indexMissing = requiresIndex(language) && !value.index; + + const handleLanguageChange = (next: OpenSearchQueryLanguage): void => { + onChange( + produce(value, (draft) => { + // Carry the live local query text (useQueryState commits on blur) so a switch can never + // discard an uncommitted edit, regardless of event ordering or when the parent applies + // onChange. On the normal path this is a no-op: blur has already committed the same value. + draft.query = query; + draft.queryLanguage = next; + }) + ); + }; + + const handleLimitChange = (e: React.ChangeEvent): void => { + onChange( + produce(value, (draft) => { + draft.limit = parseLimitInput(e.target.value); + }) + ); + }; + const handleDatasourceChange: DatasourceSelectProps['onChange'] = (newDatasourceSelection) => { if (!isVariableDatasource(newDatasourceSelection) && newDatasourceSelection.kind === DATASOURCE_KIND) { onChange( @@ -100,6 +144,29 @@ export function OpenSearchLogQueryEditor(props: OpenSearchQueryEditorProps): Rea return ( +
+ + Query language + + +
+
Datasource
@@ -156,8 +224,26 @@ export function OpenSearchLogQueryEditor(props: OpenSearchQueryEditorProps): Rea label="Disable automatic time filtering" /> + {/* `requiresIndex` is true for exactly the `_search`-based languages (lucene/dsl), which + is also where a result-count limit applies, so it doubles as the gate for this field. */} + {requiresIndex(language) && ( +
+ Limit (optional) + +
+ )} +
- PPL Query + {`${meta.label} Query`} handleQueryChange(e.target.value)} onBlur={handleQueryBlur} onKeyDown={createModEnterHandler(handleQueryExecute)} - placeholder='e.g. source=logs-* | where level="error"' + placeholder={meta.placeholder} inputProps={{ style: { fontFamily: 'monospace' } }} /> Uses OpenSearch{' '} - - PPL + + {meta.docsLabel} - . Requires the PPL plugin enabled on the OpenSearch cluster. + . {meta.usageNote}
@@ -186,7 +272,7 @@ export function OpenSearchLogQueryEditor(props: OpenSearchQueryEditorProps): Rea > Query Examples - {PPL_QUERY_EXAMPLES} + {meta.examples} ); diff --git a/opensearch/src/queries/opensearch-log-query/build/dsl.test.ts b/opensearch/src/queries/opensearch-log-query/build/dsl.test.ts new file mode 100644 index 000000000..7eb13e082 --- /dev/null +++ b/opensearch/src/queries/opensearch-log-query/build/dsl.test.ts @@ -0,0 +1,91 @@ +// Copyright The Perses Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +import { buildDslSearchBody } from './dsl'; + +const start = new Date('2025-01-01T00:00:00.000Z'); // 1735689600000 +const end = new Date('2025-01-01T01:00:00.000Z'); // 1735693200000 + +const timeFilter = { + range: { '@timestamp': { gte: 1735689600000, lte: 1735693200000, format: 'epoch_millis' } }, +}; + +describe('buildDslSearchBody', () => { + it("ANDs the user's query under bool.filter with the time range", () => { + const body = buildDslSearchBody('{"query":{"match":{"level":"error"}}}', start, end); + expect(body).toEqual({ + size: 500, + sort: [{ '@timestamp': { order: 'desc', unmapped_type: 'boolean' } }], + query: { bool: { filter: [timeFilter, { match: { level: 'error' } }] } }, + }); + }); + + it('applies only the time filter when the body has no query key', () => { + const body = buildDslSearchBody('{}', start, end); + expect(body.query).toEqual({ bool: { filter: [timeFilter] } }); + }); + + it("lets the user's own size and sort win", () => { + const body = buildDslSearchBody('{"size":7,"sort":[{"seq":"asc"}],"query":{"match_all":{}}}', start, end, { + limit: 500, + }); + expect(body.size).toBe(7); + expect(body.sort).toEqual([{ seq: 'asc' }]); + }); + + it('passes unrelated top-level keys through untouched', () => { + const body = buildDslSearchBody( + '{"_source":["message"],"aggs":{"by_level":{"terms":{"field":"level"}}}}', + start, + end + ); + expect(body._source).toEqual(['message']); + expect(body.aggs).toEqual({ by_level: { terms: { field: 'level' } } }); + }); + + it('nests a user bool query inside our filter rather than merging it', () => { + const body = buildDslSearchBody('{"query":{"bool":{"must_not":[{"term":{"level":"debug"}}]}}}', start, end); + expect(body.query).toEqual({ + bool: { filter: [timeFilter, { bool: { must_not: [{ term: { level: 'debug' } }] } }] }, + }); + }); + + it('keeps the query untouched when disableTimeFilter is set', () => { + const body = buildDslSearchBody('{"query":{"match_all":{}}}', start, end, { disableTimeFilter: true }); + expect(body.query).toEqual({ match_all: {} }); + }); + + it('defaults to match_all when there is neither a query nor a time filter', () => { + const body = buildDslSearchBody('{}', start, end, { disableTimeFilter: true }); + expect(body.query).toEqual({ match_all: {} }); + }); + + it('preserves an explicit size of 0 rather than falling back to the default', () => { + expect(buildDslSearchBody('{"size":0,"query":{"match_all":{}}}', start, end).size).toBe(0); + }); + + it('preserves an explicit empty sort array rather than falling back to the default', () => { + expect(buildDslSearchBody('{"sort":[],"query":{"match_all":{}}}', start, end).sort).toEqual([]); + }); + + it.each(['not json', '', ' ', '{"query":}', '{'])('throws a JSON syntax error for %p', (input) => { + expect(() => buildDslSearchBody(input, start, end)).toThrow(/Query DSL is not valid JSON/i); + }); + + it.each(['[1,2]', '"a string"', 'null', '42', 'true'])( + 'throws a shape error for valid JSON that is not an object: %p', + (input) => { + expect(() => buildDslSearchBody(input, start, end)).toThrow(/Query DSL must be a JSON object/i); + } + ); +}); diff --git a/opensearch/src/queries/opensearch-log-query/build/dsl.ts b/opensearch/src/queries/opensearch-log-query/build/dsl.ts new file mode 100644 index 000000000..adbee0959 --- /dev/null +++ b/opensearch/src/queries/opensearch-log-query/build/dsl.ts @@ -0,0 +1,69 @@ +// Copyright The Perses Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +import type { OpenSearchSearchRequestBody } from '../../../model/opensearch-client-types'; +import type { SearchBodyOptions } from './search'; +import { buildDefaultSort, buildTimeRangeFilter, DEFAULT_TIMESTAMP_FIELD, resolveSize } from './search'; + +function parseBody(userJson: string): Record { + let parsed: unknown; + try { + parsed = JSON.parse(userJson); + } catch (e) { + const detail = e instanceof Error ? e.message : String(e); + throw new Error(`Query DSL is not valid JSON: ${detail}`); + } + + if (parsed === null || typeof parsed !== 'object' || Array.isArray(parsed)) { + throw new Error('Query DSL must be a JSON object shaped like a _search request body, e.g. {"query": {...}}.'); + } + + return parsed as Record; +} + +/** + * Build a `_search` body from a raw Query DSL document. The user's JSON is treated as a + * complete `_search` request body — the thing you would paste out of Dev Tools — so any + * key we do not manage (`_source`, `aggs`, `highlight`, …) passes straight through. + * + * The user wins on `size` and `sort`; we only supply defaults. Their `query`, if present, + * is nested inside our `bool.filter` next to the time range, which is valid DSL and means + * exactly "AND" — we deliberately do not try to merge into their bool clause, because + * rewriting a user's query semantics is far worse than one extra level of nesting. + */ +export function buildDslSearchBody( + userJson: string, + start: Date, + end: Date, + { timestampField = DEFAULT_TIMESTAMP_FIELD, disableTimeFilter = false, limit }: SearchBodyOptions = {} +): OpenSearchSearchRequestBody { + const parsed = parseBody(userJson); + const userQuery = parsed.query as Record | undefined; + + const body: OpenSearchSearchRequestBody = { ...parsed }; + body.size = parsed.size ?? resolveSize(limit); + body.sort = parsed.sort ?? buildDefaultSort(timestampField); + + if (disableTimeFilter) { + body.query = userQuery ?? { match_all: {} }; + return body; + } + + const filter: Array> = [buildTimeRangeFilter(timestampField, start, end)]; + if (userQuery) { + filter.push(userQuery); + } + body.query = { bool: { filter } }; + + return body; +} diff --git a/opensearch/src/queries/opensearch-log-query/build/index.ts b/opensearch/src/queries/opensearch-log-query/build/index.ts new file mode 100644 index 000000000..4d8502ae1 --- /dev/null +++ b/opensearch/src/queries/opensearch-log-query/build/index.ts @@ -0,0 +1,18 @@ +// Copyright The Perses Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +export * from './ppl'; +export * from './sql'; +export * from './search'; +export * from './lucene'; +export * from './dsl'; diff --git a/opensearch/src/queries/opensearch-log-query/build/lucene.test.ts b/opensearch/src/queries/opensearch-log-query/build/lucene.test.ts new file mode 100644 index 000000000..f8d38d09b --- /dev/null +++ b/opensearch/src/queries/opensearch-log-query/build/lucene.test.ts @@ -0,0 +1,71 @@ +// Copyright The Perses Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +import { buildLuceneSearchBody } from './lucene'; + +const start = new Date('2025-01-01T00:00:00.000Z'); // 1735689600000 +const end = new Date('2025-01-01T01:00:00.000Z'); // 1735693200000 + +const timeFilter = { + range: { '@timestamp': { gte: 1735689600000, lte: 1735693200000, format: 'epoch_millis' } }, +}; + +describe('buildLuceneSearchBody', () => { + it('wraps the user query in a query_string clause beside the time range', () => { + expect(buildLuceneSearchBody('level:error AND service:api', start, end)).toEqual({ + size: 500, + sort: [{ '@timestamp': { order: 'desc', unmapped_type: 'boolean' } }], + query: { + bool: { + filter: [timeFilter, { query_string: { query: 'level:error AND service:api', analyze_wildcard: true } }], + }, + }, + }); + }); + + it('drops the query_string clause for a blank query, leaving the time window', () => { + const body = buildLuceneSearchBody(' ', start, end); + expect(body.query).toEqual({ bool: { filter: [timeFilter] } }); + }); + + it('honours limit as the search size', () => { + expect(buildLuceneSearchBody('*', start, end, { limit: 25 }).size).toBe(25); + }); + + // The CUE schema constrains `limit` to `int & >0`, so `limit: 0` cannot reach here from a + // validated dashboard. This test pins the `??` operator choice in resolveSize, not `size: 0` + // as a supported use case — do not "fix" it by switching resolveSize to `||`. + it('preserves an explicit limit of 0 rather than falling back to the default size', () => { + expect(buildLuceneSearchBody('*', start, end, { limit: 0 }).size).toBe(0); + }); + + it('sorts and filters on the configured timestamp field', () => { + const body = buildLuceneSearchBody('*', start, end, { timestampField: 'event.time' }); + expect(body.sort).toEqual([{ 'event.time': { order: 'desc', unmapped_type: 'boolean' } }]); + expect(body.query).toEqual({ + bool: { + filter: [ + { range: { 'event.time': { gte: 1735689600000, lte: 1735693200000, format: 'epoch_millis' } } }, + { query_string: { query: '*', analyze_wildcard: true } }, + ], + }, + }); + }); + + it('omits the range clause when disableTimeFilter is set', () => { + const body = buildLuceneSearchBody('level:error', start, end, { disableTimeFilter: true }); + expect(body.query).toEqual({ + bool: { filter: [{ query_string: { query: 'level:error', analyze_wildcard: true } }] }, + }); + }); +}); diff --git a/opensearch/src/queries/opensearch-log-query/build/lucene.ts b/opensearch/src/queries/opensearch-log-query/build/lucene.ts new file mode 100644 index 000000000..72f9aff12 --- /dev/null +++ b/opensearch/src/queries/opensearch-log-query/build/lucene.ts @@ -0,0 +1,44 @@ +// Copyright The Perses Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +import type { OpenSearchSearchRequestBody } from '../../../model/opensearch-client-types'; +import type { SearchBodyOptions } from './search'; +import { buildDefaultSort, buildTimeRangeFilter, DEFAULT_TIMESTAMP_FIELD, resolveSize } from './search'; + +/** + * Build a `_search` body from a Lucene (query string) query: the user's string becomes a + * `query_string` clause ANDed with the panel time range inside `bool.filter`. A blank + * query is legitimate — it means "everything in this time window". + */ +export function buildLuceneSearchBody( + userQuery: string, + start: Date, + end: Date, + { timestampField = DEFAULT_TIMESTAMP_FIELD, disableTimeFilter = false, limit }: SearchBodyOptions = {} +): OpenSearchSearchRequestBody { + const trimmed = userQuery.trim(); + + const filter: Array> = []; + if (!disableTimeFilter) { + filter.push(buildTimeRangeFilter(timestampField, start, end)); + } + if (trimmed.length > 0) { + filter.push({ query_string: { query: trimmed, analyze_wildcard: true } }); + } + + return { + size: resolveSize(limit), + sort: buildDefaultSort(timestampField), + query: { bool: { filter } }, + }; +} diff --git a/opensearch/src/queries/opensearch-log-query/build/ppl.ts b/opensearch/src/queries/opensearch-log-query/build/ppl.ts new file mode 100644 index 000000000..5b3c95a94 --- /dev/null +++ b/opensearch/src/queries/opensearch-log-query/build/ppl.ts @@ -0,0 +1,92 @@ +// Copyright The Perses Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +import { DEFAULT_TIMESTAMP_FIELD } from './search'; + +/** + * Bound the query to the panel time range using a PPL `where` clause on the + * configured timestamp field (`@timestamp` by default). + * The bound is injected immediately after the source clause so it runs before any + * pipe that drops the timestamp column from the schema (stats, fields, top, etc.). + * If the user already filters on the timestamp field themselves, PPL ANDs the two clauses. + */ +export interface BoundedPPLOptions { + index?: string; + timestampField?: string; + disableTimeFilter?: boolean; +} + +/** + * Escape a user-supplied identifier (timestamp field name) for safe interpolation + * inside a PPL backtick-quoted identifier. A backtick in the value would otherwise + * close the quoting early and let the rest of the string be parsed as PPL, producing + * an invalid — or potentially injected — query. PPL escapes a literal backtick by + * doubling it. + */ +function escapeIdentifier(name: string): string { + return name.replace(/`/g, '``'); +} + +/** + * Sanitize a user-supplied index name/pattern before embedding it in a `source=` + * clause. Unlike the timestamp field, `source=` is not backtick-quoted and must + * keep wildcards (`*`) and multi-index commas intact, so we can't just quote it. + * Instead we strip the characters that could terminate the clause and inject an + * extra PPL stage (backtick, pipe, quotes) or that are simply invalid in an index + * name (whitespace) — e.g. `logs-* | delete` would otherwise smuggle in a pipe. + */ +function sanitizeIndex(index: string): string { + return index.replace(/[`|'"\s]/g, ''); +} + +/** Join PPL stages with ` | `, dropping any empty segments so we never emit a dangling pipe. */ +function joinPPLStages(stages: string[]): string { + return stages.filter((stage) => stage.trim().length > 0).join(' | '); +} + +export function buildBoundedPPL( + userQuery: string, + start: Date, + end: Date, + { index, timestampField = DEFAULT_TIMESTAMP_FIELD, disableTimeFilter = false }: BoundedPPLOptions = {} +): string { + let trimmed = userQuery.trim(); + + if (index && !/^(?:search\s+)?source\s*=/i.test(trimmed)) { + trimmed = `source=${sanitizeIndex(index)} | ${trimmed}`; + } + + // Skip the auto-injected time-range clause when the caller manages their own time + // bounds (disableTimeFilter). + if (disableTimeFilter) { + return trimmed; + } + + const startIso = start.toISOString(); + const endIso = end.toISOString(); + // timestampField is user-editable, so escape it before embedding it in the + // backtick-quoted identifier (the ISO bounds are machine-generated and safe). + const tsField = escapeIdentifier(timestampField); + const bound = `where \`${tsField}\` >= '${startIso}' and \`${tsField}\` <= '${endIso}'`; + + const firstPipe = trimmed.indexOf('|'); + if (firstPipe === -1) { + return joinPPLStages([trimmed, bound]); + } + + const sourceClause = trimmed.slice(0, firstPipe).trimEnd(); + const rest = trimmed.slice(firstPipe + 1).trimStart(); + // `rest` can be empty (e.g. a trailing pipe or an empty query), so join through a + // filter to avoid producing an invalid `... | ` dangling stage. + return joinPPLStages([sourceClause, bound, rest]); +} diff --git a/opensearch/src/queries/opensearch-log-query/build/search.ts b/opensearch/src/queries/opensearch-log-query/build/search.ts new file mode 100644 index 000000000..f4865b46f --- /dev/null +++ b/opensearch/src/queries/opensearch-log-query/build/search.ts @@ -0,0 +1,52 @@ +// Copyright The Perses Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +import { DEFAULT_SEARCH_SIZE, DEFAULT_TIMESTAMP_FIELDS } from '../../constants'; + +export interface SearchBodyOptions { + timestampField?: string; + disableTimeFilter?: boolean; + /** `_search` size. Defaults to DEFAULT_SEARCH_SIZE. */ + limit?: number; +} + +// Single source of truth for the default timestamp field, shared with build/ppl.ts. +// Non-null assertion: DEFAULT_TIMESTAMP_FIELDS is a non-empty literal array, so index 0 +// always exists — `noUncheckedIndexedAccess` just can't see that from the `string[]` type. +export const DEFAULT_TIMESTAMP_FIELD = DEFAULT_TIMESTAMP_FIELDS[0]!; + +/** A `range` clause bounding the timestamp field to the panel window, in epoch milliseconds. */ +export function buildTimeRangeFilter(timestampField: string, start: Date, end: Date): Record { + return { + range: { + [timestampField]: { + gte: start.getTime(), + lte: end.getTime(), + format: 'epoch_millis', + }, + }, + }; +} + +/** + * Newest-first, which is what a log panel wants — log panels do not sort entries + * themselves. `unmapped_type` keeps a wildcard index pattern working when one of the + * matched indices has no such field, which would otherwise fail the shard. + */ +export function buildDefaultSort(timestampField: string): Array> { + return [{ [timestampField]: { order: 'desc', unmapped_type: 'boolean' } }]; +} + +export function resolveSize(limit?: number): number { + return limit ?? DEFAULT_SEARCH_SIZE; +} diff --git a/opensearch/src/queries/opensearch-log-query/build/sql.test.ts b/opensearch/src/queries/opensearch-log-query/build/sql.test.ts new file mode 100644 index 000000000..bbcaf6854 --- /dev/null +++ b/opensearch/src/queries/opensearch-log-query/build/sql.test.ts @@ -0,0 +1,44 @@ +// Copyright The Perses Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +import { buildSqlRequest } from './sql'; + +const start = new Date('2025-01-01T00:00:00.000Z'); // 1735689600000 +const end = new Date('2025-01-01T01:00:00.000Z'); // 1735693200000 + +describe('buildSqlRequest', () => { + it('passes the user query through untouched apart from trimming', () => { + const { query } = buildSqlRequest(' SELECT * FROM `logs-2025` ', start, end); + expect(query).toBe('SELECT * FROM `logs-2025`'); + }); + + it('bounds the time range with a filter on @timestamp by default', () => { + const { filter } = buildSqlRequest('SELECT * FROM logs', start, end); + expect(filter).toEqual({ + range: { '@timestamp': { gte: 1735689600000, lte: 1735693200000, format: 'epoch_millis' } }, + }); + }); + + it('uses the configured timestamp field', () => { + const { filter } = buildSqlRequest('SELECT * FROM logs', start, end, { timestampField: 'event.time' }); + expect(filter).toEqual({ + range: { 'event.time': { gte: 1735689600000, lte: 1735693200000, format: 'epoch_millis' } }, + }); + }); + + it('omits the filter entirely when disableTimeFilter is set', () => { + const request = buildSqlRequest('SELECT * FROM logs', start, end, { disableTimeFilter: true }); + expect(request).toEqual({ query: 'SELECT * FROM logs' }); + expect('filter' in request).toBe(false); + }); +}); diff --git a/opensearch/src/queries/opensearch-log-query/build/sql.ts b/opensearch/src/queries/opensearch-log-query/build/sql.ts new file mode 100644 index 000000000..ea59a3edd --- /dev/null +++ b/opensearch/src/queries/opensearch-log-query/build/sql.ts @@ -0,0 +1,44 @@ +// Copyright The Perses Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +import type { OpenSearchSQLParams } from '../../../model/opensearch-client'; +import { buildTimeRangeFilter, DEFAULT_TIMESTAMP_FIELD } from './search'; + +export interface SqlRequestOptions { + timestampField?: string; + disableTimeFilter?: boolean; +} + +/** + * Bound a SQL query to the panel time range using the `_sql` API's `filter` request + * field, which OpenSearch ANDs into the DSL it generates. Unlike PPL — where the bound + * has to be spliced into the query string — this leaves the user's SQL completely + * untouched, so there is nothing to escape and no parse to get wrong. + */ +export function buildSqlRequest( + userQuery: string, + start: Date, + end: Date, + { timestampField = DEFAULT_TIMESTAMP_FIELD, disableTimeFilter = false }: SqlRequestOptions = {} +): OpenSearchSQLParams { + const query = userQuery.trim(); + + if (disableTimeFilter) { + return { query }; + } + + return { + query, + filter: buildTimeRangeFilter(timestampField, start, end), + }; +} diff --git a/opensearch/src/queries/opensearch-log-query/convert/datarows-to-logs.ts b/opensearch/src/queries/opensearch-log-query/convert/datarows-to-logs.ts new file mode 100644 index 000000000..228685617 --- /dev/null +++ b/opensearch/src/queries/opensearch-log-query/convert/datarows-to-logs.ts @@ -0,0 +1,77 @@ +// Copyright The Perses Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +import type { LogEntry, LogData } from '@perses-dev/spec'; +import type { OpenSearchDatarowsResponse } from '../../../model/opensearch-client-types'; +import { DEFAULT_MESSAGE_FIELDS, DEFAULT_TIMESTAMP_FIELDS } from '../../constants'; +import { parseTimestamp } from './parse-timestamp'; + +function pickIndex(cols: Array<{ name: string }>, candidates: string[]): number { + for (const candidate of candidates) { + const idx = cols.findIndex((c) => c.name === candidate); + if (idx !== -1) return idx; + } + return -1; +} + +export interface ConvertOptions { + timestampField?: string; + messageField?: string; +} + +export function convertDatarowsToLogs(response: OpenSearchDatarowsResponse, options: ConvertOptions = {}): LogData { + const { schema = [], datarows = [] } = response; + + const tsCandidates = options.timestampField + ? [options.timestampField, ...DEFAULT_TIMESTAMP_FIELDS] + : DEFAULT_TIMESTAMP_FIELDS; + const msgCandidates = options.messageField + ? [options.messageField, ...DEFAULT_MESSAGE_FIELDS] + : DEFAULT_MESSAGE_FIELDS; + + const tsIdx = pickIndex(schema, tsCandidates); + const msgIdx = pickIndex(schema, msgCandidates); + + const entries: LogEntry[] = datarows.map((row) => { + const rawTs = tsIdx !== -1 ? row[tsIdx] : null; + const rawMsg = msgIdx !== -1 ? row[msgIdx] : null; + + const timestamp = parseTimestamp(rawTs); + const line = rawMsg !== null && rawMsg !== undefined ? String(rawMsg) : JSON.stringify(rowToObject(schema, row)); + + const labels: Record = {}; + schema.forEach((col, i) => { + if (i === tsIdx || i === msgIdx) return; + const v = row[i]; + if (v !== null && v !== undefined) labels[col.name] = String(v); + }); + + return { timestamp, line, labels }; + }); + + // A PPL response carries no separate "total hits" field, so totalCount reflects + // the number of rows actually returned by the query (bounded by any `head`/limit + // the user added), not a grand total of matching documents. + return { entries, totalCount: entries.length }; +} + +function rowToObject( + schema: OpenSearchDatarowsResponse['schema'], + row: Array +): Record { + const out: Record = {}; + schema.forEach((col, i) => { + out[col.name] = row[i]; + }); + return out; +} diff --git a/opensearch/src/queries/opensearch-log-query/convert/hits-to-logs.test.ts b/opensearch/src/queries/opensearch-log-query/convert/hits-to-logs.test.ts new file mode 100644 index 000000000..e8374b2ac --- /dev/null +++ b/opensearch/src/queries/opensearch-log-query/convert/hits-to-logs.test.ts @@ -0,0 +1,122 @@ +// Copyright The Perses Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +import { convertHitsToLogs, flattenSource } from './hits-to-logs'; + +describe('flattenSource', () => { + it('flattens nested objects to dot paths and stringifies scalars', () => { + expect(flattenSource({ a: { b: 1, c: 'x' }, d: true })).toEqual({ 'a.b': '1', 'a.c': 'x', d: 'true' }); + }); + + it('JSON-stringifies arrays rather than exploding them', () => { + expect(flattenSource({ tags: ['a', 'b'] })).toEqual({ tags: '["a","b"]' }); + }); + + it('drops null and undefined values', () => { + expect(flattenSource({ a: null, b: undefined, c: 0 })).toEqual({ c: '0' }); + }); +}); + +describe('convertHitsToLogs', () => { + const response = { + hits: { + total: { value: 2, relation: 'eq' as const }, + hits: [ + { + _index: 'logs-2025.01.01', + _id: 'abc', + _source: { + '@timestamp': '2025-01-01T00:00:00.000Z', + message: 'Error processing request', + level: 'error', + service: { name: 'api' }, + }, + }, + { + _index: 'logs-2025.01.01', + _id: 'def', + _source: { '@timestamp': '2025-01-01T00:00:01.000Z', message: 'Retrying', level: 'warn' }, + }, + ], + }, + }; + + it('maps _source to timestamp, line, and labels', () => { + const { entries } = convertHitsToLogs(response); + expect(entries).toHaveLength(2); + expect(entries[0]).toEqual({ + timestamp: 1735689600, + line: 'Error processing request', + labels: { level: 'error', 'service.name': 'api', _id: 'abc', _index: 'logs-2025.01.01' }, + }); + }); + + it('excludes the chosen timestamp and message fields from labels', () => { + const { entries } = convertHitsToLogs(response); + expect(entries[0]?.labels['@timestamp']).toBeUndefined(); + expect(entries[0]?.labels.message).toBeUndefined(); + }); + + it('reports totalCount from hits.total and no hasMore when the count is exact', () => { + const result = convertHitsToLogs(response); + expect(result.totalCount).toBe(2); + expect(result.hasMore).toBeUndefined(); + }); + + it('sets hasMore when hits.total.relation is gte', () => { + const result = convertHitsToLogs({ + hits: { total: { value: 10000, relation: 'gte' }, hits: [] }, + }); + expect(result.totalCount).toBe(10000); + expect(result.hasMore).toBe(true); + }); + + it('falls back to the entry count when hits.total is absent', () => { + const result = convertHitsToLogs({ hits: { hits: [{ _source: { message: 'a' } }] } }); + expect(result.totalCount).toBe(1); + }); + + it('honours explicit timestampField and messageField overrides', () => { + const { entries } = convertHitsToLogs( + { hits: { hits: [{ _source: { ts: 1735689600000, body: 'hello', other: 'x' } }] } }, + { timestampField: 'ts', messageField: 'body' } + ); + expect(entries[0]?.timestamp).toBe(1735689600); + expect(entries[0]?.line).toBe('hello'); + expect(entries[0]?.labels).toEqual({ other: 'x' }); + }); + + it('falls back through the default field names', () => { + const { entries } = convertHitsToLogs({ + hits: { hits: [{ _source: { time: '2025-01-01T00:00:00.000Z', body: 'from body' } }] }, + }); + expect(entries[0]?.timestamp).toBe(1735689600); + expect(entries[0]?.line).toBe('from body'); + }); + + it('stringifies the whole source when no message field matches', () => { + const { entries } = convertHitsToLogs({ hits: { hits: [{ _source: { foo: 'bar' } }] } }); + expect(entries[0]?.line).toBe(JSON.stringify({ foo: 'bar' })); + }); + + it('yields timestamp 0 and an empty-object line for a hit with no _source', () => { + const { entries } = convertHitsToLogs({ hits: { hits: [{ _id: 'x' }] } }); + expect(entries[0]?.timestamp).toBe(0); + expect(entries[0]?.line).toBe('{}'); + expect(entries[0]?.labels).toEqual({ _id: 'x' }); + }); + + it('returns no entries for an empty hits array', () => { + expect(convertHitsToLogs({ hits: { hits: [] } })).toEqual({ entries: [], totalCount: 0 }); + }); +}); diff --git a/opensearch/src/queries/opensearch-log-query/convert/hits-to-logs.ts b/opensearch/src/queries/opensearch-log-query/convert/hits-to-logs.ts new file mode 100644 index 000000000..dd2116113 --- /dev/null +++ b/opensearch/src/queries/opensearch-log-query/convert/hits-to-logs.ts @@ -0,0 +1,105 @@ +// Copyright The Perses Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +import type { LogData, LogEntry } from '@perses-dev/spec'; +import type { OpenSearchSearchResponse } from '../../../model/opensearch-client-types'; +import { DEFAULT_MESSAGE_FIELDS, DEFAULT_TIMESTAMP_FIELDS } from '../../constants'; +import type { ConvertOptions } from './datarows-to-logs'; +import { parseTimestamp } from './parse-timestamp'; + +/** + * Flatten a `_source` document to dot-separated keys, because LogEntry labels are a flat + * `Record`. Arrays are JSON-stringified rather than indexed, since + * `tags.0`/`tags.1` labels are noise; null/undefined values are dropped so they do not + * render as the strings "null"/"undefined". + */ +export function flattenSource(source: Record): Record { + const out: Record = {}; + + const walk = (value: unknown, prefix: string): void => { + if (value === null || value === undefined) return; + if (Array.isArray(value)) { + out[prefix] = JSON.stringify(value); + return; + } + if (typeof value === 'object') { + for (const [key, child] of Object.entries(value as Record)) { + walk(child, prefix ? `${prefix}.${key}` : key); + } + return; + } + out[prefix] = String(value); + }; + + for (const [key, value] of Object.entries(source)) { + walk(value, key); + } + + return out; +} + +/** First candidate present in the flattened document, or undefined if none match. */ +function pickKey(flat: Record, candidates: string[]): string | undefined { + return candidates.find((candidate) => candidate in flat); +} + +/** + * Convert a `_search` response into the same LogData shape the datarows converter produces, so + * Lucene and Query DSL results are indistinguishable from PPL/SQL results downstream. + */ +export function convertHitsToLogs(response: OpenSearchSearchResponse, options: ConvertOptions = {}): LogData { + const hits = response?.hits?.hits ?? []; + + const tsCandidates = options.timestampField + ? [options.timestampField, ...DEFAULT_TIMESTAMP_FIELDS] + : DEFAULT_TIMESTAMP_FIELDS; + const msgCandidates = options.messageField + ? [options.messageField, ...DEFAULT_MESSAGE_FIELDS] + : DEFAULT_MESSAGE_FIELDS; + + const entries: LogEntry[] = hits.map((hit) => { + const source = hit._source ?? {}; + const flat = flattenSource(source); + + const tsKey = pickKey(flat, tsCandidates); + const msgKey = pickKey(flat, msgCandidates); + + const timestamp = tsKey ? parseTimestamp(flat[tsKey]) : 0; + const line = msgKey ? (flat[msgKey] as string) : JSON.stringify(source); + + const labels: Record = {}; + for (const [key, value] of Object.entries(flat)) { + if (key === tsKey || key === msgKey) continue; + labels[key] = value; + } + // Document identity is genuinely useful: _index disambiguates a wildcard pattern and + // _id lets a user correlate a line back to the source document. + if (hit._id !== undefined) labels._id = hit._id; + if (hit._index !== undefined) labels._index = hit._index; + + return { timestamp, line, labels }; + }); + + const total = response?.hits?.total; + const result: LogData = { + entries, + totalCount: total?.value ?? entries.length, + }; + // `relation: 'gte'` means OpenSearch stopped counting (default cap 10,000), so the true + // total is at least this — surface that rather than paying for track_total_hits. + if (total?.relation === 'gte') { + result.hasMore = true; + } + + return result; +} diff --git a/opensearch/src/queries/opensearch-log-query/convert/index.ts b/opensearch/src/queries/opensearch-log-query/convert/index.ts new file mode 100644 index 000000000..e0ebc3736 --- /dev/null +++ b/opensearch/src/queries/opensearch-log-query/convert/index.ts @@ -0,0 +1,16 @@ +// Copyright The Perses Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +export * from './datarows-to-logs'; +export * from './parse-timestamp'; +export * from './hits-to-logs'; diff --git a/opensearch/src/queries/opensearch-log-query/convert/parse-timestamp.ts b/opensearch/src/queries/opensearch-log-query/convert/parse-timestamp.ts new file mode 100644 index 000000000..6b980007d --- /dev/null +++ b/opensearch/src/queries/opensearch-log-query/convert/parse-timestamp.ts @@ -0,0 +1,51 @@ +// Copyright The Perses Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +export function parseTimestamp(v: unknown): number { + if (v === null || v === undefined) return 0; + + // Numeric epochs arrive in seconds, milliseconds, microseconds, or nanoseconds + // (logs frequently carry ns precision). They come as a JS number, or — when the + // value exceeds Number.MAX_SAFE_INTEGER (a ns epoch is ~1e18, well past 2^53) — as + // a numeric string so it survives JSON parsing without silent overflow. + // We emit seconds (LogEntry is second-resolution), so we only need the correct + // magnitude, not sub-second precision: a float's rounding at the ns scale is far + // below one second and is discarded anyway, so BigInt isn't required here. + let numeric = NaN; + if (typeof v === 'number') { + numeric = v; + } else if (/^\d+$/.test(String(v))) { + numeric = Number(v); + } + if (!Number.isNaN(numeric)) { + // Detect the unit by magnitude — a value past year ~5138 for a given unit must + // really be a finer unit — and normalize to seconds. Without the µs/ns tiers + // those values would parse far in the future. + if (numeric >= 1e17) return numeric / 1e9; // nanoseconds → seconds + if (numeric >= 1e14) return numeric / 1e6; // microseconds → seconds + if (numeric >= 1e11) return numeric / 1e3; // milliseconds → seconds + return numeric; // seconds + } + + // OpenSearch PPL returns timestamp/datetime fields as space-separated UTC strings + // with no timezone designator (e.g. "2020-08-26 13:49:00"). Date.parse treats a + // zoneless date-time as *local* time, which would shift the value by the browser's + // offset, so normalize it to explicit UTC (replace the space with `T`, append `Z`) + // before parsing. Strings that already carry a zone (…Z / ±hh:mm) or are date-only + // are left untouched — the spec parses those as UTC already. + const str = String(v).trim(); + const zonelessDateTime = /^\d{4}-\d{2}-\d{2}[ T]\d{2}:\d{2}:\d{2}(\.\d+)?$/; + const iso = zonelessDateTime.test(str) ? `${str.replace(' ', 'T')}Z` : str; + const parsed = Date.parse(iso); + return Number.isNaN(parsed) ? 0 : parsed / 1000; +} diff --git a/opensearch/src/queries/opensearch-log-query/get-opensearch-log-data.test.ts b/opensearch/src/queries/opensearch-log-query/get-opensearch-log-data.test.ts index 84e892511..516e84421 100644 --- a/opensearch/src/queries/opensearch-log-query/get-opensearch-log-data.test.ts +++ b/opensearch/src/queries/opensearch-log-query/get-opensearch-log-data.test.ts @@ -16,9 +16,10 @@ import type { Mock } from 'vitest'; import { OpenSearchDatasource } from '../../datasources/opensearch-datasource'; import type { OpenSearchDatasourceSpec } from '../../datasources/opensearch-datasource/opensearch-datasource-types'; -import type { OpenSearchPPLResponse } from '../../model/opensearch-client-types'; -import { buildBoundedPPL, convertPPLToLogs, parseTimestamp } from './get-opensearch-log-data'; +import type { OpenSearchDatarowsResponse } from '../../model/opensearch-client-types'; import { OpenSearchLogQuery } from './OpenSearchLogQuery'; +import { buildBoundedPPL } from './build/ppl'; +import { convertDatarowsToLogs, parseTimestamp } from './convert'; const datasource: OpenSearchDatasourceSpec = { directUrl: '/test', @@ -27,7 +28,7 @@ const datasource: OpenSearchDatasourceSpec = { const stubClient = OpenSearchDatasource.createClient(datasource, {}); stubClient.ppl = vi.fn(async () => { - const response: OpenSearchPPLResponse = { + const response: OpenSearchDatarowsResponse = { schema: [ { name: '@timestamp', type: 'timestamp' }, { name: 'message', type: 'text' }, @@ -43,6 +44,23 @@ stubClient.ppl = vi.fn(async () => { return response; }); +stubClient.sql = vi.fn(async () => ({ + schema: [ + { name: '@timestamp', type: 'timestamp' }, + { name: 'message', type: 'text' }, + ], + datarows: [['2025-01-01T00:00:00.000Z', 'from sql']], +})); + +stubClient.search = vi.fn(async () => ({ + hits: { + total: { value: 1, relation: 'eq' as const }, + hits: [ + { _index: 'logs-2025', _id: 'h1', _source: { '@timestamp': '2025-01-01T00:00:00.000Z', message: 'from search' } }, + ], + }, +})); + const getDatasourceClient: Mock = vi.fn(() => stubClient); function createStubContext(): LogQueryContext { @@ -68,10 +86,12 @@ function createStubContext(): LogQueryContext { describe('OpenSearchLogQuery', () => { afterEach(() => { (stubClient.ppl as Mock).mockClear(); + (stubClient.sql as Mock).mockClear(); + (stubClient.search as Mock).mockClear(); }); - it('creates initial options with empty query', () => { - expect(OpenSearchLogQuery.createInitialOptions()).toEqual({ query: '' }); + it('creates initial options with an empty PPL query', () => { + expect(OpenSearchLogQuery.createInitialOptions()).toEqual({ query: '', queryLanguage: 'ppl' }); }); it('resolves variable dependencies from query and index', () => { @@ -347,9 +367,9 @@ describe('parseTimestamp', () => { }); }); -describe('convertPPLToLogs', () => { +describe('convertDatarowsToLogs', () => { it('maps @timestamp and message fields and collects remaining fields as labels', () => { - const logs = convertPPLToLogs({ + const logs = convertDatarowsToLogs({ schema: [ { name: '@timestamp', type: 'timestamp' }, { name: 'message', type: 'text' }, @@ -363,7 +383,7 @@ describe('convertPPLToLogs', () => { }); it('parses numeric epoch-millis timestamps into seconds', () => { - const logs = convertPPLToLogs({ + const logs = convertDatarowsToLogs({ schema: [ { name: '@timestamp', type: 'long' }, { name: 'message', type: 'text' }, @@ -374,7 +394,7 @@ describe('convertPPLToLogs', () => { }); it('falls back to a JSON dump of the row when no message field is present', () => { - const logs = convertPPLToLogs({ + const logs = convertDatarowsToLogs({ schema: [ { name: '@timestamp', type: 'timestamp' }, { name: 'event', type: 'text' }, @@ -385,12 +405,12 @@ describe('convertPPLToLogs', () => { }); it('handles empty datarows', () => { - const logs = convertPPLToLogs({ schema: [], datarows: [] }); + const logs = convertDatarowsToLogs({ schema: [], datarows: [] }); expect(logs).toEqual({ entries: [], totalCount: 0 }); }); it('honors explicit timestampField and messageField overrides', () => { - const logs = convertPPLToLogs( + const logs = convertDatarowsToLogs( { schema: [ { name: 'time', type: 'timestamp' }, @@ -407,7 +427,7 @@ describe('convertPPLToLogs', () => { }); it('falls back to defaults when the override field is not in the schema', () => { - const logs = convertPPLToLogs( + const logs = convertDatarowsToLogs( { schema: [ { name: '@timestamp', type: 'timestamp' }, @@ -421,7 +441,7 @@ describe('convertPPLToLogs', () => { }); it('puts trace_id columns into labels for trace pivot', () => { - const logs = convertPPLToLogs({ + const logs = convertDatarowsToLogs({ schema: [ { name: '@timestamp', type: 'timestamp' }, { name: 'message', type: 'text' }, @@ -432,3 +452,93 @@ describe('convertPPLToLogs', () => { expect(logs.entries[0]?.labels.traceId).toBe('abc123'); }); }); + +describe('getLogData language dispatch', () => { + const context = createStubContext(); + + afterEach(() => { + (stubClient.ppl as Mock).mockClear(); + (stubClient.sql as Mock).mockClear(); + (stubClient.search as Mock).mockClear(); + }); + + it('defaults to PPL when queryLanguage is absent (backward compatibility)', async () => { + await OpenSearchLogQuery.getLogData({ query: 'source=logs-*' }, context); + expect(stubClient.ppl).toHaveBeenCalledTimes(1); + expect(stubClient.sql).not.toHaveBeenCalled(); + expect(stubClient.search).not.toHaveBeenCalled(); + }); + + it('routes queryLanguage: sql to client.sql with a time filter', async () => { + const result = await OpenSearchLogQuery.getLogData({ query: 'SELECT * FROM logs', queryLanguage: 'sql' }, context); + expect(stubClient.ppl).not.toHaveBeenCalled(); + const [params] = (stubClient.sql as Mock).mock.calls[0]!; + expect(params.query).toBe('SELECT * FROM logs'); + expect(params.filter).toEqual({ + range: { '@timestamp': { gte: 1735689600000, lte: 1735693200000, format: 'epoch_millis' } }, + }); + expect(result.logs.entries[0]?.line).toBe('from sql'); + }); + + it('routes queryLanguage: lucene to client.search with the resolved index', async () => { + const result = await OpenSearchLogQuery.getLogData( + { query: 'level:error', queryLanguage: 'lucene', index: 'logs-*' }, + context + ); + const [params] = (stubClient.search as Mock).mock.calls[0]!; + expect(params.index).toBe('logs-*'); + expect(params.body.query.bool.filter).toHaveLength(2); + expect(result.logs.entries[0]?.line).toBe('from search'); + expect(result.logs.entries[0]?.labels._index).toBe('logs-2025'); + }); + + it('routes queryLanguage: dsl to client.search', async () => { + await OpenSearchLogQuery.getLogData( + { query: '{"query":{"match":{"level":"error"}}}', queryLanguage: 'dsl', index: 'logs-*' }, + context + ); + const [params] = (stubClient.search as Mock).mock.calls[0]!; + expect(params.body.query.bool.filter[1]).toEqual({ match: { level: 'error' } }); + }); + + it('passes limit through as the search size', async () => { + await OpenSearchLogQuery.getLogData( + { query: 'level:error', queryLanguage: 'lucene', index: 'logs-*', limit: 42 }, + context + ); + const [params] = (stubClient.search as Mock).mock.calls[0]!; + expect(params.body.size).toBe(42); + }); + + it('resolves variables in a DSL body', async () => { + await OpenSearchLogQuery.getLogData( + { query: '{"query":{"term":{"traceId":"$traceId"}}}', queryLanguage: 'dsl', index: 'logs-*' }, + { + ...context, + variableState: { + traceId: { value: 'abc123', loading: false }, + } as unknown as LogQueryContext['variableState'], + } + ); + const [params] = (stubClient.search as Mock).mock.calls[0]!; + expect(params.body.query.bool.filter[1]).toEqual({ term: { traceId: 'abc123' } }); + }); + + it.each(['lucene', 'dsl'] as const)('rejects %s without an index', async (queryLanguage) => { + await expect(OpenSearchLogQuery.getLogData({ query: 'level:error', queryLanguage }, context)).rejects.toThrow( + /index is required/i + ); + expect(stubClient.search).not.toHaveBeenCalled(); + }); + + it.each([ + { query: 'source=logs-*', queryLanguage: 'ppl' as const }, + { query: 'SELECT * FROM logs', queryLanguage: 'sql' as const }, + { query: 'level:error', queryLanguage: 'lucene' as const, index: 'logs-*' }, + { query: '{"query":{"match_all":{}}}', queryLanguage: 'dsl' as const, index: 'logs-*' }, + ])('records the executed request in metadata for $queryLanguage', async (spec) => { + const result = await OpenSearchLogQuery.getLogData(spec, context); + expect(typeof result.metadata?.executedQueryString).toBe('string'); + expect(result.metadata?.executedQueryString.length).toBeGreaterThan(0); + }); +}); diff --git a/opensearch/src/queries/opensearch-log-query/get-opensearch-log-data.ts b/opensearch/src/queries/opensearch-log-query/get-opensearch-log-data.ts index af085d7a6..e2c772d1f 100644 --- a/opensearch/src/queries/opensearch-log-query/get-opensearch-log-data.ts +++ b/opensearch/src/queries/opensearch-log-query/get-opensearch-log-data.ts @@ -11,229 +11,85 @@ // See the License for the specific language governing permissions and // limitations under the License. -import type { LogQueryPlugin, LogQueryContext } from '@perses-dev/plugin-system'; +import type { LogQueryContext, LogQueryPlugin } from '@perses-dev/plugin-system'; import { replaceVariables } from '@perses-dev/plugin-system'; -import type { LogEntry, LogData } from '@perses-dev/spec'; +import type { LogData } from '@perses-dev/spec'; import type { OpenSearchClient } from '../../model/opensearch-client'; -import type { OpenSearchPPLResponse } from '../../model/opensearch-client-types'; -import { DEFAULT_DATASOURCE, DEFAULT_MESSAGE_FIELDS, DEFAULT_TIMESTAMP_FIELDS } from '../constants'; +import { DEFAULT_DATASOURCE } from '../constants'; +import { buildBoundedPPL, buildDslSearchBody, buildLuceneSearchBody, buildSqlRequest } from './build'; +import { convertDatarowsToLogs, convertHitsToLogs } from './convert'; import type { OpenSearchLogQuerySpec } from './opensearch-log-query-types'; - -/** - * Bound the query to the panel time range using a PPL `where` clause on the - * configured timestamp field (`@timestamp` by default). - * The bound is injected immediately after the source clause so it runs before any - * pipe that drops the timestamp column from the schema (stats, fields, top, etc.). - * If the user already filters on the timestamp field themselves, PPL ANDs the two clauses. - */ -interface BoundedPPLOptions { - index?: string; - timestampField?: string; - disableTimeFilter?: boolean; -} - -/** - * Escape a user-supplied identifier (timestamp field name) for safe interpolation - * inside a PPL backtick-quoted identifier. A backtick in the value would otherwise - * close the quoting early and let the rest of the string be parsed as PPL, producing - * an invalid — or potentially injected — query. PPL escapes a literal backtick by - * doubling it. - */ -function escapeIdentifier(name: string): string { - return name.replace(/`/g, '``'); -} - -/** - * Sanitize a user-supplied index name/pattern before embedding it in a `source=` - * clause. Unlike the timestamp field, `source=` is not backtick-quoted and must - * keep wildcards (`*`) and multi-index commas intact, so we can't just quote it. - * Instead we strip the characters that could terminate the clause and inject an - * extra PPL stage (backtick, pipe, quotes) or that are simply invalid in an index - * name (whitespace) — e.g. `logs-* | delete` would otherwise smuggle in a pipe. - */ -function sanitizeIndex(index: string): string { - return index.replace(/[`|'"\s]/g, ''); -} - -/** Join PPL stages with ` | `, dropping any empty segments so we never emit a dangling pipe. */ -function joinPPLStages(stages: string[]): string { - return stages.filter((stage) => stage.trim().length > 0).join(' | '); -} - -export function buildBoundedPPL( - userQuery: string, - start: Date, - end: Date, - { index, timestampField = '@timestamp', disableTimeFilter = false }: BoundedPPLOptions = {}, -): string { - let trimmed = userQuery.trim(); - - if (index && !/^(?:search\s+)?source\s*=/i.test(trimmed)) { - trimmed = `source=${sanitizeIndex(index)} | ${trimmed}`; - } - - // Skip the auto-injected time-range clause when the caller manages their own time - // bounds (disableTimeFilter). - if (disableTimeFilter) { - return trimmed; - } - - const startIso = start.toISOString(); - const endIso = end.toISOString(); - // timestampField is user-editable, so escape it before embedding it in the - // backtick-quoted identifier (the ISO bounds are machine-generated and safe). - const tsField = escapeIdentifier(timestampField); - const bound = `where \`${tsField}\` >= '${startIso}' and \`${tsField}\` <= '${endIso}'`; - - const firstPipe = trimmed.indexOf('|'); - if (firstPipe === -1) { - return joinPPLStages([trimmed, bound]); - } - - const sourceClause = trimmed.slice(0, firstPipe).trimEnd(); - const rest = trimmed.slice(firstPipe + 1).trimStart(); - // `rest` can be empty (e.g. a trailing pipe or an empty query), so join through a - // filter to avoid producing an invalid `... | ` dangling stage. - return joinPPLStages([sourceClause, bound, rest]); -} - -function pickIndex(cols: Array<{ name: string }>, candidates: string[]): number { - for (const candidate of candidates) { - const idx = cols.findIndex((c) => c.name === candidate); - if (idx !== -1) return idx; - } - return -1; -} - -interface ConvertOptions { - timestampField?: string; - messageField?: string; -} - -export function convertPPLToLogs(response: OpenSearchPPLResponse, options: ConvertOptions = {}): LogData { - const { schema = [], datarows = [] } = response; - - const tsCandidates = options.timestampField - ? [options.timestampField, ...DEFAULT_TIMESTAMP_FIELDS] - : DEFAULT_TIMESTAMP_FIELDS; - const msgCandidates = options.messageField - ? [options.messageField, ...DEFAULT_MESSAGE_FIELDS] - : DEFAULT_MESSAGE_FIELDS; - - const tsIdx = pickIndex(schema, tsCandidates); - const msgIdx = pickIndex(schema, msgCandidates); - - const entries: LogEntry[] = datarows.map((row) => { - const rawTs = tsIdx !== -1 ? row[tsIdx] : null; - const rawMsg = msgIdx !== -1 ? row[msgIdx] : null; - - const timestamp = parseTimestamp(rawTs); - const line = rawMsg !== null && rawMsg !== undefined ? String(rawMsg) : JSON.stringify(rowToObject(schema, row)); - - const labels: Record = {}; - schema.forEach((col, i) => { - if (i === tsIdx || i === msgIdx) return; - const v = row[i]; - if (v !== null && v !== undefined) labels[col.name] = String(v); - }); - - return { timestamp, line, labels }; - }); - - // A PPL response carries no separate "total hits" field, so totalCount reflects - // the number of rows actually returned by the query (bounded by any `head`/limit - // the user added), not a grand total of matching documents. - return { entries, totalCount: entries.length }; -} - -export function parseTimestamp(v: unknown): number { - if (v === null || v === undefined) return 0; - - // Numeric epochs arrive in seconds, milliseconds, microseconds, or nanoseconds - // (logs frequently carry ns precision). They come as a JS number, or — when the - // value exceeds Number.MAX_SAFE_INTEGER (a ns epoch is ~1e18, well past 2^53) — as - // a numeric string so it survives JSON parsing without silent overflow. - // We emit seconds (LogEntry is second-resolution), so we only need the correct - // magnitude, not sub-second precision: a float's rounding at the ns scale is far - // below one second and is discarded anyway, so BigInt isn't required here. - let numeric = NaN; - if (typeof v === 'number') { - numeric = v; - } else if (/^\d+$/.test(String(v))) { - numeric = Number(v); - } - if (!Number.isNaN(numeric)) { - // Detect the unit by magnitude — a value past year ~5138 for a given unit must - // really be a finer unit — and normalize to seconds. Without the µs/ns tiers - // those values would parse far in the future. - if (numeric >= 1e17) return numeric / 1e9; // nanoseconds → seconds - if (numeric >= 1e14) return numeric / 1e6; // microseconds → seconds - if (numeric >= 1e11) return numeric / 1e3; // milliseconds → seconds - return numeric; // seconds - } - - // OpenSearch PPL returns timestamp/datetime fields as space-separated UTC strings - // with no timezone designator (e.g. "2020-08-26 13:49:00"). Date.parse treats a - // zoneless date-time as *local* time, which would shift the value by the browser's - // offset, so normalize it to explicit UTC (replace the space with `T`, append `Z`) - // before parsing. Strings that already carry a zone (…Z / ±hh:mm) or are date-only - // are left untouched — the spec parses those as UTC already. - const str = String(v).trim(); - const zonelessDateTime = /^\d{4}-\d{2}-\d{2}[ T]\d{2}:\d{2}:\d{2}(\.\d+)?$/; - const iso = zonelessDateTime.test(str) ? `${str.replace(' ', 'T')}Z` : str; - const parsed = Date.parse(iso); - return Number.isNaN(parsed) ? 0 : parsed / 1000; -} - -function rowToObject( - schema: OpenSearchPPLResponse['schema'], - row: Array, -): Record { - const out: Record = {}; - schema.forEach((col, i) => { - out[col.name] = row[i]; - }); - return out; -} +import { requiresIndex, resolveQueryLanguage } from './opensearch-log-query-types'; export const getOpenSearchLogData: LogQueryPlugin['getLogData'] = async ( spec: OpenSearchLogQuerySpec, context: LogQueryContext, abortSignal?: AbortSignal, ) => { - // Treat a blank or whitespace-only query as "no query": trimming it would leave - // an empty string that produces invalid PPL, so short-circuit before hitting the API. - if (!spec.query?.trim()) { - return { - logs: { entries: [], totalCount: 0 }, - timeRange: { start: context.timeRange.start, end: context.timeRange.end }, - }; + const { start, end } = context.timeRange; + const language = resolveQueryLanguage(spec); + + // Treat a blank or whitespace-only query as "no query": trimming it would leave an empty + // string that produces an invalid request, so short-circuit before hitting the API. + // A blank Lucene query is the one exception — it legitimately means "everything in range". + if (!spec.query?.trim() && language !== 'lucene') { + return { logs: { entries: [], totalCount: 0 }, timeRange: { start, end } }; } const query = replaceVariables(spec.query, context.variableState); const resolvedIndex = spec.index ? replaceVariables(spec.index, context.variableState) : undefined; + + if (requiresIndex(language) && !resolvedIndex) { + throw new Error(`An index is required when the query language is ${language}.`); + } + const client = (await context.datasourceStore.getDatasourceClient( spec.datasource ?? DEFAULT_DATASOURCE, )) as OpenSearchClient; - const { start, end } = context.timeRange; - const boundedQuery = buildBoundedPPL(query, start, end, { - index: resolvedIndex, + const convertOptions = { timestampField: spec.timestampField, messageField: spec.messageField }; + const searchOptions = { timestampField: spec.timestampField, disableTimeFilter: spec.disableTimeFilter, - }); + limit: spec.limit, + }; - const response = await client.ppl({ query: boundedQuery }, undefined, abortSignal); + let logs: LogData; + let executedQueryString: string; + + switch (language) { + case 'sql': { + const request = buildSqlRequest(query, start, end, { + timestampField: spec.timestampField, + disableTimeFilter: spec.disableTimeFilter, + }); + logs = convertDatarowsToLogs(await client.sql(request, undefined, abortSignal), convertOptions); + executedQueryString = `POST /_plugins/_sql ${JSON.stringify(request)}`; + break; + } + case 'lucene': + case 'dsl': { + const body = + language === 'lucene' + ? buildLuceneSearchBody(query, start, end, searchOptions) + : buildDslSearchBody(query, start, end, searchOptions); + const index = resolvedIndex as string; + logs = convertHitsToLogs(await client.search({ index, body }, undefined, abortSignal), convertOptions); + executedQueryString = `POST /${index}/_search ${JSON.stringify(body)}`; + break; + } + default: { + const boundedQuery = buildBoundedPPL(query, start, end, { + index: resolvedIndex, + timestampField: spec.timestampField, + disableTimeFilter: spec.disableTimeFilter, + }); + logs = convertDatarowsToLogs(await client.ppl({ query: boundedQuery }, undefined, abortSignal), convertOptions); + executedQueryString = boundedQuery; + break; + } + } - return { - logs: convertPPLToLogs(response, { - timestampField: spec.timestampField, - messageField: spec.messageField, - }), - timeRange: { start, end }, - metadata: { - executedQueryString: boundedQuery, - }, - }; + return { logs, timeRange: { start, end }, metadata: { executedQueryString } }; }; diff --git a/opensearch/src/queries/opensearch-log-query/index.ts b/opensearch/src/queries/opensearch-log-query/index.ts index 58706dec6..254a55970 100644 --- a/opensearch/src/queries/opensearch-log-query/index.ts +++ b/opensearch/src/queries/opensearch-log-query/index.ts @@ -15,3 +15,5 @@ export * from './get-opensearch-log-data'; export * from './OpenSearchLogQuery'; export * from './OpenSearchLogQueryEditor'; export * from './opensearch-log-query-types'; +export * from './build'; +export * from './convert'; diff --git a/opensearch/src/queries/opensearch-log-query/opensearch-log-query-types.test.ts b/opensearch/src/queries/opensearch-log-query/opensearch-log-query-types.test.ts new file mode 100644 index 000000000..58191ad9c --- /dev/null +++ b/opensearch/src/queries/opensearch-log-query/opensearch-log-query-types.test.ts @@ -0,0 +1,31 @@ +// Copyright The Perses Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +import type { OpenSearchQueryLanguage } from './opensearch-log-query-types'; +import { resolveQueryLanguage } from './opensearch-log-query-types'; + +describe('resolveQueryLanguage', () => { + it('defaults to ppl when queryLanguage is absent', () => { + expect(resolveQueryLanguage({})).toBe('ppl'); + }); + + it.each(['ppl', 'sql', 'lucene', 'dsl'] as const)('resolves a recognised language %s unchanged', (language) => { + expect(resolveQueryLanguage({ queryLanguage: language })).toBe(language); + }); + + it('falls back to ppl for an out-of-enum queryLanguage', () => { + // Cast through `as` since 'kql' is not a valid OpenSearchQueryLanguage — this simulates a + // hand-edited or hand-crafted spec reaching the resolver at runtime, bypassing the type system. + expect(resolveQueryLanguage({ queryLanguage: 'kql' as OpenSearchQueryLanguage })).toBe('ppl'); + }); +}); diff --git a/opensearch/src/queries/opensearch-log-query/opensearch-log-query-types.ts b/opensearch/src/queries/opensearch-log-query/opensearch-log-query-types.ts index 386360994..054312d36 100644 --- a/opensearch/src/queries/opensearch-log-query/opensearch-log-query-types.ts +++ b/opensearch/src/queries/opensearch-log-query/opensearch-log-query-types.ts @@ -13,16 +13,43 @@ import type { DatasourceSelector } from '@perses-dev/spec'; -import type { OpenSearchPPLResponse } from '../../model/opensearch-client-types'; +import type { OpenSearchDatarowsResponse } from '../../model/opensearch-client-types'; + +/** + * Query language a single log query is written in. + * `ppl` and `sql` go to the SQL-plugin endpoints and return the `datarows` format; + * `lucene` and `dsl` go to `_search` and return hits. + */ +export type OpenSearchQueryLanguage = 'ppl' | 'sql' | 'lucene' | 'dsl'; + +export const OPENSEARCH_QUERY_LANGUAGES: readonly OpenSearchQueryLanguage[] = ['ppl', 'sql', 'lucene', 'dsl']; export interface OpenSearchLogQuerySpec { query: string; + /** Defaults to `ppl` when absent, which keeps pre-existing dashboards working unchanged. */ + queryLanguage?: OpenSearchQueryLanguage; datasource?: DatasourceSelector; + /** Required for `lucene` and `dsl` (it forms the `_search` path). Ignored for `sql`, where the index lives in the FROM clause. */ index?: string; timestampField?: string; messageField?: string; - /** When true, the panel time range is NOT injected as a `where` clause on the timestamp field. */ + /** When true, the panel time range is NOT injected into the request. */ disableTimeFilter?: boolean; + /** `_search` size for `lucene` and `dsl`. Defaults to DEFAULT_SEARCH_SIZE. */ + limit?: number; +} + +export type OpenSearchLogQueryResponse = OpenSearchDatarowsResponse; + +export function resolveQueryLanguage(spec: Pick): OpenSearchQueryLanguage { + const language = spec.queryLanguage; + // Fall back to PPL for an unrecognised value as well as an absent one: the CUE schema rejects + // unknown languages on save, but a hand-edited or hand-crafted spec can still reach the editor, + // and an unknown key would otherwise resolve to `undefined` metadata and throw during render. + return language !== undefined && OPENSEARCH_QUERY_LANGUAGES.includes(language) ? language : 'ppl'; } -export type OpenSearchLogQueryResponse = OpenSearchPPLResponse; +/** Languages whose index goes into the request URL and therefore cannot be omitted. */ +export function requiresIndex(language: OpenSearchQueryLanguage): boolean { + return language === 'lucene' || language === 'dsl'; +} From 2868846b16d720a351d56deb197b086a41dc2a36 Mon Sep 17 00:00:00 2001 From: Kelechi Oliver Azorji Date: Tue, 29 Sep 2026 06:27:46 +0000 Subject: [PATCH 2/5] chore: query improvement fixes Signed-off-by: Kelechi Oliver Azorji --- .../queries/opensearch-log-query/query.cue | 10 +++- .../tests/invalid/sql-empty-query.json | 4 ++ .../tests/valid/lucene-empty-query.json | 4 ++ opensearch/sdk/go/query/log/log.go | 7 +-- opensearch/sdk/go/query/log/log_test.go | 16 +++++++ .../src/model/opensearch-client.test.ts | 24 ++++++++-- opensearch/src/model/opensearch-client.ts | 19 ++++++-- .../OpenSearchLogQueryEditor.test.tsx | 4 +- .../OpenSearchLogQueryEditor.tsx | 46 +++++++++++-------- .../opensearch-log-query/build/dsl.test.ts | 6 +-- .../queries/opensearch-log-query/build/dsl.ts | 4 +- .../opensearch-log-query/build/lucene.ts | 2 +- .../queries/opensearch-log-query/build/ppl.ts | 2 +- .../queries/opensearch-log-query/build/sql.ts | 2 +- .../convert/datarows-to-logs.ts | 3 +- .../convert/hits-to-logs.test.ts | 17 ++++++- .../convert/hits-to-logs.ts | 15 +++--- .../convert/parse-timestamp.ts | 6 ++- .../get-opensearch-log-data.test.ts | 12 ++--- 19 files changed, 146 insertions(+), 57 deletions(-) create mode 100644 opensearch/schemas/queries/opensearch-log-query/tests/invalid/sql-empty-query.json create mode 100644 opensearch/schemas/queries/opensearch-log-query/tests/valid/lucene-empty-query.json diff --git a/opensearch/schemas/queries/opensearch-log-query/query.cue b/opensearch/schemas/queries/opensearch-log-query/query.cue index 47fc1beb2..e1e1a3c10 100644 --- a/opensearch/schemas/queries/opensearch-log-query/query.cue +++ b/opensearch/schemas/queries/opensearch-log-query/query.cue @@ -22,7 +22,8 @@ kind: "OpenSearchLogQuery" // `index` is required only for the languages that put it in the request URL // (`lucene`, `dsl`, which hit `_search`); it's optional for `ppl`/`sql`, where -// the index lives in the query text itself. +// the index lives in the query text itself. `query` may be empty only for +// `lucene`, where a blank query string means "everything in the time range". // // A disjunction of two closed structs (one branch requiring `index` for // lucene/dsl, one leaving it optional for ppl/sql) does NOT work: a missing @@ -49,12 +50,17 @@ kind: "OpenSearchLogQuery" // without re-testing the comprehension against it. spec: close({ ds.#selector - query: strings.MinRunes(1) timestampField?: strings.MinRunes(1) messageField?: strings.MinRunes(1) disableTimeFilter?: bool limit?: int & >0 queryLanguage: *"ppl" | "sql" | "lucene" | "dsl" + if queryLanguage == "lucene" { + query: string + } + if queryLanguage != "lucene" { + query: strings.MinRunes(1) + } if queryLanguage == "lucene" || queryLanguage == "dsl" { index: strings.MinRunes(1) } diff --git a/opensearch/schemas/queries/opensearch-log-query/tests/invalid/sql-empty-query.json b/opensearch/schemas/queries/opensearch-log-query/tests/invalid/sql-empty-query.json new file mode 100644 index 000000000..cc6708835 --- /dev/null +++ b/opensearch/schemas/queries/opensearch-log-query/tests/invalid/sql-empty-query.json @@ -0,0 +1,4 @@ +{ + "kind": "OpenSearchLogQuery", + "spec": { "queryLanguage": "sql", "query": "" } +} diff --git a/opensearch/schemas/queries/opensearch-log-query/tests/valid/lucene-empty-query.json b/opensearch/schemas/queries/opensearch-log-query/tests/valid/lucene-empty-query.json new file mode 100644 index 000000000..34047889c --- /dev/null +++ b/opensearch/schemas/queries/opensearch-log-query/tests/valid/lucene-empty-query.json @@ -0,0 +1,4 @@ +{ + "kind": "OpenSearchLogQuery", + "spec": { "queryLanguage": "lucene", "query": "", "index": "logs-*" } +} diff --git a/opensearch/sdk/go/query/log/log.go b/opensearch/sdk/go/query/log/log.go index c5afac5c9..f982f7dfc 100644 --- a/opensearch/sdk/go/query/log/log.go +++ b/opensearch/sdk/go/query/log/log.go @@ -65,12 +65,13 @@ func (s *PluginSpec) UnmarshalYAML(unmarshal func(interface{}) error) error { var validLanguages = map[string]bool{"": true, "ppl": true, "sql": true, "lucene": true, "dsl": true} func (s *PluginSpec) validate() error { - if len(s.Query) == 0 { - return fmt.Errorf("query cannot be empty") - } if !validLanguages[s.QueryLanguage] { return fmt.Errorf("unknown query language %q: want one of ppl, sql, lucene, dsl", s.QueryLanguage) } + // A blank Lucene query string means "everything in the time range". + if len(s.Query) == 0 && s.QueryLanguage != "lucene" { + return fmt.Errorf("query cannot be empty") + } if (s.QueryLanguage == "lucene" || s.QueryLanguage == "dsl") && len(s.Index) == 0 { return fmt.Errorf("index is required when queryLanguage is %q", s.QueryLanguage) } diff --git a/opensearch/sdk/go/query/log/log_test.go b/opensearch/sdk/go/query/log/log_test.go index 278530c16..718328c85 100644 --- a/opensearch/sdk/go/query/log/log_test.go +++ b/opensearch/sdk/go/query/log/log_test.go @@ -79,6 +79,22 @@ func TestPluginSpecRejectsEmptyQueryOnUnmarshal(t *testing.T) { } } +func TestPluginSpecAcceptsEmptyLuceneQueryOnUnmarshal(t *testing.T) { + raw := []byte(`{"query":"","queryLanguage":"lucene","index":"logs-*"}`) + var spec PluginSpec + if err := json.Unmarshal(raw, &spec); err != nil { + t.Fatalf("unexpected error: %v", err) + } +} + +func TestPluginSpecRejectsEmptySQLQueryOnUnmarshal(t *testing.T) { + raw := []byte(`{"query":"","queryLanguage":"sql"}`) + var spec PluginSpec + if err := json.Unmarshal(raw, &spec); err == nil { + t.Fatalf("expected error unmarshalling sql spec with empty query, got nil") + } +} + func TestPluginSpecAcceptsNonEmptyQueryOnUnmarshal(t *testing.T) { raw := []byte(`{"query":"source=logs-*"}`) var spec PluginSpec diff --git a/opensearch/src/model/opensearch-client.test.ts b/opensearch/src/model/opensearch-client.test.ts index bd5c01438..dd93363bd 100644 --- a/opensearch/src/model/opensearch-client.test.ts +++ b/opensearch/src/model/opensearch-client.test.ts @@ -131,6 +131,24 @@ describe('opensearch-client', () => { expect(err.message).toBe('OpenSearch API request failed (400): Invalid Query'); }); + it('explains a Perses proxy 403 for an endpoint missing from allowedEndpoints', () => { + const err = new OpenSearchApiError( + 403, + '{"message":"you are not allowed to use this endpoint \\"/_plugins/_sql\\" with the HTTP method POST"}', + ); + expect(err.message).toBe( + 'OpenSearch API request was blocked by the Perses datasource proxy (403): ' + + 'you are not allowed to use this endpoint "/_plugins/_sql" with the HTTP method POST. ' + + "Add the endpoint to the datasource's proxy allowedEndpoints (POST /_plugins/_ppl, " + + '/_plugins/_sql and /.*/_search).', + ); + }); + + it('keeps the OpenSearch reason for a 403 that did not come from the proxy', () => { + const err = new OpenSearchApiError(403, '{"error":{"reason":"no permissions for [indices:data/read/search]"}}'); + expect(err.message).toBe('OpenSearch API request failed (403): no permissions for [indices:data/read/search]'); + }); + it('falls through to details when reason is an empty string', () => { const err = new OpenSearchApiError( 400, @@ -162,7 +180,7 @@ describe('opensearch-client', () => { it('labels the error message SQL', async () => { mockFetch({ ok: false, status: 400, body: '{"error":{"reason":"bad SQL"}}' }); await expect(sql({ query: 'x' }, { datasourceUrl: 'http://localhost:9200' })).rejects.toThrow( - 'OpenSearch SQL request failed (400): bad SQL' + 'OpenSearch SQL request failed (400): bad SQL', ); }); }); @@ -210,7 +228,7 @@ describe('opensearch-client', () => { ])('rejects unsafe index %p without issuing a request', async (index) => { const mock = mockFetch({ ok: true, status: 200, body: { hits: { hits: [] } } }); await expect(search({ index, body: {} }, { datasourceUrl: 'http://localhost:9200' })).rejects.toThrow( - /invalid OpenSearch index/i + /invalid OpenSearch index/i, ); expect(mock).not.toHaveBeenCalled(); }); @@ -222,7 +240,7 @@ describe('opensearch-client', () => { body: '{"error":{"type":"search_phase_execution_exception","root_cause":[{"reason":"failed to parse date field"}]},"status":400}', }); await expect(search({ index: 'logs-*', body: {} }, { datasourceUrl: 'http://localhost:9200' })).rejects.toThrow( - 'OpenSearch Search request failed (400): failed to parse date field' + 'OpenSearch Search request failed (400): failed to parse date field', ); }); }); diff --git a/opensearch/src/model/opensearch-client.ts b/opensearch/src/model/opensearch-client.ts index 3d6ba6b20..1b4dee6e1 100644 --- a/opensearch/src/model/opensearch-client.ts +++ b/opensearch/src/model/opensearch-client.ts @@ -82,7 +82,18 @@ function buildShortMessage(label: ApiLabel, status: number, body: string): strin try { const parsed = JSON.parse(body) as { error?: { reason?: string; details?: string; root_cause?: Array<{ reason?: string }> }; + message?: string; }; + // The Perses datasource proxy rejects endpoints missing from `allowedEndpoints` with a + // 403 and a top-level `message`. Datasources created before SQL/Lucene/DSL support only + // allow `/_plugins/_ppl`, so say how to fix it rather than surfacing a bare 403. + if (status === 403 && parsed?.message?.includes('not allowed to use this endpoint')) { + return ( + `OpenSearch ${label} request was blocked by the Perses datasource proxy (403): ${parsed.message}. ` + + "Add the endpoint to the datasource's proxy allowedEndpoints (POST /_plugins/_ppl, " + + '/_plugins/_sql and /.*/_search).' + ); + } const reason = parsed?.error?.reason; const details = parsed?.error?.details; // OpenSearch often puts a generic phrase in `reason` ("Invalid Query") and the @@ -163,7 +174,7 @@ export function assertSafeIndexPath(index: string): void { `invalid OpenSearch index for a _search request: ${JSON.stringify(index)}. ` + 'An index (or comma-separated list of indices) must be non-empty and may only contain ' + 'letters, digits, "_", ".", "+", "-", and "*" as path segments, with no empty or ' + - 'dots-only segment.' + 'dots-only segment.', ); }; @@ -185,7 +196,7 @@ export function assertSafeIndexPath(index: string): void { export async function ppl( params: OpenSearchPPLParams, options: OpenSearchApiOptions, - signal?: AbortSignal + signal?: AbortSignal, ): Promise { return postJson('PPL', '/_plugins/_ppl', { query: params.query }, options, signal); } @@ -193,7 +204,7 @@ export async function ppl( export async function sql( params: OpenSearchSQLParams, options: OpenSearchApiOptions, - signal?: AbortSignal + signal?: AbortSignal, ): Promise { const body: Record = { query: params.query }; if (params.filter !== undefined) { @@ -205,7 +216,7 @@ export async function sql( export async function search( params: OpenSearchSearchParams, options: OpenSearchApiOptions, - signal?: AbortSignal + signal?: AbortSignal, ): Promise { assertSafeIndexPath(params.index); return postJson('Search', `/${params.index}/_search`, params.body, options, signal); diff --git a/opensearch/src/queries/opensearch-log-query/OpenSearchLogQueryEditor.test.tsx b/opensearch/src/queries/opensearch-log-query/OpenSearchLogQueryEditor.test.tsx index d71ede9e7..6524c9c80 100644 --- a/opensearch/src/queries/opensearch-log-query/OpenSearchLogQueryEditor.test.tsx +++ b/opensearch/src/queries/opensearch-log-query/OpenSearchLogQueryEditor.test.tsx @@ -140,7 +140,7 @@ describe('OpenSearchLogQueryEditor language selector', () => { // Cast through `as` since 'kql' is not a valid OpenSearchQueryLanguage — this simulates a // hand-edited or hand-crafted spec reaching the editor at runtime, bypassing the type system. expect(() => - setup({ query: 'source=logs-*', queryLanguage: 'kql' as OpenSearchLogQuerySpec['queryLanguage'] }) + setup({ query: 'source=logs-*', queryLanguage: 'kql' as OpenSearchLogQuerySpec['queryLanguage'] }), ).not.toThrow(); expect(screen.getByLabelText('Query language')).toHaveTextContent('PPL'); }); @@ -158,7 +158,7 @@ describe('OpenSearchLogQueryEditor language selector', () => { fireEvent.click(screen.getByRole('option', { name: 'Lucene' })); expect(onChange).toHaveBeenCalledWith( - expect.objectContaining({ queryLanguage: 'lucene', query: 'source=logs-* | where level="warn"' }) + expect.objectContaining({ queryLanguage: 'lucene', query: 'source=logs-* | where level="warn"' }), ); }); }); diff --git a/opensearch/src/queries/opensearch-log-query/OpenSearchLogQueryEditor.tsx b/opensearch/src/queries/opensearch-log-query/OpenSearchLogQueryEditor.tsx index c7441d02c..89dfe6188 100644 --- a/opensearch/src/queries/opensearch-log-query/OpenSearchLogQueryEditor.tsx +++ b/opensearch/src/queries/opensearch-log-query/OpenSearchLogQueryEditor.tsx @@ -11,7 +11,18 @@ // See the License for the specific language governing permissions and // limitations under the License. -import { Box, Checkbox, FormControlLabel, InputLabel, Link, MenuItem, Select, Stack, TextField, Typography } from '@mui/material'; +import { + Box, + Checkbox, + FormControlLabel, + InputLabel, + Link, + MenuItem, + Select, + Stack, + TextField, + Typography, +} from '@mui/material'; import { createModEnterHandler } from '@perses-dev/dashboards'; import type { DatasourceSelectProps, OptionsEditorProps } from '@perses-dev/plugin-system'; import { DatasourceSelect, isVariableDatasource, useDatasourceSelectValueToSelector } from '@perses-dev/plugin-system'; @@ -46,6 +57,10 @@ export function parseLimitInput(raw: string): number | undefined { return Number.isFinite(parsed) && parsed > 0 ? parsed : undefined; } +const labelSx = { display: 'block', marginBottom: '4px', fontWeight: 500 } as const; +const languageInputProps = { 'aria-label': 'Query language' } as const; +const limitInputProps = { min: 1, 'aria-label': 'Limit (optional)' } as const; + const examplesSx = { fontSize: '11px', color: 'text.secondary', @@ -80,7 +95,7 @@ export function OpenSearchLogQueryEditor(props: OpenSearchQueryEditorProps): Rea // onChange. On the normal path this is a no-op: blur has already committed the same value. draft.query = query; draft.queryLanguage = next; - }) + }), ); }; @@ -88,7 +103,7 @@ export function OpenSearchLogQueryEditor(props: OpenSearchQueryEditorProps): Rea onChange( produce(value, (draft) => { draft.limit = parseLimitInput(e.target.value); - }) + }), ); }; @@ -145,17 +160,14 @@ export function OpenSearchLogQueryEditor(props: OpenSearchQueryEditorProps): Rea return (
- + Query language