PLT-0000 feat(app): bundle a trace explorer app with the datasource

.github/workflows/ci.yml

@@ -101,7 +101,7 @@ jobs:
           path: dist/
 
       - name: Restore executable permissions
-        run: chmod +x dist/gpx_quickwit_*
+        run: chmod +x dist/datasources/quickwit/gpx_quickwit_*
 
       - name: Setup Node.js environment
         uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # ratchet:actions/setup-node@v6

Magefile.go

@@ -8,5 +8,16 @@ import (
 	build "github.com/grafana/grafana-plugin-sdk-go/build"
 )
 
+func init() {
+	// The Go backend belongs to the nested datasource plugin, not the
+	// top-level app: read its plugin.json for the executable name and emit
+	// the binaries (and go_plugin_build_manifest) next to it.
+	_ = build.SetBeforeBuildCallback(func(cfg build.Config) (build.Config, error) {
+		cfg.PluginJSONPath = "src/datasources/quickwit"
+		cfg.OutputBinaryPath = "dist/datasources/quickwit"
+		return cfg, nil
+	})
+}
+
 // Default configures the default target.
 var Default = build.BuildAll

docker-compose.yaml

@@ -14,12 +14,14 @@ services:
     build:
       context: ./.config
       args:
-        grafana_version: ${GRAFANA_VERSION:-12.4.0}
+        grafana_version: ${GRAFANA_VERSION:-13.0.2}
         grafana_image: ${GRAFANA_IMAGE:-grafana-oss}
+    environment:
+      - GF_PLUGINS_ALLOW_LOADING_UNSIGNED_PLUGINS=quickwit-quickwit-app,quickwit-quickwit-datasource
     ports:
       - 3000:3000/tcp
     volumes:
-      - ./:/var/lib/grafana/plugins/grafana-quickwit-datasource
+      - ./dist:/var/lib/grafana/plugins/quickwit-quickwit-app
       - ./provisioning:/etc/grafana/provisioning
       - gquickwit:/var/lib/grafana
     extra_hosts:

package.json

@@ -3,8 +3,8 @@
   "version": "0.6.4",
   "description": "Quickwit datasource",
   "scripts": {
-    "build": "webpack -c ./.config/webpack/webpack.config.ts --env production",
-    "dev": "webpack -w -c ./.config/webpack/webpack.config.ts --env development",
+    "build": "webpack -c ./webpack.config.ts --env production && ./scripts/nest-backend.sh",
+    "dev": "webpack -w -c ./webpack.config.ts --env development",
     "test": "jest --watch --onlyChanged",
     "test:ci": "jest --passWithNoTests --maxWorkers 4",
     "typecheck": "tsc --noEmit",
@@ -21,10 +21,15 @@
   "license": "AGPL-3.0",
   "devDependencies": {
     "@babel/core": "^7.21.4",
-    "@grafana/e2e-selectors": "10.1.9",
+    "@grafana/data": "^13.0.2",
+    "@grafana/e2e-selectors": "^13.0.2",
     "@grafana/eslint-config": "^6.0.0",
     "@grafana/plugin-e2e": "^3.4.2",
+    "@grafana/runtime": "^13.0.2",
+    "@grafana/scenes": "^7.4.2",
+    "@grafana/schema": "^13.0.2",
     "@grafana/tsconfig": "^1.2.0-rc1",
+    "@grafana/ui": "^13.0.2",
     "@playwright/test": "^1.58.2",
     "@storybook/addon-essentials": "^8.6.14",
     "@storybook/addon-interactions": "^8.6.14",
@@ -56,6 +61,7 @@
     "jest": "^29.5.0",
     "jest-environment-jsdom": "^29.5.0",
     "prettier": "^2.8.7",
+    "react-router-dom": "^6.30.4",
     "replace-in-file-webpack-plugin": "^1.0.6",
     "sass": "1.63.2",
     "sass-loader": "13.3.1",
@@ -76,9 +82,6 @@
     "@codemirror/commands": "^6.8.1",
     "@codemirror/view": "^6.38.1",
     "@emotion/css": "^11.11",
-    "@grafana/data": "^12.1.0",
-    "@grafana/runtime": "^12.1.0",
-    "@grafana/ui": "^12.1.0",
     "@reduxjs/toolkit": "^2.0.0",
     "@uiw/react-codemirror": "^4.21.21",
     "lucene": "^2.1.1",

pkg/quickwit/data_query.go

@@ -537,7 +537,13 @@ func processTracesQuery(q *Query, b *es.SearchRequestBuilder, defaultTimeField s
 
 func processTraceSearchQuery(q *Query, b *es.SearchRequestBuilder, defaultTimeField string) {
 	metric := q.Metrics[0]
-	b.Sort(es.SortOrderDesc, defaultTimeField, "epoch_nanos_int")
+	if metric.Settings.Get("sort").MustString() == "duration" {
+		// Scan the slowest spans of the whole window (sort_by=-span_duration_millis)
+		// so "slowest" is not limited to the most recent spans.
+		b.Sort(es.SortOrderDesc, "span_duration_millis", "")
+	} else {
+		b.Sort(es.SortOrderDesc, defaultTimeField, "epoch_nanos_int")
+	}
 	b.Size(stringToIntWithDefaultValue(metric.Settings.Get("spanLimit").MustString(), 5000))
 }
 

pkg/quickwit/response_parser_traces.go

@@ -3,6 +3,7 @@ package quickwit
 import (
 	"encoding/json"
 	"fmt"
+	neturl "net/url"
 	"sort"
 	"strconv"
 	"strings"
@@ -19,7 +20,8 @@ import (
 const (
 	tracesType       = "traces"
 	traceSearchType  = "trace_search"
-	quickwitPluginID = "quickwit-quickwit-datasource"
+	quickwitPluginID    = "quickwit-quickwit-datasource"
+	quickwitAppPluginID = "quickwit-quickwit-app"
 
 	maxTraceStackTraceBytes      = 64 * 1024
 	traceStackTraceTruncatedText = "\n... truncated"
@@ -273,8 +275,32 @@ func processTraceSearchResponse(res *es.SearchResponse, target *Query, dsInfo *e
 		}
 		summaries = append(summaries, summary)
 	}
+
+	// Optional server-side ordering, applied before the limit truncation so
+	// e.g. "slowest" returns the slowest traces of the scanned window rather
+	// than a client-side re-sort of the most recent page.
+	sortKey := ""
+	if len(target.Metrics) > 0 {
+		sortKey = target.Metrics[0].Settings.Get("sort").MustString()
+	}
 	sort.SliceStable(summaries, func(i, j int) bool {
-		return summaries[i].startTimeMillis > summaries[j].startTimeMillis
+		a, b := summaries[i], summaries[j]
+		switch sortKey {
+		case "duration":
+			da, db := a.endTimeMillis-a.startTimeMillis, b.endTimeMillis-b.startTimeMillis
+			if da != db {
+				return da > db
+			}
+		case "errors":
+			if a.errorCount != b.errorCount {
+				return a.errorCount > b.errorCount
+			}
+		case "spans":
+			if a.spanCount != b.spanCount {
+				return a.spanCount > b.spanCount
+			}
+		}
+		return a.startTimeMillis > b.startTimeMillis
 	})
 
 	limit := defaultSize
@@ -975,7 +1001,19 @@ func traceSearchDataLinks(dsInfo *es.DatasourceInfo) []data.DataLink {
 		return nil
 	}
 
-	return traceInternalDataLinks("Open trace", datasourceUID, datasourceName, "trace_id:${__value.raw}", tracesType, "1000")
+	links := traceInternalDataLinks("Open trace", datasourceUID, datasourceName, "trace_id:${__value.raw}", tracesType, "1000")
+	return append(links, tracePeekDataLinks(datasourceUID)...)
+}
+
+// tracePeekDataLinks links into the bundled trace explorer app, which opens
+// the trace in a side drawer (waterfall or logs tab) without leaving the page.
+func tracePeekDataLinks(datasourceUID string) []data.DataLink {
+	base := "/a/" + quickwitAppPluginID + "/explorer?ds=" + neturl.QueryEscape(datasourceUID) +
+		"&${__url_time_range}&peek=${__value.raw}"
+	return []data.DataLink{
+		{Title: "Peek trace", URL: base},
+		{Title: "Peek logs", URL: base + "&peekTab=logs"},
+	}
 }
 
 func traceToLogsDataLinks(dsInfo *es.DatasourceInfo) []data.DataLink {

pkg/quickwit/response_parser_traces_test.go

@@ -487,7 +487,15 @@ func TestTraceDataLinks(t *testing.T) {
 		traceIDField, _ := result.Responses["A"].Frames[0].FieldByName("traceID")
 		require.NotNil(t, traceIDField)
 		require.NotNil(t, traceIDField.Config)
-		require.Len(t, traceIDField.Config.Links, 1)
+		require.Len(t, traceIDField.Config.Links, 3)
+
+		peekTrace := traceIDField.Config.Links[1]
+		require.Equal(t, "Peek trace", peekTrace.Title)
+		require.Equal(t, "/a/quickwit-quickwit-app/explorer?ds=traces-uid&${__url_time_range}&peek=${__value.raw}", peekTrace.URL)
+
+		peekLogs := traceIDField.Config.Links[2]
+		require.Equal(t, "Peek logs", peekLogs.Title)
+		require.Equal(t, "/a/quickwit-quickwit-app/explorer?ds=traces-uid&${__url_time_range}&peek=${__value.raw}&peekTab=logs", peekLogs.URL)
 
 		link := traceIDField.Config.Links[0]
 		require.Equal(t, "Open trace", link.Title)

provisioning/datasources/quickwit.yaml

@@ -0,0 +1,22 @@
+apiVersion: 1
+
+datasources:
+  - name: Quickwit Logs
+    uid: quickwit-logs
+    type: quickwit-quickwit-datasource
+    url: http://quickwit:7280/api/v1
+    jsonData:
+      index: 'otel-logs-v0_9'
+      logMessageField: body.message
+      logLevelField: severity_text
+      tracesDatasourceUid: quickwit-traces
+      tracesDatasourceName: Quickwit Traces
+
+  - name: Quickwit Traces
+    uid: quickwit-traces
+    type: quickwit-quickwit-datasource
+    url: http://quickwit:7280/api/v1
+    jsonData:
+      index: 'otel-traces-v0_9'
+      logsDatasourceUid: quickwit-logs
+      logsDatasourceName: Quickwit Logs

provisioning/plugins/quickwit-app.yaml

@@ -0,0 +1,9 @@
+apiVersion: 1
+
+apps:
+  - type: 'quickwit-quickwit-app'
+    org_id: 1
+    disabled: false
+    jsonData:
+      tracesDatasourceUid: quickwit-traces
+      logsDatasourceUid: quickwit-logs

scripts/nest-backend.sh

@@ -0,0 +1,12 @@
+#!/usr/bin/env bash
+# The Go backend belongs to the nested datasource plugin. Mage now builds it
+# straight into dist/datasources/quickwit (see Magefile.go); this script only
+# migrates binaries left at the dist root by older builds.
+set -euo pipefail
+cd "$(dirname "$0")/.."
+
+mkdir -p dist/datasources/quickwit
+for artifact in dist/gpx_quickwit* dist/go_plugin_build_manifest; do
+  [ -e "$artifact" ] && mv "$artifact" dist/datasources/quickwit/
+done
+exit 0

src/app/components/App/App.tsx

@@ -0,0 +1,64 @@
+import React from 'react';
+import { css } from '@emotion/css';
+import { SceneApp, useSceneApp } from '@grafana/scenes';
+import { AppRootProps, GrafanaTheme2 } from '@grafana/data';
+import { Alert, useStyles2 } from '@grafana/ui';
+import { PluginPropsContext } from '../../utils/utils.plugin';
+import {
+  quickwitDatasources,
+  setTraceExplorerSettings,
+  TraceExplorerSettings,
+} from '../../utils/utils.datasource';
+import { breakdownPage, spanExplorerPage, traceViewerPage } from '../../pages/TraceExplorer/traceExplorerPage';
+
+function getSceneApp() {
+  return new SceneApp({
+    pages: [traceViewerPage, breakdownPage, spanExplorerPage],
+    urlSyncOptions: {
+      updateUrlOnInit: true,
+      createBrowserHistorySteps: true,
+    },
+  });
+}
+
+function AppWithScenes() {
+  const scene = useSceneApp(getSceneApp);
+  const styles = useStyles2(getStyles);
+
+  if (quickwitDatasources().length === 0) {
+    return (
+      <Alert title="No Quickwit datasource found" severity="warning">
+        The trace explorer queries Quickwit datasources. Add one (pointing at your OTEL traces index, e.g.{' '}
+        <code>otel-traces-v0_7</code>) and configure it in the app settings.
+      </Alert>
+    );
+  }
+
+  // The pages use PageLayoutType.Custom (no standard page header), so provide
+  // the content padding the standard layout would normally add.
+  return (
+    <div className={styles.wrapper}>
+      <scene.Component model={scene} />
+    </div>
+  );
+}
+
+const getStyles = (theme: GrafanaTheme2) => ({
+  wrapper: css`
+    height: 100%;
+    padding: ${theme.spacing(2)};
+  `,
+});
+
+function App(props: AppRootProps<TraceExplorerSettings>) {
+  // Make the configured datasource UIDs available to scene construction.
+  setTraceExplorerSettings(props.meta.jsonData ?? undefined);
+
+  return (
+    <PluginPropsContext.Provider value={props}>
+      <AppWithScenes />
+    </PluginPropsContext.Provider>
+  );
+}
+
+export default App;

src/app/components/AppConfig/AppConfig.tsx

@@ -0,0 +1,89 @@
+import React, { useState } from 'react';
+import { Button, Field, FieldSet } from '@grafana/ui';
+import { PluginConfigPageProps, AppPluginMeta, PluginMeta } from '@grafana/data';
+import { DataSourcePicker, getBackendSrv, locationService } from '@grafana/runtime';
+import { lastValueFrom } from 'rxjs';
+import { testIds } from '../testIds';
+import { QUICKWIT_DS_TYPE } from '../../constants';
+import { TraceExplorerSettings } from '../../utils/utils.datasource';
+
+export interface AppConfigProps extends PluginConfigPageProps<AppPluginMeta<TraceExplorerSettings>> {}
+
+const AppConfig = ({ plugin }: AppConfigProps) => {
+  const { enabled, pinned, jsonData } = plugin.meta;
+  const [state, setState] = useState<TraceExplorerSettings>({
+    tracesDatasourceUid: jsonData?.tracesDatasourceUid,
+    logsDatasourceUid: jsonData?.logsDatasourceUid,
+  });
+
+  const onSubmit = (event: React.FormEvent) => {
+    event.preventDefault();
+    updatePluginAndReload(plugin.meta.id, {
+      enabled,
+      pinned,
+      jsonData: state,
+    });
+  };
+
+  return (
+    <form onSubmit={onSubmit} data-testid={testIds.appConfig.container}>
+      <FieldSet label="Datasources">
+        <Field
+          label="Traces datasource"
+          description="Quickwit datasource pointing at your OTEL traces index (e.g. otel-traces-v0_7)"
+        >
+          <DataSourcePicker
+            width={40}
+            inputId="traces-datasource"
+            filter={(ds) => ds.type === QUICKWIT_DS_TYPE}
+            current={state.tracesDatasourceUid}
+            onChange={(ds) => setState({ ...state, tracesDatasourceUid: ds.uid })}
+            noDefault
+          />
+        </Field>
+        <Field
+          label="Logs datasource"
+          description="Quickwit datasource pointing at your OTEL logs index (e.g. otel-logs-v0_7), used for trace log correlation"
+        >
+          <DataSourcePicker
+            width={40}
+            inputId="logs-datasource"
+            filter={(ds) => ds.type === QUICKWIT_DS_TYPE}
+            current={state.logsDatasourceUid}
+            onChange={(ds) => setState({ ...state, logsDatasourceUid: ds.uid })}
+            noDefault
+          />
+        </Field>
+        <Button type="submit" data-testid={testIds.appConfig.submit}>
+          Save settings
+        </Button>
+      </FieldSet>
+    </form>
+  );
+};
+
+export default AppConfig;
+
+const updatePluginAndReload = async (pluginId: string, data: Partial<PluginMeta<TraceExplorerSettings>>) => {
+  try {
+    await updatePlugin(pluginId, data);
+
+    // Reloading the page as the changes made here wouldn't be propagated to the actual plugin otherwise.
+    // This is not ideal, however unfortunately currently there is no supported way for updating the plugin state.
+    locationService.reload();
+  } catch (e) {
+    console.error('Error while updating the plugin', e);
+  }
+};
+
+const updatePlugin = async (pluginId: string, data: Partial<PluginMeta>) => {
+  const response = getBackendSrv().fetch({
+    url: `/api/plugins/${pluginId}/settings`,
+    method: 'POST',
+    data,
+  });
+
+  const dataResponse = await lastValueFrom(response);
+
+  return dataResponse.data;
+};