diff --git a/NEXT_CHANGELOG.md b/NEXT_CHANGELOG.md index 80cc71eba0..0a9f73a08c 100644 --- a/NEXT_CHANGELOG.md +++ b/NEXT_CHANGELOG.md @@ -8,6 +8,7 @@ * Added `--limit` flag to all paginated list commands for client-side result capping ([#4984](https://github.com/databricks/cli/pull/4984)). * Added interactive pagination for list commands that render as JSON. When stdin, stdout, and stderr are all TTYs and the command has no row template, `databricks list` now streams 50 JSON items at a time and prompts `[space] more [enter] all [q|esc] quit` on stderr. ENTER can be interrupted by `q`/`esc`/`Ctrl+C` between pages. Piped output still receives the full JSON array; early quit still produces a syntactically valid array. +* Added interactive pagination for list commands that have a row template (jobs, clusters, apps, pipelines, etc.). Same prompt, same keys as the JSON pager; colors and alignment match the existing non-paged output and column widths stay stable across pages. ### Bundles diff --git a/NOTICE b/NOTICE index 883c24ab78..baad4b0525 100644 --- a/NOTICE +++ b/NOTICE @@ -115,6 +115,10 @@ golang.org/x/sys - https://github.com/golang/sys Copyright 2009 The Go Authors. License - https://github.com/golang/sys/blob/master/LICENSE +golang.org/x/term - https://github.com/golang/term +Copyright 2009 The Go Authors. +License - https://github.com/golang/term/blob/master/LICENSE + golang.org/x/text - https://github.com/golang/text Copyright 2009 The Go Authors. License - https://github.com/golang/text/blob/master/LICENSE diff --git a/go.mod b/go.mod index efe73189d5..9aa4544205 100644 --- a/go.mod +++ b/go.mod @@ -40,12 +40,11 @@ require ( golang.org/x/oauth2 v0.36.0 // BSD-3-Clause golang.org/x/sync v0.20.0 // BSD-3-Clause golang.org/x/sys v0.43.0 // BSD-3-Clause + golang.org/x/term v0.41.0 // BSD-3-Clause golang.org/x/text v0.35.0 // BSD-3-Clause gopkg.in/ini.v1 v1.67.1 // Apache-2.0 ) -require golang.org/x/term v0.41.0 - require ( cloud.google.com/go/auth v0.18.1 // indirect cloud.google.com/go/auth/oauth2adapt v0.2.8 // indirect diff --git a/libs/cmdio/paged_template.go b/libs/cmdio/paged_template.go new file mode 100644 index 0000000000..15ff1f9311 --- /dev/null +++ b/libs/cmdio/paged_template.go @@ -0,0 +1,228 @@ +package cmdio + +import ( + "bytes" + "context" + "fmt" + "io" + "os" + "regexp" + "strings" + "text/template" + "unicode/utf8" + + "github.com/databricks/databricks-sdk-go/listing" +) + +// pagerColumnSeparator is the inter-column spacing used when emitting paged +// template output. Matches text/tabwriter.NewWriter(..., 2, ' ', ...) in +// the non-paged path, so single-batch output is visually indistinguishable +// from what renderUsingTemplate produces. +const pagerColumnSeparator = " " + +// ansiCSIPattern matches ANSI SGR escape sequences so we can compute the +// on-screen width of cells that contain colored output. We do not attempt +// to cover every ANSI escape — just the SGR color/style ones (CSI ... m) +// emitted by github.com/fatih/color, which is all our templates use today. +var ansiCSIPattern = regexp.MustCompile("\x1b\\[[0-9;]*m") + +// renderIteratorPagedTemplate streams an iterator through the existing +// template-based renderer one page at a time, prompting the user between +// pages on stderr. The rendering is intentionally identical to the non- +// paged path — same templates, same colors — only the flush cadence, +// the user-facing prompt, and column-width stability across batches +// differ. +// +// SPACE advances by one page; ENTER drains the remaining iterator (still +// interruptible by q/esc/Ctrl+C); q/esc/Ctrl+C stop immediately. +func renderIteratorPagedTemplate[T any]( + ctx context.Context, + iter listing.Iterator[T], + out io.Writer, + headerTemplate, tmpl string, +) error { + keys, restore, err := startRawStdinKeyReader(ctx) + if err != nil { + return err + } + defer restore() + return renderIteratorPagedTemplateCore( + ctx, + iter, + crlfWriter{w: out}, + crlfWriter{w: os.Stderr}, + keys, + headerTemplate, + tmpl, + pagerPageSize, + ) +} + +// renderIteratorPagedTemplateCore is the testable core of +// renderIteratorPagedTemplate: it assumes the caller has already set up +// raw stdin (or any key source) and delivered a channel of keystrokes. +// It never touches os.Stdin directly. +// +// Unlike renderUsingTemplate (the non-paged path) we do not rely on +// text/tabwriter to align columns. Tabwriter computes column widths +// per-flush and resets on Flush(), which produces a jarring +// width-shift when a short final batch follows a wider first batch. +// Here we render each page's template output into an intermediate +// buffer, split it into cells by tab, lock visual column widths from +// the first page, and pad every subsequent page to the same widths. +// The output is visually indistinguishable from tabwriter for single- +// batch lists and stays aligned across batches for longer ones. +func renderIteratorPagedTemplateCore[T any]( + ctx context.Context, + iter listing.Iterator[T], + out io.Writer, + prompts io.Writer, + keys <-chan byte, + headerTemplate, tmpl string, + pageSize int, +) error { + // Use two independent templates so parsing the row template doesn't + // overwrite the header template's parsed body (they would if they + // shared the same *template.Template instance — Parse replaces the + // body in place and returns the receiver). + headerT, err := template.New("header").Funcs(renderFuncMap).Parse(headerTemplate) + if err != nil { + return err + } + rowT, err := template.New("row").Funcs(renderFuncMap).Parse(tmpl) + if err != nil { + return err + } + + limit := limitFromContext(ctx) + drainAll := false + buf := make([]any, 0, pageSize) + total := 0 + + var lockedWidths []int + firstBatchDone := false + + flushPage := func() error { + // Nothing to emit after the first batch is out the door and the + // buffer is empty — we've already written the header. + if firstBatchDone && len(buf) == 0 { + return nil + } + + var rendered bytes.Buffer + if !firstBatchDone && headerTemplate != "" { + if err := headerT.Execute(&rendered, nil); err != nil { + return err + } + rendered.WriteByte('\n') + } + if len(buf) > 0 { + if err := rowT.Execute(&rendered, buf); err != nil { + return err + } + buf = buf[:0] + } + firstBatchDone = true + + text := strings.TrimRight(rendered.String(), "\n") + if text == "" { + return nil + } + rows := strings.Split(text, "\n") + + // Lock column widths from the first batch (header + first page). + // Every subsequent batch pads to these widths so columns do not + // shift between pages. + if lockedWidths == nil { + for _, row := range rows { + for i, cell := range strings.Split(row, "\t") { + if i >= len(lockedWidths) { + lockedWidths = append(lockedWidths, 0) + } + if w := visualWidth(cell); w > lockedWidths[i] { + lockedWidths[i] = w + } + } + } + } + + for _, row := range rows { + line := padRow(strings.Split(row, "\t"), lockedWidths) + if _, err := io.WriteString(out, line+"\n"); err != nil { + return err + } + } + return nil + } + + for iter.HasNext(ctx) { + if limit > 0 && total >= limit { + break + } + n, err := iter.Next(ctx) + if err != nil { + return err + } + buf = append(buf, n) + total++ + + if len(buf) < pageSize { + continue + } + if err := flushPage(); err != nil { + return err + } + if drainAll { + if pagerShouldQuit(keys) { + return nil + } + continue + } + // Show the prompt and wait for a key. + fmt.Fprint(prompts, pagerPromptText) + k, ok := pagerNextKey(ctx, keys) + fmt.Fprint(prompts, pagerClearLine) + if !ok { + return nil + } + switch k { + case ' ': + // Continue to the next page. + case '\r', '\n': + drainAll = true + case 'q', 'Q', pagerKeyEscape, pagerKeyCtrlC: + return nil + } + } + return flushPage() +} + +// visualWidth returns the number of runes a string would occupy on-screen +// if ANSI SGR escape sequences are respected as zero-width (which the +// terminal does). This matches what the user sees and lets us pad colored +// cells to consistent visual widths. +func visualWidth(s string) int { + return utf8.RuneCountInString(ansiCSIPattern.ReplaceAllString(s, "")) +} + +// padRow joins the given cells with pagerColumnSeparator, padding every +// cell except the last to widths[i] visual runes. Cells wider than +// widths[i] are emitted as-is — the extra content pushes subsequent +// columns right for that row only, which is the same behavior tabwriter +// would give and the same behavior the non-paged renderer has today. +func padRow(cells []string, widths []int) string { + var b strings.Builder + for i, cell := range cells { + if i > 0 { + b.WriteString(pagerColumnSeparator) + } + b.WriteString(cell) + if i < len(cells)-1 && i < len(widths) { + pad := widths[i] - visualWidth(cell) + if pad > 0 { + b.WriteString(strings.Repeat(" ", pad)) + } + } + } + return b.String() +} diff --git a/libs/cmdio/paged_template_test.go b/libs/cmdio/paged_template_test.go new file mode 100644 index 0000000000..1b61135108 --- /dev/null +++ b/libs/cmdio/paged_template_test.go @@ -0,0 +1,274 @@ +package cmdio + +import ( + "bytes" + "context" + "errors" + "fmt" + "strings" + "testing" + + "github.com/databricks/cli/libs/flags" + "github.com/databricks/databricks-sdk-go/listing" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +type numberIterator struct { + n int + pos int + err error +} + +func (it *numberIterator) HasNext(_ context.Context) bool { + return it.pos < it.n +} + +func (it *numberIterator) Next(_ context.Context) (int, error) { + if it.err != nil { + return 0, it.err + } + it.pos++ + return it.pos, nil +} + +func makeTemplateKeys(bytes ...byte) <-chan byte { + ch := make(chan byte, len(bytes)) + for _, b := range bytes { + ch <- b + } + close(ch) + return ch +} + +func runPagedTemplate(t *testing.T, n, pageSize int, keys []byte) string { + t.Helper() + var out, prompts bytes.Buffer + iter := listing.Iterator[int](&numberIterator{n: n}) + err := renderIteratorPagedTemplateCore( + t.Context(), + iter, + &out, + &prompts, + makeTemplateKeys(keys...), + "", + "{{range .}}{{.}}\n{{end}}", + pageSize, + ) + require.NoError(t, err) + return out.String() +} + +func TestPagedTemplateDrainsWhenFirstPageExhausts(t *testing.T) { + out := runPagedTemplate(t, 3, 10, nil) + require.Equal(t, "1\n2\n3\n", out) +} + +func TestPagedTemplateSpaceFetchesOneMorePage(t *testing.T) { + out := runPagedTemplate(t, 7, 3, []byte{' '}) + lines := strings.Split(strings.TrimRight(out, "\n"), "\n") + assert.Len(t, lines, 6) +} + +func TestPagedTemplateEnterDrainsIterator(t *testing.T) { + out := runPagedTemplate(t, 25, 5, []byte{'\r'}) + lines := strings.Split(strings.TrimRight(out, "\n"), "\n") + assert.Len(t, lines, 25) +} + +func TestPagedTemplateQuitKeyExits(t *testing.T) { + out := runPagedTemplate(t, 100, 5, []byte{'q'}) + lines := strings.Split(strings.TrimRight(out, "\n"), "\n") + assert.Len(t, lines, 5) +} + +func TestPagedTemplateEscExits(t *testing.T) { + out := runPagedTemplate(t, 100, 5, []byte{pagerKeyEscape}) + lines := strings.Split(strings.TrimRight(out, "\n"), "\n") + assert.Len(t, lines, 5) +} + +func TestPagedTemplateCtrlCExits(t *testing.T) { + out := runPagedTemplate(t, 100, 5, []byte{pagerKeyCtrlC}) + lines := strings.Split(strings.TrimRight(out, "\n"), "\n") + assert.Len(t, lines, 5) +} + +func TestPagedTemplateEnterInterruptibleByCtrlC(t *testing.T) { + out := runPagedTemplate(t, 20, 5, []byte{'\r', pagerKeyCtrlC}) + lines := strings.Split(strings.TrimRight(out, "\n"), "\n") + assert.Len(t, lines, 10) +} + +func TestPagedTemplateRespectsLimit(t *testing.T) { + var out, prompts bytes.Buffer + iter := listing.Iterator[int](&numberIterator{n: 200}) + ctx := WithLimit(t.Context(), 7) + err := renderIteratorPagedTemplateCore( + ctx, + iter, + &out, + &prompts, + makeTemplateKeys('\r'), + "", + "{{range .}}{{.}}\n{{end}}", + 5, + ) + require.NoError(t, err) + lines := strings.Split(strings.TrimRight(out.String(), "\n"), "\n") + assert.Len(t, lines, 7) +} + +func TestPagedTemplatePrintsHeaderOnce(t *testing.T) { + var out, prompts bytes.Buffer + iter := listing.Iterator[int](&numberIterator{n: 8}) + err := renderIteratorPagedTemplateCore( + t.Context(), + iter, + &out, + &prompts, + makeTemplateKeys(' '), + `ID`, + "{{range .}}{{.}}\n{{end}}", + 3, + ) + require.NoError(t, err) + assert.Equal(t, 1, strings.Count(out.String(), "ID\n")) + assert.True(t, strings.HasPrefix(out.String(), "ID\n")) +} + +func TestPagedTemplatePropagatesFetchError(t *testing.T) { + var out, prompts bytes.Buffer + iter := listing.Iterator[int](&numberIterator{n: 100, err: errors.New("boom")}) + err := renderIteratorPagedTemplateCore( + t.Context(), + iter, + &out, + &prompts, + makeTemplateKeys(), + "", + "{{range .}}{{.}}\n{{end}}", + 5, + ) + require.Error(t, err) + assert.Contains(t, err.Error(), "boom") +} + +func TestPagedTemplateRendersHeaderAndRowsCorrectly(t *testing.T) { + // Regression guard against the header/row template cross-pollution + // bug: if the two templates share a *template.Template receiver, + // every flush re-emits the header text where rows should be. + var out, prompts bytes.Buffer + iter := listing.Iterator[int](&numberIterator{n: 6}) + err := renderIteratorPagedTemplateCore( + t.Context(), + iter, + &out, + &prompts, + makeTemplateKeys(), + `ID Name`, + "{{range .}}{{.}} item-{{.}}\n{{end}}", + 100, + ) + require.NoError(t, err) + got := out.String() + assert.Contains(t, got, "ID") + assert.Contains(t, got, "Name") + for i := 1; i <= 6; i++ { + assert.Contains(t, got, fmt.Sprintf("item-%d", i)) + } + assert.Equal(t, 1, strings.Count(got, "ID")) +} + +func TestPagedTemplateEmptyIteratorStillFlushesHeader(t *testing.T) { + var out, prompts bytes.Buffer + iter := listing.Iterator[int](&numberIterator{n: 0}) + err := renderIteratorPagedTemplateCore( + t.Context(), + iter, + &out, + &prompts, + makeTemplateKeys(), + `ID Name`, + "{{range .}}{{.}}\n{{end}}", + 10, + ) + require.NoError(t, err) + assert.Contains(t, out.String(), "ID") + assert.Contains(t, out.String(), "Name") +} + +// slicedIterator is a tiny iterator implementation for tests that prefer +// to hand over strongly-typed row structs. +type slicedIterator[T any] struct { + data []T + pos int +} + +func (it *slicedIterator[T]) HasNext(_ context.Context) bool { return it.pos < len(it.data) } +func (it *slicedIterator[T]) Next(_ context.Context) (T, error) { + v := it.data[it.pos] + it.pos++ + return v, nil +} + +func TestPagedTemplateColumnWidthsStableAcrossBatches(t *testing.T) { + type row struct { + Name string + Tag string + } + rows := []row{ + {"wide-name-that-sets-the-width", "a"}, + {"short", "b"}, + } + iter := &slicedIterator[row]{data: rows} + var out, prompts bytes.Buffer + err := renderIteratorPagedTemplateCore( + t.Context(), + iter, + &out, + &prompts, + makeTemplateKeys(' '), + "Name Tag", + "{{range .}}{{.Name}} {{.Tag}}\n{{end}}", + 1, + ) + require.NoError(t, err) + got := out.String() + lines := strings.Split(strings.TrimRight(got, "\n"), "\n") + require.Len(t, lines, 3) + + // Column 1 must start at the same visible offset on every line. + const wantColTwoOffset = 31 + for i, line := range lines { + idx := strings.LastIndex(line, " ") + 1 + assert.Equal(t, wantColTwoOffset, idx, "line %d: col 1 expected at offset %d, got %d (line=%q)", i, wantColTwoOffset, idx, line) + } +} + +// TestPagedTemplateMatchesNonPagedForSmallList asserts that single-batch +// output is byte-identical to the non-paged template renderer, so users +// who never hit a second page see the exact same thing they used to. +func TestPagedTemplateMatchesNonPagedForSmallList(t *testing.T) { + const rows = 5 + tmpl := `{{range .}}{{green "%d" .}} {{.}} +{{end}}` + var expected bytes.Buffer + refIter := listing.Iterator[int](&numberIterator{n: rows}) + require.NoError(t, renderWithTemplate(t.Context(), newIteratorRenderer(refIter), flags.OutputText, &expected, "", tmpl)) + + var actual, prompts bytes.Buffer + pagedIter := listing.Iterator[int](&numberIterator{n: rows}) + require.NoError(t, renderIteratorPagedTemplateCore( + t.Context(), + pagedIter, + &actual, + &prompts, + makeTemplateKeys(), + "", + tmpl, + 100, + )) + assert.Equal(t, expected.String(), actual.String()) + assert.NotEmpty(t, expected.String()) +} diff --git a/libs/cmdio/render.go b/libs/cmdio/render.go index d8152260de..ef9c841384 100644 --- a/libs/cmdio/render.go +++ b/libs/cmdio/render.go @@ -272,18 +272,26 @@ func Render(ctx context.Context, v any) error { } // RenderIterator renders the items produced by i. When the terminal is -// fully interactive (stdin + stdout + stderr all TTYs) and the caller -// hasn't supplied a row template, we page through the iterator's JSON -// representation instead of dumping the full array at once: 50 items -// at a time, with a prompt on stderr between pages asking whether to -// continue. Piped output and explicit --output json against a pipe -// keep the existing non-paged behavior. Commands that do supply a row -// template also keep the existing non-paged behavior — a follow-up -// change will add paging for them. +// fully interactive (stdin + stdout + stderr all TTYs), the iterator is +// paged through one batch at a time with a prompt between pages: +// +// - For text output with a row template, we page through the existing +// template + tabwriter pipeline (same colors, same alignment as the +// non-paged path; widths are locked from the first batch so columns +// stay aligned across pages). +// - For JSON output or text output without a row template, we page +// through the JSON renderer instead. +// +// Piped output keeps the existing non-paged behavior in all cases. func RenderIterator[T any](ctx context.Context, i listing.Iterator[T]) error { c := fromContext(ctx) - if c.template == "" && c.capabilities.SupportsPager() && (c.outputFormat == flags.OutputJSON || c.outputFormat == flags.OutputText) { - return renderIteratorPagedJSON(ctx, i, c.out) + if c.capabilities.SupportsPager() { + if c.outputFormat == flags.OutputText && c.template != "" { + return renderIteratorPagedTemplate(ctx, i, c.out, c.headerTemplate, c.template) + } + if c.outputFormat == flags.OutputJSON || c.outputFormat == flags.OutputText { + return renderIteratorPagedJSON(ctx, i, c.out) + } } return renderWithTemplate(ctx, newIteratorRenderer(i), c.outputFormat, c.out, c.headerTemplate, c.template) }