diff --git a/NEXT_CHANGELOG.md b/NEXT_CHANGELOG.md index 52d764e384..80cc71eba0 100644 --- a/NEXT_CHANGELOG.md +++ b/NEXT_CHANGELOG.md @@ -7,6 +7,7 @@ ### CLI * Added `--limit` flag to all paginated list commands for client-side result capping ([#4984](https://github.com/databricks/cli/pull/4984)). +* Added interactive pagination for list commands that render as JSON. When stdin, stdout, and stderr are all TTYs and the command has no row template, `databricks list` now streams 50 JSON items at a time and prompts `[space] more [enter] all [q|esc] quit` on stderr. ENTER can be interrupted by `q`/`esc`/`Ctrl+C` between pages. Piped output still receives the full JSON array; early quit still produces a syntactically valid array. ### Bundles diff --git a/NOTICE b/NOTICE index 883c24ab78..baad4b0525 100644 --- a/NOTICE +++ b/NOTICE @@ -115,6 +115,10 @@ golang.org/x/sys - https://github.com/golang/sys Copyright 2009 The Go Authors. License - https://github.com/golang/sys/blob/master/LICENSE +golang.org/x/term - https://github.com/golang/term +Copyright 2009 The Go Authors. +License - https://github.com/golang/term/blob/master/LICENSE + golang.org/x/text - https://github.com/golang/text Copyright 2009 The Go Authors. License - https://github.com/golang/text/blob/master/LICENSE diff --git a/go.mod b/go.mod index a11ce1a599..9aa4544205 100644 --- a/go.mod +++ b/go.mod @@ -40,6 +40,7 @@ require ( golang.org/x/oauth2 v0.36.0 // BSD-3-Clause golang.org/x/sync v0.20.0 // BSD-3-Clause golang.org/x/sys v0.43.0 // BSD-3-Clause + golang.org/x/term v0.41.0 // BSD-3-Clause golang.org/x/text v0.35.0 // BSD-3-Clause gopkg.in/ini.v1 v1.67.1 // Apache-2.0 ) diff --git a/libs/cmdio/capabilities.go b/libs/cmdio/capabilities.go index 455acebc77..12b46f6150 100644 --- a/libs/cmdio/capabilities.go +++ b/libs/cmdio/capabilities.go @@ -48,6 +48,16 @@ func (c Capabilities) SupportsColor(w io.Writer) bool { return isTTY(w) && c.color } +// SupportsPager returns true when we can run an interactive pager between +// batches of output: stdin, stdout, and stderr must all be TTYs. Stdin +// carries the user's keystrokes, stdout receives the paged content, and +// stderr carries the "[space] more / [enter] all" prompt — all three +// must be visible for the interaction to make sense. Git Bash is +// excluded because raw-mode stdin reads are unreliable there. +func (c Capabilities) SupportsPager() bool { + return c.stdinIsTTY && c.stdoutIsTTY && c.stderrIsTTY && !c.isGitBash +} + // detectGitBash returns true if running in Git Bash on Windows (has broken promptui support). // We do not allow prompting in Git Bash on Windows. // Likely due to fact that Git Bash does not correctly support ANSI escape sequences, diff --git a/libs/cmdio/paged_json.go b/libs/cmdio/paged_json.go new file mode 100644 index 0000000000..5ce22d3704 --- /dev/null +++ b/libs/cmdio/paged_json.go @@ -0,0 +1,151 @@ +package cmdio + +import ( + "bytes" + "context" + "encoding/json" + "fmt" + "io" + "os" + + "github.com/databricks/databricks-sdk-go/listing" +) + +// renderIteratorPagedJSON streams the iterator as a JSON array, pausing +// for user input every pagerPageSize items. The output is always a +// syntactically valid JSON array — even when the user quits early, we +// still write the closing ']' before returning so a reader that pipes +// the output to a file gets parseable JSON. +// +// The caller is expected to have already verified that the terminal +// supports paging (Capabilities.SupportsPager). +func renderIteratorPagedJSON[T any]( + ctx context.Context, + iter listing.Iterator[T], + out io.Writer, +) error { + keys, restore, err := startRawStdinKeyReader(ctx) + if err != nil { + return err + } + defer restore() + return renderIteratorPagedJSONCore( + ctx, + iter, + crlfWriter{w: out}, + crlfWriter{w: os.Stderr}, + keys, + pagerPageSize, + ) +} + +// renderIteratorPagedJSONCore is the testable core of +// renderIteratorPagedJSON: it takes the output streams and key channel +// as dependencies and never touches os.Stdin directly. +// +// The rendering mirrors iteratorRenderer.renderJson — a pretty-printed +// JSON array with 2-space indentation — but Flush() is followed by a +// user prompt once every pageSize items. If the user says to quit, the +// closing bracket is still written so the accumulated output remains a +// valid JSON document. +func renderIteratorPagedJSONCore[T any]( + ctx context.Context, + iter listing.Iterator[T], + out io.Writer, + prompts io.Writer, + keys <-chan byte, + pageSize int, +) error { + if pageSize <= 0 { + pageSize = pagerPageSize + } + + // We render into an intermediate buffer and flush at page + // boundaries. This lets us show the user N items, prompt, then + // continue — without juggling partial writes to the underlying + // writer mid-item. + var buf bytes.Buffer + flush := func() error { + if buf.Len() == 0 { + return nil + } + if _, err := out.Write(buf.Bytes()); err != nil { + return err + } + buf.Reset() + return nil + } + + // We defer writing the opening bracket until we actually have an + // item to write. That way an empty iterator (or one that errors + // before yielding anything) produces "[]\n" — valid JSON — rather + // than a half-open "[\n " that the caller can't parse. + totalWritten := 0 + finalize := func() error { + if totalWritten == 0 { + buf.WriteString("[]\n") + } else { + buf.WriteString("\n]\n") + } + return flush() + } + + limit := limitFromContext(ctx) + drainAll := false + inPage := 0 + + for iter.HasNext(ctx) { + if limit > 0 && totalWritten >= limit { + break + } + item, err := iter.Next(ctx) + if err != nil { + _ = finalize() + return err + } + if totalWritten == 0 { + buf.WriteString("[\n ") + } else { + buf.WriteString(",\n ") + } + encoded, err := json.MarshalIndent(item, " ", " ") + if err != nil { + _ = finalize() + return err + } + buf.Write(encoded) + totalWritten++ + inPage++ + + if inPage < pageSize { + continue + } + // End of a page. Flush what we have and either prompt or + // continue the drain. + if err := flush(); err != nil { + return err + } + inPage = 0 + if drainAll { + if pagerShouldQuit(keys) { + return finalize() + } + continue + } + fmt.Fprint(prompts, pagerPromptText) + k, ok := pagerNextKey(ctx, keys) + fmt.Fprint(prompts, pagerClearLine) + if !ok { + return finalize() + } + switch k { + case ' ': + // continue with the next page + case '\r', '\n': + drainAll = true + case 'q', 'Q', pagerKeyEscape, pagerKeyCtrlC: + return finalize() + } + } + return finalize() +} diff --git a/libs/cmdio/paged_json_test.go b/libs/cmdio/paged_json_test.go new file mode 100644 index 0000000000..68849b1827 --- /dev/null +++ b/libs/cmdio/paged_json_test.go @@ -0,0 +1,159 @@ +package cmdio + +import ( + "bytes" + "context" + "encoding/json" + "errors" + "fmt" + "testing" + + "github.com/databricks/databricks-sdk-go/listing" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +type pagedJSONRow struct { + ID int `json:"id"` + Name string `json:"name"` +} + +type pagedJSONIterator struct { + data []pagedJSONRow + pos int + err error +} + +func (it *pagedJSONIterator) HasNext(_ context.Context) bool { + return it.pos < len(it.data) +} + +func (it *pagedJSONIterator) Next(_ context.Context) (pagedJSONRow, error) { + if it.err != nil { + return pagedJSONRow{}, it.err + } + v := it.data[it.pos] + it.pos++ + return v, nil +} + +func makePagedJSONRows(n int) []pagedJSONRow { + rows := make([]pagedJSONRow, n) + for i := range rows { + rows[i] = pagedJSONRow{ID: i + 1, Name: fmt.Sprintf("row-%d", i+1)} + } + return rows +} + +func makeKeyChan(keys ...byte) <-chan byte { + ch := make(chan byte, len(keys)) + for _, k := range keys { + ch <- k + } + close(ch) + return ch +} + +func runPagedJSON(t *testing.T, rows []pagedJSONRow, pageSize int, keys []byte) (string, string, error) { + t.Helper() + var out, prompts bytes.Buffer + iter := listing.Iterator[pagedJSONRow](&pagedJSONIterator{data: rows}) + err := renderIteratorPagedJSONCore(t.Context(), iter, &out, &prompts, makeKeyChan(keys...), pageSize) + return out.String(), prompts.String(), err +} + +func TestPagedJSONRendersFullArrayWhenFitsInOnePage(t *testing.T) { + got, _, err := runPagedJSON(t, makePagedJSONRows(3), 10, nil) + require.NoError(t, err) + assert.JSONEq(t, `[{"id":1,"name":"row-1"},{"id":2,"name":"row-2"},{"id":3,"name":"row-3"}]`, got) +} + +func TestPagedJSONSpaceAdvancesOneMorePage(t *testing.T) { + // 7 items, page=3: first page (3), SPACE → second page (3), then the + // key channel closes (stdin EOF) and the pager finalizes, writing + // 6 items total. Fetching item 7 would require a second keypress. + got, _, err := runPagedJSON(t, makePagedJSONRows(7), 3, []byte{' '}) + require.NoError(t, err) + var items []pagedJSONRow + require.NoError(t, json.Unmarshal([]byte(got), &items)) + assert.Len(t, items, 6) +} + +func TestPagedJSONTwoSpacesAdvanceTwoMorePages(t *testing.T) { + // 7 items, page=3, SPACE + SPACE: after the second SPACE the loop + // fetches item 7, HasNext is false, loop exits and the finalizer + // closes the array. + got, _, err := runPagedJSON(t, makePagedJSONRows(7), 3, []byte{' ', ' '}) + require.NoError(t, err) + var items []pagedJSONRow + require.NoError(t, json.Unmarshal([]byte(got), &items)) + assert.Len(t, items, 7) +} + +func TestPagedJSONEnterDrainsIterator(t *testing.T) { + got, _, err := runPagedJSON(t, makePagedJSONRows(20), 5, []byte{'\r'}) + require.NoError(t, err) + var items []pagedJSONRow + require.NoError(t, json.Unmarshal([]byte(got), &items)) + assert.Len(t, items, 20) +} + +func TestPagedJSONQuitEndsEarlyButKeepsValidArray(t *testing.T) { + got, _, err := runPagedJSON(t, makePagedJSONRows(20), 5, []byte{'q'}) + require.NoError(t, err) + var items []pagedJSONRow + require.NoError(t, json.Unmarshal([]byte(got), &items), "output must still be valid JSON after early quit: %q", got) + assert.Len(t, items, 5, "only the first page should have rendered") +} + +func TestPagedJSONEscQuitsWithValidJSON(t *testing.T) { + got, _, err := runPagedJSON(t, makePagedJSONRows(20), 5, []byte{pagerKeyEscape}) + require.NoError(t, err) + var items []pagedJSONRow + require.NoError(t, json.Unmarshal([]byte(got), &items)) + assert.Len(t, items, 5) +} + +func TestPagedJSONCtrlCInterruptsDrain(t *testing.T) { + // ENTER drains, then a buffered Ctrl+C interrupts after the next + // flush: first page (5) + second page (5) = 10 items; third page + // skipped due to quit signal. + got, _, err := runPagedJSON(t, makePagedJSONRows(20), 5, []byte{'\r', pagerKeyCtrlC}) + require.NoError(t, err) + var items []pagedJSONRow + require.NoError(t, json.Unmarshal([]byte(got), &items)) + assert.Len(t, items, 10) +} + +func TestPagedJSONEmptyIteratorProducesValidEmptyArray(t *testing.T) { + got, _, err := runPagedJSON(t, nil, 10, nil) + require.NoError(t, err) + assert.JSONEq(t, `[]`, got) +} + +func TestPagedJSONRespectsLimit(t *testing.T) { + var out, prompts bytes.Buffer + iter := listing.Iterator[pagedJSONRow](&pagedJSONIterator{data: makePagedJSONRows(100)}) + ctx := WithLimit(t.Context(), 7) + err := renderIteratorPagedJSONCore(ctx, iter, &out, &prompts, makeKeyChan('\r'), 5) + require.NoError(t, err) + var items []pagedJSONRow + require.NoError(t, json.Unmarshal(out.Bytes(), &items)) + assert.Len(t, items, 7) +} + +func TestPagedJSONFetchErrorPropagatesButStillValidJSON(t *testing.T) { + var out, prompts bytes.Buffer + iter := listing.Iterator[pagedJSONRow](&pagedJSONIterator{data: makePagedJSONRows(10), err: errors.New("boom")}) + err := renderIteratorPagedJSONCore(t.Context(), iter, &out, &prompts, makeKeyChan(), 5) + require.Error(t, err) + assert.Contains(t, err.Error(), "boom") + // Partial buffer should still produce valid JSON (empty array). + assert.JSONEq(t, `[]`, out.String()) +} + +func TestPagedJSONWritesPromptToPromptsStream(t *testing.T) { + _, promptsOut, err := runPagedJSON(t, makePagedJSONRows(20), 5, []byte{'q'}) + require.NoError(t, err) + assert.Contains(t, promptsOut, pagerPromptText) +} diff --git a/libs/cmdio/pager.go b/libs/cmdio/pager.go new file mode 100644 index 0000000000..400aee8c6a --- /dev/null +++ b/libs/cmdio/pager.go @@ -0,0 +1,133 @@ +package cmdio + +import ( + "context" + "fmt" + "io" + "os" + + "golang.org/x/term" +) + +// pagerPageSize is the number of items rendered between prompts. +const pagerPageSize = 50 + +// pagerPromptText is shown on stderr between pages. +const pagerPromptText = "[space] more [enter] all [q|esc] quit" + +// pagerClearLine is the ANSI sequence to return to column 0 and erase the +// current line. Used to remove the prompt before writing the next page. +const pagerClearLine = "\r\x1b[K" + +// Key codes we care about when reading single bytes from stdin in raw mode. +const ( + pagerKeyEscape = 0x1b + pagerKeyCtrlC = 0x03 +) + +// startRawStdinKeyReader puts stdin into raw mode and spawns a goroutine +// that publishes each keystroke as a byte on the returned channel. The +// returned restore function must be called (typically via defer) to put +// the terminal back in its original mode; it is safe to call even if +// MakeRaw failed (it's a no-op). +// +// The goroutine exits when stdin returns an error (e.g. EOF on process +// shutdown) or when ctx is cancelled, at which point the channel is +// closed. Leaking the goroutine before that is acceptable because the +// pager is only invoked by short-lived CLI commands: the process exits +// shortly after the caller returns. +// +// Note: term.MakeRaw also clears the TTY's OPOST flag on most Unixes. +// With OPOST off, outbound '\n' is not translated to '\r\n', so callers +// that write newlines while raw mode is active should wrap their output +// stream in crlfWriter to avoid staircase output. +func startRawStdinKeyReader(ctx context.Context) (<-chan byte, func(), error) { + fd := int(os.Stdin.Fd()) + oldState, err := term.MakeRaw(fd) + if err != nil { + return nil, func() {}, fmt.Errorf("failed to enter raw mode on stdin: %w", err) + } + restore := func() { _ = term.Restore(fd, oldState) } + + ch := make(chan byte, 16) + go func() { + defer close(ch) + buf := make([]byte, 1) + for { + n, err := os.Stdin.Read(buf) + if err != nil || n == 0 { + return + } + select { + case ch <- buf[0]: + case <-ctx.Done(): + return + } + } + }() + return ch, restore, nil +} + +// pagerNextKey blocks until a key arrives, the key channel closes, or the +// context is cancelled. Returns ok=false on close or cancellation. +func pagerNextKey(ctx context.Context, keys <-chan byte) (byte, bool) { + select { + case k, ok := <-keys: + return k, ok + case <-ctx.Done(): + return 0, false + } +} + +// pagerShouldQuit drains any buffered keys non-blockingly and returns true +// if one of q/Q/esc/Ctrl+C was pressed. Other keys are consumed and +// dropped. A closed channel means stdin ran out (EOF) — that's not a +// quit signal; the caller should keep draining. +func pagerShouldQuit(keys <-chan byte) bool { + for { + select { + case k, ok := <-keys: + if !ok { + return false + } + if k == 'q' || k == 'Q' || k == pagerKeyEscape || k == pagerKeyCtrlC { + return true + } + default: + return false + } + } +} + +// crlfWriter translates outbound '\n' bytes into '\r\n' so output written +// while the TTY is in raw mode (OPOST cleared) still starts at column 0. +// io.Writer semantics are preserved: the returned byte count is the +// number of bytes from p that were consumed, not the (possibly larger) +// number of bytes written to the underlying writer. +type crlfWriter struct { + w io.Writer +} + +func (c crlfWriter) Write(p []byte) (int, error) { + start := 0 + for i, b := range p { + if b != '\n' { + continue + } + if i > start { + if _, err := c.w.Write(p[start:i]); err != nil { + return start, err + } + } + if _, err := c.w.Write([]byte{'\r', '\n'}); err != nil { + return i, err + } + start = i + 1 + } + if start < len(p) { + if _, err := c.w.Write(p[start:]); err != nil { + return start, err + } + } + return len(p), nil +} diff --git a/libs/cmdio/pager_test.go b/libs/cmdio/pager_test.go new file mode 100644 index 0000000000..d2e6f00463 --- /dev/null +++ b/libs/cmdio/pager_test.go @@ -0,0 +1,72 @@ +package cmdio + +import ( + "bytes" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestCrlfWriterTranslatesNewlines(t *testing.T) { + tests := []struct { + name string + in string + want string + }{ + {"no newlines", "abc", "abc"}, + {"single newline mid", "a\nb", "a\r\nb"}, + {"newline at end", "abc\n", "abc\r\n"}, + {"newline at start", "\nabc", "\r\nabc"}, + {"consecutive newlines", "\n\n", "\r\n\r\n"}, + {"multiple lines", "one\ntwo\nthree\n", "one\r\ntwo\r\nthree\r\n"}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + var buf bytes.Buffer + w := crlfWriter{w: &buf} + n, err := w.Write([]byte(tt.in)) + require.NoError(t, err) + assert.Equal(t, len(tt.in), n, "Write must return the input byte count") + assert.Equal(t, tt.want, buf.String()) + }) + } +} + +func TestPagerShouldQuitDrainsNonQuitKeys(t *testing.T) { + ch := make(chan byte, 4) + ch <- ' ' + ch <- 'x' + ch <- 'y' + assert.False(t, pagerShouldQuit(ch), "non-quit keys must return false") + assert.Empty(t, ch, "non-quit keys must be drained from the channel") +} + +func TestPagerShouldQuitReturnsTrueForQuitKeys(t *testing.T) { + for _, k := range []byte{'q', 'Q', pagerKeyEscape, pagerKeyCtrlC} { + ch := make(chan byte, 1) + ch <- k + assert.Truef(t, pagerShouldQuit(ch), "key %q must trigger quit", k) + } +} + +func TestPagerShouldQuitClosedChannelKeepsDraining(t *testing.T) { + ch := make(chan byte) + close(ch) + assert.False(t, pagerShouldQuit(ch), "closed channel (stdin EOF) must not force quit") +} + +func TestPagerNextKeyReturnsFalseOnClosedChannel(t *testing.T) { + ch := make(chan byte) + close(ch) + _, ok := pagerNextKey(t.Context(), ch) + assert.False(t, ok) +} + +func TestPagerNextKeyReturnsKey(t *testing.T) { + ch := make(chan byte, 1) + ch <- ' ' + k, ok := pagerNextKey(t.Context(), ch) + assert.True(t, ok) + assert.Equal(t, byte(' '), k) +} diff --git a/libs/cmdio/render.go b/libs/cmdio/render.go index 733dd53fa7..d8152260de 100644 --- a/libs/cmdio/render.go +++ b/libs/cmdio/render.go @@ -271,8 +271,20 @@ func Render(ctx context.Context, v any) error { return renderWithTemplate(ctx, newRenderer(v), c.outputFormat, c.out, c.headerTemplate, c.template) } +// RenderIterator renders the items produced by i. When the terminal is +// fully interactive (stdin + stdout + stderr all TTYs) and the caller +// hasn't supplied a row template, we page through the iterator's JSON +// representation instead of dumping the full array at once: 50 items +// at a time, with a prompt on stderr between pages asking whether to +// continue. Piped output and explicit --output json against a pipe +// keep the existing non-paged behavior. Commands that do supply a row +// template also keep the existing non-paged behavior — a follow-up +// change will add paging for them. func RenderIterator[T any](ctx context.Context, i listing.Iterator[T]) error { c := fromContext(ctx) + if c.template == "" && c.capabilities.SupportsPager() && (c.outputFormat == flags.OutputJSON || c.outputFormat == flags.OutputText) { + return renderIteratorPagedJSON(ctx, i, c.out) + } return renderWithTemplate(ctx, newIteratorRenderer(i), c.outputFormat, c.out, c.headerTemplate, c.template) }