Skip to content

Commit d0c1262

Browse files
ysyneuysyneu
authored
Merge pull request #168 from flashcatcloud/fix/insight-export-truncation
fix(insight): fail loudly when incident-export CSV is truncated
2 parents 592cc71 + 9623dd4 commit d0c1262

2 files changed

Lines changed: 412 additions & 0 deletions

File tree

‎internal/cli/insight.go‎

Lines changed: 230 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,10 @@
11
package cli
22

33
import (
4+
"bytes"
5+
"encoding/csv"
46
"fmt"
7+
"io"
58

69
"github.com/flashcatcloud/go-flashduty"
710
"github.com/spf13/cobra"
@@ -16,8 +19,12 @@ func newInsightCmd() *cobra.Command {
1619
// (richer flag set: severities, *_ids, fields, aggregate-unit, …; relative
1720
// time on --start-time/--end-time). Their human tables are preserved via the
1821
// DimensionInsightItem / ResponderInsightItem entries in display_columns.go.
22+
// incident-export is curated (below) so it can verify the exported CSV
23+
// against the incident-list total; the generated twin is dropped by
24+
// genAddLeaf.
1925
cmd.AddCommand(newInsightTopAlertsCmd())
2026
cmd.AddCommand(newInsightIncidentsCmd())
27+
cmd.AddCommand(newInsightIncidentExportCmd())
2128
return cmd
2229
}
2330

@@ -143,3 +150,226 @@ func newInsightIncidentsCmd() *cobra.Command {
143150

144151
return cmd
145152
}
153+
154+
// newInsightIncidentExportCmd exports the filtered incident analytics list as
155+
// CSV. It replaces the generated incident-export command (same flags) because
156+
// the export endpoint answers a one-shot CSV with no pagination cursor and
157+
// caps its row count server-side without saying so in the payload: a wide
158+
// time window comes back silently truncated. Since the endpoint cannot be
159+
// paged, the command verifies completeness instead — it counts the CSV data
160+
// rows and compares against the incident-list total for the same filter (the
161+
// authoritative count the analytics dashboard pages over). A short export is
162+
// still written to stdout, but the command exits non-zero stating written vs
163+
// total, so a truncated CSV can never pass for a complete one.
164+
func newInsightIncidentExportCmd() *cobra.Command {
165+
var dataJSON string
166+
var fAsc bool
167+
var fChannelIDs []int
168+
var fDescriptionHTMLToText bool
169+
var fEndTime int64
170+
var fExportFields []string
171+
var fIncidentIDs []string
172+
var fIncludeEverMuted bool
173+
var fIsMyTeam bool
174+
var fOrderby string
175+
var fQuery string
176+
var fResponderIDs []int
177+
var fSecondsToAckFrom int64
178+
var fSecondsToAckTo int64
179+
var fSecondsToCloseFrom int64
180+
var fSecondsToCloseTo int64
181+
var fSeverities []string
182+
var fStartTime int64
183+
var fTeamIDs []int
184+
var fTimeZone string
185+
cmd := &cobra.Command{
186+
Use: "incident-export",
187+
Short: "Export insight incidents",
188+
Long: `Export insight incidents.
189+
190+
Export the filtered incident analytics list as a CSV file (to stdout; redirect with '> file.csv'). CSV headers and formatted values use the request locale, falling back to the member locale and then the account locale. --start-time/--end-time take Unix seconds.
191+
192+
The export endpoint returns a one-shot CSV and caps its row count server-side, so a wide window may come back truncated. After writing, this command compares the CSV data-row count against the incident-list total for the same filter: on a shortfall the partial CSV is still written, a 'rows=N' line (the actual written data-row count) goes to stderr, and the command exits non-zero stating written vs total — narrow the window and retry.
193+
194+
API: POST /insight/incident/export (insightIncidentExport)
195+
196+
Request fields:
197+
--asc bool
198+
--channel-ids []int
199+
--description-html-to-text bool
200+
--end-time int
201+
--export-fields []string
202+
--incident-ids []string
203+
--include-ever-muted bool
204+
--is-my-team bool
205+
--orderby string
206+
--query string
207+
--responder-ids []int
208+
--seconds-to-ack-from int
209+
--seconds-to-ack-to int
210+
--seconds-to-close-from int
211+
--seconds-to-close-to int
212+
--severities []string
213+
--start-time int
214+
--team-ids []int
215+
--time-zone string
216+
fields (JSON, via --data)
217+
labels (JSON, via --data)`,
218+
Example: ` flashduty insight incident-export --start-time 1712000000 --end-time 1712604800 > incidents.csv`,
219+
RunE: func(cmd *cobra.Command, args []string) error {
220+
return runCommand(cmd, args, func(ctx *RunContext) error {
221+
body, err := genAssembleBody(dataJSON, func(body map[string]any) error {
222+
if cmd.Flags().Changed("asc") {
223+
body["asc"] = fAsc
224+
}
225+
if cmd.Flags().Changed("channel-ids") {
226+
body["channel_ids"] = fChannelIDs
227+
}
228+
if cmd.Flags().Changed("description-html-to-text") {
229+
body["description_html_to_text"] = fDescriptionHTMLToText
230+
}
231+
if cmd.Flags().Changed("end-time") {
232+
body["end_time"] = fEndTime
233+
}
234+
if cmd.Flags().Changed("export-fields") {
235+
body["export_fields"] = fExportFields
236+
}
237+
if cmd.Flags().Changed("incident-ids") {
238+
body["incident_ids"] = fIncidentIDs
239+
}
240+
if cmd.Flags().Changed("include-ever-muted") {
241+
body["include_ever_muted"] = fIncludeEverMuted
242+
}
243+
if cmd.Flags().Changed("is-my-team") {
244+
body["is_my_team"] = fIsMyTeam
245+
}
246+
if cmd.Flags().Changed("orderby") {
247+
body["orderby"] = fOrderby
248+
}
249+
if cmd.Flags().Changed("query") {
250+
body["query"] = fQuery
251+
}
252+
if cmd.Flags().Changed("responder-ids") {
253+
body["responder_ids"] = fResponderIDs
254+
}
255+
if cmd.Flags().Changed("seconds-to-ack-from") {
256+
body["seconds_to_ack_from"] = fSecondsToAckFrom
257+
}
258+
if cmd.Flags().Changed("seconds-to-ack-to") {
259+
body["seconds_to_ack_to"] = fSecondsToAckTo
260+
}
261+
if cmd.Flags().Changed("seconds-to-close-from") {
262+
body["seconds_to_close_from"] = fSecondsToCloseFrom
263+
}
264+
if cmd.Flags().Changed("seconds-to-close-to") {
265+
body["seconds_to_close_to"] = fSecondsToCloseTo
266+
}
267+
if cmd.Flags().Changed("severities") {
268+
body["severities"] = fSeverities
269+
}
270+
if cmd.Flags().Changed("start-time") {
271+
body["start_time"] = fStartTime
272+
}
273+
if cmd.Flags().Changed("team-ids") {
274+
body["team_ids"] = fTeamIDs
275+
}
276+
if cmd.Flags().Changed("time-zone") {
277+
body["time_zone"] = fTimeZone
278+
}
279+
return nil
280+
})
281+
if err != nil {
282+
return err
283+
}
284+
req := new(flashduty.InsightFilter)
285+
if err := genBindBody(body, req); err != nil {
286+
return err
287+
}
288+
resp, err := ctx.Client.Analytics.IncidentExport(cmdContext(ctx.Cmd), req)
289+
if err != nil {
290+
return err
291+
}
292+
if resp == nil || len(resp.Raw) == 0 {
293+
ctx.WriteResult("OK: POST /insight/incident/export")
294+
return nil
295+
}
296+
rows, err := countCSVDataRows(resp.Raw)
297+
if err != nil {
298+
return fmt.Errorf("insight incident-export: cannot parse the exported CSV: %w", err)
299+
}
300+
if err := ctx.WriteRaw(resp.Raw); err != nil {
301+
return err
302+
}
303+
_, _ = fmt.Fprintf(ctx.Cmd.ErrOrStderr(), "rows=%d\n", rows)
304+
total, err := insightIncidentTotal(ctx, body)
305+
if err != nil {
306+
return fmt.Errorf("insight incident-export: wrote %d rows but cannot verify completeness against /insight/incident/list: %w", rows, err)
307+
}
308+
if int64(rows) < total {
309+
return fmt.Errorf("insight incident-export: incomplete export — wrote %d of %d incidents matching the filter; narrow the time window (--start-time/--end-time) and retry", rows, total)
310+
}
311+
return nil
312+
})
313+
},
314+
}
315+
cmd.Flags().BoolVar(&fAsc, "asc", false, "Request field ")
316+
cmd.Flags().IntSliceVar(&fChannelIDs, "channel-ids", nil, "Request field ")
317+
cmd.Flags().BoolVar(&fDescriptionHTMLToText, "description-html-to-text", false, "Request field ")
318+
cmd.Flags().Int64Var(&fEndTime, "end-time", 0, "Request field ")
319+
cmd.Flags().StringSliceVar(&fExportFields, "export-fields", nil, "Request field ")
320+
cmd.Flags().StringSliceVar(&fIncidentIDs, "incident-ids", nil, "Request field ")
321+
cmd.Flags().BoolVar(&fIncludeEverMuted, "include-ever-muted", false, "Request field ")
322+
cmd.Flags().BoolVar(&fIsMyTeam, "is-my-team", false, "Request field ")
323+
cmd.Flags().StringVar(&fOrderby, "orderby", "", "Request field ")
324+
cmd.Flags().StringVar(&fQuery, "query", "", "Request field ")
325+
cmd.Flags().IntSliceVar(&fResponderIDs, "responder-ids", nil, "Request field ")
326+
cmd.Flags().Int64Var(&fSecondsToAckFrom, "seconds-to-ack-from", 0, "Request field ")
327+
cmd.Flags().Int64Var(&fSecondsToAckTo, "seconds-to-ack-to", 0, "Request field ")
328+
cmd.Flags().Int64Var(&fSecondsToCloseFrom, "seconds-to-close-from", 0, "Request field ")
329+
cmd.Flags().Int64Var(&fSecondsToCloseTo, "seconds-to-close-to", 0, "Request field ")
330+
cmd.Flags().StringSliceVar(&fSeverities, "severities", nil, "Request field ")
331+
cmd.Flags().Int64Var(&fStartTime, "start-time", 0, "Request field ")
332+
cmd.Flags().IntSliceVar(&fTeamIDs, "team-ids", nil, "Request field ")
333+
cmd.Flags().StringVar(&fTimeZone, "time-zone", "", "Request field ")
334+
cmd.Flags().StringVar(&dataJSON, "data", "", "Full request body as JSON; positional arguments and typed flags override its fields. Accepts inline JSON, or - to read stdin.")
335+
return cmd
336+
}
337+
338+
// countCSVDataRows counts the data records in an exported CSV body — every
339+
// record except the header row. encoding/csv handles quoted fields with
340+
// embedded newlines, which a naive line count would over-count.
341+
func countCSVDataRows(raw []byte) (int, error) {
342+
r := csv.NewReader(bytes.NewReader(raw))
343+
r.FieldsPerRecord = -1
344+
n := 0
345+
for {
346+
if _, err := r.Read(); err == io.EOF {
347+
break
348+
} else if err != nil {
349+
return 0, err
350+
}
351+
n++
352+
}
353+
if n == 0 {
354+
return 0, nil
355+
}
356+
return n - 1, nil
357+
}
358+
359+
// insightIncidentTotal returns the number of incidents matching the export
360+
// filter, per the incident-list endpoint — the authoritative total an export
361+
// is verified against. Only the total is needed, so a single 1-item page is
362+
// fetched.
363+
func insightIncidentTotal(ctx *RunContext, body map[string]any) (int64, error) {
364+
req := new(flashduty.InsightIncidentListRequest)
365+
if err := genBindBody(body, req); err != nil {
366+
return 0, err
367+
}
368+
req.Page = 1
369+
req.Limit = 1
370+
out, _, err := ctx.Client.Analytics.IncidentList(cmdContext(ctx.Cmd), req)
371+
if err != nil {
372+
return 0, err
373+
}
374+
return out.Total, nil
375+
}

0 commit comments

Comments
 (0)