From 3555ded2791e5d35b5aef3568bc80506101035bc Mon Sep 17 00:00:00 2001 From: Noah Treuhaft Date: Fri, 21 Aug 2026 15:26:30 -0400 Subject: [PATCH] remove quiet() and special handling for error("quiet") --- book/src/SUMMARY.md | 1 - book/src/super-sql/functions/errors/quiet.md | 57 ---------- .../super-sql/functions/generics/coalesce.md | 2 +- book/src/super-sql/functions/types/nameof.md | 15 --- book/src/super-sql/operators/cut.md | 15 --- book/src/super-sql/types/error.md | 26 +---- book/src/tutorials/jq.md | 11 -- compiler/parser/ztests/unnest-expr.yaml | 4 +- compiler/rungen/vop.go | 2 - compiler/semantic/expr.go | 11 +- compiler/ztests/implied-quiet-assignment.yaml | 10 -- complex.go | 5 - context.go | 6 - runtime/sam/expr/agg.go | 5 +- runtime/sam/expr/dequiet.go | 66 ----------- runtime/sam/expr/dropper.go | 3 - runtime/sam/expr/function/function.go | 2 - runtime/sam/expr/function/has.go | 2 +- runtime/sam/expr/function/types.go | 12 -- runtime/vam/expr/dropper.go | 4 +- runtime/vam/expr/function/errors.go | 43 -------- runtime/vam/expr/function/function.go | 2 - runtime/vam/expr/function/has.go | 12 +- runtime/vam/expr/quiet.go | 103 ------------------ runtime/vam/op/aggregate/aggregate.go | 67 +----------- runtime/vam/op/aggregate/aggtable.go | 17 --- runtime/vam/op/aggregate/scalar.go | 4 - runtime/vam/op/values.go | 21 +--- runtime/ztests/expr/array-expr.yaml | 4 +- runtime/ztests/expr/f-string.yaml | 3 +- runtime/ztests/expr/function/coalesce.yaml | 6 +- runtime/ztests/expr/function/concat.yaml | 8 +- runtime/ztests/expr/function/quiet.yaml | 27 ----- runtime/ztests/expr/optional-fields.yaml | 14 --- runtime/ztests/op/aggregate/any.yaml | 6 +- runtime/ztests/op/aggregate/array_agg.yaml | 1 - runtime/ztests/op/aggregate/blend.yaml | 12 -- runtime/ztests/op/aggregate/collect.yaml | 1 - .../op/aggregate/container-partials.yaml | 7 +- runtime/ztests/op/aggregate/count.yaml | 3 +- runtime/ztests/op/aggregate/dcount.yaml | 1 - runtime/ztests/op/aggregate/distinct.yaml | 1 - runtime/ztests/op/aggregate/quiet-key.yaml | 31 ------ runtime/ztests/op/aggregate/reducers.yaml | 1 - runtime/ztests/op/cut-quiet.yaml | 64 ----------- runtime/ztests/op/drop-foo-both.yaml | 6 + runtime/ztests/op/put-quiet.yaml | 9 -- sio/bsupio/ztests/zctx-named-reset-2.yaml | 2 +- spq_test.go | 2 +- value.go | 7 -- 50 files changed, 43 insertions(+), 701 deletions(-) delete mode 100644 book/src/super-sql/functions/errors/quiet.md delete mode 100644 compiler/ztests/implied-quiet-assignment.yaml delete mode 100644 runtime/sam/expr/dequiet.go delete mode 100644 runtime/vam/expr/function/errors.go delete mode 100644 runtime/vam/expr/quiet.go delete mode 100644 runtime/ztests/expr/function/quiet.yaml delete mode 100644 runtime/ztests/op/aggregate/quiet-key.yaml delete mode 100644 runtime/ztests/op/cut-quiet.yaml delete mode 100644 runtime/ztests/op/put-quiet.yaml diff --git a/book/src/SUMMARY.md b/book/src/SUMMARY.md index dd2c6eb7d0..366b6b8c98 100644 --- a/book/src/SUMMARY.md +++ b/book/src/SUMMARY.md @@ -106,7 +106,6 @@ - [has_error](super-sql/functions/errors/has_error.md) - [is_error](super-sql/functions/errors/is_error.md) - [missing](super-sql/functions/errors/missing.md) - - [quiet](super-sql/functions/errors/quiet.md) - [Generics](super-sql/functions/generics/intro.md) - [coalesce](super-sql/functions/generics/coalesce.md) - [compare](super-sql/functions/generics/compare.md) diff --git a/book/src/super-sql/functions/errors/quiet.md b/book/src/super-sql/functions/errors/quiet.md deleted file mode 100644 index cdceed248c..0000000000 --- a/book/src/super-sql/functions/errors/quiet.md +++ /dev/null @@ -1,57 +0,0 @@ -# quiet - -quiet "missing" errors - -## Synopsis - -``` -quiet(val: any) -> any -``` - -## Description - -The `quiet` function returns its argument `val` unless `val` is -`error("missing")`, in which case it returns `error("quiet")`. -Various operators and functions treat quiet errors differently than -missing errors, in particular, dropping them instead of propagating them. -Quiet errors are ignored by operators `aggregate`, `cut`, and `values`. - -## Examples - ---- - -_A quiet error in `values` produces no output_ - -```mdtest-spq -# spq -values quiet(this) -# input -error("missing") -# expected output -``` - ---- - -_Without quiet, values produces the missing error_ - -```mdtest-spq -# spq -values this -# input -error("missing") -# expected output -error("missing") -``` - ---- - -_The `cut` operator drops quiet errors but retains missing errors_ - -```mdtest-spq -# spq -cut b:=x+1,c:=quiet(x+1),d:=quiet(a+1) -# input -{a:1} -# expected output -{b:error("missing"),d:2} -``` diff --git a/book/src/super-sql/functions/generics/coalesce.md b/book/src/super-sql/functions/generics/coalesce.md index e1b0f631aa..d677384a88 100644 --- a/book/src/super-sql/functions/generics/coalesce.md +++ b/book/src/super-sql/functions/generics/coalesce.md @@ -32,7 +32,7 @@ values coalesce(null, error("missing"), error({x:"foo"}), this) # spq values coalesce(null, error({x:"foo"}), this) # input -error("quiet") +error("bar") # expected output null ``` diff --git a/book/src/super-sql/functions/types/nameof.md b/book/src/super-sql/functions/types/nameof.md index aaa0fd6a53..ce2b3111d9 100644 --- a/book/src/super-sql/functions/types/nameof.md +++ b/book/src/super-sql/functions/types/nameof.md @@ -30,18 +30,3 @@ type port=int16 "port" error("missing") ``` - ---- - -_The missing value can be ignored with quiet_ - -```mdtest-spq -# spq -values quiet(nameof(this)) -# input -type port=int16 -80::port -80 -# expected output -"port" -``` diff --git a/book/src/super-sql/operators/cut.md b/book/src/super-sql/operators/cut.md index fefcec23a5..5c314001cd 100644 --- a/book/src/super-sql/operators/cut.md +++ b/book/src/super-sql/operators/cut.md @@ -31,9 +31,6 @@ simple field references, the _cut_ operation resembles the Unix shell command, e ``` ... | cut a,c | ... ``` -If an expression results in `error("quiet")`, the corresponding field is omitted -from the output. This allows you to wrap expressions in a `quiet()` function -to filter out missing errors. If an input value to cut is not a record, then cut still operates as defined resulting in `error("missing")` for expressions that reference fields of `this`. @@ -74,18 +71,6 @@ cut a,d --- -_The missing fields can be ignored with quiet_ -```mdtest-spq -# spq -cut a:=quiet(a),d:=quiet(d) -# input -{a:1,b:2,c:3} -# expected output -{a:1} -``` - ---- - _Non-record values generate missing errors for fields not present in a non-record `this`_ ```mdtest-spq {data-layout="stacked"} # spq diff --git a/book/src/super-sql/types/error.md b/book/src/super-sql/types/error.md index 77dc87bf09..3643274aed 100644 --- a/book/src/super-sql/types/error.md +++ b/book/src/super-sql/types/error.md @@ -92,7 +92,7 @@ but having a first-class data type to manage them all while allowing them to peacefully coexist with valid production data is a novel and useful approach that SuperSQL enables. -## Missing and Quiet +## Missing SuperDB's heterogeneous data model allows for queries that operate over different types of data whose structure and type @@ -146,16 +146,6 @@ values x error("missing") ``` -Sometimes you want missing errors to show up and sometimes you don't. -The [`quiet`](../functions/errors/quiet.md) function transforms missing errors into -"quiet errors". A quiet error is the value `error("quiet")` and is ignored -by most operators, in particular, -[`values`](../operators/values.md), e.g., -``` -values error("quiet") -``` -produces no output. - ## Examples --- @@ -205,20 +195,6 @@ typeof(1/this) --- -_The `quiet` function suppresses error values_ - -```mdtest-spq -# spq -values quiet(x) -# input -{x:1} -{y:2} -# expected output -1 -``` - ---- - _Coalesce replaces `error("missing")` values with a default value_ ```mdtest-spq diff --git a/book/src/tutorials/jq.md b/book/src/tutorials/jq.md index 7c219b8676..0cba6888b8 100644 --- a/book/src/tutorials/jq.md +++ b/book/src/tutorials/jq.md @@ -378,17 +378,6 @@ produces {val:1} {val:error("missing")} ``` -Sometimes you expect "missing" errors to occur sporadically and just want -to ignore them, which can you easily do with the -[quiet](../super-sql/functions/errors/quiet.md) function, e.g., -```mdtest-command -echo '{s:"foo", val:1}{s:"bar"}' | super -s -c 'cut quiet(val)' - -``` -produces -```mdtest-output -{val:1} -{} -``` ### Union Types diff --git a/compiler/parser/ztests/unnest-expr.yaml b/compiler/parser/ztests/unnest-expr.yaml index 4279bc4588..59aa84d0e6 100644 --- a/compiler/parser/ztests/unnest-expr.yaml +++ b/compiler/parser/ztests/unnest-expr.yaml @@ -6,7 +6,7 @@ script: | echo === expression super compile -C 'values (unnest a | ?b)' echo === function - super compile -C 'quiet((unnest a | ?b))' + super compile -C 'typeof((unnest a | ?b))' echo === grep super compile -C 'grep("regexp", (unnest a | ?b))' @@ -30,7 +30,7 @@ outputs: | search b ) === function - quiet(( + typeof(( unnest a | search b )) diff --git a/compiler/rungen/vop.go b/compiler/rungen/vop.go index c57aa6d2a7..cd008e6f45 100644 --- a/compiler/rungen/vop.go +++ b/compiler/rungen/vop.go @@ -205,7 +205,6 @@ func (b *Builder) compileVamLeaf(o dag.Op, parent vio.Puller) (vio.Puller, error if err != nil { return nil, err } - e = vamexpr.NewDequiet(b.sctx(), e) return vamop.NewValues(b.sctx(), parent, []vamexpr.Evaluator{e}), nil case *dag.DebugOp: e, err := b.compileVamExpr(o.Expr) @@ -268,7 +267,6 @@ func (b *Builder) compileVamLeaf(o dag.Op, parent vio.Puller) (vio.Puller, error if err != nil { return nil, err } - e = vamexpr.NewDequiet(b.sctx(), e) putter := vamexpr.NewPutter(b.sctx(), e) return vamop.NewValues(b.sctx(), parent, []vamexpr.Evaluator{putter}), nil case *dag.RenameOp: diff --git a/compiler/semantic/expr.go b/compiler/semantic/expr.go index 6550299964..8b98427bee 100644 --- a/compiler/semantic/expr.go +++ b/compiler/semantic/expr.go @@ -1008,14 +1008,11 @@ func deriveNameFromExpr(e ast.Expr) string { case *ast.AggFuncExpr: return e.Name case *ast.CallExpr: - var name string - if f, ok := e.Func.(*ast.FuncNameExpr); ok { - name = f.Name - } - if strings.ToLower(name) == "quiet" && len(e.Args) > 0 { - return deriveNameFromExpr(e.Args[0]) + f, ok := e.Func.(*ast.FuncNameExpr) + if !ok { + return "" } - return name + return f.Name case *ast.BinaryExpr: if name, ok := dottedName(e); ok { return name diff --git a/compiler/ztests/implied-quiet-assignment.yaml b/compiler/ztests/implied-quiet-assignment.yaml deleted file mode 100644 index 8390f20c9a..0000000000 --- a/compiler/ztests/implied-quiet-assignment.yaml +++ /dev/null @@ -1,10 +0,0 @@ -skip: needs dequiet - -spq: cut quiet(a.b) - -input: | - {a:1} - {a:{b:1}} - -output: | - {b:1} diff --git a/complex.go b/complex.go index 9a6a9a2fc4..be5114cb15 100644 --- a/complex.go +++ b/complex.go @@ -39,7 +39,6 @@ var ErrMissing = errors.New("missing") // each operator has clearly defined semantics with respect to the Missing value. // For example, "true AND MISSING" is MISSING. var Missing = scode.Bytes("missing") -var Quiet = scode.Bytes("quiet") type TypeError struct { id int @@ -62,10 +61,6 @@ func (t *TypeError) IsMissing(zv scode.Bytes) bool { return t.Type == TypeString && bytes.Equal(zv, Missing) } -func (t *TypeError) IsQuiet(zv scode.Bytes) bool { - return t.Type == TypeString && bytes.Equal(zv, Quiet) -} - type TypeEnum struct { id int Symbols []string diff --git a/context.go b/context.go index 9d4abcd43b..1c47e2794a 100644 --- a/context.go +++ b/context.go @@ -650,12 +650,6 @@ func (c *Context) Missing() Value { return NewValue(c.StringTypeError(), Missing) } -func (c *Context) Quiet() Value { - return NewValue(c.StringTypeError(), Quiet) -} - -// batch/allocator should handle these? - func (c *Context) NewErrorf(format string, args ...any) Value { return NewValue(c.StringTypeError(), fmt.Appendf(nil, format, args...)) } diff --git a/runtime/sam/expr/agg.go b/runtime/sam/expr/agg.go index 0f1f5733f9..b53e8a30d5 100644 --- a/runtime/sam/expr/agg.go +++ b/runtime/sam/expr/agg.go @@ -39,8 +39,5 @@ func (a *Aggregator) Apply(sctx *super.Context, f agg.Function, this super.Value return } } - v := a.expr.Eval(this) - if !v.IsQuiet() { - f.Consume(v) - } + f.Consume(a.expr.Eval(this)) } diff --git a/runtime/sam/expr/dequiet.go b/runtime/sam/expr/dequiet.go deleted file mode 100644 index 6708a6e8bc..0000000000 --- a/runtime/sam/expr/dequiet.go +++ /dev/null @@ -1,66 +0,0 @@ -package expr - -import ( - "github.com/brimdata/super" - "github.com/brimdata/super/scode" -) - -type Dequiet struct { - sctx *super.Context - expr Evaluator - builder scode.Builder -} - -func NewDequiet(sctx *super.Context, expr Evaluator) Evaluator { - return &Dequiet{sctx: sctx, expr: expr} -} - -func (d *Dequiet) Eval(this super.Value) super.Value { - val := d.expr.Eval(this) - if val.Type().Kind() == super.RecordKind { - d.builder.Reset() - typ := d.rec(&d.builder, val.Type(), val.Bytes()) - return super.NewValue(typ, d.builder.Bytes().Body()) - } - return val -} - -func (d *Dequiet) rec(builder *scode.Builder, typ super.Type, b scode.Bytes) super.Type { - if b == nil { - builder.Append(nil) - return typ - } - rtyp := super.TypeRecordOf(typ) - if rtyp == nil { - panic(typ) - } - var changed bool - builder.BeginContainer() - var fields []super.Field - it := b.Iter() - for _, f := range rtyp.Fields { - ftyp := d.dequiet(builder, f.Type, it.Next()) - if ftyp == nil { - changed = true - continue - } - fields = append(fields, super.NewField(f.Name, ftyp)) - } - if changed { - rtyp = d.sctx.MustLookupTypeRecord(fields) - typ = rtyp - } - builder.EndContainer() - return typ -} - -func (d *Dequiet) dequiet(builder *scode.Builder, typ super.Type, b scode.Bytes) super.Type { - if typ.Kind() == super.RecordKind { - return d.rec(builder, typ, b) - } - if errtyp, ok := typ.(*super.TypeError); ok && errtyp.IsQuiet(b) { - return nil - } - builder.Append(b) - return typ -} diff --git a/runtime/sam/expr/dropper.go b/runtime/sam/expr/dropper.go index aaa0b6b288..f6f4d47167 100644 --- a/runtime/sam/expr/dropper.go +++ b/runtime/sam/expr/dropper.go @@ -61,9 +61,6 @@ func (d *Dropper) Eval(in super.Value) super.Value { if dropType == typ { return in } - if dropType == d.emptyType { - return d.sctx.Quiet() - } b := &d.builder b.Reset() d.recode(b, typ, in.Bytes(), dropType, d.dropMap) diff --git a/runtime/sam/expr/function/function.go b/runtime/sam/expr/function/function.go index cbeb7a23da..3c63822da7 100644 --- a/runtime/sam/expr/function/function.go +++ b/runtime/sam/expr/function/function.go @@ -143,8 +143,6 @@ func New(sctx *super.Context, name string, narg int) (expr.Function, error) { argmin = 2 argmax = 2 f = &Pow{sctx: sctx} - case "quiet": - f = &Quiet{sctx: sctx} case "regexp": argmin, argmax = 2, 2 f = &Regexp{sctx: sctx} diff --git a/runtime/sam/expr/function/has.go b/runtime/sam/expr/function/has.go index 6cf10a4969..7dc854b185 100644 --- a/runtime/sam/expr/function/has.go +++ b/runtime/sam/expr/function/has.go @@ -15,7 +15,7 @@ func (h *Has) Call(args []super.Value) super.Value { } for _, val := range args { if val.IsError() { - if val.IsMissing() || val.IsQuiet() { + if val.IsMissing() { return super.False } return val diff --git a/runtime/sam/expr/function/types.go b/runtime/sam/expr/function/types.go index 3252df1e27..4ed00a2dcf 100644 --- a/runtime/sam/expr/function/types.go +++ b/runtime/sam/expr/function/types.go @@ -123,18 +123,6 @@ func (h HasError) hasError(t super.Type, b scode.Bytes) bool { } } -type Quiet struct { - sctx *super.Context -} - -func (q *Quiet) Call(args []super.Value) super.Value { - val := args[0] - if val.IsMissing() || val.IsNone() { - return q.sctx.Quiet() - } - return val -} - type Kind struct { sctx *super.Context } diff --git a/runtime/vam/expr/dropper.go b/runtime/vam/expr/dropper.go index 7283252b2e..03d3687f21 100644 --- a/runtime/vam/expr/dropper.go +++ b/runtime/vam/expr/dropper.go @@ -7,7 +7,7 @@ import ( ) // Dropper drops one or more fields in a record. If it drops all fields of a -// top-level record, the record is replaced by error("quiet"). If it drops all +// top-level record, the record is replaced by an empty record. If it drops all // fields of a nested record, the nested record is dropped. Dropper does not // modify non-records. type Dropper struct { @@ -36,7 +36,7 @@ func (d *Dropper) eval(vecs ...vector.Any) vector.Any { if vec2, ok := d.drop(vec, d.fm); ok { if vec2 == nil { // Dropped all fields. - return vector.NewStringError(d.sctx, "quiet", vec.Len()) + return vector.NewRecord(d.sctx.MustLookupTypeRecord(nil), nil, vec.Len()) } return vec2 } diff --git a/runtime/vam/expr/function/errors.go b/runtime/vam/expr/function/errors.go deleted file mode 100644 index f2701afd40..0000000000 --- a/runtime/vam/expr/function/errors.go +++ /dev/null @@ -1,43 +0,0 @@ -package function - -import ( - "github.com/brimdata/super" - "github.com/brimdata/super/vector" -) - -type Quiet struct { - sctx *super.Context -} - -func (q *Quiet) Call(args ...vector.Any) vector.Any { - return vector.Apply(vector.ApplyNone, q.call, vector.DeoptionWithMissing(q.sctx, args[0])) -} - -func (q *Quiet) call(args ...vector.Any) vector.Any { - arg, ok := args[0].(*vector.Error) - if !ok { - return args[0] - } - if _, ok := arg.Vals.Type().(*super.TypeOfString); !ok { - return args[0] - } - if _, ok := arg.Vals.(*vector.Const); ok { - // Fast path - if vector.StringValue(arg.Vals, 0) == "missing" { - return vector.NewStringError(q.sctx, "quiet", arg.Len()) - } - return args[0] - } - n := arg.Len() - vec := vector.NewStringEmpty(n) - for i := range n { - s := vector.StringValue(arg.Vals, i) - if s == "missing" { - s = "quiet" - } - vec.Append(s) - } - return vector.NewError(arg.Typ, vec) -} - -func (q *Quiet) ApplyOpt() vector.ApplyOpt { return vector.ApplyNone } diff --git a/runtime/vam/expr/function/function.go b/runtime/vam/expr/function/function.go index c933b2d362..8051c3264b 100644 --- a/runtime/vam/expr/function/function.go +++ b/runtime/vam/expr/function/function.go @@ -136,8 +136,6 @@ func New(sctx *super.Context, name string, narg int) (expr.Function, error) { argmin = 2 argmax = 2 f = &Pow{sctx} - case "quiet": - f = &Quiet{sctx} case "regexp": argmin, argmax = 2, 2 f = &Regexp{sctx: sctx} diff --git a/runtime/vam/expr/function/has.go b/runtime/vam/expr/function/has.go index 158ee3d149..c0c259e97a 100644 --- a/runtime/vam/expr/function/has.go +++ b/runtime/vam/expr/function/has.go @@ -37,7 +37,7 @@ func (m *Missing) Call(args ...vector.Any) vector.Any { for _, vec := range args { vec = vector.DeoptionWithMissing(m.sctx, vec) if err, ok := vec.(*vector.Error); ok { - b := missingOrQuiet(err) + b := isMissing(err) if b.IsEmpty() { return err } @@ -54,7 +54,7 @@ func (m *Missing) Call(args ...vector.Any) vector.Any { return vector.NewConstBool(false, args[0].Len()) } -func missingOrQuiet(verr *vector.Error) *roaring.Bitmap { +func isMissing(verr *vector.Error) *roaring.Bitmap { b := roaring.New() inner := verr.Vals if inner.Type() != super.TypeString { @@ -63,14 +63,14 @@ func missingOrQuiet(verr *vector.Error) *roaring.Bitmap { switch inner := inner.(type) { case *vector.Const: s := vector.StringValue(inner, 0) - if s == "missing" || s == "quiet" { + if s == "missing" { b.AddRange(0, uint64(inner.Len())) } case *vector.View: vec := inner.Any.(*vector.String) for i := range inner.Len() { s := vec.Value(inner.Index[i]) - if s == "missing" || s == "quiet" { + if s == "missing" { b.Add(i) } } @@ -78,14 +78,14 @@ func missingOrQuiet(verr *vector.Error) *roaring.Bitmap { vec := inner.Any.(*vector.String) for i := range inner.Len() { s := vec.Value(uint32(inner.Index[i])) - if s == "missing" || s == "quiet" { + if s == "missing" { b.Add(i) } } case *vector.String: for i := range inner.Len() { s := inner.Value(i) - if s == "missing" || s == "quiet" { + if s == "missing" { b.Add(i) } } diff --git a/runtime/vam/expr/quiet.go b/runtime/vam/expr/quiet.go deleted file mode 100644 index d1cf888508..0000000000 --- a/runtime/vam/expr/quiet.go +++ /dev/null @@ -1,103 +0,0 @@ -package expr - -import ( - "github.com/brimdata/super" - "github.com/brimdata/super/vector" -) - -func QuietMask(vec vector.Any) (vector.Any, bool) { - errvec, ok := vector.Under(vec).(*vector.Error) - if !ok || errvec.Vals.Kind() != vector.KindString { - return vector.NewConstBool(true, vec.Len()), false - } - lhs := vector.NewConstString("quiet", vec.Len()) - out := NewCompare(nil, "!=", nil, nil).Compare(lhs, errvec.Vals) - return out, true -} - -type Dequiet struct { - sctx *super.Context - expr Evaluator - rmtyp *super.TypeError -} - -func NewDequiet(sctx *super.Context, expr Evaluator) Evaluator { - return &Dequiet{ - sctx: sctx, - expr: expr, - rmtyp: sctx.LookupTypeError(sctx.MustLookupTypeRecord(nil)), - } -} - -func (d *Dequiet) Eval(this vector.Any) vector.Any { - return vector.Apply(vector.ApplyRipUnions, func(vecs ...vector.Any) vector.Any { - vec := vecs[0] - if vec.Kind() == vector.KindRecord { - vec = d.rec(vec) - } - return vec - }, d.expr.Eval(this)) -} - -func (d *Dequiet) rec(vec vector.Any) vector.Any { - vec = vector.Under(vec) - origVec := vec - var index []uint32 - if view, ok := vec.(*vector.View); ok { - index = view.Index - vec = view.Any - } - var vecs []vector.Any - rec := vec.(*vector.Record) - if len(rec.Fields) == 0 { - return origVec - } - for _, field := range rec.Fields { - vec := field - if index != nil { - vec = vector.Pick(field, index) - } - vecs = append(vecs, d.dequiet(vec)) - } - return vector.Apply(vector.ApplyNone, func(vecs ...vector.Any) vector.Any { - var fields []super.Field - var vals []vector.Any - for i, vec := range vecs { - typ := vec.Type() - if typ == d.rmtyp { - continue - } - fields = append(fields, rec.Typ.Fields[i]) - vals = append(vals, vec) - } - rtyp := d.sctx.MustLookupTypeRecord(fields) - return vector.NewRecord(rtyp, vals, vecs[0].Len()) - }, vecs...) -} - -func (d *Dequiet) dequiet(vec vector.Any) vector.Any { - if vec.Kind() == vector.KindRecord { - return d.rec(vec) - } - mask, ok := QuietMask(vec) - if !ok { - return vec - } - b, _ := BoolMask(new(Not).eval(mask)) - if b.IsEmpty() { - return vec - } - n := uint32(b.GetCardinality()) - quiet := d.quietTmp(n) - if n == vec.Len() { - return quiet - } - index := b.ToArray() - vec = vector.ReversePick(vec, index) - out := vector.Combine(vec, index, quiet).(*vector.Dynamic) - return vector.NewUnionFromDynamic(d.sctx, vector.FlattenUnions(out)) -} - -func (d *Dequiet) quietTmp(n uint32) vector.Any { - return vector.NewError(d.rmtyp, vector.NewNull(n)) -} diff --git a/runtime/vam/op/aggregate/aggregate.go b/runtime/vam/op/aggregate/aggregate.go index 7723338d03..a567436520 100644 --- a/runtime/vam/op/aggregate/aggregate.go +++ b/runtime/vam/op/aggregate/aggregate.go @@ -1,7 +1,6 @@ package aggregate import ( - "github.com/RoaringBitmap/roaring/v2" "github.com/brimdata/super" "github.com/brimdata/super/pkg/field" "github.com/brimdata/super/runtime/vam/expr" @@ -95,8 +94,7 @@ func (a *Aggregate) Pull(done bool) (vector.Any, error) { func (a *Aggregate) consume(keys []vector.Any, vals []vector.Any) { vector.ClearNoRips(keys) vector.ClearNoRips(vals) - keys, vals, ok := removeQuietRows(keys, vals) - if !ok || keys[0].Len() == 0 { + if keys[0].Len() == 0 { return } var keyTypes []super.Type @@ -112,69 +110,6 @@ func (a *Aggregate) consume(keys []vector.Any, vals []vector.Any) { table.update(keys, vals) } -// removeQuietRows removes rows in which any key is error("quiet"). It returns -// false if all rows are removed. -func removeQuietRows(keys, vals []vector.Any) ([]vector.Any, []vector.Any, bool) { - if index, ok := notQuietIndex(keys...); ok { - if len(index) == 0 { - // All slots are quiet. - return nil, nil, false - } - for i, k := range keys { - keys[i] = vector.Pick(k, index) - } - for i, v := range vals { - vals[i] = vector.Pick(v, index) - } - } - return keys, vals, true -} - -// notQuietIndex returns the slots that are not quiet across vecs (i.e., the -// slot's value is not error("quiet") in any of vecs). It returns nil, true if -// all slots are quiet and false if no slots are quiet. -func notQuietIndex(vecs ...vector.Any) ([]uint32, bool) { - rb := quietBitmap(vecs...) - if rb.IsEmpty() { - // No slots are quiet. - return nil, false - } - len := uint64(vecs[0].Len()) - if rb.GetCardinality() == len { - // All slots are quiet. - return nil, true - } - rb.Flip(0, len) - return rb.ToArray(), true -} - -// quietBitmap returns a bitmap in which the bit for each slot is set if the -// slot's value is error("quiet") in any of vecs. -func quietBitmap(vecs ...vector.Any) *roaring.Bitmap { - var rb roaring.Bitmap - for _, vec := range vecs { - errVec, ok := vec.(*vector.Error) - if !ok || errVec.Vals.Kind() != vector.KindString { - continue - } - valsVec := errVec.Vals - if _, ok := valsVec.(*vector.Const); ok { - if vector.StringValue(valsVec, 0) == string(super.Quiet) { - // Every slot is error("quiet"). - rb.AddRange(0, uint64(valsVec.Len())) - return &rb - } - continue - } - for i := range valsVec.Len() { - if vector.StringValue(valsVec, i) == string(super.Quiet) { - rb.Add(i) - } - } - } - return &rb -} - func (a *Aggregate) newAggTable(keyTypes []super.Type) aggTable { // Check if we can us an optimized table, else go slow path. if a.isCountByString(keyTypes) && len(a.aggs) == 1 && a.aggs[0].Where == nil { diff --git a/runtime/vam/op/aggregate/aggtable.go b/runtime/vam/op/aggregate/aggtable.go index c53e139951..268dc4e715 100644 --- a/runtime/vam/op/aggregate/aggtable.go +++ b/runtime/vam/op/aggregate/aggtable.go @@ -63,10 +63,6 @@ func (s *superTable) update(keys []vector.Any, args []vector.Any) { if len(m) > 1 { arg = vector.Pick(arg, index) } - arg, ok := removeQuiet(arg) - if !ok { - continue - } if s.partialsIn { row.funcs[i].ConsumeAsPartial(arg) } else { @@ -76,19 +72,6 @@ func (s *superTable) update(keys []vector.Any, args []vector.Any) { } } -// removeQuiet removes any error("quiet") values from vec. It returns false if -// all values are error("quiet"). -func removeQuiet(vec vector.Any) (vector.Any, bool) { - if index, ok := notQuietIndex(vec); ok { - if len(index) == 0 { - // Every slot is error("quiet"). - return nil, false - } - return vector.Pick(vec, index), true - } - return vec, true -} - func (s *superTable) newRow(keys []vector.Any, index []uint32) aggRow { var row aggRow for _, agg := range s.aggs { diff --git a/runtime/vam/op/aggregate/scalar.go b/runtime/vam/op/aggregate/scalar.go index 28ebad45ca..4417d275b5 100644 --- a/runtime/vam/op/aggregate/scalar.go +++ b/runtime/vam/op/aggregate/scalar.go @@ -67,10 +67,6 @@ func (s *scalarAggregate) Pull(done bool) (vector.Any, error) { func (s *scalarAggregate) consume(vecs ...vector.Any) vector.Any { vector.ClearNoRips(vecs) for i, vec := range vecs { - vec, ok := removeQuiet(vec) - if !ok { - continue - } if s.partialsIn { s.funcs[i].ConsumeAsPartial(vec) } else { diff --git a/runtime/vam/op/values.go b/runtime/vam/op/values.go index 888a492e78..ba82e7078a 100644 --- a/runtime/vam/op/values.go +++ b/runtime/vam/op/values.go @@ -31,12 +31,7 @@ func (v *Values) Pull(done bool) (vector.Any, error) { } vals := make([]vector.Any, 0, len(v.exprs)) for _, e := range v.exprs { - vec := e.Eval(val) - vec = vector.DeoptionWithMissing(v.sctx, vec) - vec = filterQuiet(vec) - if vec != nil { - vals = append(vals, vec) - } + vals = append(vals, vector.DeoptionWithMissing(v.sctx, e.Eval(val))) } if len(vals) == 1 { return vals[0], nil @@ -61,17 +56,3 @@ func interleave(vals []vector.Any) vector.Any { } return vector.NewDynamic(tags, vals) } - -func filterQuiet(vec vector.Any) vector.Any { - var filtered bool - mask := vector.Apply(vector.ApplyRipUnions, func(vecs ...vector.Any) vector.Any { - mask, hasfiltered := expr.QuietMask(vecs[0]) - filtered = filtered || hasfiltered - return mask - }, vec) - if !filtered { - return vec - } - masked, _ := applyMask(vec, mask) - return masked -} diff --git a/runtime/ztests/expr/array-expr.yaml b/runtime/ztests/expr/array-expr.yaml index 316b397d66..332885da8f 100644 --- a/runtime/ztests/expr/array-expr.yaml +++ b/runtime/ztests/expr/array-expr.yaml @@ -1,7 +1,7 @@ spq: values [a,b,c] input: | - {a:error("missing"),b:error("quiet"),c:null} + {a:error("missing"),b:error(0),c:null} {a:[1,2],b:null,c:[3,4]} {a:[1,"foo"],b:[2,"bar"],c:[3,"baz"]} {a:set[1,2],b:null,c:set[3,4]} @@ -13,7 +13,7 @@ input: | {a:0,b:1,c:fusion([2]::(null|[int64]),<[int64]>)} output: | - [error("missing"),error("quiet"),null] + [error("missing"),error(0),null] [[1,2],null,[3,4]] [[1,"foo"],[2,"bar"],[3,"baz"]] [set[1,2],null,set[3,4]] diff --git a/runtime/ztests/expr/f-string.yaml b/runtime/ztests/expr/f-string.yaml index d859b11b84..4019b1129b 100644 --- a/runtime/ztests/expr/f-string.yaml +++ b/runtime/ztests/expr/f-string.yaml @@ -7,8 +7,7 @@ input: | {a:1,b:null,c:3} {a:1::(int64|null),b:" "::(string|null),c:3.3.3.3::(ip|null)} {a:null::(int64|null),b:" "::(string|null),c:3.3.3.3::(ip|null)} - {a:1,b:error("missing"),c:error("quiet")} - {a:1,b:error("quiet"),c:error("missing")} + {a:1,b:error("missing"),c:error(0)} output: | "1 3.3.3.3" diff --git a/runtime/ztests/expr/function/coalesce.yaml b/runtime/ztests/expr/function/coalesce.yaml index 996f2213bb..5ad753e762 100644 --- a/runtime/ztests/expr/function/coalesce.yaml +++ b/runtime/ztests/expr/function/coalesce.yaml @@ -1,12 +1,12 @@ spq: coalesce(a, b, c) input: | - {a:null, b:error("quiet"), c:"foo"::(uint64|string)} - {a:"bar"::string, b:error("quiet"), c:1::uint64::(uint64|string)} + {a:null, b:error(0), c:"foo"::(uint64|string)} + {a:"bar"::string, b:error(0), c:1::uint64::(uint64|string)} type port=int64 type missing=error("string") {a:error("missing")::missing, b:2020::port, c:null} - {a:null, b:error("missing"), c:error("quiet")} + {a:null, b:error("missing"), c:error(0)} {a:null, b:"gotme", c:"ted sando"} {a:null, b:error({x:1}), c:"foo"} {a:null::(string|null), b:error(0)::(error(int64)|null), c:"foo"::(string|null)} diff --git a/runtime/ztests/expr/function/concat.yaml b/runtime/ztests/expr/function/concat.yaml index a24265bd0f..3e196d3188 100644 --- a/runtime/ztests/expr/function/concat.yaml +++ b/runtime/ztests/expr/function/concat.yaml @@ -7,8 +7,8 @@ input: | {a:"a",b:null,c:"c"} {a:"hello"::(string|null),b:" "::(string|null),c:"world"::(string|null)} {a:null::(string|null),b:"b"::(string|null),c:null::(string|null)} - {a:"a",b:2,c:error("quiet")} - {a:"a",b:error("quiet"),c:3} + {a:"a",b:2,c:error(0)} + {a:"a",b:error(0),c:3} output: | "hello world" @@ -18,6 +18,7 @@ output: | "hello world" "b" error({message:"concat: string arg required",on:2}) + error(0) --- @@ -29,7 +30,7 @@ input: | "hello"::(string|null) null::(string|null) 1 - error("quiet") + error(0) output: | "single" @@ -37,6 +38,7 @@ output: | "hello" "" error({message:"concat: string arg required",on:1}) + error(0) --- diff --git a/runtime/ztests/expr/function/quiet.yaml b/runtime/ztests/expr/function/quiet.yaml deleted file mode 100644 index 8c25b28dd4..0000000000 --- a/runtime/ztests/expr/function/quiet.yaml +++ /dev/null @@ -1,27 +0,0 @@ -spq: values quiet(this) - -input: | - 1 - [3,2,1,0] - "s"::(string|null) - error("missing") - type foo=error(string) - error("missing")::foo - type bar=string - error("missing"::bar) - null - error(null) - error("quiet") - error({x:"missing"}) - -output: | - 1 - [3,2,1,0] - "s"::(string|null) - type foo=error(string) - error("missing")::foo - type bar=string - error("missing"::bar) - null - error(null) - error({x:"missing"}) diff --git a/runtime/ztests/expr/optional-fields.yaml b/runtime/ztests/expr/optional-fields.yaml index 60d11ecce9..f6b9a7ad7b 100644 --- a/runtime/ztests/expr/optional-fields.yaml +++ b/runtime/ztests/expr/optional-fields.yaml @@ -15,20 +15,6 @@ output: | --- -spq: values quiet(b) - -input: | - {a:1,b?:"foo"} - {a:1,b?:_::string} - {a:1,b:1} - {a:1,c:1} - -output: | - "foo" - 1 - ---- - spq: values flatten(this) input: | diff --git a/runtime/ztests/op/aggregate/any.yaml b/runtime/ztests/op/aggregate/any.yaml index 276ba3121b..57b4aeefa0 100644 --- a/runtime/ztests/op/aggregate/any.yaml +++ b/runtime/ztests/op/aggregate/any.yaml @@ -5,12 +5,10 @@ input: | {k:"k1",v:1} {k:"k2",v:null} {k:"k3",v:error("missing")} - {k:"k4",v:error("quiet")} - {k:"k5",v:fusion({a:1,b?:_::int64},<{a:int64}>)} + {k:"k4",v:fusion({a:1,b?:_::int64},<{a:int64}>)} output: | {k:"k1",any:1} {k:"k2",any:null} {k:"k3",any:error("missing")} - {k:"k4",any:null} - {k:"k5",any:{a:1}} + {k:"k4",any:{a:1}} diff --git a/runtime/ztests/op/aggregate/array_agg.yaml b/runtime/ztests/op/aggregate/array_agg.yaml index f23c9e618d..b38aafcb12 100644 --- a/runtime/ztests/op/aggregate/array_agg.yaml +++ b/runtime/ztests/op/aggregate/array_agg.yaml @@ -6,7 +6,6 @@ input: | null {b:1.5} error("missing") - error("quiet") type foo=int64 1::foo fusion(1::(int64|null),) diff --git a/runtime/ztests/op/aggregate/blend.yaml b/runtime/ztests/op/aggregate/blend.yaml index 0363807e3b..8f3f4fa3f0 100644 --- a/runtime/ztests/op/aggregate/blend.yaml +++ b/runtime/ztests/op/aggregate/blend.yaml @@ -7,15 +7,3 @@ input: | output: | <{a:string,b:int32|string}|error(string)> - ---- - -spq: blend(this) - -input: | - {a:"hello",b:"world"} - {a:"goodnight",b:123::int32} - error("quiet") - -output: | - <{a:string,b:int32|string}> diff --git a/runtime/ztests/op/aggregate/collect.yaml b/runtime/ztests/op/aggregate/collect.yaml index c7405b0cb6..252601140c 100644 --- a/runtime/ztests/op/aggregate/collect.yaml +++ b/runtime/ztests/op/aggregate/collect.yaml @@ -6,7 +6,6 @@ input: | null {b:1.5} error("missing") - error("quiet") type foo=int64 1::foo fusion(1::(int64|null),) diff --git a/runtime/ztests/op/aggregate/container-partials.yaml b/runtime/ztests/op/aggregate/container-partials.yaml index c13996f2a6..0b913df29b 100644 --- a/runtime/ztests/op/aggregate/container-partials.yaml +++ b/runtime/ztests/op/aggregate/container-partials.yaml @@ -1,7 +1,5 @@ # This test exercises the partials paths in the reducers by doing an aggregate -# with a single-row limit. We also make sure the partials consumer can handle -# an empty input by including a record for key "a" with error("quiet") in the -# value field. +# with a single-row limit. script: | super -s -c "union(x) by key with -limit 1 | sort key" in.sup > union.sup super -s -c "collect(x) by key with -limit 1 | sort key" in.sup > collect.sup @@ -15,9 +13,6 @@ inputs: {key:"b",x:1::int32} {key:"a",x:8::int32} {key:"b",x:1::int32} - {key:"a",x:error("quiet")} - {key:"a",x:error("quiet")} - {key:"a",x:error("quiet")} outputs: - name: union.sup diff --git a/runtime/ztests/op/aggregate/count.yaml b/runtime/ztests/op/aggregate/count.yaml index 5e8d71884d..13cba2f4a8 100644 --- a/runtime/ztests/op/aggregate/count.yaml +++ b/runtime/ztests/op/aggregate/count.yaml @@ -12,7 +12,6 @@ input: | {_path:"conn",foo:"9"} {_path:"conn",foo:"10"} error("missing") - error("quiet") output: | - {count:12} + {count:11} diff --git a/runtime/ztests/op/aggregate/dcount.yaml b/runtime/ztests/op/aggregate/dcount.yaml index 80debf0a46..ddd943cece 100644 --- a/runtime/ztests/op/aggregate/dcount.yaml +++ b/runtime/ztests/op/aggregate/dcount.yaml @@ -7,7 +7,6 @@ input: | {x:1::int8} 1 error("missing") - error("quiet") output: | {dcount:4} diff --git a/runtime/ztests/op/aggregate/distinct.yaml b/runtime/ztests/op/aggregate/distinct.yaml index d36a4a8103..69ff55899c 100644 --- a/runtime/ztests/op/aggregate/distinct.yaml +++ b/runtime/ztests/op/aggregate/distinct.yaml @@ -21,7 +21,6 @@ input: | {key:"b",n:1,b:true} {key:"b",n:0,b:true} {key:"c",n:error("missing"),b:error("missing")} - {key:"c",n:error("quiet"),b:error("quiet")} output: | {key:"a",and:false,any:1,array_agg:["a"],avg:1.5,collect:["a"],count:2,dcount:2,max:2,min:1,or:true,sum:3,union:set[1,2]} diff --git a/runtime/ztests/op/aggregate/quiet-key.yaml b/runtime/ztests/op/aggregate/quiet-key.yaml deleted file mode 100644 index d5926b7c66..0000000000 --- a/runtime/ztests/op/aggregate/quiet-key.yaml +++ /dev/null @@ -1,31 +0,0 @@ -spq: count() by id.orig_h:=quiet(id.orig_h) | sort id - -input: | - type port=uint16 - {_path:"weird",id:{orig_h:10.47.1.152,orig_p:49562::port,resp_h:23.217.103.245,resp_p:80::port}} - {_path:"x509",id:"FYNFkU3KccxXgIuUg5"} - {_path:"weird",id:{orig_h:10.47.5.155,orig_p:40712::port,resp_h:91.189.91.23,resp_p:80::port}} - -output: | - {id:{orig_h:10.47.1.152},count:1} - {id:{orig_h:10.47.5.155},count:1} - ---- - -spq: count() by quiet(a) - -input: | - {} - ---- - -spq: count() by quiet(a), quiet(b) - -input: | - {a:1,b:2} - {a:1} - {b:2} - {} - -output: | - {a:1,b:2,count:1} diff --git a/runtime/ztests/op/aggregate/reducers.yaml b/runtime/ztests/op/aggregate/reducers.yaml index e233c88a24..a37bd31f84 100644 --- a/runtime/ztests/op/aggregate/reducers.yaml +++ b/runtime/ztests/op/aggregate/reducers.yaml @@ -5,7 +5,6 @@ input: | {key1:"a",key2:"y",n:2::int32} {key1:"b",key2:"z",n:1::int32} {key1:"c",key2:"0",n:error("missing")} - {key1:"c",key2:"1",n:error("quiet")} output: | {key1:"a",any:1::int32,sum:3,avg:1.5,min:1,max:2} diff --git a/runtime/ztests/op/cut-quiet.yaml b/runtime/ztests/op/cut-quiet.yaml deleted file mode 100644 index b69a30a8df..0000000000 --- a/runtime/ztests/op/cut-quiet.yaml +++ /dev/null @@ -1,64 +0,0 @@ -spq: cut foo:=quiet(foo), bar:=quiet(bar) - -input: | - {foo:"foo0"} - {foo:"foo1",goo:"goo1"} - {bar:"bar2"} - {goo:"goo3",bar:"bar3"} - {bar:"bar4",goo:"goo4",foo:"foo4"} - {goo:"goo5"} - -output: | - {foo:"foo0"} - {foo:"foo1"} - {bar:"bar2"} - {bar:"bar3"} - {foo:"foo4",bar:"bar4"} - {} - ---- - -# Rename fields. -spq: cut f:=quiet(foo), b:=quiet(bar) - -input: | - {foo:"foo0"} - {foo:"foo1",goo:"goo1"} - {bar:"bar2"} - {goo:"goo3",bar:"bar3"} - {bar:"bar4",goo:"goo4",foo:"foo4"} - {goo:"goo5"} - -output: | - {f:"foo0"} - {f:"foo1"} - {b:"bar2"} - {b:"bar3"} - {f:"foo4",b:"bar4"} - {} - ---- - -# Rename nested fields. -spq: cut ports:=id, resp_p:=quiet(id.resp_p) - -input: | - type port=uint16 - {id:{orig_p:1::port,resp_p:2::port}} - {id:[3::port,4::port]} - -output: | - type port=uint16 - {ports:{orig_p:1::port,resp_p:2::port},resp_p:2::port} - {ports:[3::port,4::port]} - ---- - -# Test dequiet keeps unions. -spq: cut x,quiet(y) - -input: | - {x:1::(int64|null)} - -output: | - {x:1::(int64|null)} diff --git a/runtime/ztests/op/drop-foo-both.yaml b/runtime/ztests/op/drop-foo-both.yaml index 024410b34a..9bdb32f27d 100644 --- a/runtime/ztests/op/drop-foo-both.yaml +++ b/runtime/ztests/op/drop-foo-both.yaml @@ -13,7 +13,13 @@ input: | fusion({foo?:_::string,bar:"bar4"},<{bar:string}>) output: | + {} + {} + {} {bar:"bar1"} {bar:"bar2"} {bar:"bar3"} + {} + {} + {} {bar:"bar4"} diff --git a/runtime/ztests/op/put-quiet.yaml b/runtime/ztests/op/put-quiet.yaml deleted file mode 100644 index a8c0e89375..0000000000 --- a/runtime/ztests/op/put-quiet.yaml +++ /dev/null @@ -1,9 +0,0 @@ -spq: x := quiet(y) - -input: | - {} - {x:1} - -output: | - {} - {} diff --git a/sio/bsupio/ztests/zctx-named-reset-2.yaml b/sio/bsupio/ztests/zctx-named-reset-2.yaml index ff7b9a99b3..472e035e8e 100644 --- a/sio/bsupio/ztests/zctx-named-reset-2.yaml +++ b/sio/bsupio/ztests/zctx-named-reset-2.yaml @@ -2,7 +2,7 @@ script: | super -c "head 1" in.sup > t1.bsup super -c "tail 2" in.sup > t2.bsup - cat t1.bsup t2.bsup | super -s -c "count() by proto:=quiet(proto)" - + super -s -c "where has(proto) | count() by proto" t1.bsup t2.bsup inputs: - name: in.sup diff --git a/spq_test.go b/spq_test.go index 3bff9eec16..48cd076296 100644 --- a/spq_test.go +++ b/spq_test.go @@ -280,7 +280,7 @@ func (f *fusionRemovingReader) Read() (*super.Value, error) { for { val, err := f.r.Read() if val != nil && err == nil { - if val.IsQuiet() || f.h.hasFusion(val.Type()) { + if f.h.hasFusion(val.Type()) { continue } } diff --git a/value.go b/value.go index eac6b6f7d4..78744ea1d1 100644 --- a/value.go +++ b/value.go @@ -276,13 +276,6 @@ func (v *Value) IsMissing() bool { return false } -func (v Value) IsQuiet() bool { - if typ, ok := v.Type().(*TypeError); ok { - return typ.IsQuiet(v.Bytes()) - } - return false -} - // Equal reports whether p and v have the same type and the same BSUP // representation. func (v Value) Equal(p Value) bool {