From b2604651c26b7b40d9584753d944b3889ea66dc2 Mon Sep 17 00:00:00 2001 From: shenjiayun Date: Thu, 6 Aug 2026 18:48:55 +0800 Subject: [PATCH 01/15] feat(evaluation): support async custom rpc evaluators --- .../evaluation/domain/evaluator/evaluator.go | 148 ++ .../domain/evaluator/evaluator_validator.go | 5 + .../domain/evaluator/k-evaluator.go | 108 ++ .../domain_openapi/evaluator/evaluator.go | 154 +- .../evaluator/evaluator_validator.go | 5 + .../domain_openapi/evaluator/k-evaluator.go | 108 ++ .../loop/evaluation/evalspiservice/client.go | 6 + .../evalspiservice/evalspiservice.go | 36 + .../spi/coze.loop.evaluation.spi.go | 1612 ++++++++++++++--- .../spi/coze.loop.evaluation.spi_validator.go | 26 + .../spi/evaluationspiservice/client.go | 6 + .../evaluationspiservice.go | 36 + .../spi/k-coze.loop.evaluation.spi.go | 692 +++++++ .../lospi/local_evaluationspiservice.go | 21 + .../convertor/evaluator/evaluator.go | 7 + .../convertor/evaluator/evaluator_test.go | 20 + .../convertor/evaluator/openapi.go | 4 + .../convertor/evaluator/openapi_test.go | 18 + .../application/eval_openapi_app.go | 65 +- .../application/eval_openapi_app_test.go | 167 +- .../evaluation/application/evaluator_app.go | 19 +- .../application/evaluator_app_test.go | 7 +- .../evaluation/application/wire_gen.go | 14 +- .../evaluation/domain/entity/evaluator.go | 12 +- .../domain/entity/evaluator_test.go | 24 + .../entity/evaluator_version_custom_rpc.go | 19 +- .../evaluator_version_custom_rpc_test.go | 83 + .../evaluation/domain/entity/expt_run.go | 1 + .../modules/evaluation/domain/entity/param.go | 3 + .../domain/repo/evaluator_record.go | 4 + .../modules/evaluation/domain/repo/expt.go | 4 + .../repo/mocks/evaluator_record_mock.go | 27 + .../evaluation/domain/repo/mocks/expt.go | 30 + .../evaluation/domain/service/evaluator.go | 4 +- .../domain/service/evaluator_impl.go | 192 +- .../domain/service/evaluator_impl_test.go | 283 ++- .../domain/service/expt_run_item_impl.go | 8 + .../domain/service/expt_run_item_impl_test.go | 54 + .../domain/service/expt_run_item_turn_impl.go | 62 +- .../service/expt_run_item_turn_impl_test.go | 83 +- .../service/mocks/evaluator_service_mock.go | 14 + .../repo/evaluator/evaluator_record_impl.go | 88 +- .../evaluator/evaluator_record_impl_test.go | 194 +- .../mysql/convertor/evaluator_test.go | 34 + .../repo/evaluator/mysql/evaluator_record.go | 29 + .../mysql/evaluator_record_aggr_test.go | 20 + .../mysql/mocks/evaluator_record_mock.go | 39 + .../convert/item_turn_eval_async_test.go | 11 + .../redis/dao/item_turn_eval_async.go | 72 + .../redis/dao/item_turn_eval_async_test.go | 84 + .../coze.loop.evaluation.spi.thrift | 15 + .../loop/evaluation/domain/evaluator.thrift | 4 +- .../domain_openapi/evaluator.thrift | 2 + 53 files changed, 4180 insertions(+), 603 deletions(-) create mode 100644 backend/modules/evaluation/infra/repo/experiment/redis/dao/item_turn_eval_async_test.go diff --git a/backend/kitex_gen/coze/loop/evaluation/domain/evaluator/evaluator.go b/backend/kitex_gen/coze/loop/evaluation/domain/evaluator/evaluator.go index 5e22b17d2a..e1d2339a34 100644 --- a/backend/kitex_gen/coze/loop/evaluation/domain/evaluator/evaluator.go +++ b/backend/kitex_gen/coze/loop/evaluation/domain/evaluator/evaluator.go @@ -3019,12 +3019,16 @@ type CustomRPCEvaluator struct { Cluster *string `thrift:"cluster,4,optional" frugal:"4,optional,string" form:"cluster" json:"cluster,omitempty" query:"cluster"` // 执行http信息 InvokeHTTPInfo *EvaluatorHTTPInfo `thrift:"invoke_http_info,5,optional" frugal:"5,optional,EvaluatorHTTPInfo" form:"invoke_http_info" json:"invoke_http_info,omitempty" query:"invoke_http_info"` + // 异步执行http信息 + AsyncInvokeHTTPInfo *EvaluatorHTTPInfo `thrift:"async_invoke_http_info,6,optional" frugal:"6,optional,EvaluatorHTTPInfo" form:"async_invoke_http_info" json:"async_invoke_http_info,omitempty" query:"async_invoke_http_info"` // ms Timeout *int64 `thrift:"timeout,10,optional" frugal:"10,optional,i64" form:"timeout" json:"timeout,omitempty" query:"timeout"` // 自定义评估器的限流配置 RateLimit *common.RateLimit `thrift:"rate_limit,11,optional" frugal:"11,optional,common.RateLimit" form:"rate_limit" json:"rate_limit,omitempty" query:"rate_limit"` // extra fields Ext map[string]string `thrift:"ext,12,optional" frugal:"12,optional,map" form:"ext" json:"ext,omitempty" query:"ext"` + // 是否异步执行；true 时调用 AsyncInvokeEvaluator + IsAsync *bool `thrift:"is_async,13,optional" frugal:"13,optional,bool" form:"is_async" json:"is_async,omitempty" query:"is_async"` } func NewCustomRPCEvaluator() *CustomRPCEvaluator { @@ -3089,6 +3093,18 @@ func (p *CustomRPCEvaluator) GetInvokeHTTPInfo() (v *EvaluatorHTTPInfo) { return p.InvokeHTTPInfo } +var CustomRPCEvaluator_AsyncInvokeHTTPInfo_DEFAULT *EvaluatorHTTPInfo + +func (p *CustomRPCEvaluator) GetAsyncInvokeHTTPInfo() (v *EvaluatorHTTPInfo) { + if p == nil { + return + } + if !p.IsSetAsyncInvokeHTTPInfo() { + return CustomRPCEvaluator_AsyncInvokeHTTPInfo_DEFAULT + } + return p.AsyncInvokeHTTPInfo +} + var CustomRPCEvaluator_Timeout_DEFAULT int64 func (p *CustomRPCEvaluator) GetTimeout() (v int64) { @@ -3124,6 +3140,18 @@ func (p *CustomRPCEvaluator) GetExt() (v map[string]string) { } return p.Ext } + +var CustomRPCEvaluator_IsAsync_DEFAULT bool + +func (p *CustomRPCEvaluator) GetIsAsync() (v bool) { + if p == nil { + return + } + if !p.IsSetIsAsync() { + return CustomRPCEvaluator_IsAsync_DEFAULT + } + return *p.IsAsync +} func (p *CustomRPCEvaluator) SetProviderEvaluatorCode(val *string) { p.ProviderEvaluatorCode = val } @@ -3139,6 +3167,9 @@ func (p *CustomRPCEvaluator) SetCluster(val *string) { func (p *CustomRPCEvaluator) SetInvokeHTTPInfo(val *EvaluatorHTTPInfo) { p.InvokeHTTPInfo = val } +func (p *CustomRPCEvaluator) SetAsyncInvokeHTTPInfo(val *EvaluatorHTTPInfo) { + p.AsyncInvokeHTTPInfo = val +} func (p *CustomRPCEvaluator) SetTimeout(val *int64) { p.Timeout = val } @@ -3148,6 +3179,9 @@ func (p *CustomRPCEvaluator) SetRateLimit(val *common.RateLimit) { func (p *CustomRPCEvaluator) SetExt(val map[string]string) { p.Ext = val } +func (p *CustomRPCEvaluator) SetIsAsync(val *bool) { + p.IsAsync = val +} var fieldIDToName_CustomRPCEvaluator = map[int16]string{ 1: "provider_evaluator_code", @@ -3155,9 +3189,11 @@ var fieldIDToName_CustomRPCEvaluator = map[int16]string{ 3: "service_name", 4: "cluster", 5: "invoke_http_info", + 6: "async_invoke_http_info", 10: "timeout", 11: "rate_limit", 12: "ext", + 13: "is_async", } func (p *CustomRPCEvaluator) IsSetProviderEvaluatorCode() bool { @@ -3176,6 +3212,10 @@ func (p *CustomRPCEvaluator) IsSetInvokeHTTPInfo() bool { return p.InvokeHTTPInfo != nil } +func (p *CustomRPCEvaluator) IsSetAsyncInvokeHTTPInfo() bool { + return p.AsyncInvokeHTTPInfo != nil +} + func (p *CustomRPCEvaluator) IsSetTimeout() bool { return p.Timeout != nil } @@ -3188,6 +3228,10 @@ func (p *CustomRPCEvaluator) IsSetExt() bool { return p.Ext != nil } +func (p *CustomRPCEvaluator) IsSetIsAsync() bool { + return p.IsAsync != nil +} + func (p *CustomRPCEvaluator) Read(iprot thrift.TProtocol) (err error) { var fieldTypeId thrift.TType var fieldId int16 @@ -3248,6 +3292,14 @@ func (p *CustomRPCEvaluator) Read(iprot thrift.TProtocol) (err error) { } else if err = iprot.Skip(fieldTypeId); err != nil { goto SkipFieldError } + case 6: + if fieldTypeId == thrift.STRUCT { + if err = p.ReadField6(iprot); err != nil { + goto ReadFieldError + } + } else if err = iprot.Skip(fieldTypeId); err != nil { + goto SkipFieldError + } case 10: if fieldTypeId == thrift.I64 { if err = p.ReadField10(iprot); err != nil { @@ -3272,6 +3324,14 @@ func (p *CustomRPCEvaluator) Read(iprot thrift.TProtocol) (err error) { } else if err = iprot.Skip(fieldTypeId); err != nil { goto SkipFieldError } + case 13: + if fieldTypeId == thrift.BOOL { + if err = p.ReadField13(iprot); err != nil { + goto ReadFieldError + } + } else if err = iprot.Skip(fieldTypeId); err != nil { + goto SkipFieldError + } default: if err = iprot.Skip(fieldTypeId); err != nil { goto SkipFieldError @@ -3359,6 +3419,14 @@ func (p *CustomRPCEvaluator) ReadField5(iprot thrift.TProtocol) error { p.InvokeHTTPInfo = _field return nil } +func (p *CustomRPCEvaluator) ReadField6(iprot thrift.TProtocol) error { + _field := NewEvaluatorHTTPInfo() + if err := _field.Read(iprot); err != nil { + return err + } + p.AsyncInvokeHTTPInfo = _field + return nil +} func (p *CustomRPCEvaluator) ReadField10(iprot thrift.TProtocol) error { var _field *int64 @@ -3407,6 +3475,17 @@ func (p *CustomRPCEvaluator) ReadField12(iprot thrift.TProtocol) error { p.Ext = _field return nil } +func (p *CustomRPCEvaluator) ReadField13(iprot thrift.TProtocol) error { + + var _field *bool + if v, err := iprot.ReadBool(); err != nil { + return err + } else { + _field = &v + } + p.IsAsync = _field + return nil +} func (p *CustomRPCEvaluator) Write(oprot thrift.TProtocol) (err error) { var fieldId int16 @@ -3434,6 +3513,10 @@ func (p *CustomRPCEvaluator) Write(oprot thrift.TProtocol) (err error) { fieldId = 5 goto WriteFieldError } + if err = p.writeField6(oprot); err != nil { + fieldId = 6 + goto WriteFieldError + } if err = p.writeField10(oprot); err != nil { fieldId = 10 goto WriteFieldError @@ -3446,6 +3529,10 @@ func (p *CustomRPCEvaluator) Write(oprot thrift.TProtocol) (err error) { fieldId = 12 goto WriteFieldError } + if err = p.writeField13(oprot); err != nil { + fieldId = 13 + goto WriteFieldError + } } if err = oprot.WriteFieldStop(); err != nil { goto WriteFieldStopError @@ -3552,6 +3639,24 @@ WriteFieldBeginError: WriteFieldEndError: return thrift.PrependError(fmt.Sprintf("%T write field 5 end error: ", p), err) } +func (p *CustomRPCEvaluator) writeField6(oprot thrift.TProtocol) (err error) { + if p.IsSetAsyncInvokeHTTPInfo() { + if err = oprot.WriteFieldBegin("async_invoke_http_info", thrift.STRUCT, 6); err != nil { + goto WriteFieldBeginError + } + if err := p.AsyncInvokeHTTPInfo.Write(oprot); err != nil { + return err + } + if err = oprot.WriteFieldEnd(); err != nil { + goto WriteFieldEndError + } + } + return nil +WriteFieldBeginError: + return thrift.PrependError(fmt.Sprintf("%T write field 6 begin error: ", p), err) +WriteFieldEndError: + return thrift.PrependError(fmt.Sprintf("%T write field 6 end error: ", p), err) +} func (p *CustomRPCEvaluator) writeField10(oprot thrift.TProtocol) (err error) { if p.IsSetTimeout() { if err = oprot.WriteFieldBegin("timeout", thrift.I64, 10); err != nil { @@ -3617,6 +3722,24 @@ WriteFieldBeginError: WriteFieldEndError: return thrift.PrependError(fmt.Sprintf("%T write field 12 end error: ", p), err) } +func (p *CustomRPCEvaluator) writeField13(oprot thrift.TProtocol) (err error) { + if p.IsSetIsAsync() { + if err = oprot.WriteFieldBegin("is_async", thrift.BOOL, 13); err != nil { + goto WriteFieldBeginError + } + if err := oprot.WriteBool(*p.IsAsync); err != nil { + return err + } + if err = oprot.WriteFieldEnd(); err != nil { + goto WriteFieldEndError + } + } + return nil +WriteFieldBeginError: + return thrift.PrependError(fmt.Sprintf("%T write field 13 begin error: ", p), err) +WriteFieldEndError: + return thrift.PrependError(fmt.Sprintf("%T write field 13 end error: ", p), err) +} func (p *CustomRPCEvaluator) String() string { if p == nil { @@ -3647,6 +3770,9 @@ func (p *CustomRPCEvaluator) DeepEqual(ano *CustomRPCEvaluator) bool { if !p.Field5DeepEqual(ano.InvokeHTTPInfo) { return false } + if !p.Field6DeepEqual(ano.AsyncInvokeHTTPInfo) { + return false + } if !p.Field10DeepEqual(ano.Timeout) { return false } @@ -3656,6 +3782,9 @@ func (p *CustomRPCEvaluator) DeepEqual(ano *CustomRPCEvaluator) bool { if !p.Field12DeepEqual(ano.Ext) { return false } + if !p.Field13DeepEqual(ano.IsAsync) { + return false + } return true } @@ -3709,6 +3838,13 @@ func (p *CustomRPCEvaluator) Field5DeepEqual(src *EvaluatorHTTPInfo) bool { } return true } +func (p *CustomRPCEvaluator) Field6DeepEqual(src *EvaluatorHTTPInfo) bool { + + if !p.AsyncInvokeHTTPInfo.DeepEqual(src) { + return false + } + return true +} func (p *CustomRPCEvaluator) Field10DeepEqual(src *int64) bool { if p.Timeout == src { @@ -3741,6 +3877,18 @@ func (p *CustomRPCEvaluator) Field12DeepEqual(src map[string]string) bool { } return true } +func (p *CustomRPCEvaluator) Field13DeepEqual(src *bool) bool { + + if p.IsAsync == src { + return true + } else if p.IsAsync == nil || src == nil { + return false + } + if *p.IsAsync != *src { + return false + } + return true +} type EvaluatorVersion struct { // 版本id diff --git a/backend/kitex_gen/coze/loop/evaluation/domain/evaluator/evaluator_validator.go b/backend/kitex_gen/coze/loop/evaluation/domain/evaluator/evaluator_validator.go index cd922a567b..671949d65f 100644 --- a/backend/kitex_gen/coze/loop/evaluation/domain/evaluator/evaluator_validator.go +++ b/backend/kitex_gen/coze/loop/evaluation/domain/evaluator/evaluator_validator.go @@ -93,6 +93,11 @@ func (p *CustomRPCEvaluator) IsValid() error { return fmt.Errorf("field InvokeHTTPInfo not valid, %w", err) } } + if p.AsyncInvokeHTTPInfo != nil { + if err := p.AsyncInvokeHTTPInfo.IsValid(); err != nil { + return fmt.Errorf("field AsyncInvokeHTTPInfo not valid, %w", err) + } + } if p.RateLimit != nil { if err := p.RateLimit.IsValid(); err != nil { return fmt.Errorf("field RateLimit not valid, %w", err) diff --git a/backend/kitex_gen/coze/loop/evaluation/domain/evaluator/k-evaluator.go b/backend/kitex_gen/coze/loop/evaluation/domain/evaluator/k-evaluator.go index 50ab60fef0..1f69d99c79 100644 --- a/backend/kitex_gen/coze/loop/evaluation/domain/evaluator/k-evaluator.go +++ b/backend/kitex_gen/coze/loop/evaluation/domain/evaluator/k-evaluator.go @@ -2087,6 +2087,20 @@ func (p *CustomRPCEvaluator) FastRead(buf []byte) (int, error) { goto SkipFieldError } } + case 6: + if fieldTypeId == thrift.STRUCT { + l, err = p.FastReadField6(buf[offset:]) + offset += l + if err != nil { + goto ReadFieldError + } + } else { + l, err = thrift.Binary.Skip(buf[offset:], fieldTypeId) + offset += l + if err != nil { + goto SkipFieldError + } + } case 10: if fieldTypeId == thrift.I64 { l, err = p.FastReadField10(buf[offset:]) @@ -2129,6 +2143,20 @@ func (p *CustomRPCEvaluator) FastRead(buf []byte) (int, error) { goto SkipFieldError } } + case 13: + if fieldTypeId == thrift.BOOL { + l, err = p.FastReadField13(buf[offset:]) + offset += l + if err != nil { + goto ReadFieldError + } + } else { + l, err = thrift.Binary.Skip(buf[offset:], fieldTypeId) + offset += l + if err != nil { + goto SkipFieldError + } + } default: l, err = thrift.Binary.Skip(buf[offset:], fieldTypeId) offset += l @@ -2221,6 +2249,18 @@ func (p *CustomRPCEvaluator) FastReadField5(buf []byte) (int, error) { return offset, nil } +func (p *CustomRPCEvaluator) FastReadField6(buf []byte) (int, error) { + offset := 0 + _field := NewEvaluatorHTTPInfo() + if l, err := _field.FastRead(buf[offset:]); err != nil { + return offset, err + } else { + offset += l + } + p.AsyncInvokeHTTPInfo = _field + return offset, nil +} + func (p *CustomRPCEvaluator) FastReadField10(buf []byte) (int, error) { offset := 0 @@ -2279,6 +2319,20 @@ func (p *CustomRPCEvaluator) FastReadField12(buf []byte) (int, error) { return offset, nil } +func (p *CustomRPCEvaluator) FastReadField13(buf []byte) (int, error) { + offset := 0 + + var _field *bool + if v, l, err := thrift.Binary.ReadBool(buf[offset:]); err != nil { + return offset, err + } else { + offset += l + _field = &v + } + p.IsAsync = _field + return offset, nil +} + func (p *CustomRPCEvaluator) FastWrite(buf []byte) int { return p.FastWriteNocopy(buf, nil) } @@ -2287,11 +2341,13 @@ func (p *CustomRPCEvaluator) FastWriteNocopy(buf []byte, w thrift.NocopyWriter) offset := 0 if p != nil { offset += p.fastWriteField10(buf[offset:], w) + offset += p.fastWriteField13(buf[offset:], w) offset += p.fastWriteField1(buf[offset:], w) offset += p.fastWriteField2(buf[offset:], w) offset += p.fastWriteField3(buf[offset:], w) offset += p.fastWriteField4(buf[offset:], w) offset += p.fastWriteField5(buf[offset:], w) + offset += p.fastWriteField6(buf[offset:], w) offset += p.fastWriteField11(buf[offset:], w) offset += p.fastWriteField12(buf[offset:], w) } @@ -2307,9 +2363,11 @@ func (p *CustomRPCEvaluator) BLength() int { l += p.field3Length() l += p.field4Length() l += p.field5Length() + l += p.field6Length() l += p.field10Length() l += p.field11Length() l += p.field12Length() + l += p.field13Length() } l += thrift.Binary.FieldStopLength() return l @@ -2358,6 +2416,15 @@ func (p *CustomRPCEvaluator) fastWriteField5(buf []byte, w thrift.NocopyWriter) return offset } +func (p *CustomRPCEvaluator) fastWriteField6(buf []byte, w thrift.NocopyWriter) int { + offset := 0 + if p.IsSetAsyncInvokeHTTPInfo() { + offset += thrift.Binary.WriteFieldBegin(buf[offset:], thrift.STRUCT, 6) + offset += p.AsyncInvokeHTTPInfo.FastWriteNocopy(buf[offset:], w) + } + return offset +} + func (p *CustomRPCEvaluator) fastWriteField10(buf []byte, w thrift.NocopyWriter) int { offset := 0 if p.IsSetTimeout() { @@ -2393,6 +2460,15 @@ func (p *CustomRPCEvaluator) fastWriteField12(buf []byte, w thrift.NocopyWriter) return offset } +func (p *CustomRPCEvaluator) fastWriteField13(buf []byte, w thrift.NocopyWriter) int { + offset := 0 + if p.IsSetIsAsync() { + offset += thrift.Binary.WriteFieldBegin(buf[offset:], thrift.BOOL, 13) + offset += thrift.Binary.WriteBool(buf[offset:], *p.IsAsync) + } + return offset +} + func (p *CustomRPCEvaluator) field1Length() int { l := 0 if p.IsSetProviderEvaluatorCode() { @@ -2436,6 +2512,15 @@ func (p *CustomRPCEvaluator) field5Length() int { return l } +func (p *CustomRPCEvaluator) field6Length() int { + l := 0 + if p.IsSetAsyncInvokeHTTPInfo() { + l += thrift.Binary.FieldBeginLength() + l += p.AsyncInvokeHTTPInfo.BLength() + } + return l +} + func (p *CustomRPCEvaluator) field10Length() int { l := 0 if p.IsSetTimeout() { @@ -2469,6 +2554,15 @@ func (p *CustomRPCEvaluator) field12Length() int { return l } +func (p *CustomRPCEvaluator) field13Length() int { + l := 0 + if p.IsSetIsAsync() { + l += thrift.Binary.FieldBeginLength() + l += thrift.Binary.BoolLength() + } + return l +} + func (p *CustomRPCEvaluator) DeepCopy(s interface{}) error { src, ok := s.(*CustomRPCEvaluator) if !ok { @@ -2510,6 +2604,15 @@ func (p *CustomRPCEvaluator) DeepCopy(s interface{}) error { } p.InvokeHTTPInfo = _invokeHTTPInfo + var _asyncInvokeHTTPInfo *EvaluatorHTTPInfo + if src.AsyncInvokeHTTPInfo != nil { + _asyncInvokeHTTPInfo = &EvaluatorHTTPInfo{} + if err := _asyncInvokeHTTPInfo.DeepCopy(src.AsyncInvokeHTTPInfo); err != nil { + return err + } + } + p.AsyncInvokeHTTPInfo = _asyncInvokeHTTPInfo + if src.Timeout != nil { tmp := *src.Timeout p.Timeout = &tmp @@ -2541,6 +2644,11 @@ func (p *CustomRPCEvaluator) DeepCopy(s interface{}) error { } } + if src.IsAsync != nil { + tmp := *src.IsAsync + p.IsAsync = &tmp + } + return nil } diff --git a/backend/kitex_gen/coze/loop/evaluation/domain_openapi/evaluator/evaluator.go b/backend/kitex_gen/coze/loop/evaluation/domain_openapi/evaluator/evaluator.go index 542816fc21..5d8d8fb2e6 100644 --- a/backend/kitex_gen/coze/loop/evaluation/domain_openapi/evaluator/evaluator.go +++ b/backend/kitex_gen/coze/loop/evaluation/domain_openapi/evaluator/evaluator.go @@ -858,14 +858,16 @@ type CustomRPCEvaluator struct { // 自定义评估器编码 ProviderEvaluatorCode *string `thrift:"provider_evaluator_code,1,optional" frugal:"1,optional,string" form:"provider_evaluator_code" json:"provider_evaluator_code,omitempty" query:"provider_evaluator_code"` // rpc / faas_http - AccessProtocol *EvaluatorAccessProtocol `thrift:"access_protocol,2,optional" frugal:"2,optional,string" form:"access_protocol" json:"access_protocol,omitempty" query:"access_protocol"` - ServiceName *string `thrift:"service_name,3,optional" frugal:"3,optional,string" form:"service_name" json:"service_name,omitempty" query:"service_name"` - Cluster *string `thrift:"cluster,4,optional" frugal:"4,optional,string" form:"cluster" json:"cluster,omitempty" query:"cluster"` - InvokeHTTPInfo *EvaluatorHTTPInfo `thrift:"invoke_http_info,5,optional" frugal:"5,optional,EvaluatorHTTPInfo" form:"invoke_http_info" json:"invoke_http_info,omitempty" query:"invoke_http_info"` + AccessProtocol *EvaluatorAccessProtocol `thrift:"access_protocol,2,optional" frugal:"2,optional,string" form:"access_protocol" json:"access_protocol,omitempty" query:"access_protocol"` + ServiceName *string `thrift:"service_name,3,optional" frugal:"3,optional,string" form:"service_name" json:"service_name,omitempty" query:"service_name"` + Cluster *string `thrift:"cluster,4,optional" frugal:"4,optional,string" form:"cluster" json:"cluster,omitempty" query:"cluster"` + InvokeHTTPInfo *EvaluatorHTTPInfo `thrift:"invoke_http_info,5,optional" frugal:"5,optional,EvaluatorHTTPInfo" form:"invoke_http_info" json:"invoke_http_info,omitempty" query:"invoke_http_info"` + AsyncInvokeHTTPInfo *EvaluatorHTTPInfo `thrift:"async_invoke_http_info,6,optional" frugal:"6,optional,EvaluatorHTTPInfo" form:"async_invoke_http_info" json:"async_invoke_http_info,omitempty" query:"async_invoke_http_info"` // ms Timeout *int64 `thrift:"timeout,10,optional" frugal:"10,optional,i64" form:"timeout" json:"timeout,omitempty" query:"timeout"` RateLimit *common.RateLimit `thrift:"rate_limit,11,optional" frugal:"11,optional,common.RateLimit" form:"rate_limit" json:"rate_limit,omitempty" query:"rate_limit"` Ext map[string]string `thrift:"ext,12,optional" frugal:"12,optional,map" form:"ext" json:"ext,omitempty" query:"ext"` + IsAsync *bool `thrift:"is_async,13,optional" frugal:"13,optional,bool" form:"is_async" json:"is_async,omitempty" query:"is_async"` } func NewCustomRPCEvaluator() *CustomRPCEvaluator { @@ -935,6 +937,18 @@ func (p *CustomRPCEvaluator) GetInvokeHTTPInfo() (v *EvaluatorHTTPInfo) { return p.InvokeHTTPInfo } +var CustomRPCEvaluator_AsyncInvokeHTTPInfo_DEFAULT *EvaluatorHTTPInfo + +func (p *CustomRPCEvaluator) GetAsyncInvokeHTTPInfo() (v *EvaluatorHTTPInfo) { + if p == nil { + return + } + if !p.IsSetAsyncInvokeHTTPInfo() { + return CustomRPCEvaluator_AsyncInvokeHTTPInfo_DEFAULT + } + return p.AsyncInvokeHTTPInfo +} + var CustomRPCEvaluator_Timeout_DEFAULT int64 func (p *CustomRPCEvaluator) GetTimeout() (v int64) { @@ -970,6 +984,18 @@ func (p *CustomRPCEvaluator) GetExt() (v map[string]string) { } return p.Ext } + +var CustomRPCEvaluator_IsAsync_DEFAULT bool + +func (p *CustomRPCEvaluator) GetIsAsync() (v bool) { + if p == nil { + return + } + if !p.IsSetIsAsync() { + return CustomRPCEvaluator_IsAsync_DEFAULT + } + return *p.IsAsync +} func (p *CustomRPCEvaluator) SetProviderEvaluatorCode(val *string) { p.ProviderEvaluatorCode = val } @@ -985,6 +1011,9 @@ func (p *CustomRPCEvaluator) SetCluster(val *string) { func (p *CustomRPCEvaluator) SetInvokeHTTPInfo(val *EvaluatorHTTPInfo) { p.InvokeHTTPInfo = val } +func (p *CustomRPCEvaluator) SetAsyncInvokeHTTPInfo(val *EvaluatorHTTPInfo) { + p.AsyncInvokeHTTPInfo = val +} func (p *CustomRPCEvaluator) SetTimeout(val *int64) { p.Timeout = val } @@ -994,6 +1023,9 @@ func (p *CustomRPCEvaluator) SetRateLimit(val *common.RateLimit) { func (p *CustomRPCEvaluator) SetExt(val map[string]string) { p.Ext = val } +func (p *CustomRPCEvaluator) SetIsAsync(val *bool) { + p.IsAsync = val +} var fieldIDToName_CustomRPCEvaluator = map[int16]string{ 1: "provider_evaluator_code", @@ -1001,9 +1033,11 @@ var fieldIDToName_CustomRPCEvaluator = map[int16]string{ 3: "service_name", 4: "cluster", 5: "invoke_http_info", + 6: "async_invoke_http_info", 10: "timeout", 11: "rate_limit", 12: "ext", + 13: "is_async", } func (p *CustomRPCEvaluator) IsSetProviderEvaluatorCode() bool { @@ -1026,6 +1060,10 @@ func (p *CustomRPCEvaluator) IsSetInvokeHTTPInfo() bool { return p.InvokeHTTPInfo != nil } +func (p *CustomRPCEvaluator) IsSetAsyncInvokeHTTPInfo() bool { + return p.AsyncInvokeHTTPInfo != nil +} + func (p *CustomRPCEvaluator) IsSetTimeout() bool { return p.Timeout != nil } @@ -1038,6 +1076,10 @@ func (p *CustomRPCEvaluator) IsSetExt() bool { return p.Ext != nil } +func (p *CustomRPCEvaluator) IsSetIsAsync() bool { + return p.IsAsync != nil +} + func (p *CustomRPCEvaluator) Read(iprot thrift.TProtocol) (err error) { var fieldTypeId thrift.TType var fieldId int16 @@ -1096,6 +1138,14 @@ func (p *CustomRPCEvaluator) Read(iprot thrift.TProtocol) (err error) { } else if err = iprot.Skip(fieldTypeId); err != nil { goto SkipFieldError } + case 6: + if fieldTypeId == thrift.STRUCT { + if err = p.ReadField6(iprot); err != nil { + goto ReadFieldError + } + } else if err = iprot.Skip(fieldTypeId); err != nil { + goto SkipFieldError + } case 10: if fieldTypeId == thrift.I64 { if err = p.ReadField10(iprot); err != nil { @@ -1120,6 +1170,14 @@ func (p *CustomRPCEvaluator) Read(iprot thrift.TProtocol) (err error) { } else if err = iprot.Skip(fieldTypeId); err != nil { goto SkipFieldError } + case 13: + if fieldTypeId == thrift.BOOL { + if err = p.ReadField13(iprot); err != nil { + goto ReadFieldError + } + } else if err = iprot.Skip(fieldTypeId); err != nil { + goto SkipFieldError + } default: if err = iprot.Skip(fieldTypeId); err != nil { goto SkipFieldError @@ -1201,6 +1259,14 @@ func (p *CustomRPCEvaluator) ReadField5(iprot thrift.TProtocol) error { p.InvokeHTTPInfo = _field return nil } +func (p *CustomRPCEvaluator) ReadField6(iprot thrift.TProtocol) error { + _field := NewEvaluatorHTTPInfo() + if err := _field.Read(iprot); err != nil { + return err + } + p.AsyncInvokeHTTPInfo = _field + return nil +} func (p *CustomRPCEvaluator) ReadField10(iprot thrift.TProtocol) error { var _field *int64 @@ -1249,6 +1315,17 @@ func (p *CustomRPCEvaluator) ReadField12(iprot thrift.TProtocol) error { p.Ext = _field return nil } +func (p *CustomRPCEvaluator) ReadField13(iprot thrift.TProtocol) error { + + var _field *bool + if v, err := iprot.ReadBool(); err != nil { + return err + } else { + _field = &v + } + p.IsAsync = _field + return nil +} func (p *CustomRPCEvaluator) Write(oprot thrift.TProtocol) (err error) { var fieldId int16 @@ -1276,6 +1353,10 @@ func (p *CustomRPCEvaluator) Write(oprot thrift.TProtocol) (err error) { fieldId = 5 goto WriteFieldError } + if err = p.writeField6(oprot); err != nil { + fieldId = 6 + goto WriteFieldError + } if err = p.writeField10(oprot); err != nil { fieldId = 10 goto WriteFieldError @@ -1288,6 +1369,10 @@ func (p *CustomRPCEvaluator) Write(oprot thrift.TProtocol) (err error) { fieldId = 12 goto WriteFieldError } + if err = p.writeField13(oprot); err != nil { + fieldId = 13 + goto WriteFieldError + } } if err = oprot.WriteFieldStop(); err != nil { goto WriteFieldStopError @@ -1396,6 +1481,24 @@ WriteFieldBeginError: WriteFieldEndError: return thrift.PrependError(fmt.Sprintf("%T write field 5 end error: ", p), err) } +func (p *CustomRPCEvaluator) writeField6(oprot thrift.TProtocol) (err error) { + if p.IsSetAsyncInvokeHTTPInfo() { + if err = oprot.WriteFieldBegin("async_invoke_http_info", thrift.STRUCT, 6); err != nil { + goto WriteFieldBeginError + } + if err := p.AsyncInvokeHTTPInfo.Write(oprot); err != nil { + return err + } + if err = oprot.WriteFieldEnd(); err != nil { + goto WriteFieldEndError + } + } + return nil +WriteFieldBeginError: + return thrift.PrependError(fmt.Sprintf("%T write field 6 begin error: ", p), err) +WriteFieldEndError: + return thrift.PrependError(fmt.Sprintf("%T write field 6 end error: ", p), err) +} func (p *CustomRPCEvaluator) writeField10(oprot thrift.TProtocol) (err error) { if p.IsSetTimeout() { if err = oprot.WriteFieldBegin("timeout", thrift.I64, 10); err != nil { @@ -1461,6 +1564,24 @@ WriteFieldBeginError: WriteFieldEndError: return thrift.PrependError(fmt.Sprintf("%T write field 12 end error: ", p), err) } +func (p *CustomRPCEvaluator) writeField13(oprot thrift.TProtocol) (err error) { + if p.IsSetIsAsync() { + if err = oprot.WriteFieldBegin("is_async", thrift.BOOL, 13); err != nil { + goto WriteFieldBeginError + } + if err := oprot.WriteBool(*p.IsAsync); err != nil { + return err + } + if err = oprot.WriteFieldEnd(); err != nil { + goto WriteFieldEndError + } + } + return nil +WriteFieldBeginError: + return thrift.PrependError(fmt.Sprintf("%T write field 13 begin error: ", p), err) +WriteFieldEndError: + return thrift.PrependError(fmt.Sprintf("%T write field 13 end error: ", p), err) +} func (p *CustomRPCEvaluator) String() string { if p == nil { @@ -1491,6 +1612,9 @@ func (p *CustomRPCEvaluator) DeepEqual(ano *CustomRPCEvaluator) bool { if !p.Field5DeepEqual(ano.InvokeHTTPInfo) { return false } + if !p.Field6DeepEqual(ano.AsyncInvokeHTTPInfo) { + return false + } if !p.Field10DeepEqual(ano.Timeout) { return false } @@ -1500,6 +1624,9 @@ func (p *CustomRPCEvaluator) DeepEqual(ano *CustomRPCEvaluator) bool { if !p.Field12DeepEqual(ano.Ext) { return false } + if !p.Field13DeepEqual(ano.IsAsync) { + return false + } return true } @@ -1558,6 +1685,13 @@ func (p *CustomRPCEvaluator) Field5DeepEqual(src *EvaluatorHTTPInfo) bool { } return true } +func (p *CustomRPCEvaluator) Field6DeepEqual(src *EvaluatorHTTPInfo) bool { + + if !p.AsyncInvokeHTTPInfo.DeepEqual(src) { + return false + } + return true +} func (p *CustomRPCEvaluator) Field10DeepEqual(src *int64) bool { if p.Timeout == src { @@ -1590,6 +1724,18 @@ func (p *CustomRPCEvaluator) Field12DeepEqual(src map[string]string) bool { } return true } +func (p *CustomRPCEvaluator) Field13DeepEqual(src *bool) bool { + + if p.IsAsync == src { + return true + } else if p.IsAsync == nil || src == nil { + return false + } + if *p.IsAsync != *src { + return false + } + return true +} // Agent评估器Prompt配置输出规则 type AgentEvaluatorPromptConfigOutputRules struct { diff --git a/backend/kitex_gen/coze/loop/evaluation/domain_openapi/evaluator/evaluator_validator.go b/backend/kitex_gen/coze/loop/evaluation/domain_openapi/evaluator/evaluator_validator.go index d6a5859832..61284f796f 100644 --- a/backend/kitex_gen/coze/loop/evaluation/domain_openapi/evaluator/evaluator_validator.go +++ b/backend/kitex_gen/coze/loop/evaluation/domain_openapi/evaluator/evaluator_validator.go @@ -41,6 +41,11 @@ func (p *CustomRPCEvaluator) IsValid() error { return fmt.Errorf("field InvokeHTTPInfo not valid, %w", err) } } + if p.AsyncInvokeHTTPInfo != nil { + if err := p.AsyncInvokeHTTPInfo.IsValid(); err != nil { + return fmt.Errorf("field AsyncInvokeHTTPInfo not valid, %w", err) + } + } if p.RateLimit != nil { if err := p.RateLimit.IsValid(); err != nil { return fmt.Errorf("field RateLimit not valid, %w", err) diff --git a/backend/kitex_gen/coze/loop/evaluation/domain_openapi/evaluator/k-evaluator.go b/backend/kitex_gen/coze/loop/evaluation/domain_openapi/evaluator/k-evaluator.go index f9ec501820..136e75e1cf 100644 --- a/backend/kitex_gen/coze/loop/evaluation/domain_openapi/evaluator/k-evaluator.go +++ b/backend/kitex_gen/coze/loop/evaluation/domain_openapi/evaluator/k-evaluator.go @@ -664,6 +664,20 @@ func (p *CustomRPCEvaluator) FastRead(buf []byte) (int, error) { goto SkipFieldError } } + case 6: + if fieldTypeId == thrift.STRUCT { + l, err = p.FastReadField6(buf[offset:]) + offset += l + if err != nil { + goto ReadFieldError + } + } else { + l, err = thrift.Binary.Skip(buf[offset:], fieldTypeId) + offset += l + if err != nil { + goto SkipFieldError + } + } case 10: if fieldTypeId == thrift.I64 { l, err = p.FastReadField10(buf[offset:]) @@ -706,6 +720,20 @@ func (p *CustomRPCEvaluator) FastRead(buf []byte) (int, error) { goto SkipFieldError } } + case 13: + if fieldTypeId == thrift.BOOL { + l, err = p.FastReadField13(buf[offset:]) + offset += l + if err != nil { + goto ReadFieldError + } + } else { + l, err = thrift.Binary.Skip(buf[offset:], fieldTypeId) + offset += l + if err != nil { + goto SkipFieldError + } + } default: l, err = thrift.Binary.Skip(buf[offset:], fieldTypeId) offset += l @@ -792,6 +820,18 @@ func (p *CustomRPCEvaluator) FastReadField5(buf []byte) (int, error) { return offset, nil } +func (p *CustomRPCEvaluator) FastReadField6(buf []byte) (int, error) { + offset := 0 + _field := NewEvaluatorHTTPInfo() + if l, err := _field.FastRead(buf[offset:]); err != nil { + return offset, err + } else { + offset += l + } + p.AsyncInvokeHTTPInfo = _field + return offset, nil +} + func (p *CustomRPCEvaluator) FastReadField10(buf []byte) (int, error) { offset := 0 @@ -850,6 +890,20 @@ func (p *CustomRPCEvaluator) FastReadField12(buf []byte) (int, error) { return offset, nil } +func (p *CustomRPCEvaluator) FastReadField13(buf []byte) (int, error) { + offset := 0 + + var _field *bool + if v, l, err := thrift.Binary.ReadBool(buf[offset:]); err != nil { + return offset, err + } else { + offset += l + _field = &v + } + p.IsAsync = _field + return offset, nil +} + func (p *CustomRPCEvaluator) FastWrite(buf []byte) int { return p.FastWriteNocopy(buf, nil) } @@ -858,11 +912,13 @@ func (p *CustomRPCEvaluator) FastWriteNocopy(buf []byte, w thrift.NocopyWriter) offset := 0 if p != nil { offset += p.fastWriteField10(buf[offset:], w) + offset += p.fastWriteField13(buf[offset:], w) offset += p.fastWriteField1(buf[offset:], w) offset += p.fastWriteField2(buf[offset:], w) offset += p.fastWriteField3(buf[offset:], w) offset += p.fastWriteField4(buf[offset:], w) offset += p.fastWriteField5(buf[offset:], w) + offset += p.fastWriteField6(buf[offset:], w) offset += p.fastWriteField11(buf[offset:], w) offset += p.fastWriteField12(buf[offset:], w) } @@ -878,9 +934,11 @@ func (p *CustomRPCEvaluator) BLength() int { l += p.field3Length() l += p.field4Length() l += p.field5Length() + l += p.field6Length() l += p.field10Length() l += p.field11Length() l += p.field12Length() + l += p.field13Length() } l += thrift.Binary.FieldStopLength() return l @@ -931,6 +989,15 @@ func (p *CustomRPCEvaluator) fastWriteField5(buf []byte, w thrift.NocopyWriter) return offset } +func (p *CustomRPCEvaluator) fastWriteField6(buf []byte, w thrift.NocopyWriter) int { + offset := 0 + if p.IsSetAsyncInvokeHTTPInfo() { + offset += thrift.Binary.WriteFieldBegin(buf[offset:], thrift.STRUCT, 6) + offset += p.AsyncInvokeHTTPInfo.FastWriteNocopy(buf[offset:], w) + } + return offset +} + func (p *CustomRPCEvaluator) fastWriteField10(buf []byte, w thrift.NocopyWriter) int { offset := 0 if p.IsSetTimeout() { @@ -966,6 +1033,15 @@ func (p *CustomRPCEvaluator) fastWriteField12(buf []byte, w thrift.NocopyWriter) return offset } +func (p *CustomRPCEvaluator) fastWriteField13(buf []byte, w thrift.NocopyWriter) int { + offset := 0 + if p.IsSetIsAsync() { + offset += thrift.Binary.WriteFieldBegin(buf[offset:], thrift.BOOL, 13) + offset += thrift.Binary.WriteBool(buf[offset:], *p.IsAsync) + } + return offset +} + func (p *CustomRPCEvaluator) field1Length() int { l := 0 if p.IsSetProviderEvaluatorCode() { @@ -1011,6 +1087,15 @@ func (p *CustomRPCEvaluator) field5Length() int { return l } +func (p *CustomRPCEvaluator) field6Length() int { + l := 0 + if p.IsSetAsyncInvokeHTTPInfo() { + l += thrift.Binary.FieldBeginLength() + l += p.AsyncInvokeHTTPInfo.BLength() + } + return l +} + func (p *CustomRPCEvaluator) field10Length() int { l := 0 if p.IsSetTimeout() { @@ -1044,6 +1129,15 @@ func (p *CustomRPCEvaluator) field12Length() int { return l } +func (p *CustomRPCEvaluator) field13Length() int { + l := 0 + if p.IsSetIsAsync() { + l += thrift.Binary.FieldBeginLength() + l += thrift.Binary.BoolLength() + } + return l +} + func (p *CustomRPCEvaluator) DeepCopy(s interface{}) error { src, ok := s.(*CustomRPCEvaluator) if !ok { @@ -1088,6 +1182,15 @@ func (p *CustomRPCEvaluator) DeepCopy(s interface{}) error { } p.InvokeHTTPInfo = _invokeHTTPInfo + var _asyncInvokeHTTPInfo *EvaluatorHTTPInfo + if src.AsyncInvokeHTTPInfo != nil { + _asyncInvokeHTTPInfo = &EvaluatorHTTPInfo{} + if err := _asyncInvokeHTTPInfo.DeepCopy(src.AsyncInvokeHTTPInfo); err != nil { + return err + } + } + p.AsyncInvokeHTTPInfo = _asyncInvokeHTTPInfo + if src.Timeout != nil { tmp := *src.Timeout p.Timeout = &tmp @@ -1119,6 +1222,11 @@ func (p *CustomRPCEvaluator) DeepCopy(s interface{}) error { } } + if src.IsAsync != nil { + tmp := *src.IsAsync + p.IsAsync = &tmp + } + return nil } diff --git a/backend/kitex_gen/coze/loop/evaluation/evalspiservice/client.go b/backend/kitex_gen/coze/loop/evaluation/evalspiservice/client.go index 2e942e8f8b..9011a7434c 100644 --- a/backend/kitex_gen/coze/loop/evaluation/evalspiservice/client.go +++ b/backend/kitex_gen/coze/loop/evaluation/evalspiservice/client.go @@ -15,6 +15,7 @@ type Client interface { InvokeEvalTarget(ctx context.Context, req *spi.InvokeEvalTargetRequest, callOptions ...callopt.Option) (r *spi.InvokeEvalTargetResponse, err error) AsyncInvokeEvalTarget(ctx context.Context, req *spi.AsyncInvokeEvalTargetRequest, callOptions ...callopt.Option) (r *spi.AsyncInvokeEvalTargetResponse, err error) InvokeEvaluator(ctx context.Context, req *spi.InvokeEvaluatorRequest, callOptions ...callopt.Option) (r *spi.InvokeEvaluatorResponse, err error) + AsyncInvokeEvaluator(ctx context.Context, req *spi.AsyncInvokeEvaluatorRequest, callOptions ...callopt.Option) (r *spi.AsyncInvokeEvaluatorResponse, err error) } // NewClient creates a client for the service defined in IDL. @@ -65,3 +66,8 @@ func (p *kEvalSPIServiceClient) InvokeEvaluator(ctx context.Context, req *spi.In ctx = client.NewCtxWithCallOptions(ctx, callOptions) return p.kClient.InvokeEvaluator(ctx, req) } + +func (p *kEvalSPIServiceClient) AsyncInvokeEvaluator(ctx context.Context, req *spi.AsyncInvokeEvaluatorRequest, callOptions ...callopt.Option) (r *spi.AsyncInvokeEvaluatorResponse, err error) { + ctx = client.NewCtxWithCallOptions(ctx, callOptions) + return p.kClient.AsyncInvokeEvaluator(ctx, req) +} diff --git a/backend/kitex_gen/coze/loop/evaluation/evalspiservice/evalspiservice.go b/backend/kitex_gen/coze/loop/evaluation/evalspiservice/evalspiservice.go index bb533a1387..6e5fa9c27b 100644 --- a/backend/kitex_gen/coze/loop/evaluation/evalspiservice/evalspiservice.go +++ b/backend/kitex_gen/coze/loop/evaluation/evalspiservice/evalspiservice.go @@ -42,6 +42,13 @@ var serviceMethods = map[string]kitex.MethodInfo{ false, kitex.WithStreamingMode(kitex.StreamingNone), ), + "AsyncInvokeEvaluator": kitex.NewMethodInfo( + asyncInvokeEvaluatorHandler, + newEvaluationSPIServiceAsyncInvokeEvaluatorArgs, + newEvaluationSPIServiceAsyncInvokeEvaluatorResult, + false, + kitex.WithStreamingMode(kitex.StreamingNone), + ), } var ( @@ -151,6 +158,25 @@ func newEvaluationSPIServiceInvokeEvaluatorResult() interface{} { return spi.NewEvaluationSPIServiceInvokeEvaluatorResult() } +func asyncInvokeEvaluatorHandler(ctx context.Context, handler interface{}, arg, result interface{}) error { + realArg := arg.(*spi.EvaluationSPIServiceAsyncInvokeEvaluatorArgs) + realResult := result.(*spi.EvaluationSPIServiceAsyncInvokeEvaluatorResult) + success, err := handler.(spi.EvaluationSPIService).AsyncInvokeEvaluator(ctx, realArg.Req) + if err != nil { + return err + } + realResult.Success = success + return nil +} + +func newEvaluationSPIServiceAsyncInvokeEvaluatorArgs() interface{} { + return spi.NewEvaluationSPIServiceAsyncInvokeEvaluatorArgs() +} + +func newEvaluationSPIServiceAsyncInvokeEvaluatorResult() interface{} { + return spi.NewEvaluationSPIServiceAsyncInvokeEvaluatorResult() +} + type kClient struct { c client.Client sc client.Streaming @@ -202,3 +228,13 @@ func (p *kClient) InvokeEvaluator(ctx context.Context, req *spi.InvokeEvaluatorR } return _result.GetSuccess(), nil } + +func (p *kClient) AsyncInvokeEvaluator(ctx context.Context, req *spi.AsyncInvokeEvaluatorRequest) (r *spi.AsyncInvokeEvaluatorResponse, err error) { + var _args spi.EvaluationSPIServiceAsyncInvokeEvaluatorArgs + _args.Req = req + var _result spi.EvaluationSPIServiceAsyncInvokeEvaluatorResult + if err = p.c.Call(ctx, "AsyncInvokeEvaluator", &_args, &_result); err != nil { + return + } + return _result.GetSuccess(), nil +} diff --git a/backend/kitex_gen/coze/loop/evaluation/spi/coze.loop.evaluation.spi.go b/backend/kitex_gen/coze/loop/evaluation/spi/coze.loop.evaluation.spi.go index c345ce4a04..7496522663 100644 --- a/backend/kitex_gen/coze/loop/evaluation/spi/coze.loop.evaluation.spi.go +++ b/backend/kitex_gen/coze/loop/evaluation/spi/coze.loop.evaluation.spi.go @@ -7977,6 +7977,644 @@ func (p *InvokeEvaluatorResponse) Field255DeepEqual(src *base.BaseResp) bool { return true } +// asynchronously invoke custom evaluator request +type AsyncInvokeEvaluatorRequest struct { + WorkspaceID *int64 `thrift:"workspace_id,1,optional" frugal:"1,optional,i64" form:"workspace_id" json:"workspace_id,string,omitempty" query:"workspace_id"` + // execution id, report result with the same id + InvokeID *int64 `thrift:"invoke_id,2,optional" frugal:"2,optional,i64" form:"invoke_id" json:"invoke_id,string,omitempty" query:"invoke_id"` + Evaluator *InvokeCustomEvaluator `thrift:"evaluator,3,optional" frugal:"3,optional,InvokeCustomEvaluator" form:"evaluator" json:"evaluator,omitempty" query:"evaluator"` + InputData *InvokeEvaluatorInputData `thrift:"input_data,4,optional" frugal:"4,optional,InvokeEvaluatorInputData" form:"input_data" json:"input_data,omitempty" query:"input_data"` + Base *base.Base `thrift:"Base,255,optional" frugal:"255,optional,base.Base" form:"Base" json:"Base,omitempty" query:"Base"` +} + +func NewAsyncInvokeEvaluatorRequest() *AsyncInvokeEvaluatorRequest { + return &AsyncInvokeEvaluatorRequest{} +} + +func (p *AsyncInvokeEvaluatorRequest) InitDefault() { +} + +var AsyncInvokeEvaluatorRequest_WorkspaceID_DEFAULT int64 + +func (p *AsyncInvokeEvaluatorRequest) GetWorkspaceID() (v int64) { + if p == nil { + return + } + if !p.IsSetWorkspaceID() { + return AsyncInvokeEvaluatorRequest_WorkspaceID_DEFAULT + } + return *p.WorkspaceID +} + +var AsyncInvokeEvaluatorRequest_InvokeID_DEFAULT int64 + +func (p *AsyncInvokeEvaluatorRequest) GetInvokeID() (v int64) { + if p == nil { + return + } + if !p.IsSetInvokeID() { + return AsyncInvokeEvaluatorRequest_InvokeID_DEFAULT + } + return *p.InvokeID +} + +var AsyncInvokeEvaluatorRequest_Evaluator_DEFAULT *InvokeCustomEvaluator + +func (p *AsyncInvokeEvaluatorRequest) GetEvaluator() (v *InvokeCustomEvaluator) { + if p == nil { + return + } + if !p.IsSetEvaluator() { + return AsyncInvokeEvaluatorRequest_Evaluator_DEFAULT + } + return p.Evaluator +} + +var AsyncInvokeEvaluatorRequest_InputData_DEFAULT *InvokeEvaluatorInputData + +func (p *AsyncInvokeEvaluatorRequest) GetInputData() (v *InvokeEvaluatorInputData) { + if p == nil { + return + } + if !p.IsSetInputData() { + return AsyncInvokeEvaluatorRequest_InputData_DEFAULT + } + return p.InputData +} + +var AsyncInvokeEvaluatorRequest_Base_DEFAULT *base.Base + +func (p *AsyncInvokeEvaluatorRequest) GetBase() (v *base.Base) { + if p == nil { + return + } + if !p.IsSetBase() { + return AsyncInvokeEvaluatorRequest_Base_DEFAULT + } + return p.Base +} +func (p *AsyncInvokeEvaluatorRequest) SetWorkspaceID(val *int64) { + p.WorkspaceID = val +} +func (p *AsyncInvokeEvaluatorRequest) SetInvokeID(val *int64) { + p.InvokeID = val +} +func (p *AsyncInvokeEvaluatorRequest) SetEvaluator(val *InvokeCustomEvaluator) { + p.Evaluator = val +} +func (p *AsyncInvokeEvaluatorRequest) SetInputData(val *InvokeEvaluatorInputData) { + p.InputData = val +} +func (p *AsyncInvokeEvaluatorRequest) SetBase(val *base.Base) { + p.Base = val +} + +var fieldIDToName_AsyncInvokeEvaluatorRequest = map[int16]string{ + 1: "workspace_id", + 2: "invoke_id", + 3: "evaluator", + 4: "input_data", + 255: "Base", +} + +func (p *AsyncInvokeEvaluatorRequest) IsSetWorkspaceID() bool { + return p.WorkspaceID != nil +} + +func (p *AsyncInvokeEvaluatorRequest) IsSetInvokeID() bool { + return p.InvokeID != nil +} + +func (p *AsyncInvokeEvaluatorRequest) IsSetEvaluator() bool { + return p.Evaluator != nil +} + +func (p *AsyncInvokeEvaluatorRequest) IsSetInputData() bool { + return p.InputData != nil +} + +func (p *AsyncInvokeEvaluatorRequest) IsSetBase() bool { + return p.Base != nil +} + +func (p *AsyncInvokeEvaluatorRequest) Read(iprot thrift.TProtocol) (err error) { + var fieldTypeId thrift.TType + var fieldId int16 + + if _, err = iprot.ReadStructBegin(); err != nil { + goto ReadStructBeginError + } + + for { + _, fieldTypeId, fieldId, err = iprot.ReadFieldBegin() + if err != nil { + goto ReadFieldBeginError + } + if fieldTypeId == thrift.STOP { + break + } + + switch fieldId { + case 1: + if fieldTypeId == thrift.I64 { + if err = p.ReadField1(iprot); err != nil { + goto ReadFieldError + } + } else if err = iprot.Skip(fieldTypeId); err != nil { + goto SkipFieldError + } + case 2: + if fieldTypeId == thrift.I64 { + if err = p.ReadField2(iprot); err != nil { + goto ReadFieldError + } + } else if err = iprot.Skip(fieldTypeId); err != nil { + goto SkipFieldError + } + case 3: + if fieldTypeId == thrift.STRUCT { + if err = p.ReadField3(iprot); err != nil { + goto ReadFieldError + } + } else if err = iprot.Skip(fieldTypeId); err != nil { + goto SkipFieldError + } + case 4: + if fieldTypeId == thrift.STRUCT { + if err = p.ReadField4(iprot); err != nil { + goto ReadFieldError + } + } else if err = iprot.Skip(fieldTypeId); err != nil { + goto SkipFieldError + } + case 255: + if fieldTypeId == thrift.STRUCT { + if err = p.ReadField255(iprot); err != nil { + goto ReadFieldError + } + } else if err = iprot.Skip(fieldTypeId); err != nil { + goto SkipFieldError + } + default: + if err = iprot.Skip(fieldTypeId); err != nil { + goto SkipFieldError + } + } + if err = iprot.ReadFieldEnd(); err != nil { + goto ReadFieldEndError + } + } + if err = iprot.ReadStructEnd(); err != nil { + goto ReadStructEndError + } + + return nil +ReadStructBeginError: + return thrift.PrependError(fmt.Sprintf("%T read struct begin error: ", p), err) +ReadFieldBeginError: + return thrift.PrependError(fmt.Sprintf("%T read field %d begin error: ", p, fieldId), err) +ReadFieldError: + return thrift.PrependError(fmt.Sprintf("%T read field %d '%s' error: ", p, fieldId, fieldIDToName_AsyncInvokeEvaluatorRequest[fieldId]), err) +SkipFieldError: + return thrift.PrependError(fmt.Sprintf("%T field %d skip type %d error: ", p, fieldId, fieldTypeId), err) + +ReadFieldEndError: + return thrift.PrependError(fmt.Sprintf("%T read field end error", p), err) +ReadStructEndError: + return thrift.PrependError(fmt.Sprintf("%T read struct end error: ", p), err) +} + +func (p *AsyncInvokeEvaluatorRequest) ReadField1(iprot thrift.TProtocol) error { + + var _field *int64 + if v, err := iprot.ReadI64(); err != nil { + return err + } else { + _field = &v + } + p.WorkspaceID = _field + return nil +} +func (p *AsyncInvokeEvaluatorRequest) ReadField2(iprot thrift.TProtocol) error { + + var _field *int64 + if v, err := iprot.ReadI64(); err != nil { + return err + } else { + _field = &v + } + p.InvokeID = _field + return nil +} +func (p *AsyncInvokeEvaluatorRequest) ReadField3(iprot thrift.TProtocol) error { + _field := NewInvokeCustomEvaluator() + if err := _field.Read(iprot); err != nil { + return err + } + p.Evaluator = _field + return nil +} +func (p *AsyncInvokeEvaluatorRequest) ReadField4(iprot thrift.TProtocol) error { + _field := NewInvokeEvaluatorInputData() + if err := _field.Read(iprot); err != nil { + return err + } + p.InputData = _field + return nil +} +func (p *AsyncInvokeEvaluatorRequest) ReadField255(iprot thrift.TProtocol) error { + _field := base.NewBase() + if err := _field.Read(iprot); err != nil { + return err + } + p.Base = _field + return nil +} + +func (p *AsyncInvokeEvaluatorRequest) Write(oprot thrift.TProtocol) (err error) { + var fieldId int16 + if err = oprot.WriteStructBegin("AsyncInvokeEvaluatorRequest"); err != nil { + goto WriteStructBeginError + } + if p != nil { + if err = p.writeField1(oprot); err != nil { + fieldId = 1 + goto WriteFieldError + } + if err = p.writeField2(oprot); err != nil { + fieldId = 2 + goto WriteFieldError + } + if err = p.writeField3(oprot); err != nil { + fieldId = 3 + goto WriteFieldError + } + if err = p.writeField4(oprot); err != nil { + fieldId = 4 + goto WriteFieldError + } + if err = p.writeField255(oprot); err != nil { + fieldId = 255 + goto WriteFieldError + } + } + if err = oprot.WriteFieldStop(); err != nil { + goto WriteFieldStopError + } + if err = oprot.WriteStructEnd(); err != nil { + goto WriteStructEndError + } + return nil +WriteStructBeginError: + return thrift.PrependError(fmt.Sprintf("%T write struct begin error: ", p), err) +WriteFieldError: + return thrift.PrependError(fmt.Sprintf("%T write field %d error: ", p, fieldId), err) +WriteFieldStopError: + return thrift.PrependError(fmt.Sprintf("%T write field stop error: ", p), err) +WriteStructEndError: + return thrift.PrependError(fmt.Sprintf("%T write struct end error: ", p), err) +} + +func (p *AsyncInvokeEvaluatorRequest) writeField1(oprot thrift.TProtocol) (err error) { + if p.IsSetWorkspaceID() { + if err = oprot.WriteFieldBegin("workspace_id", thrift.I64, 1); err != nil { + goto WriteFieldBeginError + } + if err := oprot.WriteI64(*p.WorkspaceID); err != nil { + return err + } + if err = oprot.WriteFieldEnd(); err != nil { + goto WriteFieldEndError + } + } + return nil +WriteFieldBeginError: + return thrift.PrependError(fmt.Sprintf("%T write field 1 begin error: ", p), err) +WriteFieldEndError: + return thrift.PrependError(fmt.Sprintf("%T write field 1 end error: ", p), err) +} +func (p *AsyncInvokeEvaluatorRequest) writeField2(oprot thrift.TProtocol) (err error) { + if p.IsSetInvokeID() { + if err = oprot.WriteFieldBegin("invoke_id", thrift.I64, 2); err != nil { + goto WriteFieldBeginError + } + if err := oprot.WriteI64(*p.InvokeID); err != nil { + return err + } + if err = oprot.WriteFieldEnd(); err != nil { + goto WriteFieldEndError + } + } + return nil +WriteFieldBeginError: + return thrift.PrependError(fmt.Sprintf("%T write field 2 begin error: ", p), err) +WriteFieldEndError: + return thrift.PrependError(fmt.Sprintf("%T write field 2 end error: ", p), err) +} +func (p *AsyncInvokeEvaluatorRequest) writeField3(oprot thrift.TProtocol) (err error) { + if p.IsSetEvaluator() { + if err = oprot.WriteFieldBegin("evaluator", thrift.STRUCT, 3); err != nil { + goto WriteFieldBeginError + } + if err := p.Evaluator.Write(oprot); err != nil { + return err + } + if err = oprot.WriteFieldEnd(); err != nil { + goto WriteFieldEndError + } + } + return nil +WriteFieldBeginError: + return thrift.PrependError(fmt.Sprintf("%T write field 3 begin error: ", p), err) +WriteFieldEndError: + return thrift.PrependError(fmt.Sprintf("%T write field 3 end error: ", p), err) +} +func (p *AsyncInvokeEvaluatorRequest) writeField4(oprot thrift.TProtocol) (err error) { + if p.IsSetInputData() { + if err = oprot.WriteFieldBegin("input_data", thrift.STRUCT, 4); err != nil { + goto WriteFieldBeginError + } + if err := p.InputData.Write(oprot); err != nil { + return err + } + if err = oprot.WriteFieldEnd(); err != nil { + goto WriteFieldEndError + } + } + return nil +WriteFieldBeginError: + return thrift.PrependError(fmt.Sprintf("%T write field 4 begin error: ", p), err) +WriteFieldEndError: + return thrift.PrependError(fmt.Sprintf("%T write field 4 end error: ", p), err) +} +func (p *AsyncInvokeEvaluatorRequest) writeField255(oprot thrift.TProtocol) (err error) { + if p.IsSetBase() { + if err = oprot.WriteFieldBegin("Base", thrift.STRUCT, 255); err != nil { + goto WriteFieldBeginError + } + if err := p.Base.Write(oprot); err != nil { + return err + } + if err = oprot.WriteFieldEnd(); err != nil { + goto WriteFieldEndError + } + } + return nil +WriteFieldBeginError: + return thrift.PrependError(fmt.Sprintf("%T write field 255 begin error: ", p), err) +WriteFieldEndError: + return thrift.PrependError(fmt.Sprintf("%T write field 255 end error: ", p), err) +} + +func (p *AsyncInvokeEvaluatorRequest) String() string { + if p == nil { + return "" + } + return fmt.Sprintf("AsyncInvokeEvaluatorRequest(%+v)", *p) + +} + +func (p *AsyncInvokeEvaluatorRequest) DeepEqual(ano *AsyncInvokeEvaluatorRequest) bool { + if p == ano { + return true + } else if p == nil || ano == nil { + return false + } + if !p.Field1DeepEqual(ano.WorkspaceID) { + return false + } + if !p.Field2DeepEqual(ano.InvokeID) { + return false + } + if !p.Field3DeepEqual(ano.Evaluator) { + return false + } + if !p.Field4DeepEqual(ano.InputData) { + return false + } + if !p.Field255DeepEqual(ano.Base) { + return false + } + return true +} + +func (p *AsyncInvokeEvaluatorRequest) Field1DeepEqual(src *int64) bool { + + if p.WorkspaceID == src { + return true + } else if p.WorkspaceID == nil || src == nil { + return false + } + if *p.WorkspaceID != *src { + return false + } + return true +} +func (p *AsyncInvokeEvaluatorRequest) Field2DeepEqual(src *int64) bool { + + if p.InvokeID == src { + return true + } else if p.InvokeID == nil || src == nil { + return false + } + if *p.InvokeID != *src { + return false + } + return true +} +func (p *AsyncInvokeEvaluatorRequest) Field3DeepEqual(src *InvokeCustomEvaluator) bool { + + if !p.Evaluator.DeepEqual(src) { + return false + } + return true +} +func (p *AsyncInvokeEvaluatorRequest) Field4DeepEqual(src *InvokeEvaluatorInputData) bool { + + if !p.InputData.DeepEqual(src) { + return false + } + return true +} +func (p *AsyncInvokeEvaluatorRequest) Field255DeepEqual(src *base.Base) bool { + + if !p.Base.DeepEqual(src) { + return false + } + return true +} + +type AsyncInvokeEvaluatorResponse struct { + BaseResp *base.BaseResp `thrift:"BaseResp,255" frugal:"255,default,base.BaseResp" form:"BaseResp" json:"BaseResp" query:"BaseResp"` +} + +func NewAsyncInvokeEvaluatorResponse() *AsyncInvokeEvaluatorResponse { + return &AsyncInvokeEvaluatorResponse{} +} + +func (p *AsyncInvokeEvaluatorResponse) InitDefault() { +} + +var AsyncInvokeEvaluatorResponse_BaseResp_DEFAULT *base.BaseResp + +func (p *AsyncInvokeEvaluatorResponse) GetBaseResp() (v *base.BaseResp) { + if p == nil { + return + } + if !p.IsSetBaseResp() { + return AsyncInvokeEvaluatorResponse_BaseResp_DEFAULT + } + return p.BaseResp +} +func (p *AsyncInvokeEvaluatorResponse) SetBaseResp(val *base.BaseResp) { + p.BaseResp = val +} + +var fieldIDToName_AsyncInvokeEvaluatorResponse = map[int16]string{ + 255: "BaseResp", +} + +func (p *AsyncInvokeEvaluatorResponse) IsSetBaseResp() bool { + return p.BaseResp != nil +} + +func (p *AsyncInvokeEvaluatorResponse) Read(iprot thrift.TProtocol) (err error) { + var fieldTypeId thrift.TType + var fieldId int16 + + if _, err = iprot.ReadStructBegin(); err != nil { + goto ReadStructBeginError + } + + for { + _, fieldTypeId, fieldId, err = iprot.ReadFieldBegin() + if err != nil { + goto ReadFieldBeginError + } + if fieldTypeId == thrift.STOP { + break + } + + switch fieldId { + case 255: + if fieldTypeId == thrift.STRUCT { + if err = p.ReadField255(iprot); err != nil { + goto ReadFieldError + } + } else if err = iprot.Skip(fieldTypeId); err != nil { + goto SkipFieldError + } + default: + if err = iprot.Skip(fieldTypeId); err != nil { + goto SkipFieldError + } + } + if err = iprot.ReadFieldEnd(); err != nil { + goto ReadFieldEndError + } + } + if err = iprot.ReadStructEnd(); err != nil { + goto ReadStructEndError + } + + return nil +ReadStructBeginError: + return thrift.PrependError(fmt.Sprintf("%T read struct begin error: ", p), err) +ReadFieldBeginError: + return thrift.PrependError(fmt.Sprintf("%T read field %d begin error: ", p, fieldId), err) +ReadFieldError: + return thrift.PrependError(fmt.Sprintf("%T read field %d '%s' error: ", p, fieldId, fieldIDToName_AsyncInvokeEvaluatorResponse[fieldId]), err) +SkipFieldError: + return thrift.PrependError(fmt.Sprintf("%T field %d skip type %d error: ", p, fieldId, fieldTypeId), err) + +ReadFieldEndError: + return thrift.PrependError(fmt.Sprintf("%T read field end error", p), err) +ReadStructEndError: + return thrift.PrependError(fmt.Sprintf("%T read struct end error: ", p), err) +} + +func (p *AsyncInvokeEvaluatorResponse) ReadField255(iprot thrift.TProtocol) error { + _field := base.NewBaseResp() + if err := _field.Read(iprot); err != nil { + return err + } + p.BaseResp = _field + return nil +} + +func (p *AsyncInvokeEvaluatorResponse) Write(oprot thrift.TProtocol) (err error) { + var fieldId int16 + if err = oprot.WriteStructBegin("AsyncInvokeEvaluatorResponse"); err != nil { + goto WriteStructBeginError + } + if p != nil { + if err = p.writeField255(oprot); err != nil { + fieldId = 255 + goto WriteFieldError + } + } + if err = oprot.WriteFieldStop(); err != nil { + goto WriteFieldStopError + } + if err = oprot.WriteStructEnd(); err != nil { + goto WriteStructEndError + } + return nil +WriteStructBeginError: + return thrift.PrependError(fmt.Sprintf("%T write struct begin error: ", p), err) +WriteFieldError: + return thrift.PrependError(fmt.Sprintf("%T write field %d error: ", p, fieldId), err) +WriteFieldStopError: + return thrift.PrependError(fmt.Sprintf("%T write field stop error: ", p), err) +WriteStructEndError: + return thrift.PrependError(fmt.Sprintf("%T write struct end error: ", p), err) +} + +func (p *AsyncInvokeEvaluatorResponse) writeField255(oprot thrift.TProtocol) (err error) { + if err = oprot.WriteFieldBegin("BaseResp", thrift.STRUCT, 255); err != nil { + goto WriteFieldBeginError + } + if err := p.BaseResp.Write(oprot); err != nil { + return err + } + if err = oprot.WriteFieldEnd(); err != nil { + goto WriteFieldEndError + } + return nil +WriteFieldBeginError: + return thrift.PrependError(fmt.Sprintf("%T write field 255 begin error: ", p), err) +WriteFieldEndError: + return thrift.PrependError(fmt.Sprintf("%T write field 255 end error: ", p), err) +} + +func (p *AsyncInvokeEvaluatorResponse) String() string { + if p == nil { + return "" + } + return fmt.Sprintf("AsyncInvokeEvaluatorResponse(%+v)", *p) + +} + +func (p *AsyncInvokeEvaluatorResponse) DeepEqual(ano *AsyncInvokeEvaluatorResponse) bool { + if p == ano { + return true + } else if p == nil || ano == nil { + return false + } + if !p.Field255DeepEqual(ano.BaseResp) { + return false + } + return true +} + +func (p *AsyncInvokeEvaluatorResponse) Field255DeepEqual(src *base.BaseResp) bool { + + if !p.BaseResp.DeepEqual(src) { + return false + } + return true +} + type EvaluationSPIService interface { SearchEvalTarget(ctx context.Context, req *SearchEvalTargetRequest) (r *SearchEvalTargetResponse, err error) @@ -7985,6 +8623,8 @@ type EvaluationSPIService interface { AsyncInvokeEvalTarget(ctx context.Context, req *AsyncInvokeEvalTargetRequest) (r *AsyncInvokeEvalTargetResponse, err error) // invoke custom evaluator InvokeEvaluator(ctx context.Context, req *InvokeEvaluatorRequest) (r *InvokeEvaluatorResponse, err error) + + AsyncInvokeEvaluator(ctx context.Context, req *AsyncInvokeEvaluatorRequest) (r *AsyncInvokeEvaluatorResponse, err error) } type EvaluationSPIServiceClient struct { @@ -8049,6 +8689,15 @@ func (p *EvaluationSPIServiceClient) InvokeEvaluator(ctx context.Context, req *I } return _result.GetSuccess(), nil } +func (p *EvaluationSPIServiceClient) AsyncInvokeEvaluator(ctx context.Context, req *AsyncInvokeEvaluatorRequest) (r *AsyncInvokeEvaluatorResponse, err error) { + var _args EvaluationSPIServiceAsyncInvokeEvaluatorArgs + _args.Req = req + var _result EvaluationSPIServiceAsyncInvokeEvaluatorResult + if err = p.Client_().Call(ctx, "AsyncInvokeEvaluator", &_args, &_result); err != nil { + return + } + return _result.GetSuccess(), nil +} type EvaluationSPIServiceProcessor struct { processorMap map[string]thrift.TProcessorFunction @@ -8074,6 +8723,7 @@ func NewEvaluationSPIServiceProcessor(handler EvaluationSPIService) *EvaluationS self.AddToProcessorMap("InvokeEvalTarget", &evaluationSPIServiceProcessorInvokeEvalTarget{handler: handler}) self.AddToProcessorMap("AsyncInvokeEvalTarget", &evaluationSPIServiceProcessorAsyncInvokeEvalTarget{handler: handler}) self.AddToProcessorMap("InvokeEvaluator", &evaluationSPIServiceProcessorInvokeEvaluator{handler: handler}) + self.AddToProcessorMap("AsyncInvokeEvaluator", &evaluationSPIServiceProcessorAsyncInvokeEvaluator{handler: handler}) return self } func (p *EvaluationSPIServiceProcessor) Process(ctx context.Context, iprot, oprot thrift.TProtocol) (success bool, err thrift.TException) { @@ -8081,29 +8731,173 @@ func (p *EvaluationSPIServiceProcessor) Process(ctx context.Context, iprot, opro if err != nil { return false, err } - if processor, ok := p.GetProcessorFunction(name); ok { - return processor.Process(ctx, seqId, iprot, oprot) + if processor, ok := p.GetProcessorFunction(name); ok { + return processor.Process(ctx, seqId, iprot, oprot) + } + iprot.Skip(thrift.STRUCT) + iprot.ReadMessageEnd() + x := thrift.NewTApplicationException(thrift.UNKNOWN_METHOD, "Unknown function "+name) + oprot.WriteMessageBegin(name, thrift.EXCEPTION, seqId) + x.Write(oprot) + oprot.WriteMessageEnd() + oprot.Flush(ctx) + return false, x +} + +type evaluationSPIServiceProcessorSearchEvalTarget struct { + handler EvaluationSPIService +} + +func (p *evaluationSPIServiceProcessorSearchEvalTarget) Process(ctx context.Context, seqId int32, iprot, oprot thrift.TProtocol) (success bool, err thrift.TException) { + args := EvaluationSPIServiceSearchEvalTargetArgs{} + if err = args.Read(iprot); err != nil { + iprot.ReadMessageEnd() + x := thrift.NewTApplicationException(thrift.PROTOCOL_ERROR, err.Error()) + oprot.WriteMessageBegin("SearchEvalTarget", thrift.EXCEPTION, seqId) + x.Write(oprot) + oprot.WriteMessageEnd() + oprot.Flush(ctx) + return false, err + } + + iprot.ReadMessageEnd() + var err2 error + result := EvaluationSPIServiceSearchEvalTargetResult{} + var retval *SearchEvalTargetResponse + if retval, err2 = p.handler.SearchEvalTarget(ctx, args.Req); err2 != nil { + x := thrift.NewTApplicationException(thrift.INTERNAL_ERROR, "Internal error processing SearchEvalTarget: "+err2.Error()) + oprot.WriteMessageBegin("SearchEvalTarget", thrift.EXCEPTION, seqId) + x.Write(oprot) + oprot.WriteMessageEnd() + oprot.Flush(ctx) + return true, err2 + } else { + result.Success = retval + } + if err2 = oprot.WriteMessageBegin("SearchEvalTarget", thrift.REPLY, seqId); err2 != nil { + err = err2 + } + if err2 = result.Write(oprot); err == nil && err2 != nil { + err = err2 + } + if err2 = oprot.WriteMessageEnd(); err == nil && err2 != nil { + err = err2 + } + if err2 = oprot.Flush(ctx); err == nil && err2 != nil { + err = err2 + } + if err != nil { + return + } + return true, err +} + +type evaluationSPIServiceProcessorInvokeEvalTarget struct { + handler EvaluationSPIService +} + +func (p *evaluationSPIServiceProcessorInvokeEvalTarget) Process(ctx context.Context, seqId int32, iprot, oprot thrift.TProtocol) (success bool, err thrift.TException) { + args := EvaluationSPIServiceInvokeEvalTargetArgs{} + if err = args.Read(iprot); err != nil { + iprot.ReadMessageEnd() + x := thrift.NewTApplicationException(thrift.PROTOCOL_ERROR, err.Error()) + oprot.WriteMessageBegin("InvokeEvalTarget", thrift.EXCEPTION, seqId) + x.Write(oprot) + oprot.WriteMessageEnd() + oprot.Flush(ctx) + return false, err + } + + iprot.ReadMessageEnd() + var err2 error + result := EvaluationSPIServiceInvokeEvalTargetResult{} + var retval *InvokeEvalTargetResponse + if retval, err2 = p.handler.InvokeEvalTarget(ctx, args.Req); err2 != nil { + x := thrift.NewTApplicationException(thrift.INTERNAL_ERROR, "Internal error processing InvokeEvalTarget: "+err2.Error()) + oprot.WriteMessageBegin("InvokeEvalTarget", thrift.EXCEPTION, seqId) + x.Write(oprot) + oprot.WriteMessageEnd() + oprot.Flush(ctx) + return true, err2 + } else { + result.Success = retval + } + if err2 = oprot.WriteMessageBegin("InvokeEvalTarget", thrift.REPLY, seqId); err2 != nil { + err = err2 + } + if err2 = result.Write(oprot); err == nil && err2 != nil { + err = err2 + } + if err2 = oprot.WriteMessageEnd(); err == nil && err2 != nil { + err = err2 + } + if err2 = oprot.Flush(ctx); err == nil && err2 != nil { + err = err2 + } + if err != nil { + return + } + return true, err +} + +type evaluationSPIServiceProcessorAsyncInvokeEvalTarget struct { + handler EvaluationSPIService +} + +func (p *evaluationSPIServiceProcessorAsyncInvokeEvalTarget) Process(ctx context.Context, seqId int32, iprot, oprot thrift.TProtocol) (success bool, err thrift.TException) { + args := EvaluationSPIServiceAsyncInvokeEvalTargetArgs{} + if err = args.Read(iprot); err != nil { + iprot.ReadMessageEnd() + x := thrift.NewTApplicationException(thrift.PROTOCOL_ERROR, err.Error()) + oprot.WriteMessageBegin("AsyncInvokeEvalTarget", thrift.EXCEPTION, seqId) + x.Write(oprot) + oprot.WriteMessageEnd() + oprot.Flush(ctx) + return false, err + } + + iprot.ReadMessageEnd() + var err2 error + result := EvaluationSPIServiceAsyncInvokeEvalTargetResult{} + var retval *AsyncInvokeEvalTargetResponse + if retval, err2 = p.handler.AsyncInvokeEvalTarget(ctx, args.Req); err2 != nil { + x := thrift.NewTApplicationException(thrift.INTERNAL_ERROR, "Internal error processing AsyncInvokeEvalTarget: "+err2.Error()) + oprot.WriteMessageBegin("AsyncInvokeEvalTarget", thrift.EXCEPTION, seqId) + x.Write(oprot) + oprot.WriteMessageEnd() + oprot.Flush(ctx) + return true, err2 + } else { + result.Success = retval + } + if err2 = oprot.WriteMessageBegin("AsyncInvokeEvalTarget", thrift.REPLY, seqId); err2 != nil { + err = err2 } - iprot.Skip(thrift.STRUCT) - iprot.ReadMessageEnd() - x := thrift.NewTApplicationException(thrift.UNKNOWN_METHOD, "Unknown function "+name) - oprot.WriteMessageBegin(name, thrift.EXCEPTION, seqId) - x.Write(oprot) - oprot.WriteMessageEnd() - oprot.Flush(ctx) - return false, x + if err2 = result.Write(oprot); err == nil && err2 != nil { + err = err2 + } + if err2 = oprot.WriteMessageEnd(); err == nil && err2 != nil { + err = err2 + } + if err2 = oprot.Flush(ctx); err == nil && err2 != nil { + err = err2 + } + if err != nil { + return + } + return true, err } -type evaluationSPIServiceProcessorSearchEvalTarget struct { +type evaluationSPIServiceProcessorInvokeEvaluator struct { handler EvaluationSPIService } -func (p *evaluationSPIServiceProcessorSearchEvalTarget) Process(ctx context.Context, seqId int32, iprot, oprot thrift.TProtocol) (success bool, err thrift.TException) { - args := EvaluationSPIServiceSearchEvalTargetArgs{} +func (p *evaluationSPIServiceProcessorInvokeEvaluator) Process(ctx context.Context, seqId int32, iprot, oprot thrift.TProtocol) (success bool, err thrift.TException) { + args := EvaluationSPIServiceInvokeEvaluatorArgs{} if err = args.Read(iprot); err != nil { iprot.ReadMessageEnd() x := thrift.NewTApplicationException(thrift.PROTOCOL_ERROR, err.Error()) - oprot.WriteMessageBegin("SearchEvalTarget", thrift.EXCEPTION, seqId) + oprot.WriteMessageBegin("InvokeEvaluator", thrift.EXCEPTION, seqId) x.Write(oprot) oprot.WriteMessageEnd() oprot.Flush(ctx) @@ -8112,11 +8906,11 @@ func (p *evaluationSPIServiceProcessorSearchEvalTarget) Process(ctx context.Cont iprot.ReadMessageEnd() var err2 error - result := EvaluationSPIServiceSearchEvalTargetResult{} - var retval *SearchEvalTargetResponse - if retval, err2 = p.handler.SearchEvalTarget(ctx, args.Req); err2 != nil { - x := thrift.NewTApplicationException(thrift.INTERNAL_ERROR, "Internal error processing SearchEvalTarget: "+err2.Error()) - oprot.WriteMessageBegin("SearchEvalTarget", thrift.EXCEPTION, seqId) + result := EvaluationSPIServiceInvokeEvaluatorResult{} + var retval *InvokeEvaluatorResponse + if retval, err2 = p.handler.InvokeEvaluator(ctx, args.Req); err2 != nil { + x := thrift.NewTApplicationException(thrift.INTERNAL_ERROR, "Internal error processing InvokeEvaluator: "+err2.Error()) + oprot.WriteMessageBegin("InvokeEvaluator", thrift.EXCEPTION, seqId) x.Write(oprot) oprot.WriteMessageEnd() oprot.Flush(ctx) @@ -8124,7 +8918,7 @@ func (p *evaluationSPIServiceProcessorSearchEvalTarget) Process(ctx context.Cont } else { result.Success = retval } - if err2 = oprot.WriteMessageBegin("SearchEvalTarget", thrift.REPLY, seqId); err2 != nil { + if err2 = oprot.WriteMessageBegin("InvokeEvaluator", thrift.REPLY, seqId); err2 != nil { err = err2 } if err2 = result.Write(oprot); err == nil && err2 != nil { @@ -8142,16 +8936,16 @@ func (p *evaluationSPIServiceProcessorSearchEvalTarget) Process(ctx context.Cont return true, err } -type evaluationSPIServiceProcessorInvokeEvalTarget struct { +type evaluationSPIServiceProcessorAsyncInvokeEvaluator struct { handler EvaluationSPIService } -func (p *evaluationSPIServiceProcessorInvokeEvalTarget) Process(ctx context.Context, seqId int32, iprot, oprot thrift.TProtocol) (success bool, err thrift.TException) { - args := EvaluationSPIServiceInvokeEvalTargetArgs{} +func (p *evaluationSPIServiceProcessorAsyncInvokeEvaluator) Process(ctx context.Context, seqId int32, iprot, oprot thrift.TProtocol) (success bool, err thrift.TException) { + args := EvaluationSPIServiceAsyncInvokeEvaluatorArgs{} if err = args.Read(iprot); err != nil { iprot.ReadMessageEnd() x := thrift.NewTApplicationException(thrift.PROTOCOL_ERROR, err.Error()) - oprot.WriteMessageBegin("InvokeEvalTarget", thrift.EXCEPTION, seqId) + oprot.WriteMessageBegin("AsyncInvokeEvaluator", thrift.EXCEPTION, seqId) x.Write(oprot) oprot.WriteMessageEnd() oprot.Flush(ctx) @@ -8160,11 +8954,11 @@ func (p *evaluationSPIServiceProcessorInvokeEvalTarget) Process(ctx context.Cont iprot.ReadMessageEnd() var err2 error - result := EvaluationSPIServiceInvokeEvalTargetResult{} - var retval *InvokeEvalTargetResponse - if retval, err2 = p.handler.InvokeEvalTarget(ctx, args.Req); err2 != nil { - x := thrift.NewTApplicationException(thrift.INTERNAL_ERROR, "Internal error processing InvokeEvalTarget: "+err2.Error()) - oprot.WriteMessageBegin("InvokeEvalTarget", thrift.EXCEPTION, seqId) + result := EvaluationSPIServiceAsyncInvokeEvaluatorResult{} + var retval *AsyncInvokeEvaluatorResponse + if retval, err2 = p.handler.AsyncInvokeEvaluator(ctx, args.Req); err2 != nil { + x := thrift.NewTApplicationException(thrift.INTERNAL_ERROR, "Internal error processing AsyncInvokeEvaluator: "+err2.Error()) + oprot.WriteMessageBegin("AsyncInvokeEvaluator", thrift.EXCEPTION, seqId) x.Write(oprot) oprot.WriteMessageEnd() oprot.Flush(ctx) @@ -8172,7 +8966,7 @@ func (p *evaluationSPIServiceProcessorInvokeEvalTarget) Process(ctx context.Cont } else { result.Success = retval } - if err2 = oprot.WriteMessageBegin("InvokeEvalTarget", thrift.REPLY, seqId); err2 != nil { + if err2 = oprot.WriteMessageBegin("AsyncInvokeEvaluator", thrift.REPLY, seqId); err2 != nil { err = err2 } if err2 = result.Write(oprot); err == nil && err2 != nil { @@ -8190,137 +8984,385 @@ func (p *evaluationSPIServiceProcessorInvokeEvalTarget) Process(ctx context.Cont return true, err } -type evaluationSPIServiceProcessorAsyncInvokeEvalTarget struct { - handler EvaluationSPIService +type EvaluationSPIServiceSearchEvalTargetArgs struct { + Req *SearchEvalTargetRequest `thrift:"req,1" frugal:"1,default,SearchEvalTargetRequest"` +} + +func NewEvaluationSPIServiceSearchEvalTargetArgs() *EvaluationSPIServiceSearchEvalTargetArgs { + return &EvaluationSPIServiceSearchEvalTargetArgs{} +} + +func (p *EvaluationSPIServiceSearchEvalTargetArgs) InitDefault() { +} + +var EvaluationSPIServiceSearchEvalTargetArgs_Req_DEFAULT *SearchEvalTargetRequest + +func (p *EvaluationSPIServiceSearchEvalTargetArgs) GetReq() (v *SearchEvalTargetRequest) { + if p == nil { + return + } + if !p.IsSetReq() { + return EvaluationSPIServiceSearchEvalTargetArgs_Req_DEFAULT + } + return p.Req +} +func (p *EvaluationSPIServiceSearchEvalTargetArgs) SetReq(val *SearchEvalTargetRequest) { + p.Req = val +} + +var fieldIDToName_EvaluationSPIServiceSearchEvalTargetArgs = map[int16]string{ + 1: "req", +} + +func (p *EvaluationSPIServiceSearchEvalTargetArgs) IsSetReq() bool { + return p.Req != nil +} + +func (p *EvaluationSPIServiceSearchEvalTargetArgs) Read(iprot thrift.TProtocol) (err error) { + var fieldTypeId thrift.TType + var fieldId int16 + + if _, err = iprot.ReadStructBegin(); err != nil { + goto ReadStructBeginError + } + + for { + _, fieldTypeId, fieldId, err = iprot.ReadFieldBegin() + if err != nil { + goto ReadFieldBeginError + } + if fieldTypeId == thrift.STOP { + break + } + + switch fieldId { + case 1: + if fieldTypeId == thrift.STRUCT { + if err = p.ReadField1(iprot); err != nil { + goto ReadFieldError + } + } else if err = iprot.Skip(fieldTypeId); err != nil { + goto SkipFieldError + } + default: + if err = iprot.Skip(fieldTypeId); err != nil { + goto SkipFieldError + } + } + if err = iprot.ReadFieldEnd(); err != nil { + goto ReadFieldEndError + } + } + if err = iprot.ReadStructEnd(); err != nil { + goto ReadStructEndError + } + + return nil +ReadStructBeginError: + return thrift.PrependError(fmt.Sprintf("%T read struct begin error: ", p), err) +ReadFieldBeginError: + return thrift.PrependError(fmt.Sprintf("%T read field %d begin error: ", p, fieldId), err) +ReadFieldError: + return thrift.PrependError(fmt.Sprintf("%T read field %d '%s' error: ", p, fieldId, fieldIDToName_EvaluationSPIServiceSearchEvalTargetArgs[fieldId]), err) +SkipFieldError: + return thrift.PrependError(fmt.Sprintf("%T field %d skip type %d error: ", p, fieldId, fieldTypeId), err) + +ReadFieldEndError: + return thrift.PrependError(fmt.Sprintf("%T read field end error", p), err) +ReadStructEndError: + return thrift.PrependError(fmt.Sprintf("%T read struct end error: ", p), err) +} + +func (p *EvaluationSPIServiceSearchEvalTargetArgs) ReadField1(iprot thrift.TProtocol) error { + _field := NewSearchEvalTargetRequest() + if err := _field.Read(iprot); err != nil { + return err + } + p.Req = _field + return nil +} + +func (p *EvaluationSPIServiceSearchEvalTargetArgs) Write(oprot thrift.TProtocol) (err error) { + var fieldId int16 + if err = oprot.WriteStructBegin("SearchEvalTarget_args"); err != nil { + goto WriteStructBeginError + } + if p != nil { + if err = p.writeField1(oprot); err != nil { + fieldId = 1 + goto WriteFieldError + } + } + if err = oprot.WriteFieldStop(); err != nil { + goto WriteFieldStopError + } + if err = oprot.WriteStructEnd(); err != nil { + goto WriteStructEndError + } + return nil +WriteStructBeginError: + return thrift.PrependError(fmt.Sprintf("%T write struct begin error: ", p), err) +WriteFieldError: + return thrift.PrependError(fmt.Sprintf("%T write field %d error: ", p, fieldId), err) +WriteFieldStopError: + return thrift.PrependError(fmt.Sprintf("%T write field stop error: ", p), err) +WriteStructEndError: + return thrift.PrependError(fmt.Sprintf("%T write struct end error: ", p), err) +} + +func (p *EvaluationSPIServiceSearchEvalTargetArgs) writeField1(oprot thrift.TProtocol) (err error) { + if err = oprot.WriteFieldBegin("req", thrift.STRUCT, 1); err != nil { + goto WriteFieldBeginError + } + if err := p.Req.Write(oprot); err != nil { + return err + } + if err = oprot.WriteFieldEnd(); err != nil { + goto WriteFieldEndError + } + return nil +WriteFieldBeginError: + return thrift.PrependError(fmt.Sprintf("%T write field 1 begin error: ", p), err) +WriteFieldEndError: + return thrift.PrependError(fmt.Sprintf("%T write field 1 end error: ", p), err) +} + +func (p *EvaluationSPIServiceSearchEvalTargetArgs) String() string { + if p == nil { + return "" + } + return fmt.Sprintf("EvaluationSPIServiceSearchEvalTargetArgs(%+v)", *p) + +} + +func (p *EvaluationSPIServiceSearchEvalTargetArgs) DeepEqual(ano *EvaluationSPIServiceSearchEvalTargetArgs) bool { + if p == ano { + return true + } else if p == nil || ano == nil { + return false + } + if !p.Field1DeepEqual(ano.Req) { + return false + } + return true +} + +func (p *EvaluationSPIServiceSearchEvalTargetArgs) Field1DeepEqual(src *SearchEvalTargetRequest) bool { + + if !p.Req.DeepEqual(src) { + return false + } + return true +} + +type EvaluationSPIServiceSearchEvalTargetResult struct { + Success *SearchEvalTargetResponse `thrift:"success,0,optional" frugal:"0,optional,SearchEvalTargetResponse"` +} + +func NewEvaluationSPIServiceSearchEvalTargetResult() *EvaluationSPIServiceSearchEvalTargetResult { + return &EvaluationSPIServiceSearchEvalTargetResult{} +} + +func (p *EvaluationSPIServiceSearchEvalTargetResult) InitDefault() { +} + +var EvaluationSPIServiceSearchEvalTargetResult_Success_DEFAULT *SearchEvalTargetResponse + +func (p *EvaluationSPIServiceSearchEvalTargetResult) GetSuccess() (v *SearchEvalTargetResponse) { + if p == nil { + return + } + if !p.IsSetSuccess() { + return EvaluationSPIServiceSearchEvalTargetResult_Success_DEFAULT + } + return p.Success +} +func (p *EvaluationSPIServiceSearchEvalTargetResult) SetSuccess(x interface{}) { + p.Success = x.(*SearchEvalTargetResponse) +} + +var fieldIDToName_EvaluationSPIServiceSearchEvalTargetResult = map[int16]string{ + 0: "success", +} + +func (p *EvaluationSPIServiceSearchEvalTargetResult) IsSetSuccess() bool { + return p.Success != nil } -func (p *evaluationSPIServiceProcessorAsyncInvokeEvalTarget) Process(ctx context.Context, seqId int32, iprot, oprot thrift.TProtocol) (success bool, err thrift.TException) { - args := EvaluationSPIServiceAsyncInvokeEvalTargetArgs{} - if err = args.Read(iprot); err != nil { - iprot.ReadMessageEnd() - x := thrift.NewTApplicationException(thrift.PROTOCOL_ERROR, err.Error()) - oprot.WriteMessageBegin("AsyncInvokeEvalTarget", thrift.EXCEPTION, seqId) - x.Write(oprot) - oprot.WriteMessageEnd() - oprot.Flush(ctx) - return false, err +func (p *EvaluationSPIServiceSearchEvalTargetResult) Read(iprot thrift.TProtocol) (err error) { + var fieldTypeId thrift.TType + var fieldId int16 + + if _, err = iprot.ReadStructBegin(); err != nil { + goto ReadStructBeginError } - iprot.ReadMessageEnd() - var err2 error - result := EvaluationSPIServiceAsyncInvokeEvalTargetResult{} - var retval *AsyncInvokeEvalTargetResponse - if retval, err2 = p.handler.AsyncInvokeEvalTarget(ctx, args.Req); err2 != nil { - x := thrift.NewTApplicationException(thrift.INTERNAL_ERROR, "Internal error processing AsyncInvokeEvalTarget: "+err2.Error()) - oprot.WriteMessageBegin("AsyncInvokeEvalTarget", thrift.EXCEPTION, seqId) - x.Write(oprot) - oprot.WriteMessageEnd() - oprot.Flush(ctx) - return true, err2 - } else { - result.Success = retval + for { + _, fieldTypeId, fieldId, err = iprot.ReadFieldBegin() + if err != nil { + goto ReadFieldBeginError + } + if fieldTypeId == thrift.STOP { + break + } + + switch fieldId { + case 0: + if fieldTypeId == thrift.STRUCT { + if err = p.ReadField0(iprot); err != nil { + goto ReadFieldError + } + } else if err = iprot.Skip(fieldTypeId); err != nil { + goto SkipFieldError + } + default: + if err = iprot.Skip(fieldTypeId); err != nil { + goto SkipFieldError + } + } + if err = iprot.ReadFieldEnd(); err != nil { + goto ReadFieldEndError + } } - if err2 = oprot.WriteMessageBegin("AsyncInvokeEvalTarget", thrift.REPLY, seqId); err2 != nil { - err = err2 + if err = iprot.ReadStructEnd(); err != nil { + goto ReadStructEndError } - if err2 = result.Write(oprot); err == nil && err2 != nil { - err = err2 + + return nil +ReadStructBeginError: + return thrift.PrependError(fmt.Sprintf("%T read struct begin error: ", p), err) +ReadFieldBeginError: + return thrift.PrependError(fmt.Sprintf("%T read field %d begin error: ", p, fieldId), err) +ReadFieldError: + return thrift.PrependError(fmt.Sprintf("%T read field %d '%s' error: ", p, fieldId, fieldIDToName_EvaluationSPIServiceSearchEvalTargetResult[fieldId]), err) +SkipFieldError: + return thrift.PrependError(fmt.Sprintf("%T field %d skip type %d error: ", p, fieldId, fieldTypeId), err) + +ReadFieldEndError: + return thrift.PrependError(fmt.Sprintf("%T read field end error", p), err) +ReadStructEndError: + return thrift.PrependError(fmt.Sprintf("%T read struct end error: ", p), err) +} + +func (p *EvaluationSPIServiceSearchEvalTargetResult) ReadField0(iprot thrift.TProtocol) error { + _field := NewSearchEvalTargetResponse() + if err := _field.Read(iprot); err != nil { + return err } - if err2 = oprot.WriteMessageEnd(); err == nil && err2 != nil { - err = err2 + p.Success = _field + return nil +} + +func (p *EvaluationSPIServiceSearchEvalTargetResult) Write(oprot thrift.TProtocol) (err error) { + var fieldId int16 + if err = oprot.WriteStructBegin("SearchEvalTarget_result"); err != nil { + goto WriteStructBeginError } - if err2 = oprot.Flush(ctx); err == nil && err2 != nil { - err = err2 + if p != nil { + if err = p.writeField0(oprot); err != nil { + fieldId = 0 + goto WriteFieldError + } } - if err != nil { - return + if err = oprot.WriteFieldStop(); err != nil { + goto WriteFieldStopError } - return true, err + if err = oprot.WriteStructEnd(); err != nil { + goto WriteStructEndError + } + return nil +WriteStructBeginError: + return thrift.PrependError(fmt.Sprintf("%T write struct begin error: ", p), err) +WriteFieldError: + return thrift.PrependError(fmt.Sprintf("%T write field %d error: ", p, fieldId), err) +WriteFieldStopError: + return thrift.PrependError(fmt.Sprintf("%T write field stop error: ", p), err) +WriteStructEndError: + return thrift.PrependError(fmt.Sprintf("%T write struct end error: ", p), err) } -type evaluationSPIServiceProcessorInvokeEvaluator struct { - handler EvaluationSPIService +func (p *EvaluationSPIServiceSearchEvalTargetResult) writeField0(oprot thrift.TProtocol) (err error) { + if p.IsSetSuccess() { + if err = oprot.WriteFieldBegin("success", thrift.STRUCT, 0); err != nil { + goto WriteFieldBeginError + } + if err := p.Success.Write(oprot); err != nil { + return err + } + if err = oprot.WriteFieldEnd(); err != nil { + goto WriteFieldEndError + } + } + return nil +WriteFieldBeginError: + return thrift.PrependError(fmt.Sprintf("%T write field 0 begin error: ", p), err) +WriteFieldEndError: + return thrift.PrependError(fmt.Sprintf("%T write field 0 end error: ", p), err) } -func (p *evaluationSPIServiceProcessorInvokeEvaluator) Process(ctx context.Context, seqId int32, iprot, oprot thrift.TProtocol) (success bool, err thrift.TException) { - args := EvaluationSPIServiceInvokeEvaluatorArgs{} - if err = args.Read(iprot); err != nil { - iprot.ReadMessageEnd() - x := thrift.NewTApplicationException(thrift.PROTOCOL_ERROR, err.Error()) - oprot.WriteMessageBegin("InvokeEvaluator", thrift.EXCEPTION, seqId) - x.Write(oprot) - oprot.WriteMessageEnd() - oprot.Flush(ctx) - return false, err +func (p *EvaluationSPIServiceSearchEvalTargetResult) String() string { + if p == nil { + return "" } + return fmt.Sprintf("EvaluationSPIServiceSearchEvalTargetResult(%+v)", *p) - iprot.ReadMessageEnd() - var err2 error - result := EvaluationSPIServiceInvokeEvaluatorResult{} - var retval *InvokeEvaluatorResponse - if retval, err2 = p.handler.InvokeEvaluator(ctx, args.Req); err2 != nil { - x := thrift.NewTApplicationException(thrift.INTERNAL_ERROR, "Internal error processing InvokeEvaluator: "+err2.Error()) - oprot.WriteMessageBegin("InvokeEvaluator", thrift.EXCEPTION, seqId) - x.Write(oprot) - oprot.WriteMessageEnd() - oprot.Flush(ctx) - return true, err2 - } else { - result.Success = retval - } - if err2 = oprot.WriteMessageBegin("InvokeEvaluator", thrift.REPLY, seqId); err2 != nil { - err = err2 - } - if err2 = result.Write(oprot); err == nil && err2 != nil { - err = err2 - } - if err2 = oprot.WriteMessageEnd(); err == nil && err2 != nil { - err = err2 +} + +func (p *EvaluationSPIServiceSearchEvalTargetResult) DeepEqual(ano *EvaluationSPIServiceSearchEvalTargetResult) bool { + if p == ano { + return true + } else if p == nil || ano == nil { + return false } - if err2 = oprot.Flush(ctx); err == nil && err2 != nil { - err = err2 + if !p.Field0DeepEqual(ano.Success) { + return false } - if err != nil { - return + return true +} + +func (p *EvaluationSPIServiceSearchEvalTargetResult) Field0DeepEqual(src *SearchEvalTargetResponse) bool { + + if !p.Success.DeepEqual(src) { + return false } - return true, err + return true } -type EvaluationSPIServiceSearchEvalTargetArgs struct { - Req *SearchEvalTargetRequest `thrift:"req,1" frugal:"1,default,SearchEvalTargetRequest"` +type EvaluationSPIServiceInvokeEvalTargetArgs struct { + Req *InvokeEvalTargetRequest `thrift:"req,1" frugal:"1,default,InvokeEvalTargetRequest"` } -func NewEvaluationSPIServiceSearchEvalTargetArgs() *EvaluationSPIServiceSearchEvalTargetArgs { - return &EvaluationSPIServiceSearchEvalTargetArgs{} +func NewEvaluationSPIServiceInvokeEvalTargetArgs() *EvaluationSPIServiceInvokeEvalTargetArgs { + return &EvaluationSPIServiceInvokeEvalTargetArgs{} } -func (p *EvaluationSPIServiceSearchEvalTargetArgs) InitDefault() { +func (p *EvaluationSPIServiceInvokeEvalTargetArgs) InitDefault() { } -var EvaluationSPIServiceSearchEvalTargetArgs_Req_DEFAULT *SearchEvalTargetRequest +var EvaluationSPIServiceInvokeEvalTargetArgs_Req_DEFAULT *InvokeEvalTargetRequest -func (p *EvaluationSPIServiceSearchEvalTargetArgs) GetReq() (v *SearchEvalTargetRequest) { +func (p *EvaluationSPIServiceInvokeEvalTargetArgs) GetReq() (v *InvokeEvalTargetRequest) { if p == nil { return } if !p.IsSetReq() { - return EvaluationSPIServiceSearchEvalTargetArgs_Req_DEFAULT + return EvaluationSPIServiceInvokeEvalTargetArgs_Req_DEFAULT } return p.Req } -func (p *EvaluationSPIServiceSearchEvalTargetArgs) SetReq(val *SearchEvalTargetRequest) { +func (p *EvaluationSPIServiceInvokeEvalTargetArgs) SetReq(val *InvokeEvalTargetRequest) { p.Req = val } -var fieldIDToName_EvaluationSPIServiceSearchEvalTargetArgs = map[int16]string{ +var fieldIDToName_EvaluationSPIServiceInvokeEvalTargetArgs = map[int16]string{ 1: "req", } -func (p *EvaluationSPIServiceSearchEvalTargetArgs) IsSetReq() bool { +func (p *EvaluationSPIServiceInvokeEvalTargetArgs) IsSetReq() bool { return p.Req != nil } -func (p *EvaluationSPIServiceSearchEvalTargetArgs) Read(iprot thrift.TProtocol) (err error) { +func (p *EvaluationSPIServiceInvokeEvalTargetArgs) Read(iprot thrift.TProtocol) (err error) { var fieldTypeId thrift.TType var fieldId int16 @@ -8365,7 +9407,7 @@ ReadStructBeginError: ReadFieldBeginError: return thrift.PrependError(fmt.Sprintf("%T read field %d begin error: ", p, fieldId), err) ReadFieldError: - return thrift.PrependError(fmt.Sprintf("%T read field %d '%s' error: ", p, fieldId, fieldIDToName_EvaluationSPIServiceSearchEvalTargetArgs[fieldId]), err) + return thrift.PrependError(fmt.Sprintf("%T read field %d '%s' error: ", p, fieldId, fieldIDToName_EvaluationSPIServiceInvokeEvalTargetArgs[fieldId]), err) SkipFieldError: return thrift.PrependError(fmt.Sprintf("%T field %d skip type %d error: ", p, fieldId, fieldTypeId), err) @@ -8375,8 +9417,8 @@ ReadStructEndError: return thrift.PrependError(fmt.Sprintf("%T read struct end error: ", p), err) } -func (p *EvaluationSPIServiceSearchEvalTargetArgs) ReadField1(iprot thrift.TProtocol) error { - _field := NewSearchEvalTargetRequest() +func (p *EvaluationSPIServiceInvokeEvalTargetArgs) ReadField1(iprot thrift.TProtocol) error { + _field := NewInvokeEvalTargetRequest() if err := _field.Read(iprot); err != nil { return err } @@ -8384,9 +9426,9 @@ func (p *EvaluationSPIServiceSearchEvalTargetArgs) ReadField1(iprot thrift.TProt return nil } -func (p *EvaluationSPIServiceSearchEvalTargetArgs) Write(oprot thrift.TProtocol) (err error) { +func (p *EvaluationSPIServiceInvokeEvalTargetArgs) Write(oprot thrift.TProtocol) (err error) { var fieldId int16 - if err = oprot.WriteStructBegin("SearchEvalTarget_args"); err != nil { + if err = oprot.WriteStructBegin("InvokeEvalTarget_args"); err != nil { goto WriteStructBeginError } if p != nil { @@ -8412,7 +9454,7 @@ WriteStructEndError: return thrift.PrependError(fmt.Sprintf("%T write struct end error: ", p), err) } -func (p *EvaluationSPIServiceSearchEvalTargetArgs) writeField1(oprot thrift.TProtocol) (err error) { +func (p *EvaluationSPIServiceInvokeEvalTargetArgs) writeField1(oprot thrift.TProtocol) (err error) { if err = oprot.WriteFieldBegin("req", thrift.STRUCT, 1); err != nil { goto WriteFieldBeginError } @@ -8429,15 +9471,15 @@ WriteFieldEndError: return thrift.PrependError(fmt.Sprintf("%T write field 1 end error: ", p), err) } -func (p *EvaluationSPIServiceSearchEvalTargetArgs) String() string { +func (p *EvaluationSPIServiceInvokeEvalTargetArgs) String() string { if p == nil { return "" } - return fmt.Sprintf("EvaluationSPIServiceSearchEvalTargetArgs(%+v)", *p) + return fmt.Sprintf("EvaluationSPIServiceInvokeEvalTargetArgs(%+v)", *p) } -func (p *EvaluationSPIServiceSearchEvalTargetArgs) DeepEqual(ano *EvaluationSPIServiceSearchEvalTargetArgs) bool { +func (p *EvaluationSPIServiceInvokeEvalTargetArgs) DeepEqual(ano *EvaluationSPIServiceInvokeEvalTargetArgs) bool { if p == ano { return true } else if p == nil || ano == nil { @@ -8449,7 +9491,7 @@ func (p *EvaluationSPIServiceSearchEvalTargetArgs) DeepEqual(ano *EvaluationSPIS return true } -func (p *EvaluationSPIServiceSearchEvalTargetArgs) Field1DeepEqual(src *SearchEvalTargetRequest) bool { +func (p *EvaluationSPIServiceInvokeEvalTargetArgs) Field1DeepEqual(src *InvokeEvalTargetRequest) bool { if !p.Req.DeepEqual(src) { return false @@ -8457,41 +9499,41 @@ func (p *EvaluationSPIServiceSearchEvalTargetArgs) Field1DeepEqual(src *SearchEv return true } -type EvaluationSPIServiceSearchEvalTargetResult struct { - Success *SearchEvalTargetResponse `thrift:"success,0,optional" frugal:"0,optional,SearchEvalTargetResponse"` +type EvaluationSPIServiceInvokeEvalTargetResult struct { + Success *InvokeEvalTargetResponse `thrift:"success,0,optional" frugal:"0,optional,InvokeEvalTargetResponse"` } -func NewEvaluationSPIServiceSearchEvalTargetResult() *EvaluationSPIServiceSearchEvalTargetResult { - return &EvaluationSPIServiceSearchEvalTargetResult{} +func NewEvaluationSPIServiceInvokeEvalTargetResult() *EvaluationSPIServiceInvokeEvalTargetResult { + return &EvaluationSPIServiceInvokeEvalTargetResult{} } -func (p *EvaluationSPIServiceSearchEvalTargetResult) InitDefault() { +func (p *EvaluationSPIServiceInvokeEvalTargetResult) InitDefault() { } -var EvaluationSPIServiceSearchEvalTargetResult_Success_DEFAULT *SearchEvalTargetResponse +var EvaluationSPIServiceInvokeEvalTargetResult_Success_DEFAULT *InvokeEvalTargetResponse -func (p *EvaluationSPIServiceSearchEvalTargetResult) GetSuccess() (v *SearchEvalTargetResponse) { +func (p *EvaluationSPIServiceInvokeEvalTargetResult) GetSuccess() (v *InvokeEvalTargetResponse) { if p == nil { return } if !p.IsSetSuccess() { - return EvaluationSPIServiceSearchEvalTargetResult_Success_DEFAULT + return EvaluationSPIServiceInvokeEvalTargetResult_Success_DEFAULT } return p.Success } -func (p *EvaluationSPIServiceSearchEvalTargetResult) SetSuccess(x interface{}) { - p.Success = x.(*SearchEvalTargetResponse) +func (p *EvaluationSPIServiceInvokeEvalTargetResult) SetSuccess(x interface{}) { + p.Success = x.(*InvokeEvalTargetResponse) } -var fieldIDToName_EvaluationSPIServiceSearchEvalTargetResult = map[int16]string{ +var fieldIDToName_EvaluationSPIServiceInvokeEvalTargetResult = map[int16]string{ 0: "success", } -func (p *EvaluationSPIServiceSearchEvalTargetResult) IsSetSuccess() bool { +func (p *EvaluationSPIServiceInvokeEvalTargetResult) IsSetSuccess() bool { return p.Success != nil } -func (p *EvaluationSPIServiceSearchEvalTargetResult) Read(iprot thrift.TProtocol) (err error) { +func (p *EvaluationSPIServiceInvokeEvalTargetResult) Read(iprot thrift.TProtocol) (err error) { var fieldTypeId thrift.TType var fieldId int16 @@ -8536,7 +9578,7 @@ ReadStructBeginError: ReadFieldBeginError: return thrift.PrependError(fmt.Sprintf("%T read field %d begin error: ", p, fieldId), err) ReadFieldError: - return thrift.PrependError(fmt.Sprintf("%T read field %d '%s' error: ", p, fieldId, fieldIDToName_EvaluationSPIServiceSearchEvalTargetResult[fieldId]), err) + return thrift.PrependError(fmt.Sprintf("%T read field %d '%s' error: ", p, fieldId, fieldIDToName_EvaluationSPIServiceInvokeEvalTargetResult[fieldId]), err) SkipFieldError: return thrift.PrependError(fmt.Sprintf("%T field %d skip type %d error: ", p, fieldId, fieldTypeId), err) @@ -8546,8 +9588,8 @@ ReadStructEndError: return thrift.PrependError(fmt.Sprintf("%T read struct end error: ", p), err) } -func (p *EvaluationSPIServiceSearchEvalTargetResult) ReadField0(iprot thrift.TProtocol) error { - _field := NewSearchEvalTargetResponse() +func (p *EvaluationSPIServiceInvokeEvalTargetResult) ReadField0(iprot thrift.TProtocol) error { + _field := NewInvokeEvalTargetResponse() if err := _field.Read(iprot); err != nil { return err } @@ -8555,9 +9597,9 @@ func (p *EvaluationSPIServiceSearchEvalTargetResult) ReadField0(iprot thrift.TPr return nil } -func (p *EvaluationSPIServiceSearchEvalTargetResult) Write(oprot thrift.TProtocol) (err error) { +func (p *EvaluationSPIServiceInvokeEvalTargetResult) Write(oprot thrift.TProtocol) (err error) { var fieldId int16 - if err = oprot.WriteStructBegin("SearchEvalTarget_result"); err != nil { + if err = oprot.WriteStructBegin("InvokeEvalTarget_result"); err != nil { goto WriteStructBeginError } if p != nil { @@ -8583,7 +9625,7 @@ WriteStructEndError: return thrift.PrependError(fmt.Sprintf("%T write struct end error: ", p), err) } -func (p *EvaluationSPIServiceSearchEvalTargetResult) writeField0(oprot thrift.TProtocol) (err error) { +func (p *EvaluationSPIServiceInvokeEvalTargetResult) writeField0(oprot thrift.TProtocol) (err error) { if p.IsSetSuccess() { if err = oprot.WriteFieldBegin("success", thrift.STRUCT, 0); err != nil { goto WriteFieldBeginError @@ -8602,15 +9644,15 @@ WriteFieldEndError: return thrift.PrependError(fmt.Sprintf("%T write field 0 end error: ", p), err) } -func (p *EvaluationSPIServiceSearchEvalTargetResult) String() string { +func (p *EvaluationSPIServiceInvokeEvalTargetResult) String() string { if p == nil { return "" } - return fmt.Sprintf("EvaluationSPIServiceSearchEvalTargetResult(%+v)", *p) + return fmt.Sprintf("EvaluationSPIServiceInvokeEvalTargetResult(%+v)", *p) } -func (p *EvaluationSPIServiceSearchEvalTargetResult) DeepEqual(ano *EvaluationSPIServiceSearchEvalTargetResult) bool { +func (p *EvaluationSPIServiceInvokeEvalTargetResult) DeepEqual(ano *EvaluationSPIServiceInvokeEvalTargetResult) bool { if p == ano { return true } else if p == nil || ano == nil { @@ -8622,7 +9664,7 @@ func (p *EvaluationSPIServiceSearchEvalTargetResult) DeepEqual(ano *EvaluationSP return true } -func (p *EvaluationSPIServiceSearchEvalTargetResult) Field0DeepEqual(src *SearchEvalTargetResponse) bool { +func (p *EvaluationSPIServiceInvokeEvalTargetResult) Field0DeepEqual(src *InvokeEvalTargetResponse) bool { if !p.Success.DeepEqual(src) { return false @@ -8630,41 +9672,41 @@ func (p *EvaluationSPIServiceSearchEvalTargetResult) Field0DeepEqual(src *Search return true } -type EvaluationSPIServiceInvokeEvalTargetArgs struct { - Req *InvokeEvalTargetRequest `thrift:"req,1" frugal:"1,default,InvokeEvalTargetRequest"` +type EvaluationSPIServiceAsyncInvokeEvalTargetArgs struct { + Req *AsyncInvokeEvalTargetRequest `thrift:"req,1" frugal:"1,default,AsyncInvokeEvalTargetRequest"` } -func NewEvaluationSPIServiceInvokeEvalTargetArgs() *EvaluationSPIServiceInvokeEvalTargetArgs { - return &EvaluationSPIServiceInvokeEvalTargetArgs{} +func NewEvaluationSPIServiceAsyncInvokeEvalTargetArgs() *EvaluationSPIServiceAsyncInvokeEvalTargetArgs { + return &EvaluationSPIServiceAsyncInvokeEvalTargetArgs{} } -func (p *EvaluationSPIServiceInvokeEvalTargetArgs) InitDefault() { +func (p *EvaluationSPIServiceAsyncInvokeEvalTargetArgs) InitDefault() { } -var EvaluationSPIServiceInvokeEvalTargetArgs_Req_DEFAULT *InvokeEvalTargetRequest +var EvaluationSPIServiceAsyncInvokeEvalTargetArgs_Req_DEFAULT *AsyncInvokeEvalTargetRequest -func (p *EvaluationSPIServiceInvokeEvalTargetArgs) GetReq() (v *InvokeEvalTargetRequest) { +func (p *EvaluationSPIServiceAsyncInvokeEvalTargetArgs) GetReq() (v *AsyncInvokeEvalTargetRequest) { if p == nil { return } if !p.IsSetReq() { - return EvaluationSPIServiceInvokeEvalTargetArgs_Req_DEFAULT + return EvaluationSPIServiceAsyncInvokeEvalTargetArgs_Req_DEFAULT } return p.Req } -func (p *EvaluationSPIServiceInvokeEvalTargetArgs) SetReq(val *InvokeEvalTargetRequest) { +func (p *EvaluationSPIServiceAsyncInvokeEvalTargetArgs) SetReq(val *AsyncInvokeEvalTargetRequest) { p.Req = val } -var fieldIDToName_EvaluationSPIServiceInvokeEvalTargetArgs = map[int16]string{ +var fieldIDToName_EvaluationSPIServiceAsyncInvokeEvalTargetArgs = map[int16]string{ 1: "req", } -func (p *EvaluationSPIServiceInvokeEvalTargetArgs) IsSetReq() bool { +func (p *EvaluationSPIServiceAsyncInvokeEvalTargetArgs) IsSetReq() bool { return p.Req != nil } -func (p *EvaluationSPIServiceInvokeEvalTargetArgs) Read(iprot thrift.TProtocol) (err error) { +func (p *EvaluationSPIServiceAsyncInvokeEvalTargetArgs) Read(iprot thrift.TProtocol) (err error) { var fieldTypeId thrift.TType var fieldId int16 @@ -8709,7 +9751,7 @@ ReadStructBeginError: ReadFieldBeginError: return thrift.PrependError(fmt.Sprintf("%T read field %d begin error: ", p, fieldId), err) ReadFieldError: - return thrift.PrependError(fmt.Sprintf("%T read field %d '%s' error: ", p, fieldId, fieldIDToName_EvaluationSPIServiceInvokeEvalTargetArgs[fieldId]), err) + return thrift.PrependError(fmt.Sprintf("%T read field %d '%s' error: ", p, fieldId, fieldIDToName_EvaluationSPIServiceAsyncInvokeEvalTargetArgs[fieldId]), err) SkipFieldError: return thrift.PrependError(fmt.Sprintf("%T field %d skip type %d error: ", p, fieldId, fieldTypeId), err) @@ -8719,8 +9761,8 @@ ReadStructEndError: return thrift.PrependError(fmt.Sprintf("%T read struct end error: ", p), err) } -func (p *EvaluationSPIServiceInvokeEvalTargetArgs) ReadField1(iprot thrift.TProtocol) error { - _field := NewInvokeEvalTargetRequest() +func (p *EvaluationSPIServiceAsyncInvokeEvalTargetArgs) ReadField1(iprot thrift.TProtocol) error { + _field := NewAsyncInvokeEvalTargetRequest() if err := _field.Read(iprot); err != nil { return err } @@ -8728,9 +9770,9 @@ func (p *EvaluationSPIServiceInvokeEvalTargetArgs) ReadField1(iprot thrift.TProt return nil } -func (p *EvaluationSPIServiceInvokeEvalTargetArgs) Write(oprot thrift.TProtocol) (err error) { +func (p *EvaluationSPIServiceAsyncInvokeEvalTargetArgs) Write(oprot thrift.TProtocol) (err error) { var fieldId int16 - if err = oprot.WriteStructBegin("InvokeEvalTarget_args"); err != nil { + if err = oprot.WriteStructBegin("AsyncInvokeEvalTarget_args"); err != nil { goto WriteStructBeginError } if p != nil { @@ -8756,7 +9798,7 @@ WriteStructEndError: return thrift.PrependError(fmt.Sprintf("%T write struct end error: ", p), err) } -func (p *EvaluationSPIServiceInvokeEvalTargetArgs) writeField1(oprot thrift.TProtocol) (err error) { +func (p *EvaluationSPIServiceAsyncInvokeEvalTargetArgs) writeField1(oprot thrift.TProtocol) (err error) { if err = oprot.WriteFieldBegin("req", thrift.STRUCT, 1); err != nil { goto WriteFieldBeginError } @@ -8773,15 +9815,15 @@ WriteFieldEndError: return thrift.PrependError(fmt.Sprintf("%T write field 1 end error: ", p), err) } -func (p *EvaluationSPIServiceInvokeEvalTargetArgs) String() string { +func (p *EvaluationSPIServiceAsyncInvokeEvalTargetArgs) String() string { if p == nil { return "" } - return fmt.Sprintf("EvaluationSPIServiceInvokeEvalTargetArgs(%+v)", *p) + return fmt.Sprintf("EvaluationSPIServiceAsyncInvokeEvalTargetArgs(%+v)", *p) } -func (p *EvaluationSPIServiceInvokeEvalTargetArgs) DeepEqual(ano *EvaluationSPIServiceInvokeEvalTargetArgs) bool { +func (p *EvaluationSPIServiceAsyncInvokeEvalTargetArgs) DeepEqual(ano *EvaluationSPIServiceAsyncInvokeEvalTargetArgs) bool { if p == ano { return true } else if p == nil || ano == nil { @@ -8793,7 +9835,7 @@ func (p *EvaluationSPIServiceInvokeEvalTargetArgs) DeepEqual(ano *EvaluationSPIS return true } -func (p *EvaluationSPIServiceInvokeEvalTargetArgs) Field1DeepEqual(src *InvokeEvalTargetRequest) bool { +func (p *EvaluationSPIServiceAsyncInvokeEvalTargetArgs) Field1DeepEqual(src *AsyncInvokeEvalTargetRequest) bool { if !p.Req.DeepEqual(src) { return false @@ -8801,41 +9843,41 @@ func (p *EvaluationSPIServiceInvokeEvalTargetArgs) Field1DeepEqual(src *InvokeEv return true } -type EvaluationSPIServiceInvokeEvalTargetResult struct { - Success *InvokeEvalTargetResponse `thrift:"success,0,optional" frugal:"0,optional,InvokeEvalTargetResponse"` +type EvaluationSPIServiceAsyncInvokeEvalTargetResult struct { + Success *AsyncInvokeEvalTargetResponse `thrift:"success,0,optional" frugal:"0,optional,AsyncInvokeEvalTargetResponse"` } -func NewEvaluationSPIServiceInvokeEvalTargetResult() *EvaluationSPIServiceInvokeEvalTargetResult { - return &EvaluationSPIServiceInvokeEvalTargetResult{} +func NewEvaluationSPIServiceAsyncInvokeEvalTargetResult() *EvaluationSPIServiceAsyncInvokeEvalTargetResult { + return &EvaluationSPIServiceAsyncInvokeEvalTargetResult{} } -func (p *EvaluationSPIServiceInvokeEvalTargetResult) InitDefault() { +func (p *EvaluationSPIServiceAsyncInvokeEvalTargetResult) InitDefault() { } -var EvaluationSPIServiceInvokeEvalTargetResult_Success_DEFAULT *InvokeEvalTargetResponse +var EvaluationSPIServiceAsyncInvokeEvalTargetResult_Success_DEFAULT *AsyncInvokeEvalTargetResponse -func (p *EvaluationSPIServiceInvokeEvalTargetResult) GetSuccess() (v *InvokeEvalTargetResponse) { +func (p *EvaluationSPIServiceAsyncInvokeEvalTargetResult) GetSuccess() (v *AsyncInvokeEvalTargetResponse) { if p == nil { return } if !p.IsSetSuccess() { - return EvaluationSPIServiceInvokeEvalTargetResult_Success_DEFAULT + return EvaluationSPIServiceAsyncInvokeEvalTargetResult_Success_DEFAULT } return p.Success } -func (p *EvaluationSPIServiceInvokeEvalTargetResult) SetSuccess(x interface{}) { - p.Success = x.(*InvokeEvalTargetResponse) +func (p *EvaluationSPIServiceAsyncInvokeEvalTargetResult) SetSuccess(x interface{}) { + p.Success = x.(*AsyncInvokeEvalTargetResponse) } -var fieldIDToName_EvaluationSPIServiceInvokeEvalTargetResult = map[int16]string{ +var fieldIDToName_EvaluationSPIServiceAsyncInvokeEvalTargetResult = map[int16]string{ 0: "success", } -func (p *EvaluationSPIServiceInvokeEvalTargetResult) IsSetSuccess() bool { +func (p *EvaluationSPIServiceAsyncInvokeEvalTargetResult) IsSetSuccess() bool { return p.Success != nil } -func (p *EvaluationSPIServiceInvokeEvalTargetResult) Read(iprot thrift.TProtocol) (err error) { +func (p *EvaluationSPIServiceAsyncInvokeEvalTargetResult) Read(iprot thrift.TProtocol) (err error) { var fieldTypeId thrift.TType var fieldId int16 @@ -8880,7 +9922,7 @@ ReadStructBeginError: ReadFieldBeginError: return thrift.PrependError(fmt.Sprintf("%T read field %d begin error: ", p, fieldId), err) ReadFieldError: - return thrift.PrependError(fmt.Sprintf("%T read field %d '%s' error: ", p, fieldId, fieldIDToName_EvaluationSPIServiceInvokeEvalTargetResult[fieldId]), err) + return thrift.PrependError(fmt.Sprintf("%T read field %d '%s' error: ", p, fieldId, fieldIDToName_EvaluationSPIServiceAsyncInvokeEvalTargetResult[fieldId]), err) SkipFieldError: return thrift.PrependError(fmt.Sprintf("%T field %d skip type %d error: ", p, fieldId, fieldTypeId), err) @@ -8890,8 +9932,8 @@ ReadStructEndError: return thrift.PrependError(fmt.Sprintf("%T read struct end error: ", p), err) } -func (p *EvaluationSPIServiceInvokeEvalTargetResult) ReadField0(iprot thrift.TProtocol) error { - _field := NewInvokeEvalTargetResponse() +func (p *EvaluationSPIServiceAsyncInvokeEvalTargetResult) ReadField0(iprot thrift.TProtocol) error { + _field := NewAsyncInvokeEvalTargetResponse() if err := _field.Read(iprot); err != nil { return err } @@ -8899,9 +9941,9 @@ func (p *EvaluationSPIServiceInvokeEvalTargetResult) ReadField0(iprot thrift.TPr return nil } -func (p *EvaluationSPIServiceInvokeEvalTargetResult) Write(oprot thrift.TProtocol) (err error) { +func (p *EvaluationSPIServiceAsyncInvokeEvalTargetResult) Write(oprot thrift.TProtocol) (err error) { var fieldId int16 - if err = oprot.WriteStructBegin("InvokeEvalTarget_result"); err != nil { + if err = oprot.WriteStructBegin("AsyncInvokeEvalTarget_result"); err != nil { goto WriteStructBeginError } if p != nil { @@ -8927,7 +9969,7 @@ WriteStructEndError: return thrift.PrependError(fmt.Sprintf("%T write struct end error: ", p), err) } -func (p *EvaluationSPIServiceInvokeEvalTargetResult) writeField0(oprot thrift.TProtocol) (err error) { +func (p *EvaluationSPIServiceAsyncInvokeEvalTargetResult) writeField0(oprot thrift.TProtocol) (err error) { if p.IsSetSuccess() { if err = oprot.WriteFieldBegin("success", thrift.STRUCT, 0); err != nil { goto WriteFieldBeginError @@ -8946,15 +9988,15 @@ WriteFieldEndError: return thrift.PrependError(fmt.Sprintf("%T write field 0 end error: ", p), err) } -func (p *EvaluationSPIServiceInvokeEvalTargetResult) String() string { +func (p *EvaluationSPIServiceAsyncInvokeEvalTargetResult) String() string { if p == nil { return "" } - return fmt.Sprintf("EvaluationSPIServiceInvokeEvalTargetResult(%+v)", *p) + return fmt.Sprintf("EvaluationSPIServiceAsyncInvokeEvalTargetResult(%+v)", *p) } -func (p *EvaluationSPIServiceInvokeEvalTargetResult) DeepEqual(ano *EvaluationSPIServiceInvokeEvalTargetResult) bool { +func (p *EvaluationSPIServiceAsyncInvokeEvalTargetResult) DeepEqual(ano *EvaluationSPIServiceAsyncInvokeEvalTargetResult) bool { if p == ano { return true } else if p == nil || ano == nil { @@ -8966,7 +10008,7 @@ func (p *EvaluationSPIServiceInvokeEvalTargetResult) DeepEqual(ano *EvaluationSP return true } -func (p *EvaluationSPIServiceInvokeEvalTargetResult) Field0DeepEqual(src *InvokeEvalTargetResponse) bool { +func (p *EvaluationSPIServiceAsyncInvokeEvalTargetResult) Field0DeepEqual(src *AsyncInvokeEvalTargetResponse) bool { if !p.Success.DeepEqual(src) { return false @@ -8974,41 +10016,41 @@ func (p *EvaluationSPIServiceInvokeEvalTargetResult) Field0DeepEqual(src *Invoke return true } -type EvaluationSPIServiceAsyncInvokeEvalTargetArgs struct { - Req *AsyncInvokeEvalTargetRequest `thrift:"req,1" frugal:"1,default,AsyncInvokeEvalTargetRequest"` +type EvaluationSPIServiceInvokeEvaluatorArgs struct { + Req *InvokeEvaluatorRequest `thrift:"req,1" frugal:"1,default,InvokeEvaluatorRequest"` } -func NewEvaluationSPIServiceAsyncInvokeEvalTargetArgs() *EvaluationSPIServiceAsyncInvokeEvalTargetArgs { - return &EvaluationSPIServiceAsyncInvokeEvalTargetArgs{} +func NewEvaluationSPIServiceInvokeEvaluatorArgs() *EvaluationSPIServiceInvokeEvaluatorArgs { + return &EvaluationSPIServiceInvokeEvaluatorArgs{} } -func (p *EvaluationSPIServiceAsyncInvokeEvalTargetArgs) InitDefault() { +func (p *EvaluationSPIServiceInvokeEvaluatorArgs) InitDefault() { } -var EvaluationSPIServiceAsyncInvokeEvalTargetArgs_Req_DEFAULT *AsyncInvokeEvalTargetRequest +var EvaluationSPIServiceInvokeEvaluatorArgs_Req_DEFAULT *InvokeEvaluatorRequest -func (p *EvaluationSPIServiceAsyncInvokeEvalTargetArgs) GetReq() (v *AsyncInvokeEvalTargetRequest) { +func (p *EvaluationSPIServiceInvokeEvaluatorArgs) GetReq() (v *InvokeEvaluatorRequest) { if p == nil { return } if !p.IsSetReq() { - return EvaluationSPIServiceAsyncInvokeEvalTargetArgs_Req_DEFAULT + return EvaluationSPIServiceInvokeEvaluatorArgs_Req_DEFAULT } return p.Req } -func (p *EvaluationSPIServiceAsyncInvokeEvalTargetArgs) SetReq(val *AsyncInvokeEvalTargetRequest) { +func (p *EvaluationSPIServiceInvokeEvaluatorArgs) SetReq(val *InvokeEvaluatorRequest) { p.Req = val } -var fieldIDToName_EvaluationSPIServiceAsyncInvokeEvalTargetArgs = map[int16]string{ +var fieldIDToName_EvaluationSPIServiceInvokeEvaluatorArgs = map[int16]string{ 1: "req", } -func (p *EvaluationSPIServiceAsyncInvokeEvalTargetArgs) IsSetReq() bool { +func (p *EvaluationSPIServiceInvokeEvaluatorArgs) IsSetReq() bool { return p.Req != nil } -func (p *EvaluationSPIServiceAsyncInvokeEvalTargetArgs) Read(iprot thrift.TProtocol) (err error) { +func (p *EvaluationSPIServiceInvokeEvaluatorArgs) Read(iprot thrift.TProtocol) (err error) { var fieldTypeId thrift.TType var fieldId int16 @@ -9053,7 +10095,7 @@ ReadStructBeginError: ReadFieldBeginError: return thrift.PrependError(fmt.Sprintf("%T read field %d begin error: ", p, fieldId), err) ReadFieldError: - return thrift.PrependError(fmt.Sprintf("%T read field %d '%s' error: ", p, fieldId, fieldIDToName_EvaluationSPIServiceAsyncInvokeEvalTargetArgs[fieldId]), err) + return thrift.PrependError(fmt.Sprintf("%T read field %d '%s' error: ", p, fieldId, fieldIDToName_EvaluationSPIServiceInvokeEvaluatorArgs[fieldId]), err) SkipFieldError: return thrift.PrependError(fmt.Sprintf("%T field %d skip type %d error: ", p, fieldId, fieldTypeId), err) @@ -9063,8 +10105,8 @@ ReadStructEndError: return thrift.PrependError(fmt.Sprintf("%T read struct end error: ", p), err) } -func (p *EvaluationSPIServiceAsyncInvokeEvalTargetArgs) ReadField1(iprot thrift.TProtocol) error { - _field := NewAsyncInvokeEvalTargetRequest() +func (p *EvaluationSPIServiceInvokeEvaluatorArgs) ReadField1(iprot thrift.TProtocol) error { + _field := NewInvokeEvaluatorRequest() if err := _field.Read(iprot); err != nil { return err } @@ -9072,9 +10114,9 @@ func (p *EvaluationSPIServiceAsyncInvokeEvalTargetArgs) ReadField1(iprot thrift. return nil } -func (p *EvaluationSPIServiceAsyncInvokeEvalTargetArgs) Write(oprot thrift.TProtocol) (err error) { +func (p *EvaluationSPIServiceInvokeEvaluatorArgs) Write(oprot thrift.TProtocol) (err error) { var fieldId int16 - if err = oprot.WriteStructBegin("AsyncInvokeEvalTarget_args"); err != nil { + if err = oprot.WriteStructBegin("InvokeEvaluator_args"); err != nil { goto WriteStructBeginError } if p != nil { @@ -9100,7 +10142,7 @@ WriteStructEndError: return thrift.PrependError(fmt.Sprintf("%T write struct end error: ", p), err) } -func (p *EvaluationSPIServiceAsyncInvokeEvalTargetArgs) writeField1(oprot thrift.TProtocol) (err error) { +func (p *EvaluationSPIServiceInvokeEvaluatorArgs) writeField1(oprot thrift.TProtocol) (err error) { if err = oprot.WriteFieldBegin("req", thrift.STRUCT, 1); err != nil { goto WriteFieldBeginError } @@ -9117,15 +10159,15 @@ WriteFieldEndError: return thrift.PrependError(fmt.Sprintf("%T write field 1 end error: ", p), err) } -func (p *EvaluationSPIServiceAsyncInvokeEvalTargetArgs) String() string { +func (p *EvaluationSPIServiceInvokeEvaluatorArgs) String() string { if p == nil { return "" } - return fmt.Sprintf("EvaluationSPIServiceAsyncInvokeEvalTargetArgs(%+v)", *p) + return fmt.Sprintf("EvaluationSPIServiceInvokeEvaluatorArgs(%+v)", *p) } -func (p *EvaluationSPIServiceAsyncInvokeEvalTargetArgs) DeepEqual(ano *EvaluationSPIServiceAsyncInvokeEvalTargetArgs) bool { +func (p *EvaluationSPIServiceInvokeEvaluatorArgs) DeepEqual(ano *EvaluationSPIServiceInvokeEvaluatorArgs) bool { if p == ano { return true } else if p == nil || ano == nil { @@ -9137,7 +10179,7 @@ func (p *EvaluationSPIServiceAsyncInvokeEvalTargetArgs) DeepEqual(ano *Evaluatio return true } -func (p *EvaluationSPIServiceAsyncInvokeEvalTargetArgs) Field1DeepEqual(src *AsyncInvokeEvalTargetRequest) bool { +func (p *EvaluationSPIServiceInvokeEvaluatorArgs) Field1DeepEqual(src *InvokeEvaluatorRequest) bool { if !p.Req.DeepEqual(src) { return false @@ -9145,41 +10187,41 @@ func (p *EvaluationSPIServiceAsyncInvokeEvalTargetArgs) Field1DeepEqual(src *Asy return true } -type EvaluationSPIServiceAsyncInvokeEvalTargetResult struct { - Success *AsyncInvokeEvalTargetResponse `thrift:"success,0,optional" frugal:"0,optional,AsyncInvokeEvalTargetResponse"` +type EvaluationSPIServiceInvokeEvaluatorResult struct { + Success *InvokeEvaluatorResponse `thrift:"success,0,optional" frugal:"0,optional,InvokeEvaluatorResponse"` } -func NewEvaluationSPIServiceAsyncInvokeEvalTargetResult() *EvaluationSPIServiceAsyncInvokeEvalTargetResult { - return &EvaluationSPIServiceAsyncInvokeEvalTargetResult{} +func NewEvaluationSPIServiceInvokeEvaluatorResult() *EvaluationSPIServiceInvokeEvaluatorResult { + return &EvaluationSPIServiceInvokeEvaluatorResult{} } -func (p *EvaluationSPIServiceAsyncInvokeEvalTargetResult) InitDefault() { +func (p *EvaluationSPIServiceInvokeEvaluatorResult) InitDefault() { } -var EvaluationSPIServiceAsyncInvokeEvalTargetResult_Success_DEFAULT *AsyncInvokeEvalTargetResponse +var EvaluationSPIServiceInvokeEvaluatorResult_Success_DEFAULT *InvokeEvaluatorResponse -func (p *EvaluationSPIServiceAsyncInvokeEvalTargetResult) GetSuccess() (v *AsyncInvokeEvalTargetResponse) { +func (p *EvaluationSPIServiceInvokeEvaluatorResult) GetSuccess() (v *InvokeEvaluatorResponse) { if p == nil { return } if !p.IsSetSuccess() { - return EvaluationSPIServiceAsyncInvokeEvalTargetResult_Success_DEFAULT + return EvaluationSPIServiceInvokeEvaluatorResult_Success_DEFAULT } return p.Success } -func (p *EvaluationSPIServiceAsyncInvokeEvalTargetResult) SetSuccess(x interface{}) { - p.Success = x.(*AsyncInvokeEvalTargetResponse) +func (p *EvaluationSPIServiceInvokeEvaluatorResult) SetSuccess(x interface{}) { + p.Success = x.(*InvokeEvaluatorResponse) } -var fieldIDToName_EvaluationSPIServiceAsyncInvokeEvalTargetResult = map[int16]string{ +var fieldIDToName_EvaluationSPIServiceInvokeEvaluatorResult = map[int16]string{ 0: "success", } -func (p *EvaluationSPIServiceAsyncInvokeEvalTargetResult) IsSetSuccess() bool { +func (p *EvaluationSPIServiceInvokeEvaluatorResult) IsSetSuccess() bool { return p.Success != nil } -func (p *EvaluationSPIServiceAsyncInvokeEvalTargetResult) Read(iprot thrift.TProtocol) (err error) { +func (p *EvaluationSPIServiceInvokeEvaluatorResult) Read(iprot thrift.TProtocol) (err error) { var fieldTypeId thrift.TType var fieldId int16 @@ -9224,7 +10266,7 @@ ReadStructBeginError: ReadFieldBeginError: return thrift.PrependError(fmt.Sprintf("%T read field %d begin error: ", p, fieldId), err) ReadFieldError: - return thrift.PrependError(fmt.Sprintf("%T read field %d '%s' error: ", p, fieldId, fieldIDToName_EvaluationSPIServiceAsyncInvokeEvalTargetResult[fieldId]), err) + return thrift.PrependError(fmt.Sprintf("%T read field %d '%s' error: ", p, fieldId, fieldIDToName_EvaluationSPIServiceInvokeEvaluatorResult[fieldId]), err) SkipFieldError: return thrift.PrependError(fmt.Sprintf("%T field %d skip type %d error: ", p, fieldId, fieldTypeId), err) @@ -9234,8 +10276,8 @@ ReadStructEndError: return thrift.PrependError(fmt.Sprintf("%T read struct end error: ", p), err) } -func (p *EvaluationSPIServiceAsyncInvokeEvalTargetResult) ReadField0(iprot thrift.TProtocol) error { - _field := NewAsyncInvokeEvalTargetResponse() +func (p *EvaluationSPIServiceInvokeEvaluatorResult) ReadField0(iprot thrift.TProtocol) error { + _field := NewInvokeEvaluatorResponse() if err := _field.Read(iprot); err != nil { return err } @@ -9243,9 +10285,9 @@ func (p *EvaluationSPIServiceAsyncInvokeEvalTargetResult) ReadField0(iprot thrif return nil } -func (p *EvaluationSPIServiceAsyncInvokeEvalTargetResult) Write(oprot thrift.TProtocol) (err error) { +func (p *EvaluationSPIServiceInvokeEvaluatorResult) Write(oprot thrift.TProtocol) (err error) { var fieldId int16 - if err = oprot.WriteStructBegin("AsyncInvokeEvalTarget_result"); err != nil { + if err = oprot.WriteStructBegin("InvokeEvaluator_result"); err != nil { goto WriteStructBeginError } if p != nil { @@ -9271,7 +10313,7 @@ WriteStructEndError: return thrift.PrependError(fmt.Sprintf("%T write struct end error: ", p), err) } -func (p *EvaluationSPIServiceAsyncInvokeEvalTargetResult) writeField0(oprot thrift.TProtocol) (err error) { +func (p *EvaluationSPIServiceInvokeEvaluatorResult) writeField0(oprot thrift.TProtocol) (err error) { if p.IsSetSuccess() { if err = oprot.WriteFieldBegin("success", thrift.STRUCT, 0); err != nil { goto WriteFieldBeginError @@ -9290,15 +10332,15 @@ WriteFieldEndError: return thrift.PrependError(fmt.Sprintf("%T write field 0 end error: ", p), err) } -func (p *EvaluationSPIServiceAsyncInvokeEvalTargetResult) String() string { +func (p *EvaluationSPIServiceInvokeEvaluatorResult) String() string { if p == nil { return "" } - return fmt.Sprintf("EvaluationSPIServiceAsyncInvokeEvalTargetResult(%+v)", *p) + return fmt.Sprintf("EvaluationSPIServiceInvokeEvaluatorResult(%+v)", *p) } -func (p *EvaluationSPIServiceAsyncInvokeEvalTargetResult) DeepEqual(ano *EvaluationSPIServiceAsyncInvokeEvalTargetResult) bool { +func (p *EvaluationSPIServiceInvokeEvaluatorResult) DeepEqual(ano *EvaluationSPIServiceInvokeEvaluatorResult) bool { if p == ano { return true } else if p == nil || ano == nil { @@ -9310,7 +10352,7 @@ func (p *EvaluationSPIServiceAsyncInvokeEvalTargetResult) DeepEqual(ano *Evaluat return true } -func (p *EvaluationSPIServiceAsyncInvokeEvalTargetResult) Field0DeepEqual(src *AsyncInvokeEvalTargetResponse) bool { +func (p *EvaluationSPIServiceInvokeEvaluatorResult) Field0DeepEqual(src *InvokeEvaluatorResponse) bool { if !p.Success.DeepEqual(src) { return false @@ -9318,41 +10360,41 @@ func (p *EvaluationSPIServiceAsyncInvokeEvalTargetResult) Field0DeepEqual(src *A return true } -type EvaluationSPIServiceInvokeEvaluatorArgs struct { - Req *InvokeEvaluatorRequest `thrift:"req,1" frugal:"1,default,InvokeEvaluatorRequest"` +type EvaluationSPIServiceAsyncInvokeEvaluatorArgs struct { + Req *AsyncInvokeEvaluatorRequest `thrift:"req,1" frugal:"1,default,AsyncInvokeEvaluatorRequest"` } -func NewEvaluationSPIServiceInvokeEvaluatorArgs() *EvaluationSPIServiceInvokeEvaluatorArgs { - return &EvaluationSPIServiceInvokeEvaluatorArgs{} +func NewEvaluationSPIServiceAsyncInvokeEvaluatorArgs() *EvaluationSPIServiceAsyncInvokeEvaluatorArgs { + return &EvaluationSPIServiceAsyncInvokeEvaluatorArgs{} } -func (p *EvaluationSPIServiceInvokeEvaluatorArgs) InitDefault() { +func (p *EvaluationSPIServiceAsyncInvokeEvaluatorArgs) InitDefault() { } -var EvaluationSPIServiceInvokeEvaluatorArgs_Req_DEFAULT *InvokeEvaluatorRequest +var EvaluationSPIServiceAsyncInvokeEvaluatorArgs_Req_DEFAULT *AsyncInvokeEvaluatorRequest -func (p *EvaluationSPIServiceInvokeEvaluatorArgs) GetReq() (v *InvokeEvaluatorRequest) { +func (p *EvaluationSPIServiceAsyncInvokeEvaluatorArgs) GetReq() (v *AsyncInvokeEvaluatorRequest) { if p == nil { return } if !p.IsSetReq() { - return EvaluationSPIServiceInvokeEvaluatorArgs_Req_DEFAULT + return EvaluationSPIServiceAsyncInvokeEvaluatorArgs_Req_DEFAULT } return p.Req } -func (p *EvaluationSPIServiceInvokeEvaluatorArgs) SetReq(val *InvokeEvaluatorRequest) { +func (p *EvaluationSPIServiceAsyncInvokeEvaluatorArgs) SetReq(val *AsyncInvokeEvaluatorRequest) { p.Req = val } -var fieldIDToName_EvaluationSPIServiceInvokeEvaluatorArgs = map[int16]string{ +var fieldIDToName_EvaluationSPIServiceAsyncInvokeEvaluatorArgs = map[int16]string{ 1: "req", } -func (p *EvaluationSPIServiceInvokeEvaluatorArgs) IsSetReq() bool { +func (p *EvaluationSPIServiceAsyncInvokeEvaluatorArgs) IsSetReq() bool { return p.Req != nil } -func (p *EvaluationSPIServiceInvokeEvaluatorArgs) Read(iprot thrift.TProtocol) (err error) { +func (p *EvaluationSPIServiceAsyncInvokeEvaluatorArgs) Read(iprot thrift.TProtocol) (err error) { var fieldTypeId thrift.TType var fieldId int16 @@ -9397,7 +10439,7 @@ ReadStructBeginError: ReadFieldBeginError: return thrift.PrependError(fmt.Sprintf("%T read field %d begin error: ", p, fieldId), err) ReadFieldError: - return thrift.PrependError(fmt.Sprintf("%T read field %d '%s' error: ", p, fieldId, fieldIDToName_EvaluationSPIServiceInvokeEvaluatorArgs[fieldId]), err) + return thrift.PrependError(fmt.Sprintf("%T read field %d '%s' error: ", p, fieldId, fieldIDToName_EvaluationSPIServiceAsyncInvokeEvaluatorArgs[fieldId]), err) SkipFieldError: return thrift.PrependError(fmt.Sprintf("%T field %d skip type %d error: ", p, fieldId, fieldTypeId), err) @@ -9407,8 +10449,8 @@ ReadStructEndError: return thrift.PrependError(fmt.Sprintf("%T read struct end error: ", p), err) } -func (p *EvaluationSPIServiceInvokeEvaluatorArgs) ReadField1(iprot thrift.TProtocol) error { - _field := NewInvokeEvaluatorRequest() +func (p *EvaluationSPIServiceAsyncInvokeEvaluatorArgs) ReadField1(iprot thrift.TProtocol) error { + _field := NewAsyncInvokeEvaluatorRequest() if err := _field.Read(iprot); err != nil { return err } @@ -9416,9 +10458,9 @@ func (p *EvaluationSPIServiceInvokeEvaluatorArgs) ReadField1(iprot thrift.TProto return nil } -func (p *EvaluationSPIServiceInvokeEvaluatorArgs) Write(oprot thrift.TProtocol) (err error) { +func (p *EvaluationSPIServiceAsyncInvokeEvaluatorArgs) Write(oprot thrift.TProtocol) (err error) { var fieldId int16 - if err = oprot.WriteStructBegin("InvokeEvaluator_args"); err != nil { + if err = oprot.WriteStructBegin("AsyncInvokeEvaluator_args"); err != nil { goto WriteStructBeginError } if p != nil { @@ -9444,7 +10486,7 @@ WriteStructEndError: return thrift.PrependError(fmt.Sprintf("%T write struct end error: ", p), err) } -func (p *EvaluationSPIServiceInvokeEvaluatorArgs) writeField1(oprot thrift.TProtocol) (err error) { +func (p *EvaluationSPIServiceAsyncInvokeEvaluatorArgs) writeField1(oprot thrift.TProtocol) (err error) { if err = oprot.WriteFieldBegin("req", thrift.STRUCT, 1); err != nil { goto WriteFieldBeginError } @@ -9461,15 +10503,15 @@ WriteFieldEndError: return thrift.PrependError(fmt.Sprintf("%T write field 1 end error: ", p), err) } -func (p *EvaluationSPIServiceInvokeEvaluatorArgs) String() string { +func (p *EvaluationSPIServiceAsyncInvokeEvaluatorArgs) String() string { if p == nil { return "" } - return fmt.Sprintf("EvaluationSPIServiceInvokeEvaluatorArgs(%+v)", *p) + return fmt.Sprintf("EvaluationSPIServiceAsyncInvokeEvaluatorArgs(%+v)", *p) } -func (p *EvaluationSPIServiceInvokeEvaluatorArgs) DeepEqual(ano *EvaluationSPIServiceInvokeEvaluatorArgs) bool { +func (p *EvaluationSPIServiceAsyncInvokeEvaluatorArgs) DeepEqual(ano *EvaluationSPIServiceAsyncInvokeEvaluatorArgs) bool { if p == ano { return true } else if p == nil || ano == nil { @@ -9481,7 +10523,7 @@ func (p *EvaluationSPIServiceInvokeEvaluatorArgs) DeepEqual(ano *EvaluationSPISe return true } -func (p *EvaluationSPIServiceInvokeEvaluatorArgs) Field1DeepEqual(src *InvokeEvaluatorRequest) bool { +func (p *EvaluationSPIServiceAsyncInvokeEvaluatorArgs) Field1DeepEqual(src *AsyncInvokeEvaluatorRequest) bool { if !p.Req.DeepEqual(src) { return false @@ -9489,41 +10531,41 @@ func (p *EvaluationSPIServiceInvokeEvaluatorArgs) Field1DeepEqual(src *InvokeEva return true } -type EvaluationSPIServiceInvokeEvaluatorResult struct { - Success *InvokeEvaluatorResponse `thrift:"success,0,optional" frugal:"0,optional,InvokeEvaluatorResponse"` +type EvaluationSPIServiceAsyncInvokeEvaluatorResult struct { + Success *AsyncInvokeEvaluatorResponse `thrift:"success,0,optional" frugal:"0,optional,AsyncInvokeEvaluatorResponse"` } -func NewEvaluationSPIServiceInvokeEvaluatorResult() *EvaluationSPIServiceInvokeEvaluatorResult { - return &EvaluationSPIServiceInvokeEvaluatorResult{} +func NewEvaluationSPIServiceAsyncInvokeEvaluatorResult() *EvaluationSPIServiceAsyncInvokeEvaluatorResult { + return &EvaluationSPIServiceAsyncInvokeEvaluatorResult{} } -func (p *EvaluationSPIServiceInvokeEvaluatorResult) InitDefault() { +func (p *EvaluationSPIServiceAsyncInvokeEvaluatorResult) InitDefault() { } -var EvaluationSPIServiceInvokeEvaluatorResult_Success_DEFAULT *InvokeEvaluatorResponse +var EvaluationSPIServiceAsyncInvokeEvaluatorResult_Success_DEFAULT *AsyncInvokeEvaluatorResponse -func (p *EvaluationSPIServiceInvokeEvaluatorResult) GetSuccess() (v *InvokeEvaluatorResponse) { +func (p *EvaluationSPIServiceAsyncInvokeEvaluatorResult) GetSuccess() (v *AsyncInvokeEvaluatorResponse) { if p == nil { return } if !p.IsSetSuccess() { - return EvaluationSPIServiceInvokeEvaluatorResult_Success_DEFAULT + return EvaluationSPIServiceAsyncInvokeEvaluatorResult_Success_DEFAULT } return p.Success } -func (p *EvaluationSPIServiceInvokeEvaluatorResult) SetSuccess(x interface{}) { - p.Success = x.(*InvokeEvaluatorResponse) +func (p *EvaluationSPIServiceAsyncInvokeEvaluatorResult) SetSuccess(x interface{}) { + p.Success = x.(*AsyncInvokeEvaluatorResponse) } -var fieldIDToName_EvaluationSPIServiceInvokeEvaluatorResult = map[int16]string{ +var fieldIDToName_EvaluationSPIServiceAsyncInvokeEvaluatorResult = map[int16]string{ 0: "success", } -func (p *EvaluationSPIServiceInvokeEvaluatorResult) IsSetSuccess() bool { +func (p *EvaluationSPIServiceAsyncInvokeEvaluatorResult) IsSetSuccess() bool { return p.Success != nil } -func (p *EvaluationSPIServiceInvokeEvaluatorResult) Read(iprot thrift.TProtocol) (err error) { +func (p *EvaluationSPIServiceAsyncInvokeEvaluatorResult) Read(iprot thrift.TProtocol) (err error) { var fieldTypeId thrift.TType var fieldId int16 @@ -9568,7 +10610,7 @@ ReadStructBeginError: ReadFieldBeginError: return thrift.PrependError(fmt.Sprintf("%T read field %d begin error: ", p, fieldId), err) ReadFieldError: - return thrift.PrependError(fmt.Sprintf("%T read field %d '%s' error: ", p, fieldId, fieldIDToName_EvaluationSPIServiceInvokeEvaluatorResult[fieldId]), err) + return thrift.PrependError(fmt.Sprintf("%T read field %d '%s' error: ", p, fieldId, fieldIDToName_EvaluationSPIServiceAsyncInvokeEvaluatorResult[fieldId]), err) SkipFieldError: return thrift.PrependError(fmt.Sprintf("%T field %d skip type %d error: ", p, fieldId, fieldTypeId), err) @@ -9578,8 +10620,8 @@ ReadStructEndError: return thrift.PrependError(fmt.Sprintf("%T read struct end error: ", p), err) } -func (p *EvaluationSPIServiceInvokeEvaluatorResult) ReadField0(iprot thrift.TProtocol) error { - _field := NewInvokeEvaluatorResponse() +func (p *EvaluationSPIServiceAsyncInvokeEvaluatorResult) ReadField0(iprot thrift.TProtocol) error { + _field := NewAsyncInvokeEvaluatorResponse() if err := _field.Read(iprot); err != nil { return err } @@ -9587,9 +10629,9 @@ func (p *EvaluationSPIServiceInvokeEvaluatorResult) ReadField0(iprot thrift.TPro return nil } -func (p *EvaluationSPIServiceInvokeEvaluatorResult) Write(oprot thrift.TProtocol) (err error) { +func (p *EvaluationSPIServiceAsyncInvokeEvaluatorResult) Write(oprot thrift.TProtocol) (err error) { var fieldId int16 - if err = oprot.WriteStructBegin("InvokeEvaluator_result"); err != nil { + if err = oprot.WriteStructBegin("AsyncInvokeEvaluator_result"); err != nil { goto WriteStructBeginError } if p != nil { @@ -9615,7 +10657,7 @@ WriteStructEndError: return thrift.PrependError(fmt.Sprintf("%T write struct end error: ", p), err) } -func (p *EvaluationSPIServiceInvokeEvaluatorResult) writeField0(oprot thrift.TProtocol) (err error) { +func (p *EvaluationSPIServiceAsyncInvokeEvaluatorResult) writeField0(oprot thrift.TProtocol) (err error) { if p.IsSetSuccess() { if err = oprot.WriteFieldBegin("success", thrift.STRUCT, 0); err != nil { goto WriteFieldBeginError @@ -9634,15 +10676,15 @@ WriteFieldEndError: return thrift.PrependError(fmt.Sprintf("%T write field 0 end error: ", p), err) } -func (p *EvaluationSPIServiceInvokeEvaluatorResult) String() string { +func (p *EvaluationSPIServiceAsyncInvokeEvaluatorResult) String() string { if p == nil { return "" } - return fmt.Sprintf("EvaluationSPIServiceInvokeEvaluatorResult(%+v)", *p) + return fmt.Sprintf("EvaluationSPIServiceAsyncInvokeEvaluatorResult(%+v)", *p) } -func (p *EvaluationSPIServiceInvokeEvaluatorResult) DeepEqual(ano *EvaluationSPIServiceInvokeEvaluatorResult) bool { +func (p *EvaluationSPIServiceAsyncInvokeEvaluatorResult) DeepEqual(ano *EvaluationSPIServiceAsyncInvokeEvaluatorResult) bool { if p == ano { return true } else if p == nil || ano == nil { @@ -9654,7 +10696,7 @@ func (p *EvaluationSPIServiceInvokeEvaluatorResult) DeepEqual(ano *EvaluationSPI return true } -func (p *EvaluationSPIServiceInvokeEvaluatorResult) Field0DeepEqual(src *InvokeEvaluatorResponse) bool { +func (p *EvaluationSPIServiceAsyncInvokeEvaluatorResult) Field0DeepEqual(src *AsyncInvokeEvaluatorResponse) bool { if !p.Success.DeepEqual(src) { return false diff --git a/backend/kitex_gen/coze/loop/evaluation/spi/coze.loop.evaluation.spi_validator.go b/backend/kitex_gen/coze/loop/evaluation/spi/coze.loop.evaluation.spi_validator.go index 71c67f0875..9aea1586a6 100644 --- a/backend/kitex_gen/coze/loop/evaluation/spi/coze.loop.evaluation.spi_validator.go +++ b/backend/kitex_gen/coze/loop/evaluation/spi/coze.loop.evaluation.spi_validator.go @@ -215,3 +215,29 @@ func (p *InvokeEvaluatorResponse) IsValid() error { } return nil } +func (p *AsyncInvokeEvaluatorRequest) IsValid() error { + if p.Evaluator != nil { + if err := p.Evaluator.IsValid(); err != nil { + return fmt.Errorf("field Evaluator not valid, %w", err) + } + } + if p.InputData != nil { + if err := p.InputData.IsValid(); err != nil { + return fmt.Errorf("field InputData not valid, %w", err) + } + } + if p.Base != nil { + if err := p.Base.IsValid(); err != nil { + return fmt.Errorf("field Base not valid, %w", err) + } + } + return nil +} +func (p *AsyncInvokeEvaluatorResponse) IsValid() error { + if p.BaseResp != nil { + if err := p.BaseResp.IsValid(); err != nil { + return fmt.Errorf("field BaseResp not valid, %w", err) + } + } + return nil +} diff --git a/backend/kitex_gen/coze/loop/evaluation/spi/evaluationspiservice/client.go b/backend/kitex_gen/coze/loop/evaluation/spi/evaluationspiservice/client.go index 955c4f6486..09a2f7832a 100644 --- a/backend/kitex_gen/coze/loop/evaluation/spi/evaluationspiservice/client.go +++ b/backend/kitex_gen/coze/loop/evaluation/spi/evaluationspiservice/client.go @@ -15,6 +15,7 @@ type Client interface { InvokeEvalTarget(ctx context.Context, req *spi.InvokeEvalTargetRequest, callOptions ...callopt.Option) (r *spi.InvokeEvalTargetResponse, err error) AsyncInvokeEvalTarget(ctx context.Context, req *spi.AsyncInvokeEvalTargetRequest, callOptions ...callopt.Option) (r *spi.AsyncInvokeEvalTargetResponse, err error) InvokeEvaluator(ctx context.Context, req *spi.InvokeEvaluatorRequest, callOptions ...callopt.Option) (r *spi.InvokeEvaluatorResponse, err error) + AsyncInvokeEvaluator(ctx context.Context, req *spi.AsyncInvokeEvaluatorRequest, callOptions ...callopt.Option) (r *spi.AsyncInvokeEvaluatorResponse, err error) } // NewClient creates a client for the service defined in IDL. @@ -65,3 +66,8 @@ func (p *kEvaluationSPIServiceClient) InvokeEvaluator(ctx context.Context, req * ctx = client.NewCtxWithCallOptions(ctx, callOptions) return p.kClient.InvokeEvaluator(ctx, req) } + +func (p *kEvaluationSPIServiceClient) AsyncInvokeEvaluator(ctx context.Context, req *spi.AsyncInvokeEvaluatorRequest, callOptions ...callopt.Option) (r *spi.AsyncInvokeEvaluatorResponse, err error) { + ctx = client.NewCtxWithCallOptions(ctx, callOptions) + return p.kClient.AsyncInvokeEvaluator(ctx, req) +} diff --git a/backend/kitex_gen/coze/loop/evaluation/spi/evaluationspiservice/evaluationspiservice.go b/backend/kitex_gen/coze/loop/evaluation/spi/evaluationspiservice/evaluationspiservice.go index da3aadc192..96a301cfe2 100644 --- a/backend/kitex_gen/coze/loop/evaluation/spi/evaluationspiservice/evaluationspiservice.go +++ b/backend/kitex_gen/coze/loop/evaluation/spi/evaluationspiservice/evaluationspiservice.go @@ -41,6 +41,13 @@ var serviceMethods = map[string]kitex.MethodInfo{ false, kitex.WithStreamingMode(kitex.StreamingNone), ), + "AsyncInvokeEvaluator": kitex.NewMethodInfo( + asyncInvokeEvaluatorHandler, + newEvaluationSPIServiceAsyncInvokeEvaluatorArgs, + newEvaluationSPIServiceAsyncInvokeEvaluatorResult, + false, + kitex.WithStreamingMode(kitex.StreamingNone), + ), } var ( @@ -150,6 +157,25 @@ func newEvaluationSPIServiceInvokeEvaluatorResult() interface{} { return spi.NewEvaluationSPIServiceInvokeEvaluatorResult() } +func asyncInvokeEvaluatorHandler(ctx context.Context, handler interface{}, arg, result interface{}) error { + realArg := arg.(*spi.EvaluationSPIServiceAsyncInvokeEvaluatorArgs) + realResult := result.(*spi.EvaluationSPIServiceAsyncInvokeEvaluatorResult) + success, err := handler.(spi.EvaluationSPIService).AsyncInvokeEvaluator(ctx, realArg.Req) + if err != nil { + return err + } + realResult.Success = success + return nil +} + +func newEvaluationSPIServiceAsyncInvokeEvaluatorArgs() interface{} { + return spi.NewEvaluationSPIServiceAsyncInvokeEvaluatorArgs() +} + +func newEvaluationSPIServiceAsyncInvokeEvaluatorResult() interface{} { + return spi.NewEvaluationSPIServiceAsyncInvokeEvaluatorResult() +} + type kClient struct { c client.Client sc client.Streaming @@ -201,3 +227,13 @@ func (p *kClient) InvokeEvaluator(ctx context.Context, req *spi.InvokeEvaluatorR } return _result.GetSuccess(), nil } + +func (p *kClient) AsyncInvokeEvaluator(ctx context.Context, req *spi.AsyncInvokeEvaluatorRequest) (r *spi.AsyncInvokeEvaluatorResponse, err error) { + var _args spi.EvaluationSPIServiceAsyncInvokeEvaluatorArgs + _args.Req = req + var _result spi.EvaluationSPIServiceAsyncInvokeEvaluatorResult + if err = p.c.Call(ctx, "AsyncInvokeEvaluator", &_args, &_result); err != nil { + return + } + return _result.GetSuccess(), nil +} diff --git a/backend/kitex_gen/coze/loop/evaluation/spi/k-coze.loop.evaluation.spi.go b/backend/kitex_gen/coze/loop/evaluation/spi/k-coze.loop.evaluation.spi.go index 514485dd13..f9c0ec33b2 100644 --- a/backend/kitex_gen/coze/loop/evaluation/spi/k-coze.loop.evaluation.spi.go +++ b/backend/kitex_gen/coze/loop/evaluation/spi/k-coze.loop.evaluation.spi.go @@ -5722,6 +5722,456 @@ func (p *InvokeEvaluatorResponse) DeepCopy(s interface{}) error { return nil } +func (p *AsyncInvokeEvaluatorRequest) FastRead(buf []byte) (int, error) { + + var err error + var offset int + var l int + var fieldTypeId thrift.TType + var fieldId int16 + for { + fieldTypeId, fieldId, l, err = thrift.Binary.ReadFieldBegin(buf[offset:]) + offset += l + if err != nil { + goto ReadFieldBeginError + } + if fieldTypeId == thrift.STOP { + break + } + switch fieldId { + case 1: + if fieldTypeId == thrift.I64 { + l, err = p.FastReadField1(buf[offset:]) + offset += l + if err != nil { + goto ReadFieldError + } + } else { + l, err = thrift.Binary.Skip(buf[offset:], fieldTypeId) + offset += l + if err != nil { + goto SkipFieldError + } + } + case 2: + if fieldTypeId == thrift.I64 { + l, err = p.FastReadField2(buf[offset:]) + offset += l + if err != nil { + goto ReadFieldError + } + } else { + l, err = thrift.Binary.Skip(buf[offset:], fieldTypeId) + offset += l + if err != nil { + goto SkipFieldError + } + } + case 3: + if fieldTypeId == thrift.STRUCT { + l, err = p.FastReadField3(buf[offset:]) + offset += l + if err != nil { + goto ReadFieldError + } + } else { + l, err = thrift.Binary.Skip(buf[offset:], fieldTypeId) + offset += l + if err != nil { + goto SkipFieldError + } + } + case 4: + if fieldTypeId == thrift.STRUCT { + l, err = p.FastReadField4(buf[offset:]) + offset += l + if err != nil { + goto ReadFieldError + } + } else { + l, err = thrift.Binary.Skip(buf[offset:], fieldTypeId) + offset += l + if err != nil { + goto SkipFieldError + } + } + case 255: + if fieldTypeId == thrift.STRUCT { + l, err = p.FastReadField255(buf[offset:]) + offset += l + if err != nil { + goto ReadFieldError + } + } else { + l, err = thrift.Binary.Skip(buf[offset:], fieldTypeId) + offset += l + if err != nil { + goto SkipFieldError + } + } + default: + l, err = thrift.Binary.Skip(buf[offset:], fieldTypeId) + offset += l + if err != nil { + goto SkipFieldError + } + } + } + + return offset, nil +ReadFieldBeginError: + return offset, thrift.PrependError(fmt.Sprintf("%T read field %d begin error: ", p, fieldId), err) +ReadFieldError: + return offset, thrift.PrependError(fmt.Sprintf("%T read field %d '%s' error: ", p, fieldId, fieldIDToName_AsyncInvokeEvaluatorRequest[fieldId]), err) +SkipFieldError: + return offset, thrift.PrependError(fmt.Sprintf("%T field %d skip type %d error: ", p, fieldId, fieldTypeId), err) +} + +func (p *AsyncInvokeEvaluatorRequest) FastReadField1(buf []byte) (int, error) { + offset := 0 + + var _field *int64 + if v, l, err := thrift.Binary.ReadI64(buf[offset:]); err != nil { + return offset, err + } else { + offset += l + _field = &v + } + p.WorkspaceID = _field + return offset, nil +} + +func (p *AsyncInvokeEvaluatorRequest) FastReadField2(buf []byte) (int, error) { + offset := 0 + + var _field *int64 + if v, l, err := thrift.Binary.ReadI64(buf[offset:]); err != nil { + return offset, err + } else { + offset += l + _field = &v + } + p.InvokeID = _field + return offset, nil +} + +func (p *AsyncInvokeEvaluatorRequest) FastReadField3(buf []byte) (int, error) { + offset := 0 + _field := NewInvokeCustomEvaluator() + if l, err := _field.FastRead(buf[offset:]); err != nil { + return offset, err + } else { + offset += l + } + p.Evaluator = _field + return offset, nil +} + +func (p *AsyncInvokeEvaluatorRequest) FastReadField4(buf []byte) (int, error) { + offset := 0 + _field := NewInvokeEvaluatorInputData() + if l, err := _field.FastRead(buf[offset:]); err != nil { + return offset, err + } else { + offset += l + } + p.InputData = _field + return offset, nil +} + +func (p *AsyncInvokeEvaluatorRequest) FastReadField255(buf []byte) (int, error) { + offset := 0 + _field := base.NewBase() + if l, err := _field.FastRead(buf[offset:]); err != nil { + return offset, err + } else { + offset += l + } + p.Base = _field + return offset, nil +} + +func (p *AsyncInvokeEvaluatorRequest) FastWrite(buf []byte) int { + return p.FastWriteNocopy(buf, nil) +} + +func (p *AsyncInvokeEvaluatorRequest) FastWriteNocopy(buf []byte, w thrift.NocopyWriter) int { + offset := 0 + if p != nil { + offset += p.fastWriteField1(buf[offset:], w) + offset += p.fastWriteField2(buf[offset:], w) + offset += p.fastWriteField3(buf[offset:], w) + offset += p.fastWriteField4(buf[offset:], w) + offset += p.fastWriteField255(buf[offset:], w) + } + offset += thrift.Binary.WriteFieldStop(buf[offset:]) + return offset +} + +func (p *AsyncInvokeEvaluatorRequest) BLength() int { + l := 0 + if p != nil { + l += p.field1Length() + l += p.field2Length() + l += p.field3Length() + l += p.field4Length() + l += p.field255Length() + } + l += thrift.Binary.FieldStopLength() + return l +} + +func (p *AsyncInvokeEvaluatorRequest) fastWriteField1(buf []byte, w thrift.NocopyWriter) int { + offset := 0 + if p.IsSetWorkspaceID() { + offset += thrift.Binary.WriteFieldBegin(buf[offset:], thrift.I64, 1) + offset += thrift.Binary.WriteI64(buf[offset:], *p.WorkspaceID) + } + return offset +} + +func (p *AsyncInvokeEvaluatorRequest) fastWriteField2(buf []byte, w thrift.NocopyWriter) int { + offset := 0 + if p.IsSetInvokeID() { + offset += thrift.Binary.WriteFieldBegin(buf[offset:], thrift.I64, 2) + offset += thrift.Binary.WriteI64(buf[offset:], *p.InvokeID) + } + return offset +} + +func (p *AsyncInvokeEvaluatorRequest) fastWriteField3(buf []byte, w thrift.NocopyWriter) int { + offset := 0 + if p.IsSetEvaluator() { + offset += thrift.Binary.WriteFieldBegin(buf[offset:], thrift.STRUCT, 3) + offset += p.Evaluator.FastWriteNocopy(buf[offset:], w) + } + return offset +} + +func (p *AsyncInvokeEvaluatorRequest) fastWriteField4(buf []byte, w thrift.NocopyWriter) int { + offset := 0 + if p.IsSetInputData() { + offset += thrift.Binary.WriteFieldBegin(buf[offset:], thrift.STRUCT, 4) + offset += p.InputData.FastWriteNocopy(buf[offset:], w) + } + return offset +} + +func (p *AsyncInvokeEvaluatorRequest) fastWriteField255(buf []byte, w thrift.NocopyWriter) int { + offset := 0 + if p.IsSetBase() { + offset += thrift.Binary.WriteFieldBegin(buf[offset:], thrift.STRUCT, 255) + offset += p.Base.FastWriteNocopy(buf[offset:], w) + } + return offset +} + +func (p *AsyncInvokeEvaluatorRequest) field1Length() int { + l := 0 + if p.IsSetWorkspaceID() { + l += thrift.Binary.FieldBeginLength() + l += thrift.Binary.I64Length() + } + return l +} + +func (p *AsyncInvokeEvaluatorRequest) field2Length() int { + l := 0 + if p.IsSetInvokeID() { + l += thrift.Binary.FieldBeginLength() + l += thrift.Binary.I64Length() + } + return l +} + +func (p *AsyncInvokeEvaluatorRequest) field3Length() int { + l := 0 + if p.IsSetEvaluator() { + l += thrift.Binary.FieldBeginLength() + l += p.Evaluator.BLength() + } + return l +} + +func (p *AsyncInvokeEvaluatorRequest) field4Length() int { + l := 0 + if p.IsSetInputData() { + l += thrift.Binary.FieldBeginLength() + l += p.InputData.BLength() + } + return l +} + +func (p *AsyncInvokeEvaluatorRequest) field255Length() int { + l := 0 + if p.IsSetBase() { + l += thrift.Binary.FieldBeginLength() + l += p.Base.BLength() + } + return l +} + +func (p *AsyncInvokeEvaluatorRequest) DeepCopy(s interface{}) error { + src, ok := s.(*AsyncInvokeEvaluatorRequest) + if !ok { + return fmt.Errorf("%T's type not matched %T", s, p) + } + + if src.WorkspaceID != nil { + tmp := *src.WorkspaceID + p.WorkspaceID = &tmp + } + + if src.InvokeID != nil { + tmp := *src.InvokeID + p.InvokeID = &tmp + } + + var _evaluator *InvokeCustomEvaluator + if src.Evaluator != nil { + _evaluator = &InvokeCustomEvaluator{} + if err := _evaluator.DeepCopy(src.Evaluator); err != nil { + return err + } + } + p.Evaluator = _evaluator + + var _inputData *InvokeEvaluatorInputData + if src.InputData != nil { + _inputData = &InvokeEvaluatorInputData{} + if err := _inputData.DeepCopy(src.InputData); err != nil { + return err + } + } + p.InputData = _inputData + + var _base *base.Base + if src.Base != nil { + _base = &base.Base{} + if err := _base.DeepCopy(src.Base); err != nil { + return err + } + } + p.Base = _base + + return nil +} + +func (p *AsyncInvokeEvaluatorResponse) FastRead(buf []byte) (int, error) { + + var err error + var offset int + var l int + var fieldTypeId thrift.TType + var fieldId int16 + for { + fieldTypeId, fieldId, l, err = thrift.Binary.ReadFieldBegin(buf[offset:]) + offset += l + if err != nil { + goto ReadFieldBeginError + } + if fieldTypeId == thrift.STOP { + break + } + switch fieldId { + case 255: + if fieldTypeId == thrift.STRUCT { + l, err = p.FastReadField255(buf[offset:]) + offset += l + if err != nil { + goto ReadFieldError + } + } else { + l, err = thrift.Binary.Skip(buf[offset:], fieldTypeId) + offset += l + if err != nil { + goto SkipFieldError + } + } + default: + l, err = thrift.Binary.Skip(buf[offset:], fieldTypeId) + offset += l + if err != nil { + goto SkipFieldError + } + } + } + + return offset, nil +ReadFieldBeginError: + return offset, thrift.PrependError(fmt.Sprintf("%T read field %d begin error: ", p, fieldId), err) +ReadFieldError: + return offset, thrift.PrependError(fmt.Sprintf("%T read field %d '%s' error: ", p, fieldId, fieldIDToName_AsyncInvokeEvaluatorResponse[fieldId]), err) +SkipFieldError: + return offset, thrift.PrependError(fmt.Sprintf("%T field %d skip type %d error: ", p, fieldId, fieldTypeId), err) +} + +func (p *AsyncInvokeEvaluatorResponse) FastReadField255(buf []byte) (int, error) { + offset := 0 + _field := base.NewBaseResp() + if l, err := _field.FastRead(buf[offset:]); err != nil { + return offset, err + } else { + offset += l + } + p.BaseResp = _field + return offset, nil +} + +func (p *AsyncInvokeEvaluatorResponse) FastWrite(buf []byte) int { + return p.FastWriteNocopy(buf, nil) +} + +func (p *AsyncInvokeEvaluatorResponse) FastWriteNocopy(buf []byte, w thrift.NocopyWriter) int { + offset := 0 + if p != nil { + offset += p.fastWriteField255(buf[offset:], w) + } + offset += thrift.Binary.WriteFieldStop(buf[offset:]) + return offset +} + +func (p *AsyncInvokeEvaluatorResponse) BLength() int { + l := 0 + if p != nil { + l += p.field255Length() + } + l += thrift.Binary.FieldStopLength() + return l +} + +func (p *AsyncInvokeEvaluatorResponse) fastWriteField255(buf []byte, w thrift.NocopyWriter) int { + offset := 0 + offset += thrift.Binary.WriteFieldBegin(buf[offset:], thrift.STRUCT, 255) + offset += p.BaseResp.FastWriteNocopy(buf[offset:], w) + return offset +} + +func (p *AsyncInvokeEvaluatorResponse) field255Length() int { + l := 0 + l += thrift.Binary.FieldBeginLength() + l += p.BaseResp.BLength() + return l +} + +func (p *AsyncInvokeEvaluatorResponse) DeepCopy(s interface{}) error { + src, ok := s.(*AsyncInvokeEvaluatorResponse) + if !ok { + return fmt.Errorf("%T's type not matched %T", s, p) + } + + var _baseResp *base.BaseResp + if src.BaseResp != nil { + _baseResp = &base.BaseResp{} + if err := _baseResp.DeepCopy(src.BaseResp); err != nil { + return err + } + } + p.BaseResp = _baseResp + + return nil +} + func (p *EvaluationSPIServiceSearchEvalTargetArgs) FastRead(buf []byte) (int, error) { var err error @@ -6658,6 +7108,240 @@ func (p *EvaluationSPIServiceInvokeEvaluatorResult) DeepCopy(s interface{}) erro return nil } +func (p *EvaluationSPIServiceAsyncInvokeEvaluatorArgs) FastRead(buf []byte) (int, error) { + + var err error + var offset int + var l int + var fieldTypeId thrift.TType + var fieldId int16 + for { + fieldTypeId, fieldId, l, err = thrift.Binary.ReadFieldBegin(buf[offset:]) + offset += l + if err != nil { + goto ReadFieldBeginError + } + if fieldTypeId == thrift.STOP { + break + } + switch fieldId { + case 1: + if fieldTypeId == thrift.STRUCT { + l, err = p.FastReadField1(buf[offset:]) + offset += l + if err != nil { + goto ReadFieldError + } + } else { + l, err = thrift.Binary.Skip(buf[offset:], fieldTypeId) + offset += l + if err != nil { + goto SkipFieldError + } + } + default: + l, err = thrift.Binary.Skip(buf[offset:], fieldTypeId) + offset += l + if err != nil { + goto SkipFieldError + } + } + } + + return offset, nil +ReadFieldBeginError: + return offset, thrift.PrependError(fmt.Sprintf("%T read field %d begin error: ", p, fieldId), err) +ReadFieldError: + return offset, thrift.PrependError(fmt.Sprintf("%T read field %d '%s' error: ", p, fieldId, fieldIDToName_EvaluationSPIServiceAsyncInvokeEvaluatorArgs[fieldId]), err) +SkipFieldError: + return offset, thrift.PrependError(fmt.Sprintf("%T field %d skip type %d error: ", p, fieldId, fieldTypeId), err) +} + +func (p *EvaluationSPIServiceAsyncInvokeEvaluatorArgs) FastReadField1(buf []byte) (int, error) { + offset := 0 + _field := NewAsyncInvokeEvaluatorRequest() + if l, err := _field.FastRead(buf[offset:]); err != nil { + return offset, err + } else { + offset += l + } + p.Req = _field + return offset, nil +} + +func (p *EvaluationSPIServiceAsyncInvokeEvaluatorArgs) FastWrite(buf []byte) int { + return p.FastWriteNocopy(buf, nil) +} + +func (p *EvaluationSPIServiceAsyncInvokeEvaluatorArgs) FastWriteNocopy(buf []byte, w thrift.NocopyWriter) int { + offset := 0 + if p != nil { + offset += p.fastWriteField1(buf[offset:], w) + } + offset += thrift.Binary.WriteFieldStop(buf[offset:]) + return offset +} + +func (p *EvaluationSPIServiceAsyncInvokeEvaluatorArgs) BLength() int { + l := 0 + if p != nil { + l += p.field1Length() + } + l += thrift.Binary.FieldStopLength() + return l +} + +func (p *EvaluationSPIServiceAsyncInvokeEvaluatorArgs) fastWriteField1(buf []byte, w thrift.NocopyWriter) int { + offset := 0 + offset += thrift.Binary.WriteFieldBegin(buf[offset:], thrift.STRUCT, 1) + offset += p.Req.FastWriteNocopy(buf[offset:], w) + return offset +} + +func (p *EvaluationSPIServiceAsyncInvokeEvaluatorArgs) field1Length() int { + l := 0 + l += thrift.Binary.FieldBeginLength() + l += p.Req.BLength() + return l +} + +func (p *EvaluationSPIServiceAsyncInvokeEvaluatorArgs) DeepCopy(s interface{}) error { + src, ok := s.(*EvaluationSPIServiceAsyncInvokeEvaluatorArgs) + if !ok { + return fmt.Errorf("%T's type not matched %T", s, p) + } + + var _req *AsyncInvokeEvaluatorRequest + if src.Req != nil { + _req = &AsyncInvokeEvaluatorRequest{} + if err := _req.DeepCopy(src.Req); err != nil { + return err + } + } + p.Req = _req + + return nil +} + +func (p *EvaluationSPIServiceAsyncInvokeEvaluatorResult) FastRead(buf []byte) (int, error) { + + var err error + var offset int + var l int + var fieldTypeId thrift.TType + var fieldId int16 + for { + fieldTypeId, fieldId, l, err = thrift.Binary.ReadFieldBegin(buf[offset:]) + offset += l + if err != nil { + goto ReadFieldBeginError + } + if fieldTypeId == thrift.STOP { + break + } + switch fieldId { + case 0: + if fieldTypeId == thrift.STRUCT { + l, err = p.FastReadField0(buf[offset:]) + offset += l + if err != nil { + goto ReadFieldError + } + } else { + l, err = thrift.Binary.Skip(buf[offset:], fieldTypeId) + offset += l + if err != nil { + goto SkipFieldError + } + } + default: + l, err = thrift.Binary.Skip(buf[offset:], fieldTypeId) + offset += l + if err != nil { + goto SkipFieldError + } + } + } + + return offset, nil +ReadFieldBeginError: + return offset, thrift.PrependError(fmt.Sprintf("%T read field %d begin error: ", p, fieldId), err) +ReadFieldError: + return offset, thrift.PrependError(fmt.Sprintf("%T read field %d '%s' error: ", p, fieldId, fieldIDToName_EvaluationSPIServiceAsyncInvokeEvaluatorResult[fieldId]), err) +SkipFieldError: + return offset, thrift.PrependError(fmt.Sprintf("%T field %d skip type %d error: ", p, fieldId, fieldTypeId), err) +} + +func (p *EvaluationSPIServiceAsyncInvokeEvaluatorResult) FastReadField0(buf []byte) (int, error) { + offset := 0 + _field := NewAsyncInvokeEvaluatorResponse() + if l, err := _field.FastRead(buf[offset:]); err != nil { + return offset, err + } else { + offset += l + } + p.Success = _field + return offset, nil +} + +func (p *EvaluationSPIServiceAsyncInvokeEvaluatorResult) FastWrite(buf []byte) int { + return p.FastWriteNocopy(buf, nil) +} + +func (p *EvaluationSPIServiceAsyncInvokeEvaluatorResult) FastWriteNocopy(buf []byte, w thrift.NocopyWriter) int { + offset := 0 + if p != nil { + offset += p.fastWriteField0(buf[offset:], w) + } + offset += thrift.Binary.WriteFieldStop(buf[offset:]) + return offset +} + +func (p *EvaluationSPIServiceAsyncInvokeEvaluatorResult) BLength() int { + l := 0 + if p != nil { + l += p.field0Length() + } + l += thrift.Binary.FieldStopLength() + return l +} + +func (p *EvaluationSPIServiceAsyncInvokeEvaluatorResult) fastWriteField0(buf []byte, w thrift.NocopyWriter) int { + offset := 0 + if p.IsSetSuccess() { + offset += thrift.Binary.WriteFieldBegin(buf[offset:], thrift.STRUCT, 0) + offset += p.Success.FastWriteNocopy(buf[offset:], w) + } + return offset +} + +func (p *EvaluationSPIServiceAsyncInvokeEvaluatorResult) field0Length() int { + l := 0 + if p.IsSetSuccess() { + l += thrift.Binary.FieldBeginLength() + l += p.Success.BLength() + } + return l +} + +func (p *EvaluationSPIServiceAsyncInvokeEvaluatorResult) DeepCopy(s interface{}) error { + src, ok := s.(*EvaluationSPIServiceAsyncInvokeEvaluatorResult) + if !ok { + return fmt.Errorf("%T's type not matched %T", s, p) + } + + var _success *AsyncInvokeEvaluatorResponse + if src.Success != nil { + _success = &AsyncInvokeEvaluatorResponse{} + if err := _success.DeepCopy(src.Success); err != nil { + return err + } + } + p.Success = _success + + return nil +} + func (p *EvaluationSPIServiceSearchEvalTargetArgs) GetFirstArgument() interface{} { return p.Req } @@ -6689,3 +7373,11 @@ func (p *EvaluationSPIServiceInvokeEvaluatorArgs) GetFirstArgument() interface{} func (p *EvaluationSPIServiceInvokeEvaluatorResult) GetResult() interface{} { return p.Success } + +func (p *EvaluationSPIServiceAsyncInvokeEvaluatorArgs) GetFirstArgument() interface{} { + return p.Req +} + +func (p *EvaluationSPIServiceAsyncInvokeEvaluatorResult) GetResult() interface{} { + return p.Success +} diff --git a/backend/loop_gen/coze/loop/evaluation/lospi/local_evaluationspiservice.go b/backend/loop_gen/coze/loop/evaluation/lospi/local_evaluationspiservice.go index d803d4a568..e1fe9fed39 100644 --- a/backend/loop_gen/coze/loop/evaluation/lospi/local_evaluationspiservice.go +++ b/backend/loop_gen/coze/loop/evaluation/lospi/local_evaluationspiservice.go @@ -108,6 +108,27 @@ func (l *LocalEvaluationSPIService) InvokeEvaluator(ctx context.Context, req *sp return result.GetSuccess(), nil } +func (l *LocalEvaluationSPIService) AsyncInvokeEvaluator(ctx context.Context, req *spi.AsyncInvokeEvaluatorRequest, callOptions ...callopt.Option) (*spi.AsyncInvokeEvaluatorResponse, error) { + chain := l.mds(func(ctx context.Context, in, out interface{}) error { + arg := in.(*spi.EvaluationSPIServiceAsyncInvokeEvaluatorArgs) + result := out.(*spi.EvaluationSPIServiceAsyncInvokeEvaluatorResult) + resp, err := l.impl.AsyncInvokeEvaluator(ctx, arg.Req) + if err != nil { + return err + } + result.SetSuccess(resp) + return nil + }) + + arg := &spi.EvaluationSPIServiceAsyncInvokeEvaluatorArgs{Req: req} + result := &spi.EvaluationSPIServiceAsyncInvokeEvaluatorResult{} + ctx = l.injectRPCInfo(ctx, "AsyncInvokeEvaluator") + if err := chain(ctx, arg, result); err != nil { + return nil, err + } + return result.GetSuccess(), nil +} + func (l *LocalEvaluationSPIService) injectRPCInfo(ctx context.Context, method string) context.Context { rpcStats := rpcinfo.AsMutableRPCStats(rpcinfo.NewRPCStats()) ri := rpcinfo.NewRPCInfo( diff --git a/backend/modules/evaluation/application/convertor/evaluator/evaluator.go b/backend/modules/evaluation/application/convertor/evaluator/evaluator.go index 6a67053bf9..82341a50f0 100644 --- a/backend/modules/evaluation/application/convertor/evaluator/evaluator.go +++ b/backend/modules/evaluation/application/convertor/evaluator/evaluator.go @@ -327,7 +327,10 @@ func ConvertEvaluatorContent2DO(content *evaluatordto.EvaluatorContent, evaluato AccessProtocol: content.CustomRPCEvaluator.AccessProtocol, ServiceName: content.CustomRPCEvaluator.ServiceName, Cluster: content.CustomRPCEvaluator.Cluster, + InvokeHTTPInfo: ConvertEvaluatorHTTPInfoDTO2DO(content.CustomRPCEvaluator.InvokeHTTPInfo), + AsyncInvokeHTTPInfo: ConvertEvaluatorHTTPInfoDTO2DO(content.CustomRPCEvaluator.AsyncInvokeHTTPInfo), Timeout: content.CustomRPCEvaluator.Timeout, + IsAsync: content.CustomRPCEvaluator.GetIsAsync(), } if content.CustomRPCEvaluator.RateLimit != nil { rateLimit, err := commonconvertor.ConvertRateLimitDTO2DO(content.CustomRPCEvaluator.RateLimit) @@ -551,7 +554,9 @@ func ConvertCustomRPCEvaluatorVersionDTO2DO(evaluatorID, spaceID int64, dto *eva customRPCEvaluatorVersion.ServiceName = dto.EvaluatorContent.CustomRPCEvaluator.ServiceName customRPCEvaluatorVersion.Cluster = dto.EvaluatorContent.CustomRPCEvaluator.Cluster customRPCEvaluatorVersion.InvokeHTTPInfo = ConvertEvaluatorHTTPInfoDTO2DO(dto.EvaluatorContent.CustomRPCEvaluator.InvokeHTTPInfo) + customRPCEvaluatorVersion.AsyncInvokeHTTPInfo = ConvertEvaluatorHTTPInfoDTO2DO(dto.EvaluatorContent.CustomRPCEvaluator.AsyncInvokeHTTPInfo) customRPCEvaluatorVersion.Timeout = dto.EvaluatorContent.CustomRPCEvaluator.Timeout + customRPCEvaluatorVersion.IsAsync = dto.EvaluatorContent.CustomRPCEvaluator.GetIsAsync() if dto.EvaluatorContent.CustomRPCEvaluator.RateLimit != nil { rateLimit, err := commonconvertor.ConvertRateLimitDTO2DO(dto.EvaluatorContent.CustomRPCEvaluator.RateLimit) if err != nil { @@ -585,8 +590,10 @@ func ConvertCustomRPCEvaluatorVersionDO2DTO(do *evaluatordo.CustomRPCEvaluatorVe Cluster: do.Cluster, Timeout: do.Timeout, InvokeHTTPInfo: ConvertEvaluatorHTTPInfoDO2DTO(do.InvokeHTTPInfo), + AsyncInvokeHTTPInfo: ConvertEvaluatorHTTPInfoDO2DTO(do.AsyncInvokeHTTPInfo), RateLimit: commonconvertor.ConvertRateLimitDO2DTO(do.RateLimit), Ext: do.Ext, + IsAsync: gptr.Of(do.IsAsync), }, }, } diff --git a/backend/modules/evaluation/application/convertor/evaluator/evaluator_test.go b/backend/modules/evaluation/application/convertor/evaluator/evaluator_test.go index 685094ca8d..17b2ba8bc4 100755 --- a/backend/modules/evaluation/application/convertor/evaluator/evaluator_test.go +++ b/backend/modules/evaluation/application/convertor/evaluator/evaluator_test.go @@ -1362,6 +1362,8 @@ func TestConvertEvaluatorLangTags_SkipNilInnerMap(t *testing.T) { // TestConvertCustomRPCEvaluatorVersionDTO2DO 测试将 CustomRPC EvaluatorVersion DTO 转换为 DO func TestConvertCustomRPCEvaluatorVersionDTO2DO(t *testing.T) { t.Parallel() + post := evaluatordto.EvaluatorHTTPMethod(evaluatordo.EvaluatorHTTPMethodPost) + asyncPath := "/async_invoke_evaluator" tests := []struct { name string @@ -1397,6 +1399,11 @@ func TestConvertCustomRPCEvaluatorVersionDTO2DO(t *testing.T) { ServiceName: gptr.Of("test_service"), Cluster: gptr.Of("test_cluster"), Timeout: gptr.Of(int64(5000)), + AsyncInvokeHTTPInfo: &evaluatordto.EvaluatorHTTPInfo{ + Method: gptr.Of(post), + Path: gptr.Of(asyncPath), + }, + IsAsync: gptr.Of(true), }, InputSchemas: []*commondto.ArgsSchema{ { @@ -1432,6 +1439,10 @@ func TestConvertCustomRPCEvaluatorVersionDTO2DO(t *testing.T) { assert.Equal(t, "test_cluster", *result.Cluster) assert.NotNil(t, result.Timeout) assert.Equal(t, int64(5000), *result.Timeout) + assert.True(t, result.IsAsync) + if assert.NotNil(t, result.AsyncInvokeHTTPInfo) { + assert.Equal(t, asyncPath, gptr.Indirect(result.AsyncInvokeHTTPInfo.Path)) + } assert.NotNil(t, result.InputSchemas) assert.Len(t, result.InputSchemas, 1) assert.NotNil(t, result.OutputSchemas) @@ -1476,6 +1487,8 @@ func TestConvertCustomRPCEvaluatorVersionDTO2DO(t *testing.T) { // TestConvertCustomRPCEvaluatorVersionDO2DTO 测试将 CustomRPC EvaluatorVersion DO 转换为 DTO func TestConvertCustomRPCEvaluatorVersionDO2DTO(t *testing.T) { t.Parallel() + post := evaluatordo.EvaluatorHTTPMethodPost + asyncPath := "/async_invoke_evaluator" tests := []struct { name string @@ -1505,6 +1518,11 @@ func TestConvertCustomRPCEvaluatorVersionDO2DTO(t *testing.T) { ServiceName: gptr.Of("test_service"), Cluster: gptr.Of("test_cluster"), Timeout: gptr.Of(int64(5000)), + AsyncInvokeHTTPInfo: &evaluatordo.EvaluatorHTTPInfo{ + Method: &post, + Path: &asyncPath, + }, + IsAsync: true, InputSchemas: []*evaluatordo.ArgsSchema{ { Key: gptr.Of("input1"), @@ -1532,6 +1550,8 @@ func TestConvertCustomRPCEvaluatorVersionDO2DTO(t *testing.T) { assert.Equal(t, "test_service", *result.EvaluatorContent.CustomRPCEvaluator.ServiceName) assert.Equal(t, "test_cluster", *result.EvaluatorContent.CustomRPCEvaluator.Cluster) assert.Equal(t, int64(5000), *result.EvaluatorContent.CustomRPCEvaluator.Timeout) + assert.True(t, result.EvaluatorContent.CustomRPCEvaluator.GetIsAsync()) + assert.Equal(t, asyncPath, result.EvaluatorContent.CustomRPCEvaluator.GetAsyncInvokeHTTPInfo().GetPath()) assert.NotNil(t, result.EvaluatorContent.InputSchemas) assert.Len(t, result.EvaluatorContent.InputSchemas, 1) assert.NotNil(t, result.EvaluatorContent.OutputSchemas) diff --git a/backend/modules/evaluation/application/convertor/evaluator/openapi.go b/backend/modules/evaluation/application/convertor/evaluator/openapi.go index 5848e6ffa5..aad147d40f 100644 --- a/backend/modules/evaluation/application/convertor/evaluator/openapi.go +++ b/backend/modules/evaluation/application/convertor/evaluator/openapi.go @@ -170,9 +170,11 @@ func OpenAPIEvaluatorContentDO2DTO(do *entity.Evaluator) *openapiEvaluator.Evalu ServiceName: v.ServiceName, Cluster: v.Cluster, InvokeHTTPInfo: OpenAPIEvaluatorHTTPInfoDO2DTO(v.InvokeHTTPInfo), + AsyncInvokeHTTPInfo: OpenAPIEvaluatorHTTPInfoDO2DTO(v.AsyncInvokeHTTPInfo), Timeout: v.Timeout, RateLimit: common_convertor.OpenAPIRateLimitDO2DTO(v.RateLimit), Ext: v.Ext, + IsAsync: gptr.Of(v.IsAsync), } } case entity.EvaluatorTypeAgent: @@ -529,9 +531,11 @@ func OpenAPIEvaluatorContentDTO2DO(dto *openapiEvaluator.EvaluatorContent, evalT res.CustomRPCEvaluatorVersion.ServiceName = gptr.Of(c.GetServiceName()) res.CustomRPCEvaluatorVersion.Cluster = gptr.Of(c.GetCluster()) res.CustomRPCEvaluatorVersion.InvokeHTTPInfo = OpenAPIEvaluatorHTTPInfoDTO2DO(c.GetInvokeHTTPInfo()) + res.CustomRPCEvaluatorVersion.AsyncInvokeHTTPInfo = OpenAPIEvaluatorHTTPInfoDTO2DO(c.GetAsyncInvokeHTTPInfo()) if c.IsSetTimeout() { res.CustomRPCEvaluatorVersion.Timeout = gptr.Of(c.GetTimeout()) } + res.CustomRPCEvaluatorVersion.IsAsync = c.GetIsAsync() if c.IsSetExt() && len(c.GetExt()) > 0 { res.CustomRPCEvaluatorVersion.Ext = c.GetExt() } diff --git a/backend/modules/evaluation/application/convertor/evaluator/openapi_test.go b/backend/modules/evaluation/application/convertor/evaluator/openapi_test.go index ceee0950fb..127de66202 100644 --- a/backend/modules/evaluation/application/convertor/evaluator/openapi_test.go +++ b/backend/modules/evaluation/application/convertor/evaluator/openapi_test.go @@ -468,10 +468,17 @@ func TestOpenAPIEvaluatorContentDTO2DO(t *testing.T) { }) t.Run("custom rpc type", func(t *testing.T) { + post := openapiEvaluator.EvaluatorHTTPMethodPost + asyncPath := "/async_invoke_evaluator" dto := &openapiEvaluator.EvaluatorContent{ CustomRPCEvaluator: &openapiEvaluator.CustomRPCEvaluator{ ServiceName: gptr.Of("svc"), Cluster: gptr.Of("cls"), + IsAsync: gptr.Of(true), + AsyncInvokeHTTPInfo: &openapiEvaluator.EvaluatorHTTPInfo{ + Method: gptr.Of(post), + Path: gptr.Of(asyncPath), + }, }, } do, err := OpenAPIEvaluatorContentDTO2DO(dto, entity.EvaluatorTypeCustomRPC) @@ -479,6 +486,8 @@ func TestOpenAPIEvaluatorContentDTO2DO(t *testing.T) { assert.NotNil(t, do) assert.Equal(t, "svc", *do.CustomRPCEvaluatorVersion.ServiceName) assert.Equal(t, "cls", *do.CustomRPCEvaluatorVersion.Cluster) + assert.True(t, do.CustomRPCEvaluatorVersion.IsAsync) + assert.Equal(t, asyncPath, gptr.Indirect(do.CustomRPCEvaluatorVersion.AsyncInvokeHTTPInfo.Path)) }) t.Run("agent type", func(t *testing.T) { @@ -637,15 +646,24 @@ func TestOpenAPIEvaluatorContentDO2DTO(t *testing.T) { }) t.Run("custom rpc type", func(t *testing.T) { + post := entity.EvaluatorHTTPMethodPost + asyncPath := "/async_invoke_evaluator" do := &entity.Evaluator{ EvaluatorType: entity.EvaluatorTypeCustomRPC, CustomRPCEvaluatorVersion: &entity.CustomRPCEvaluatorVersion{ ServiceName: gptr.Of("svc"), + IsAsync: true, + AsyncInvokeHTTPInfo: &entity.EvaluatorHTTPInfo{ + Method: gptr.Of(post), + Path: gptr.Of(asyncPath), + }, }, } dto := OpenAPIEvaluatorContentDO2DTO(do) assert.NotNil(t, dto) assert.Equal(t, "svc", dto.CustomRPCEvaluator.GetServiceName()) + assert.True(t, dto.CustomRPCEvaluator.GetIsAsync()) + assert.Equal(t, asyncPath, dto.CustomRPCEvaluator.GetAsyncInvokeHTTPInfo().GetPath()) }) t.Run("agent type", func(t *testing.T) { diff --git a/backend/modules/evaluation/application/eval_openapi_app.go b/backend/modules/evaluation/application/eval_openapi_app.go index 766f5ccded..771c84ecc4 100644 --- a/backend/modules/evaluation/application/eval_openapi_app.go +++ b/backend/modules/evaluation/application/eval_openapi_app.go @@ -3001,24 +3001,17 @@ func (e *EvalOpenAPIApplication) AsyncRunEvaluatorOApi(ctx context.Context, req InputData: inputData, EvaluatorRunConf: runConf, Ext: req.Ext, + AsyncCtx: &entity.EvalAsyncCtx{ + Session: &entity.Session{UserID: usersession.UserIDInCtxOrEmpty(ctx)}, + CallbackURL: req.GetCallbackURL(), + ResumeReady: true, + AsyncUnixMS: startTime.UnixMilli(), + }, }) if err != nil { return nil, err } - // 写入异步上下文供 ReportEvaluatorInvokeResult 回调读取；独立调用 Event 留空 - asyncCtxKey := fmt.Sprintf("evaluator:%d", record.ID) - if err = e.asyncRepo.SetEvalAsyncCtx(ctx, asyncCtxKey, &entity.EvalAsyncCtx{ - RecordID: record.ID, - AsyncUnixMS: startTime.UnixMilli(), - Session: &entity.Session{UserID: usersession.UserIDInCtxOrEmpty(ctx)}, - EvaluatorVersionID: req.GetEvaluatorVersionID(), - CallbackURL: req.GetCallbackURL(), - }); err != nil { - logs.CtxError(ctx, "[AsyncRunEvaluatorOApi] SetEvalAsyncCtx fail, invokeID: %d, err: %v", record.ID, err) - return nil, err - } - return &openapi.AsyncRunEvaluatorOApiResponse{ Data: &openapi.AsyncRunEvaluatorOpenAPIData{ InvokeID: gptr.Of(record.ID), @@ -3530,7 +3523,26 @@ func (e *EvalOpenAPIApplication) ListExptTemplatesOApi(ctx context.Context, req } func (e *EvalOpenAPIApplication) ReportEvaluatorInvokeResult_(ctx context.Context, req *openapi.ReportEvaluatorInvokeResultRequest) (r *openapi.ReportEvaluatorInvokeResultResponse, err error) { - logs.CtxInfo(ctx, "ReportEvaluatorInvokeResult receive req: %v", json.Jsonify(req)) + if req == nil { + return nil, errorx.NewByCode(errno.CommonInvalidParamCode, errorx.WithExtraMsg("req is nil")) + } + if req.GetWorkspaceID() <= 0 || req.GetInvokeID() <= 0 { + return nil, errorx.NewByCode(errno.CommonInvalidParamCode, errorx.WithExtraMsg("workspace_id and invoke_id are required")) + } + if req.GetStatus() != spi.InvokeEvaluatorRunStatus_SUCCESS && req.GetStatus() != spi.InvokeEvaluatorRunStatus_FAILED { + return nil, errorx.NewByCode(errno.CommonInvalidParamCode, errorx.WithExtraMsg("status must be SUCCESS or FAILED")) + } + if req.GetStatus() == spi.InvokeEvaluatorRunStatus_SUCCESS { + if req.Output == nil || req.Output.EvaluatorResult_ == nil || req.Output.EvaluatorResult_.Score == nil { + return nil, errorx.NewByCode(errno.CommonInvalidParamCode, errorx.WithExtraMsg("evaluator_result.score is required for SUCCESS")) + } + } + if req.GetStatus() == spi.InvokeEvaluatorRunStatus_FAILED { + if req.Output == nil || req.Output.EvaluatorRunError == nil || strings.TrimSpace(req.Output.EvaluatorRunError.GetMessage()) == "" { + return nil, errorx.NewByCode(errno.CommonInvalidParamCode, errorx.WithExtraMsg("evaluator_run_error.message is required for FAILED")) + } + } + logs.CtxInfo(ctx, "ReportEvaluatorInvokeResult receive, workspace_id: %d, invoke_id: %d, status: %v", req.GetWorkspaceID(), req.GetInvokeID(), req.GetStatus()) err = e.auth.Authorization(ctx, &rpc.AuthorizationParam{ ObjectID: strconv.FormatInt(req.GetWorkspaceID(), 10), @@ -3542,7 +3554,7 @@ func (e *EvalOpenAPIApplication) ReportEvaluatorInvokeResult_(ctx context.Contex } asyncCtxKey := fmt.Sprintf("evaluator:%d", req.GetInvokeID()) - actx, err := e.asyncRepo.GetEvalAsyncCtx(ctx, asyncCtxKey) + actx, err := e.asyncRepo.GetEvalAsyncCtxStrong(ctx, asyncCtxKey) if err != nil { return nil, err } @@ -3555,9 +3567,6 @@ func (e *EvalOpenAPIApplication) ReportEvaluatorInvokeResult_(ctx context.Contex req.GetInvokeID(), actx.EvaluatorVersionID, req.GetWorkspaceID(), actx.Event.GetExptID(), actx.Event.GetExptRunID(), actx.Event.GetEvalSetItemID(), req.GetStatus()) outputData := evaluator_convertor.ToInvokeEvaluatorOutputDataDO(req.GetOutput(), req.GetStatus()) - if outputData != nil { - outputData.TimeConsumingMS = time.Now().UnixMilli() - actx.AsyncUnixMS - } runStatus := evaluator_convertor.ToEvaluatorRunStatusDO(req.GetStatus()) if err := e.evaluatorService.ReportEvaluatorInvokeResult(ctx, &entity.ReportEvaluatorRecordParam{ @@ -3568,12 +3577,22 @@ func (e *EvalOpenAPIApplication) ReportEvaluatorInvokeResult_(ctx context.Contex }); err != nil { return nil, err } - if actx.Event != nil { - if err := e.publisher.PublishExptRecordEvalEvent(ctx, actx.Event, gptr.Of(time.Second*3), func(event *entity.ExptItemEvalEvent) { - event.AsyncEvaluatorReportTrigger = true - }); err != nil { - return nil, err + if !actx.ResumeReady { + latestCtx, ctxErr := e.asyncRepo.GetEvalAsyncCtxStrong(ctx, asyncCtxKey) + if ctxErr != nil { + return nil, ctxErr + } + if latestCtx != nil { + actx = latestCtx + } + } + if actx.ResumeReady { + if err := e.publisher.PublishExptRecordEvalEvent(ctx, actx.Event, gptr.Of(time.Second*3), func(event *entity.ExptItemEvalEvent) { + event.AsyncEvaluatorReportTrigger = true + }); err != nil { + return nil, err + } } } diff --git a/backend/modules/evaluation/application/eval_openapi_app_test.go b/backend/modules/evaluation/application/eval_openapi_app_test.go index c451c5f5e4..622f721565 100755 --- a/backend/modules/evaluation/application/eval_openapi_app_test.go +++ b/backend/modules/evaluation/application/eval_openapi_app_test.go @@ -6555,12 +6555,56 @@ func TestEvalOpenAPIApplication_ReportEvaluatorInvokeResult(t *testing.T) { setup func(auth *rpcmocks.MockIAuthProvider, asyncRepo *repomocks.MockIEvalAsyncRepo, evaluatorSvc *servicemocks.MockEvaluatorService, publisher *eventmocks.MockExptEventPublisher, dispatcher *servicemocks.MockIEvaluatorCallbackDispatcher) wantErr int32 }{ + { + name: "nil request", + req: nil, + setup: func(*rpcmocks.MockIAuthProvider, *repomocks.MockIEvalAsyncRepo, *servicemocks.MockEvaluatorService, *eventmocks.MockExptEventPublisher, *servicemocks.MockIEvaluatorCallbackDispatcher) { + }, + wantErr: errno.CommonInvalidParamCode, + }, + { + name: "unknown status rejected before auth", + req: &openapi.ReportEvaluatorInvokeResultRequest{ + WorkspaceID: gptr.Of(workspaceID), + InvokeID: gptr.Of(invokeID), + Status: gptr.Of(spi.InvokeEvaluatorRunStatus_UNKNOWN), + }, + setup: func(*rpcmocks.MockIAuthProvider, *repomocks.MockIEvalAsyncRepo, *servicemocks.MockEvaluatorService, *eventmocks.MockExptEventPublisher, *servicemocks.MockIEvaluatorCallbackDispatcher) { + }, + wantErr: errno.CommonInvalidParamCode, + }, + + { + name: "success without score rejected", + req: &openapi.ReportEvaluatorInvokeResultRequest{ + WorkspaceID: gptr.Of(workspaceID), + InvokeID: gptr.Of(invokeID), + Status: gptr.Of(spi.InvokeEvaluatorRunStatus_SUCCESS), + }, + setup: func(*rpcmocks.MockIAuthProvider, *repomocks.MockIEvalAsyncRepo, *servicemocks.MockEvaluatorService, *eventmocks.MockExptEventPublisher, *servicemocks.MockIEvaluatorCallbackDispatcher) { + }, + wantErr: errno.CommonInvalidParamCode, + }, + { + name: "failed without error message rejected", + req: &openapi.ReportEvaluatorInvokeResultRequest{ + WorkspaceID: gptr.Of(workspaceID), + InvokeID: gptr.Of(invokeID), + Status: gptr.Of(spi.InvokeEvaluatorRunStatus_FAILED), + Output: &spi.InvokeEvaluatorOutputData{EvaluatorRunError: &spi.InvokeEvaluatorRunError{}}, + }, + setup: func(*rpcmocks.MockIAuthProvider, *repomocks.MockIEvalAsyncRepo, *servicemocks.MockEvaluatorService, *eventmocks.MockExptEventPublisher, *servicemocks.MockIEvaluatorCallbackDispatcher) { + }, + wantErr: errno.CommonInvalidParamCode, + }, + { name: "auth failed", req: &openapi.ReportEvaluatorInvokeResultRequest{ WorkspaceID: gptr.Of(workspaceID), InvokeID: gptr.Of(invokeID), Status: gptr.Of(spi.InvokeEvaluatorRunStatus_SUCCESS), + Output: &spi.InvokeEvaluatorOutputData{EvaluatorResult_: &spi.InvokeEvaluatorResult_{Score: gptr.Of(float64(0.9))}}, }, setup: func(auth *rpcmocks.MockIAuthProvider, _ *repomocks.MockIEvalAsyncRepo, _ *servicemocks.MockEvaluatorService, _ *eventmocks.MockExptEventPublisher, _ *servicemocks.MockIEvaluatorCallbackDispatcher) { auth.EXPECT().Authorization(gomock.Any(), gomock.Any()).Return(errorx.NewByCode(errno.CommonNoPermissionCode)) @@ -6573,10 +6617,11 @@ func TestEvalOpenAPIApplication_ReportEvaluatorInvokeResult(t *testing.T) { WorkspaceID: gptr.Of(workspaceID), InvokeID: gptr.Of(invokeID), Status: gptr.Of(spi.InvokeEvaluatorRunStatus_SUCCESS), + Output: &spi.InvokeEvaluatorOutputData{EvaluatorResult_: &spi.InvokeEvaluatorResult_{Score: gptr.Of(float64(0.9))}}, }, setup: func(auth *rpcmocks.MockIAuthProvider, asyncRepo *repomocks.MockIEvalAsyncRepo, _ *servicemocks.MockEvaluatorService, _ *eventmocks.MockExptEventPublisher, _ *servicemocks.MockIEvaluatorCallbackDispatcher) { auth.EXPECT().Authorization(gomock.Any(), gomock.Any()).Return(nil) - asyncRepo.EXPECT().GetEvalAsyncCtx(gomock.Any(), "evaluator:2002").Return(nil, errors.New("get failed")) + asyncRepo.EXPECT().GetEvalAsyncCtxStrong(gomock.Any(), "evaluator:2002").Return(nil, errors.New("get failed")) }, wantErr: -1, }, @@ -6586,10 +6631,11 @@ func TestEvalOpenAPIApplication_ReportEvaluatorInvokeResult(t *testing.T) { WorkspaceID: gptr.Of(workspaceID), InvokeID: gptr.Of(invokeID), Status: gptr.Of(spi.InvokeEvaluatorRunStatus_SUCCESS), + Output: &spi.InvokeEvaluatorOutputData{EvaluatorResult_: &spi.InvokeEvaluatorResult_{Score: gptr.Of(float64(0.9))}}, }, setup: func(auth *rpcmocks.MockIAuthProvider, asyncRepo *repomocks.MockIEvalAsyncRepo, _ *servicemocks.MockEvaluatorService, _ *eventmocks.MockExptEventPublisher, _ *servicemocks.MockIEvaluatorCallbackDispatcher) { auth.EXPECT().Authorization(gomock.Any(), gomock.Any()).Return(nil) - asyncRepo.EXPECT().GetEvalAsyncCtx(gomock.Any(), "evaluator:2002").Return(nil, nil) + asyncRepo.EXPECT().GetEvalAsyncCtxStrong(gomock.Any(), "evaluator:2002").Return(nil, nil) }, wantErr: -1, }, @@ -6605,8 +6651,9 @@ func TestEvalOpenAPIApplication_ReportEvaluatorInvokeResult(t *testing.T) { }, setup: func(auth *rpcmocks.MockIAuthProvider, asyncRepo *repomocks.MockIEvalAsyncRepo, evaluatorSvc *servicemocks.MockEvaluatorService, _ *eventmocks.MockExptEventPublisher, _ *servicemocks.MockIEvaluatorCallbackDispatcher) { auth.EXPECT().Authorization(gomock.Any(), gomock.Any()).Return(nil) - asyncRepo.EXPECT().GetEvalAsyncCtx(gomock.Any(), "evaluator:2002").Return(&entity.EvalAsyncCtx{ + asyncRepo.EXPECT().GetEvalAsyncCtxStrong(gomock.Any(), "evaluator:2002").Return(&entity.EvalAsyncCtx{ Event: event, + ResumeReady: true, AsyncUnixMS: time.Now().UnixMilli() - 10, EvaluatorVersionID: 9, }, nil) @@ -6627,8 +6674,9 @@ func TestEvalOpenAPIApplication_ReportEvaluatorInvokeResult(t *testing.T) { }, setup: func(auth *rpcmocks.MockIAuthProvider, asyncRepo *repomocks.MockIEvalAsyncRepo, evaluatorSvc *servicemocks.MockEvaluatorService, publisher *eventmocks.MockExptEventPublisher, _ *servicemocks.MockIEvaluatorCallbackDispatcher) { auth.EXPECT().Authorization(gomock.Any(), gomock.Any()).Return(nil) - asyncRepo.EXPECT().GetEvalAsyncCtx(gomock.Any(), "evaluator:2002").Return(&entity.EvalAsyncCtx{ + asyncRepo.EXPECT().GetEvalAsyncCtxStrong(gomock.Any(), "evaluator:2002").Return(&entity.EvalAsyncCtx{ Event: event, + ResumeReady: true, AsyncUnixMS: time.Now().UnixMilli() - 10, EvaluatorVersionID: 9, }, nil) @@ -6656,7 +6704,7 @@ func TestEvalOpenAPIApplication_ReportEvaluatorInvokeResult(t *testing.T) { }, setup: func(auth *rpcmocks.MockIAuthProvider, asyncRepo *repomocks.MockIEvalAsyncRepo, evaluatorSvc *servicemocks.MockEvaluatorService, _ *eventmocks.MockExptEventPublisher, _ *servicemocks.MockIEvaluatorCallbackDispatcher) { auth.EXPECT().Authorization(gomock.Any(), gomock.Any()).Return(nil) - asyncRepo.EXPECT().GetEvalAsyncCtx(gomock.Any(), "evaluator:2002").Return(&entity.EvalAsyncCtx{ + asyncRepo.EXPECT().GetEvalAsyncCtxStrong(gomock.Any(), "evaluator:2002").Return(&entity.EvalAsyncCtx{ Event: nil, AsyncUnixMS: time.Now().UnixMilli() - 10, EvaluatorVersionID: 9, @@ -6664,6 +6712,61 @@ func TestEvalOpenAPIApplication_ReportEvaluatorInvokeResult(t *testing.T) { evaluatorSvc.EXPECT().ReportEvaluatorInvokeResult(gomock.Any(), gomock.Any()).Return(nil) }, }, + { + name: "success before resume ready skips experiment publish", + req: &openapi.ReportEvaluatorInvokeResultRequest{ + WorkspaceID: gptr.Of(workspaceID), + InvokeID: gptr.Of(invokeID), + Status: gptr.Of(spi.InvokeEvaluatorRunStatus_SUCCESS), + Output: &spi.InvokeEvaluatorOutputData{ + EvaluatorResult_: &spi.InvokeEvaluatorResult_{Score: gptr.Of(float64(0.9))}, + }, + }, + setup: func(auth *rpcmocks.MockIAuthProvider, asyncRepo *repomocks.MockIEvalAsyncRepo, evaluatorSvc *servicemocks.MockEvaluatorService, publisher *eventmocks.MockExptEventPublisher, _ *servicemocks.MockIEvaluatorCallbackDispatcher) { + auth.EXPECT().Authorization(gomock.Any(), gomock.Any()).Return(nil) + asyncRepo.EXPECT().GetEvalAsyncCtxStrong(gomock.Any(), "evaluator:2002").Return(&entity.EvalAsyncCtx{ + Event: event, + ResumeReady: false, + EvaluatorVersionID: 9, + }, nil) + evaluatorSvc.EXPECT().ReportEvaluatorInvokeResult(gomock.Any(), gomock.Any()).Return(nil) + asyncRepo.EXPECT().GetEvalAsyncCtxStrong(gomock.Any(), "evaluator:2002").Return(&entity.EvalAsyncCtx{ + Event: event, + ResumeReady: false, + EvaluatorVersionID: 9, + }, nil) + publisher.EXPECT().PublishExptRecordEvalEvent(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Times(0) + }, + }, + { + name: "resume becomes ready between initial read and terminal CAS", + req: &openapi.ReportEvaluatorInvokeResultRequest{ + WorkspaceID: gptr.Of(workspaceID), + InvokeID: gptr.Of(invokeID), + Status: gptr.Of(spi.InvokeEvaluatorRunStatus_SUCCESS), + Output: &spi.InvokeEvaluatorOutputData{ + EvaluatorResult_: &spi.InvokeEvaluatorResult_{Score: gptr.Of(float64(0.9))}, + }, + }, + setup: func(auth *rpcmocks.MockIAuthProvider, asyncRepo *repomocks.MockIEvalAsyncRepo, evaluatorSvc *servicemocks.MockEvaluatorService, publisher *eventmocks.MockExptEventPublisher, _ *servicemocks.MockIEvaluatorCallbackDispatcher) { + auth.EXPECT().Authorization(gomock.Any(), gomock.Any()).Return(nil) + gomock.InOrder( + asyncRepo.EXPECT().GetEvalAsyncCtxStrong(gomock.Any(), "evaluator:2002").Return(&entity.EvalAsyncCtx{ + Event: event, + ResumeReady: false, + EvaluatorVersionID: 9, + }, nil), + evaluatorSvc.EXPECT().ReportEvaluatorInvokeResult(gomock.Any(), gomock.Any()).Return(nil), + asyncRepo.EXPECT().GetEvalAsyncCtxStrong(gomock.Any(), "evaluator:2002").Return(&entity.EvalAsyncCtx{ + Event: event, + ResumeReady: true, + EvaluatorVersionID: 9, + }, nil), + ) + publisher.EXPECT().PublishExptRecordEvalEvent(gomock.Any(), event, gomock.Not(gomock.Nil()), gomock.Any()).Return(nil) + }, + }, + { name: "success", req: &openapi.ReportEvaluatorInvokeResultRequest{ @@ -6677,8 +6780,9 @@ func TestEvalOpenAPIApplication_ReportEvaluatorInvokeResult(t *testing.T) { }, setup: func(auth *rpcmocks.MockIAuthProvider, asyncRepo *repomocks.MockIEvalAsyncRepo, evaluatorSvc *servicemocks.MockEvaluatorService, publisher *eventmocks.MockExptEventPublisher, _ *servicemocks.MockIEvaluatorCallbackDispatcher) { auth.EXPECT().Authorization(gomock.Any(), gomock.Any()).Return(nil) - asyncRepo.EXPECT().GetEvalAsyncCtx(gomock.Any(), "evaluator:2002").Return(&entity.EvalAsyncCtx{ + asyncRepo.EXPECT().GetEvalAsyncCtxStrong(gomock.Any(), "evaluator:2002").Return(&entity.EvalAsyncCtx{ Event: event, + ResumeReady: true, AsyncUnixMS: time.Now().UnixMilli() - 50, EvaluatorVersionID: 9, }, nil) @@ -6713,7 +6817,7 @@ func TestEvalOpenAPIApplication_ReportEvaluatorInvokeResult(t *testing.T) { }, setup: func(auth *rpcmocks.MockIAuthProvider, asyncRepo *repomocks.MockIEvalAsyncRepo, evaluatorSvc *servicemocks.MockEvaluatorService, _ *eventmocks.MockExptEventPublisher, dispatcher *servicemocks.MockIEvaluatorCallbackDispatcher) { auth.EXPECT().Authorization(gomock.Any(), gomock.Any()).Return(nil) - asyncRepo.EXPECT().GetEvalAsyncCtx(gomock.Any(), "evaluator:2002").Return(&entity.EvalAsyncCtx{ + asyncRepo.EXPECT().GetEvalAsyncCtxStrong(gomock.Any(), "evaluator:2002").Return(&entity.EvalAsyncCtx{ Event: nil, AsyncUnixMS: time.Now().UnixMilli() - 10, EvaluatorVersionID: 9, @@ -6742,7 +6846,7 @@ func TestEvalOpenAPIApplication_ReportEvaluatorInvokeResult(t *testing.T) { }, setup: func(auth *rpcmocks.MockIAuthProvider, asyncRepo *repomocks.MockIEvalAsyncRepo, evaluatorSvc *servicemocks.MockEvaluatorService, _ *eventmocks.MockExptEventPublisher, dispatcher *servicemocks.MockIEvaluatorCallbackDispatcher) { auth.EXPECT().Authorization(gomock.Any(), gomock.Any()).Return(nil) - asyncRepo.EXPECT().GetEvalAsyncCtx(gomock.Any(), "evaluator:2002").Return(&entity.EvalAsyncCtx{ + asyncRepo.EXPECT().GetEvalAsyncCtxStrong(gomock.Any(), "evaluator:2002").Return(&entity.EvalAsyncCtx{ Event: nil, AsyncUnixMS: time.Now().UnixMilli() - 10, EvaluatorVersionID: 9, @@ -6768,7 +6872,7 @@ func TestEvalOpenAPIApplication_ReportEvaluatorInvokeResult(t *testing.T) { }, setup: func(auth *rpcmocks.MockIAuthProvider, asyncRepo *repomocks.MockIEvalAsyncRepo, evaluatorSvc *servicemocks.MockEvaluatorService, _ *eventmocks.MockExptEventPublisher, dispatcher *servicemocks.MockIEvaluatorCallbackDispatcher) { auth.EXPECT().Authorization(gomock.Any(), gomock.Any()).Return(nil) - asyncRepo.EXPECT().GetEvalAsyncCtx(gomock.Any(), "evaluator:2002").Return(&entity.EvalAsyncCtx{ + asyncRepo.EXPECT().GetEvalAsyncCtxStrong(gomock.Any(), "evaluator:2002").Return(&entity.EvalAsyncCtx{ Event: nil, AsyncUnixMS: time.Now().UnixMilli() - 10, EvaluatorVersionID: 9, @@ -6790,7 +6894,7 @@ func TestEvalOpenAPIApplication_ReportEvaluatorInvokeResult(t *testing.T) { }, setup: func(auth *rpcmocks.MockIAuthProvider, asyncRepo *repomocks.MockIEvalAsyncRepo, evaluatorSvc *servicemocks.MockEvaluatorService, _ *eventmocks.MockExptEventPublisher, dispatcher *servicemocks.MockIEvaluatorCallbackDispatcher) { auth.EXPECT().Authorization(gomock.Any(), gomock.Any()).Return(nil) - asyncRepo.EXPECT().GetEvalAsyncCtx(gomock.Any(), "evaluator:2002").Return(&entity.EvalAsyncCtx{ + asyncRepo.EXPECT().GetEvalAsyncCtxStrong(gomock.Any(), "evaluator:2002").Return(&entity.EvalAsyncCtx{ Event: nil, AsyncUnixMS: time.Now().UnixMilli() - 10, EvaluatorVersionID: 9, @@ -8414,21 +8518,20 @@ func TestEvalOpenAPIApplication_AsyncRunEvaluatorOApi(t *testing.T) { wantErr: -1, }, { - name: "set async ctx failed", + name: "coordinator context setup failed", req: &openapi.AsyncRunEvaluatorOApiRequest{ WorkspaceID: gptr.Of(workspaceID), EvaluatorVersionID: gptr.Of(evaluatorVersionID), }, - setup: func(auth *rpcmocks.MockIAuthProvider, evaluatorSvc *servicemocks.MockEvaluatorService, asyncRepo *repomocks.MockIEvalAsyncRepo) { + setup: func(auth *rpcmocks.MockIAuthProvider, evaluatorSvc *servicemocks.MockEvaluatorService, _ *repomocks.MockIEvalAsyncRepo) { evaluator := &entity.Evaluator{ ID: evaluatorVersionID, SpaceID: workspaceID, BaseInfo: &entity.BaseInfo{CreatedBy: &entity.UserInfo{UserID: gptr.Of("owner")}}, } - record := &entity.EvaluatorRecord{ID: invokeID, Status: entity.EvaluatorRunStatusAsyncInvoking} + record := &entity.EvaluatorRecord{ID: invokeID, Status: entity.EvaluatorRunStatusFail} evaluatorSvc.EXPECT().GetEvaluatorVersion(gomock.Any(), gomock.Any(), evaluatorVersionID, false, false).Return(evaluator, nil) auth.EXPECT().AuthorizationWithoutSPI(gomock.Any(), gomock.Any()).Return(nil) - evaluatorSvc.EXPECT().AsyncRunEvaluator(gomock.Any(), gomock.Any()).Return(record, nil) - asyncRepo.EXPECT().SetEvalAsyncCtx(gomock.Any(), gomock.Any(), gomock.Any()).Return(errors.New("redis error")) + evaluatorSvc.EXPECT().AsyncRunEvaluator(gomock.Any(), gomock.Any()).Return(record, errors.New("redis error")) }, wantErr: -1, }, @@ -8439,7 +8542,7 @@ func TestEvalOpenAPIApplication_AsyncRunEvaluatorOApi(t *testing.T) { EvaluatorVersionID: gptr.Of(evaluatorVersionID), CallbackURL: gptr.Of("https://example.com/hook"), }, - setup: func(auth *rpcmocks.MockIAuthProvider, evaluatorSvc *servicemocks.MockEvaluatorService, asyncRepo *repomocks.MockIEvalAsyncRepo) { + setup: func(auth *rpcmocks.MockIAuthProvider, evaluatorSvc *servicemocks.MockEvaluatorService, _ *repomocks.MockIEvalAsyncRepo) { evaluator := &entity.Evaluator{ ID: evaluatorVersionID, SpaceID: workspaceID, Name: "agent-eval", BaseInfo: &entity.BaseInfo{CreatedBy: &entity.UserInfo{UserID: gptr.Of("owner")}}, @@ -8447,15 +8550,12 @@ func TestEvalOpenAPIApplication_AsyncRunEvaluatorOApi(t *testing.T) { record := &entity.EvaluatorRecord{ID: invokeID, Status: entity.EvaluatorRunStatusAsyncInvoking} evaluatorSvc.EXPECT().GetEvaluatorVersion(gomock.Any(), gomock.Any(), evaluatorVersionID, false, false).Return(evaluator, nil) auth.EXPECT().AuthorizationWithoutSPI(gomock.Any(), gomock.Any()).Return(nil) - evaluatorSvc.EXPECT().AsyncRunEvaluator(gomock.Any(), gomock.Any()).Return(record, nil) - asyncRepo.EXPECT().SetEvalAsyncCtx(gomock.Any(), "evaluator:4004", gomock.Any()). - DoAndReturn(func(_ context.Context, _ string, actx *entity.EvalAsyncCtx) error { - assert.Equal(t, invokeID, actx.RecordID) - assert.Equal(t, evaluatorVersionID, actx.EvaluatorVersionID) - assert.Nil(t, actx.Event) - assert.Equal(t, "https://example.com/hook", actx.CallbackURL) - return nil - }) + evaluatorSvc.EXPECT().AsyncRunEvaluator(gomock.Any(), gomock.Any()).DoAndReturn(func(_ context.Context, req *entity.AsyncRunEvaluatorRequest) (*entity.EvaluatorRecord, error) { + require.NotNil(t, req.AsyncCtx) + assert.True(t, req.AsyncCtx.ResumeReady) + assert.Nil(t, req.AsyncCtx.Event) + return record, nil + }) }, }, { @@ -8464,19 +8564,16 @@ func TestEvalOpenAPIApplication_AsyncRunEvaluatorOApi(t *testing.T) { WorkspaceID: gptr.Of(workspaceID), EvaluatorVersionID: gptr.Of(evaluatorVersionID), }, - setup: func(_ *rpcmocks.MockIAuthProvider, evaluatorSvc *servicemocks.MockEvaluatorService, asyncRepo *repomocks.MockIEvalAsyncRepo) { + setup: func(_ *rpcmocks.MockIAuthProvider, evaluatorSvc *servicemocks.MockEvaluatorService, _ *repomocks.MockIEvalAsyncRepo) { evaluator := &entity.Evaluator{ID: evaluatorVersionID, SpaceID: workspaceID + 999, Builtin: true, Name: "builtin-agent"} record := &entity.EvaluatorRecord{ID: invokeID, Status: entity.EvaluatorRunStatusAsyncInvoking} evaluatorSvc.EXPECT().GetEvaluatorVersion(gomock.Any(), gomock.Any(), evaluatorVersionID, false, false).Return(evaluator, nil) - evaluatorSvc.EXPECT().AsyncRunEvaluator(gomock.Any(), gomock.Any()).Return(record, nil) - asyncRepo.EXPECT().SetEvalAsyncCtx(gomock.Any(), "evaluator:4004", gomock.Any()). - DoAndReturn(func(_ context.Context, _ string, actx *entity.EvalAsyncCtx) error { - assert.Equal(t, invokeID, actx.RecordID) - assert.Equal(t, evaluatorVersionID, actx.EvaluatorVersionID) - assert.Nil(t, actx.Event) - assert.Equal(t, "", actx.CallbackURL) - return nil - }) + evaluatorSvc.EXPECT().AsyncRunEvaluator(gomock.Any(), gomock.Any()).DoAndReturn(func(_ context.Context, req *entity.AsyncRunEvaluatorRequest) (*entity.EvaluatorRecord, error) { + require.NotNil(t, req.AsyncCtx) + assert.True(t, req.AsyncCtx.ResumeReady) + assert.Nil(t, req.AsyncCtx.Event) + return record, nil + }) }, }, } diff --git a/backend/modules/evaluation/application/evaluator_app.go b/backend/modules/evaluation/application/evaluator_app.go index 05e0486765..a636635026 100644 --- a/backend/modules/evaluation/application/evaluator_app.go +++ b/backend/modules/evaluation/application/evaluator_app.go @@ -2085,7 +2085,6 @@ func (e *EvaluatorHandlerImpl) ListEvaluatorTags(ctx context.Context, request *e } func (e *EvaluatorHandlerImpl) AsyncRunEvaluator(ctx context.Context, req *evaluatorservice.AsyncRunEvaluatorRequest) (r *evaluatorservice.AsyncRunEvaluatorResponse, err error) { - startTime := time.Now() evaluatorDO, err := e.evaluatorService.GetEvaluatorVersion(ctx, nil, req.GetEvaluatorVersionID(), false, false) if err != nil { return nil, err @@ -2103,19 +2102,13 @@ func (e *EvaluatorHandlerImpl) AsyncRunEvaluator(ctx context.Context, req *evalu return nil, err } } - resp, err := e.evaluatorService.AsyncRunEvaluator(ctx, buildAsyncRunEvaluatorRequest(evaluatorDO.Name, req)) - if err != nil { - return nil, err + asyncReq := buildAsyncRunEvaluatorRequest(evaluatorDO.Name, req) + asyncReq.AsyncCtx = &entity.EvalAsyncCtx{ + Session: &entity.Session{UserID: session.UserIDInCtxOrEmpty(ctx)}, + ResumeReady: true, } - - asyncCtxKey := fmt.Sprintf("evaluator:%d", resp.ID) - if err := e.evalAsyncRepo.SetEvalAsyncCtx(ctx, asyncCtxKey, &entity.EvalAsyncCtx{ - RecordID: resp.ID, - AsyncUnixMS: startTime.UnixMilli(), - Session: &entity.Session{UserID: session.UserIDInCtxOrEmpty(ctx)}, - EvaluatorVersionID: req.GetEvaluatorVersionID(), - }); err != nil { - logs.CtxError(ctx, "[AsyncRunEvaluator] SetEvalAsyncCtx fail, invokeID: %d, err: %v", resp.ID, err) + resp, err := e.evaluatorService.AsyncRunEvaluator(ctx, asyncReq) + if err != nil { return nil, err } diff --git a/backend/modules/evaluation/application/evaluator_app_test.go b/backend/modules/evaluation/application/evaluator_app_test.go index b2113359c8..14fc2e5d65 100644 --- a/backend/modules/evaluation/application/evaluator_app_test.go +++ b/backend/modules/evaluation/application/evaluator_app_test.go @@ -9028,7 +9028,6 @@ func TestEvaluatorHandlerImpl_AsyncRunEvaluator(t *testing.T) { mockEvaluatorService.EXPECT().GetEvaluatorVersion(gomock.Any(), nil, int64(101), false, false).Return(evaluatorDO, nil) mockAuth.EXPECT().Authorization(gomock.Any(), gomock.Any()).Return(nil) mockEvaluatorService.EXPECT().AsyncRunEvaluator(gomock.Any(), gomock.Any()).Return(&entity.EvaluatorRecord{ID: 999}, nil) - mockEvalAsyncRepo.EXPECT().SetEvalAsyncCtx(gomock.Any(), "evaluator:999", gomock.Any()).Return(nil) }, wantErr: false, }, @@ -9058,12 +9057,11 @@ func TestEvaluatorHandlerImpl_AsyncRunEvaluator(t *testing.T) { wantErr: true, }, { - name: "失败 - SetEvalAsyncCtx失败", + name: "失败 - coordinator context setup失败", setupMocks: func() { mockEvaluatorService.EXPECT().GetEvaluatorVersion(gomock.Any(), nil, int64(101), false, false).Return(evaluatorDO, nil) mockAuth.EXPECT().Authorization(gomock.Any(), gomock.Any()).Return(nil) - mockEvaluatorService.EXPECT().AsyncRunEvaluator(gomock.Any(), gomock.Any()).Return(&entity.EvaluatorRecord{ID: 999}, nil) - mockEvalAsyncRepo.EXPECT().SetEvalAsyncCtx(gomock.Any(), "evaluator:999", gomock.Any()).Return(errors.New("set ctx failed")) + mockEvaluatorService.EXPECT().AsyncRunEvaluator(gomock.Any(), gomock.Any()).Return(&entity.EvaluatorRecord{ID: 999, Status: entity.EvaluatorRunStatusFail}, errors.New("set ctx failed")) }, wantErr: true, }, @@ -9257,7 +9255,6 @@ func TestEvaluatorHandlerImpl_AsyncRunEvaluator_Builtin_Agent(t *testing.T) { mockEvaluatorService.EXPECT().GetEvaluatorVersion(gomock.Any(), nil, int64(101), false, false).Return(evaluatorDO, nil) // 不调用 Authorization mockEvaluatorService.EXPECT().AsyncRunEvaluator(gomock.Any(), gomock.Any()).Return(&entity.EvaluatorRecord{ID: 999}, nil) - mockEvalAsyncRepo.EXPECT().SetEvalAsyncCtx(gomock.Any(), "evaluator:999", gomock.Any()).Return(nil) resp, err := handler.AsyncRunEvaluator(ctx, req) assert.NoError(t, err) diff --git a/backend/modules/evaluation/application/wire_gen.go b/backend/modules/evaluation/application/wire_gen.go index 36c5a57aa4..e19e2e74b7 100644 --- a/backend/modules/evaluation/application/wire_gen.go +++ b/backend/modules/evaluation/application/wire_gen.go @@ -108,11 +108,13 @@ func InitExperimentApplication(ctx context.Context, idgen2 idgen.IIDGenerator, d codeBuilderFactory := service.NewCodeBuilderFactory() v := service.NewEvaluatorSourceServices(illmProvider, evaluatorExecMetrics, iConfiger, iRuntimeManager, codeBuilderFactory) iPlainRateLimiter := evaluator.NewPlainRateLimiterImpl(plainLimiterFactory) - serviceEvaluatorService := service.NewEvaluatorServiceImpl(idgen2, rateLimiter, rmqFactory, iEvaluatorRepo, iEvaluatorRecordRepo, idempotentService, iConfiger, v, iPlainRateLimiter, componentIConfiger) + iEvalAsyncDAO := dao.NewEvalAsyncDAO(cmdable) + iEvalAsyncRepo := experiment.NewEvalAsyncRepo(iEvalAsyncDAO) exptEventPublisher, err := producer.NewExptEventPublisher(ctx, configFactory, rmqFactory) if err != nil { return nil, err } + serviceEvaluatorService := service.NewEvaluatorServiceImpl(idgen2, rateLimiter, rmqFactory, iEvaluatorRepo, iEvaluatorRecordRepo, idempotentService, iConfiger, v, iPlainRateLimiter, componentIConfiger, iEvalAsyncRepo, exptEventPublisher) evaluatorEventPublisher, err := producer.NewEvaluatorEventPublisher(ctx, configFactory, rmqFactory) if err != nil { return nil, err @@ -176,8 +178,6 @@ func InitExperimentApplication(ctx context.Context, idgen2 idgen.IIDGenerator, d sandboxAgentNotifier := service.NewSandboxAgentNotifier(iNotifyRPCAdapter, iUserProvider, iExptStatsRepo, iLocker, componentIConfiger) schedulerModeFactory := service.NewSchedulerModeFactory(iExptManager, iExptItemResultRepo, iExptStatsRepo, iExptTurnResultRepo, idgen2, evaluationSetItemService, iExperimentRepo, iExptItemRefRepo, idempotentService, componentIConfiger, exptEventPublisher, evaluatorRecordService, exptResultService, iExptTemplateManager, iExptRunLogRepo, iLocker, sandboxAgentNotifier) exptSchedulerEvent := service.NewExptSchedulerSvc(iExptManager, iExperimentRepo, iExptItemResultRepo, iExptTurnResultRepo, iEvaluatorRecordRepo, iExptStatsRepo, iExptRunLogRepo, idempotentService, componentIConfiger, quotaRepo, iLocker, exptEventPublisher, auditClient, exptMetric, exptResultService, idgen2, evaluationSetItemService, schedulerModeFactory, iEvalTargetService, sandboxAgentMetrics, sandboxAgentNotifier) - iEvalAsyncDAO := dao.NewEvalAsyncDAO(cmdable) - iEvalAsyncRepo := experiment.NewEvalAsyncRepo(iEvalAsyncDAO) iItemCompletePublisher := service.ProvideNilItemCompletePublisher() exptItemEvalEvent := service.NewExptRecordEvalService(iExptManager, componentIConfiger, exptEventPublisher, iExptItemResultRepo, iExptTurnResultRepo, iExptStatsRepo, iExperimentRepo, iExptItemRefRepo, quotaRepo, iLocker, idempotentService, auditClient, exptMetric, exptResultService, iEvalTargetService, evaluationSetItemService, evaluatorRecordService, serviceEvaluatorService, idgen2, benefitSvc, iEvalAsyncRepo, iItemCompletePublisher, sandboxAgentNotifier) iExptAnnotateService := service.NewExptAnnotateService(db2, iExptAnnotateRepo, iExptTurnResultRepo, exptEventPublisher, evaluationSetItemService, iExperimentRepo, exptResultService, iExptTurnResultFilterRepo, iExptAggrResultRepo) @@ -226,11 +226,13 @@ func InitEvaluatorApplication(ctx context.Context, idgen2 idgen.IIDGenerator, au codeBuilderFactory := service.NewCodeBuilderFactory() v := service.NewEvaluatorSourceServices(illmProvider, evaluatorExecMetrics, iConfiger, iRuntimeManager, codeBuilderFactory) iPlainRateLimiter := evaluator.NewPlainRateLimiterImpl(plainLimiterFactory) - evaluatorService := service.NewEvaluatorServiceImpl(idgen2, rateLimiter, rmqFactory, iEvaluatorRepo, iEvaluatorRecordRepo, idempotentService, iConfiger, v, iPlainRateLimiter, componentIConfiger) + iEvalAsyncDAO := dao.NewEvalAsyncDAO(cmdable) + iEvalAsyncRepo := experiment.NewEvalAsyncRepo(iEvalAsyncDAO) exptEventPublisher, err := producer.NewExptEventPublisher(ctx, configFactory, rmqFactory) if err != nil { return nil, err } + evaluatorService := service.NewEvaluatorServiceImpl(idgen2, rateLimiter, rmqFactory, iEvaluatorRepo, iEvaluatorRecordRepo, idempotentService, iConfiger, v, iPlainRateLimiter, componentIConfiger, iEvalAsyncRepo, exptEventPublisher) evaluatorEventPublisher, err := producer.NewEvaluatorEventPublisher(ctx, configFactory, rmqFactory) if err != nil { return nil, err @@ -281,8 +283,6 @@ func InitEvaluatorApplication(ctx context.Context, idgen2 idgen.IIDGenerator, au iTagRPCAdapter := tag.NewTagRPCProvider(tagClient) iEvaluationAnalysisService := service.NewEvaluationAnalysisService() exptResultService := service.NewExptResultService(iExptItemResultRepo, iExptTurnResultRepo, iExptAnnotateRepo, iExptStatsRepo, iExperimentRepo, exptMetric, iLatestWriteTracker, idgen2, iExptTurnResultFilterRepo, evaluatorService, iEvalTargetService, evaluationSetVersionService, iEvaluationSetService, evaluatorRecordService, evaluationSetItemService, exptEventPublisher, iTagRPCAdapter, iEvaluationAnalysisService, iFileProvider, iEvaluatorScoreCalculator) - iEvalAsyncDAO := dao.NewEvalAsyncDAO(cmdable) - iEvalAsyncRepo := experiment.NewEvalAsyncRepo(iEvalAsyncDAO) evaluationEvaluatorService := NewEvaluatorHandlerImpl(idgen2, iConfiger, iAuthProvider, evaluatorService, evaluatorRecordService, evaluatorTemplateService, evaluatorExecMetrics, userInfoService, auditClient, benefitSvc, iFileProvider, v, exptResultService, iEvalAsyncRepo) return evaluationEvaluatorService, nil } @@ -400,7 +400,7 @@ func InitEvalOpenAPIApplication(ctx context.Context, configFactory conf.IConfigL codeBuilderFactory := service.NewCodeBuilderFactory() v2 := service.NewEvaluatorSourceServices(illmProvider, evaluatorExecMetrics, confIConfiger, iRuntimeManager, codeBuilderFactory) iPlainRateLimiter := evaluator.NewPlainRateLimiterImpl(plainLimiterFactory) - evaluatorService := service.NewEvaluatorServiceImpl(idgen2, rateLimiter, rmqFactory, iEvaluatorRepo, iEvaluatorRecordRepo, idempotentService, confIConfiger, v2, iPlainRateLimiter, iConfiger) + evaluatorService := service.NewEvaluatorServiceImpl(idgen2, rateLimiter, rmqFactory, iEvaluatorRepo, iEvaluatorRecordRepo, idempotentService, confIConfiger, v2, iPlainRateLimiter, iConfiger, iEvalAsyncRepo, exptEventPublisher) evaluatorEventPublisher, err := producer.NewEvaluatorEventPublisher(ctx, configFactory, rmqFactory) if err != nil { return nil, err diff --git a/backend/modules/evaluation/domain/entity/evaluator.go b/backend/modules/evaluation/domain/entity/evaluator.go index c52edd7303..e3ce7a6731 100644 --- a/backend/modules/evaluation/domain/entity/evaluator.go +++ b/backend/modules/evaluation/domain/entity/evaluator.go @@ -65,7 +65,17 @@ var EvaluatorTypeSet = map[EvaluatorType]struct{}{ } func (e *Evaluator) IsAsync() bool { - return e.EvaluatorType == EvaluatorTypeAgent + if e == nil { + return false + } + switch e.EvaluatorType { + case EvaluatorTypeAgent: + return true + case EvaluatorTypeCustomRPC: + return e.CustomRPCEvaluatorVersion != nil && e.CustomRPCEvaluatorVersion.IsAsync + default: + return false + } } // UpdateEvaluatorMetaRequest 用于更新评估器元信息的参数 diff --git a/backend/modules/evaluation/domain/entity/evaluator_test.go b/backend/modules/evaluation/domain/entity/evaluator_test.go index f9a66f9cdf..fa859bd2d7 100644 --- a/backend/modules/evaluation/domain/entity/evaluator_test.go +++ b/backend/modules/evaluation/domain/entity/evaluator_test.go @@ -2378,3 +2378,27 @@ func TestEvaluator_SetEvaluatorVersion_CustomRPCAndAgent(t *testing.T) { }) } } + +func TestEvaluator_IsAsync_CustomRPC(t *testing.T) { + t.Parallel() + + tests := []struct { + name string + eval *Evaluator + want bool + }{ + {name: "agent remains async", eval: &Evaluator{EvaluatorType: EvaluatorTypeAgent}, want: true}, + {name: "custom rpc async version", eval: &Evaluator{EvaluatorType: EvaluatorTypeCustomRPC, CustomRPCEvaluatorVersion: &CustomRPCEvaluatorVersion{IsAsync: true}}, want: true}, + {name: "custom rpc sync version", eval: &Evaluator{EvaluatorType: EvaluatorTypeCustomRPC, CustomRPCEvaluatorVersion: &CustomRPCEvaluatorVersion{}}, want: false}, + {name: "custom rpc missing version", eval: &Evaluator{EvaluatorType: EvaluatorTypeCustomRPC}, want: false}, + {name: "prompt remains sync", eval: &Evaluator{EvaluatorType: EvaluatorTypePrompt}, want: false}, + {name: "code remains sync", eval: &Evaluator{EvaluatorType: EvaluatorTypeCode}, want: false}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + assert.Equal(t, tt.want, tt.eval.IsAsync()) + }) + } +} diff --git a/backend/modules/evaluation/domain/entity/evaluator_version_custom_rpc.go b/backend/modules/evaluation/domain/entity/evaluator_version_custom_rpc.go index 4e99e30a7b..5aadce290a 100644 --- a/backend/modules/evaluation/domain/entity/evaluator_version_custom_rpc.go +++ b/backend/modules/evaluation/domain/entity/evaluator_version_custom_rpc.go @@ -57,8 +57,10 @@ type CustomRPCEvaluatorVersion struct { ServiceName *string `json:"service_name"` Cluster *string `json:"cluster"` InvokeHTTPInfo *EvaluatorHTTPInfo `json:"invoke_http_info,omitempty"` // invoke http info - Timeout *int64 `json:"timeout"` // timeout duration in milliseconds(ms) + AsyncInvokeHTTPInfo *EvaluatorHTTPInfo `json:"async_invoke_http_info,omitempty"` + Timeout *int64 `json:"timeout"` // timeout duration in milliseconds(ms) RateLimit *RateLimit `json:"rate_limit,omitempty"` + IsAsync bool `json:"is_async,omitempty"` // extra fields Ext map[string]string `json:"ext,omitempty"` @@ -150,6 +152,21 @@ func (do *CustomRPCEvaluatorVersion) ValidateBaseInfo() error { if do.ServiceName == nil || lo.IsEmpty(*do.ServiceName) { return errorx.NewByCode(errno.InvalidServiceNameCode, errorx.WithExtraMsg("service_name is empty")) } + if do.IsAsync { + switch do.AccessProtocol { + case EvaluatorAccessProtocolRPC: + return nil + case EvaluatorAccessProtocolFaasHTTP: + if do.AsyncInvokeHTTPInfo == nil || do.AsyncInvokeHTTPInfo.Path == nil || lo.IsEmpty(*do.AsyncInvokeHTTPInfo.Path) { + return errorx.NewByCode(errno.InvalidEvaluatorConfigurationCode, errorx.WithExtraMsg("async_invoke_http_info.path is empty")) + } + return nil + case EvaluatorAccessProtocolRPCOld, EvaluatorAccessProtocolFaasHTTPOld: + return errorx.NewByCode(errno.InvalidAccessProtocolCode, errorx.WithExtraMsg("legacy access protocol does not support async evaluator")) + default: + return errorx.NewByCode(errno.InvalidAccessProtocolCode, errorx.WithExtraMsg("access protocol does not support async evaluator")) + } + } return nil } diff --git a/backend/modules/evaluation/domain/entity/evaluator_version_custom_rpc_test.go b/backend/modules/evaluation/domain/entity/evaluator_version_custom_rpc_test.go index 88ee330b8b..72203fbdc5 100644 --- a/backend/modules/evaluation/domain/entity/evaluator_version_custom_rpc_test.go +++ b/backend/modules/evaluation/domain/entity/evaluator_version_custom_rpc_test.go @@ -382,3 +382,86 @@ func TestCustomRPCEvaluatorVersion_GettersAndSetters(t *testing.T) { evaluator.SetBaseInfo(baseInfo) assert.Equal(t, baseInfo, evaluator.GetBaseInfo()) } + +func TestCustomRPCEvaluatorVersion_ValidateBaseInfo_Async(t *testing.T) { + t.Parallel() + + post := EvaluatorHTTPMethodPost + path := "/async_invoke_evaluator" + tests := []struct { + name string + version *CustomRPCEvaluatorVersion + wantErr bool + }{ + { + name: "rpc async does not require http info", + version: &CustomRPCEvaluatorVersion{ + AccessProtocol: EvaluatorAccessProtocolRPC, + ServiceName: gptr.Of("trae.work.evaluator"), + IsAsync: true, + }, + }, + { + name: "faas http async requires async http info", + version: &CustomRPCEvaluatorVersion{ + AccessProtocol: EvaluatorAccessProtocolFaasHTTP, + ServiceName: gptr.Of("trae.work.evaluator"), + IsAsync: true, + }, + wantErr: true, + }, + { + name: "faas http async accepts async http info", + version: &CustomRPCEvaluatorVersion{ + AccessProtocol: EvaluatorAccessProtocolFaasHTTP, + ServiceName: gptr.Of("trae.work.evaluator"), + IsAsync: true, + AsyncInvokeHTTPInfo: &EvaluatorHTTPInfo{ + Method: &post, + Path: &path, + }, + }, + }, + { + name: "old rpc async is rejected", + version: &CustomRPCEvaluatorVersion{ + AccessProtocol: EvaluatorAccessProtocolRPCOld, + ServiceName: gptr.Of("trae.work.evaluator"), + IsAsync: true, + }, + wantErr: true, + }, + { + name: "old http async is rejected", + version: &CustomRPCEvaluatorVersion{ + AccessProtocol: EvaluatorAccessProtocolFaasHTTPOld, + ServiceName: gptr.Of("trae.work.evaluator"), + IsAsync: true, + AsyncInvokeHTTPInfo: &EvaluatorHTTPInfo{ + Method: &post, + Path: &path, + }, + }, + wantErr: true, + }, + { + name: "sync faas http keeps existing behavior", + version: &CustomRPCEvaluatorVersion{ + AccessProtocol: EvaluatorAccessProtocolFaasHTTP, + ServiceName: gptr.Of("trae.work.evaluator"), + }, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + err := tt.version.ValidateBaseInfo() + if tt.wantErr { + assert.Error(t, err) + return + } + assert.NoError(t, err) + }) + } +} diff --git a/backend/modules/evaluation/domain/entity/expt_run.go b/backend/modules/evaluation/domain/entity/expt_run.go index 85021a03eb..a5bbb2f558 100644 --- a/backend/modules/evaluation/domain/entity/expt_run.go +++ b/backend/modules/evaluation/domain/entity/expt_run.go @@ -649,6 +649,7 @@ type EvalAsyncCtx struct { Callee string EvaluatorVersionID int64 // evaluator version id, used for evaluator async scenario EnableExtractTrajectory *bool + ResumeReady bool `json:"resume_ready,omitempty"` // experiment turn refs are durable and callback may resume scheduling CallbackURL string `json:"callback_url,omitempty"` // 异步执行完成后回调通知的 URL，为空则不回调 // 下述字段用于沙箱内部 step 上报的 tag 反查, 由 target async 写入位点从 etec 填充, // 调试场景 (无实验上下文) 保留零值, 由上报侧回退为占位符. diff --git a/backend/modules/evaluation/domain/entity/param.go b/backend/modules/evaluation/domain/entity/param.go index e1094fe9e6..4a16676aa4 100644 --- a/backend/modules/evaluation/domain/entity/param.go +++ b/backend/modules/evaluation/domain/entity/param.go @@ -326,6 +326,9 @@ type AsyncRunEvaluatorRequest struct { // ★ alias 多实例: 同步与 RunEvaluatorRequest Alias string `json:"alias,omitempty"` SourceType EvaluatorRecordSourceType `json:"source_type,omitempty"` + // AsyncCtx is persisted after the AsyncInvoking record is created and before the provider is dispatched. + // Experiment calls set ResumeReady=false; direct calls set it true. + AsyncCtx *EvalAsyncCtx `json:"-"` } type AsyncRunEvaluatorResponse struct { diff --git a/backend/modules/evaluation/domain/repo/evaluator_record.go b/backend/modules/evaluation/domain/repo/evaluator_record.go index 3c3285541c..d3a4bc6e3e 100644 --- a/backend/modules/evaluation/domain/repo/evaluator_record.go +++ b/backend/modules/evaluation/domain/repo/evaluator_record.go @@ -20,4 +20,8 @@ type IEvaluatorRecordRepo interface { // 三个大字段的查询与反序列化，只返回 status=Success 且 score 非 NULL 的行（与内存聚合 contributing 集一致）。 BatchGetEvaluatorRecordForAggr(ctx context.Context, evaluatorRecordIDs []int64) ([]*entity.EvaluatorRecordAggr, error) UpdateEvaluatorRecordResult(ctx context.Context, recordID int64, status entity.EvaluatorRunStatus, outputData *entity.EvaluatorOutputData) error + // CompareAndSwapEvaluatorRecordResult updates a record only when id/space/status still match. + // It is the terminal-state guard for async callbacks and dispatch compensation. + CompareAndSwapEvaluatorRecordResult(ctx context.Context, recordID, spaceID int64, fromStatus, toStatus entity.EvaluatorRunStatus, outputData *entity.EvaluatorOutputData) (bool, error) + UpdateEvaluatorRecordAsyncDispatch(ctx context.Context, recordID, spaceID int64, traceID string, outputData *entity.EvaluatorOutputData) error } diff --git a/backend/modules/evaluation/domain/repo/expt.go b/backend/modules/evaluation/domain/repo/expt.go index aa7b8ce9bc..a3bc06c95c 100644 --- a/backend/modules/evaluation/domain/repo/expt.go +++ b/backend/modules/evaluation/domain/repo/expt.go @@ -146,7 +146,11 @@ type IExptResultExportRecordRepo interface { type IEvalAsyncRepo interface { GetEvalAsyncCtx(ctx context.Context, invokeID string) (*entity.EvalAsyncCtx, error) + // GetEvalAsyncCtxStrong uses the shared direct Redis client and retries a transient missing key at 50/100/200ms. + GetEvalAsyncCtxStrong(ctx context.Context, invokeID string) (*entity.EvalAsyncCtx, error) SetEvalAsyncCtx(ctx context.Context, invokeID string, actx *entity.EvalAsyncCtx) error + // MarkEvalAsyncResumeReady atomically persists ResumeReady=true and returns the latest context. + MarkEvalAsyncResumeReady(ctx context.Context, invokeID string) (*entity.EvalAsyncCtx, error) } type IExptInsightAnalysisRecordRepo interface { diff --git a/backend/modules/evaluation/domain/repo/mocks/evaluator_record_mock.go b/backend/modules/evaluation/domain/repo/mocks/evaluator_record_mock.go index 0a93e30bfc..552d3a59e0 100644 --- a/backend/modules/evaluation/domain/repo/mocks/evaluator_record_mock.go +++ b/backend/modules/evaluation/domain/repo/mocks/evaluator_record_mock.go @@ -75,6 +75,33 @@ func (mr *MockIEvaluatorRecordRepoMockRecorder) BatchGetEvaluatorRecordForAggr(a return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "BatchGetEvaluatorRecordForAggr", reflect.TypeOf((*MockIEvaluatorRecordRepo)(nil).BatchGetEvaluatorRecordForAggr), arg0, arg1) } +// CompareAndSwapEvaluatorRecordResult mocks base method. +func (m *MockIEvaluatorRecordRepo) CompareAndSwapEvaluatorRecordResult(arg0 context.Context, arg1, arg2 int64, arg3, arg4 entity.EvaluatorRunStatus, arg5 *entity.EvaluatorOutputData) (bool, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "CompareAndSwapEvaluatorRecordResult", arg0, arg1, arg2, arg3, arg4, arg5) + ret0, _ := ret[0].(bool) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +func (mr *MockIEvaluatorRecordRepoMockRecorder) CompareAndSwapEvaluatorRecordResult(arg0, arg1, arg2, arg3, arg4, arg5 any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "CompareAndSwapEvaluatorRecordResult", reflect.TypeOf((*MockIEvaluatorRecordRepo)(nil).CompareAndSwapEvaluatorRecordResult), arg0, arg1, arg2, arg3, arg4, arg5) +} + +// UpdateEvaluatorRecordAsyncDispatch mocks base method. +func (m *MockIEvaluatorRecordRepo) UpdateEvaluatorRecordAsyncDispatch(arg0 context.Context, arg1, arg2 int64, arg3 string, arg4 *entity.EvaluatorOutputData) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "UpdateEvaluatorRecordAsyncDispatch", arg0, arg1, arg2, arg3, arg4) + ret0, _ := ret[0].(error) + return ret0 +} + +func (mr *MockIEvaluatorRecordRepoMockRecorder) UpdateEvaluatorRecordAsyncDispatch(arg0, arg1, arg2, arg3, arg4 any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "UpdateEvaluatorRecordAsyncDispatch", reflect.TypeOf((*MockIEvaluatorRecordRepo)(nil).UpdateEvaluatorRecordAsyncDispatch), arg0, arg1, arg2, arg3, arg4) +} + // CorrectEvaluatorRecord mocks base method. func (m *MockIEvaluatorRecordRepo) CorrectEvaluatorRecord(arg0 context.Context, arg1 *entity.EvaluatorRecord) error { m.ctrl.T.Helper() diff --git a/backend/modules/evaluation/domain/repo/mocks/expt.go b/backend/modules/evaluation/domain/repo/mocks/expt.go index ee7f202bb6..833fa13d31 100644 --- a/backend/modules/evaluation/domain/repo/mocks/expt.go +++ b/backend/modules/evaluation/domain/repo/mocks/expt.go @@ -1737,6 +1737,36 @@ func (mr *MockIEvalAsyncRepoMockRecorder) GetEvalAsyncCtx(arg0, arg1 any) *gomoc return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetEvalAsyncCtx", reflect.TypeOf((*MockIEvalAsyncRepo)(nil).GetEvalAsyncCtx), arg0, arg1) } +// GetEvalAsyncCtxStrong mocks base method. +func (m *MockIEvalAsyncRepo) GetEvalAsyncCtxStrong(ctx context.Context, invokeID string) (*entity.EvalAsyncCtx, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "GetEvalAsyncCtxStrong", ctx, invokeID) + ret0, _ := ret[0].(*entity.EvalAsyncCtx) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// GetEvalAsyncCtxStrong indicates an expected call of GetEvalAsyncCtxStrong. +func (mr *MockIEvalAsyncRepoMockRecorder) GetEvalAsyncCtxStrong(ctx, invokeID any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetEvalAsyncCtxStrong", reflect.TypeOf((*MockIEvalAsyncRepo)(nil).GetEvalAsyncCtxStrong), ctx, invokeID) +} + +// MarkEvalAsyncResumeReady mocks base method. +func (m *MockIEvalAsyncRepo) MarkEvalAsyncResumeReady(ctx context.Context, invokeID string) (*entity.EvalAsyncCtx, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "MarkEvalAsyncResumeReady", ctx, invokeID) + ret0, _ := ret[0].(*entity.EvalAsyncCtx) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// MarkEvalAsyncResumeReady indicates an expected call of MarkEvalAsyncResumeReady. +func (mr *MockIEvalAsyncRepoMockRecorder) MarkEvalAsyncResumeReady(ctx, invokeID any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "MarkEvalAsyncResumeReady", reflect.TypeOf((*MockIEvalAsyncRepo)(nil).MarkEvalAsyncResumeReady), ctx, invokeID) +} + // SetEvalAsyncCtx mocks base method. func (m *MockIEvalAsyncRepo) SetEvalAsyncCtx(arg0 context.Context, arg1 string, arg2 *entity.EvalAsyncCtx) error { m.ctrl.T.Helper() diff --git a/backend/modules/evaluation/domain/service/evaluator.go b/backend/modules/evaluation/domain/service/evaluator.go index 1fa36fcb84..2b4fb146b3 100644 --- a/backend/modules/evaluation/domain/service/evaluator.go +++ b/backend/modules/evaluation/domain/service/evaluator.go @@ -65,8 +65,10 @@ type EvaluatorService interface { CheckNameExist(ctx context.Context, spaceID, evaluatorID int64, name string) (bool, error) // ListEvaluatorTags 根据 tagType 聚合标签，并按字母序返回 ListEvaluatorTags(ctx context.Context, tagType entity.EvaluatorTagKeyType) (map[entity.EvaluatorTagKey][]string, error) - // ReportEvaluatorInvokeResult 上报评估器异步执行结果 + // ReportEvaluatorInvokeResult 上报评估器异步执行结果. Duplicate/conflicting terminal callbacks are ignored. ReportEvaluatorInvokeResult(ctx context.Context, param *entity.ReportEvaluatorRecordParam) error + // ArmEvaluatorResume marks ResumeReady after turn refs are durable and republishes recovery when the record is already terminal. + ArmEvaluatorResume(ctx context.Context, recordID int64) error } //go:generate mockgen -destination mocks/evaluator_record_service_mock.go -package mocks . EvaluatorRecordService diff --git a/backend/modules/evaluation/domain/service/evaluator_impl.go b/backend/modules/evaluation/domain/service/evaluator_impl.go index 9d411cfa99..6bc335b7b0 100644 --- a/backend/modules/evaluation/domain/service/evaluator_impl.go +++ b/backend/modules/evaluation/domain/service/evaluator_impl.go @@ -20,8 +20,10 @@ import ( "github.com/coze-dev/coze-loop/backend/modules/evaluation/domain/component" "github.com/coze-dev/coze-loop/backend/modules/evaluation/domain/component/idem" "github.com/coze-dev/coze-loop/backend/modules/evaluation/domain/entity" + "github.com/coze-dev/coze-loop/backend/modules/evaluation/domain/events" "github.com/coze-dev/coze-loop/backend/modules/evaluation/domain/repo" "github.com/coze-dev/coze-loop/backend/modules/evaluation/pkg/conf" + "github.com/coze-dev/coze-loop/backend/modules/evaluation/pkg/contexts" "github.com/coze-dev/coze-loop/backend/modules/evaluation/pkg/errno" "github.com/coze-dev/coze-loop/backend/modules/evaluation/pkg/utils" "github.com/coze-dev/coze-loop/backend/pkg/errorx" @@ -46,6 +48,8 @@ func NewEvaluatorServiceImpl( evaluatorSourceServices map[entity.EvaluatorType]EvaluatorSourceService, plainRateLimiter repo.IPlainRateLimiter, cConfiger component.IConfiger, + evalAsyncRepo repo.IEvalAsyncRepo, + exptEventPublisher events.ExptEventPublisher, ) EvaluatorService { onceEvaluatorService.Do(func() { singletonEvaluatorService = &EvaluatorServiceImpl{ @@ -58,6 +62,8 @@ func NewEvaluatorServiceImpl( configer: configer, evaluatorSourceServices: evaluatorSourceServices, plainRateLimiter: plainRateLimiter, + evalAsyncRepo: evalAsyncRepo, + exptEventPublisher: exptEventPublisher, cConfiger: cConfiger, } }) @@ -75,6 +81,8 @@ type EvaluatorServiceImpl struct { configer conf.IConfiger evaluatorSourceServices map[entity.EvaluatorType]EvaluatorSourceService plainRateLimiter repo.IPlainRateLimiter + evalAsyncRepo repo.IEvalAsyncRepo + exptEventPublisher events.ExptEventPublisher cConfiger component.IConfiger } @@ -956,7 +964,7 @@ func (e *EvaluatorServiceImpl) CreateEvaluatorRunFailRecord(ctx context.Context, return recordDO, nil } -// AsyncRunEvaluator Agent evaluator_version 异步运行 +// AsyncRunEvaluator coordinates evaluator async kickoff in the strict order Record -> Context -> Provider. func (e *EvaluatorServiceImpl) AsyncRunEvaluator(ctx context.Context, request *entity.AsyncRunEvaluatorRequest) (*entity.EvaluatorRecord, error) { evaluatorDOList, err := e.evaluatorRepo.BatchGetEvaluatorByVersionID(ctx, nil, []int64{request.EvaluatorVersionID}, false, false) if err != nil { @@ -966,13 +974,14 @@ func (e *EvaluatorServiceImpl) AsyncRunEvaluator(ctx context.Context, request *e return nil, errorx.NewByCode(errno.EvaluatorVersionNotFoundCode, errorx.WithExtraMsg("evaluator_version version not found")) } evaluatorDO := evaluatorDOList[0] - if evaluatorDO.EvaluatorType != entity.EvaluatorTypeAgent { - return nil, errorx.NewByCode(errno.InvalidEvaluatorTypeCode, errorx.WithExtraMsg("async run only supports Agent evaluator type")) + if evaluatorDO.EvaluatorType == entity.EvaluatorTypeCustomRPC && evaluatorDO.Builtin { + return nil, errorx.NewByCode(errno.InvalidEvaluatorTypeCode, errorx.WithExtraMsg("builtin CustomRPC evaluator does not support async run")) } - if !evaluatorDO.Builtin { - if evaluatorDO.SpaceID != request.SpaceID { - return nil, errorx.NewByCode(errno.EvaluatorVersionNotFoundCode, errorx.WithExtraMsg("evaluator_version not found in current space")) - } + if !evaluatorDO.IsAsync() { + return nil, errorx.NewByCode(errno.InvalidEvaluatorTypeCode, errorx.WithExtraMsg("evaluator does not support async run")) + } + if !evaluatorDO.Builtin && evaluatorDO.SpaceID != request.SpaceID { + return nil, errorx.NewByCode(errno.EvaluatorVersionNotFoundCode, errorx.WithExtraMsg("evaluator_version not found in current space")) } if allow := e.limiter.AllowInvoke(ctx, request.SpaceID); !allow { return nil, errorx.NewByCode(errno.EvaluatorQPSLimitCode, errorx.WithExtraMsg("evaluator throttled due to space-level rate limit")) @@ -986,20 +995,11 @@ func (e *EvaluatorServiceImpl) AsyncRunEvaluator(ctx context.Context, request *e } evaluatorSourceService, ok := e.evaluatorSourceServices[evaluatorDO.EvaluatorType] if !ok { - return nil, errorx.NewByCode(errno.InvalidEvaluatorTypeCode, errorx.WithExtraMsg("evaluator source service not found for agent type")) - } - asyncRunExt, traceID, err := evaluatorSourceService.AsyncRun(ctx, evaluatorDO, request.InputData, request.EvaluatorRunConf, request.SpaceID, invokeID) - if err != nil { - logs.CtxError(ctx, "[AsyncRunEvaluator] AsyncRun fail, invokeID: %d, err: %v", invokeID, err) - return nil, err + return nil, errorx.NewByCode(errno.InvalidEvaluatorTypeCode, errorx.WithExtraMsg("evaluator source service not found for async type")) } + now := time.Now().UnixMilli() userIDInContext := session.UserIDInCtxOrEmpty(ctx) - logID := logs.GetLogID(ctx) - status := entity.EvaluatorRunStatusAsyncInvoking - outputData := &entity.EvaluatorOutputData{ - Ext: asyncRunExt, - } recordDO := &entity.EvaluatorRecord{ ID: invokeID, SpaceID: request.SpaceID, @@ -1010,21 +1010,16 @@ func (e *EvaluatorServiceImpl) AsyncRunEvaluator(ctx context.Context, request *e EvaluatorVersionID: request.EvaluatorVersionID, Alias: request.Alias, SourceType: normalizeEvaluatorRecordSourceType(request.SourceType), - TraceID: traceID, - LogID: logID, + LogID: logs.GetLogID(ctx), EvaluatorInputData: request.InputData, - EvaluatorOutputData: outputData, - Status: status, + EvaluatorOutputData: &entity.EvaluatorOutputData{}, + Status: entity.EvaluatorRunStatusAsyncInvoking, Ext: request.Ext, BaseInfo: &entity.BaseInfo{ - CreatedBy: &entity.UserInfo{ - UserID: gptr.Of(userIDInContext), - }, - UpdatedBy: &entity.UserInfo{ - UserID: gptr.Of(userIDInContext), - }, - CreatedAt: gptr.Of(time.Now().UnixMilli()), - UpdatedAt: gptr.Of(time.Now().UnixMilli()), + CreatedBy: &entity.UserInfo{UserID: gptr.Of(userIDInContext)}, + UpdatedBy: &entity.UserInfo{UserID: gptr.Of(userIDInContext)}, + CreatedAt: gptr.Of(now), + UpdatedAt: gptr.Of(now), }, } if err := e.evaluatorRecordRepo.CreateEvaluatorRecord(ctx, recordDO); err != nil { @@ -1032,11 +1027,68 @@ func (e *EvaluatorServiceImpl) AsyncRunEvaluator(ctx context.Context, request *e return nil, err } + asyncCtx := request.AsyncCtx + if asyncCtx == nil { + asyncCtx = &entity.EvalAsyncCtx{ResumeReady: true} + } + asyncCtx.RecordID = invokeID + asyncCtx.EvaluatorVersionID = request.EvaluatorVersionID + if asyncCtx.AsyncUnixMS == 0 { + asyncCtx.AsyncUnixMS = now + } + if asyncCtx.Session == nil { + asyncCtx.Session = &entity.Session{UserID: userIDInContext} + } + if e.evalAsyncRepo == nil { + return e.failAsyncEvaluatorRecord(ctx, recordDO, errorx.New("eval async repo is nil")) + } + asyncCtxKey := fmt.Sprintf("evaluator:%d", invokeID) + if err := e.evalAsyncRepo.SetEvalAsyncCtx(ctx, asyncCtxKey, asyncCtx); err != nil { + return e.failAsyncEvaluatorRecord(ctx, recordDO, err) + } + + asyncRunExt, traceID, err := evaluatorSourceService.AsyncRun(ctx, evaluatorDO, request.InputData, request.EvaluatorRunConf, request.SpaceID, invokeID) + if err != nil { + logs.CtxError(ctx, "[AsyncRunEvaluator] AsyncRun fail, invokeID: %d, err: %v", invokeID, err) + return e.failAsyncEvaluatorRecord(ctx, recordDO, err) + } + recordDO.TraceID = traceID + recordDO.EvaluatorOutputData.Ext = asyncRunExt + if traceID != "" || len(asyncRunExt) > 0 { + if err := e.evaluatorRecordRepo.UpdateEvaluatorRecordAsyncDispatch(ctx, recordDO.ID, recordDO.SpaceID, traceID, recordDO.EvaluatorOutputData); err != nil { + // The provider has already accepted the work. Treat dispatch metadata as best-effort; + // marking the record failed here would create an uncertain-state split brain and reject a later valid callback. + logs.CtxError(ctx, "[AsyncRunEvaluator] persist dispatch metadata fail, keep record async invoking, invokeID: %d, err: %v", invokeID, err) + } + } logs.CtxInfo(ctx, "[AsyncRunEvaluator] invokeID: %d, evaluatorVersionID: %d, spaceID: %d, record_ext: %v", invokeID, request.EvaluatorVersionID, request.SpaceID, json.Jsonify(recordDO.Ext)) return recordDO, nil } +func (e *EvaluatorServiceImpl) failAsyncEvaluatorRecord(ctx context.Context, record *entity.EvaluatorRecord, runErr error) (*entity.EvaluatorRecord, error) { + if record == nil { + return nil, runErr + } + errMsg := "evaluator async run failed" + if runErr != nil { + errMsg = errorx.ErrorWithoutStack(runErr) + } + output := &entity.EvaluatorOutputData{EvaluatorRunError: &entity.EvaluatorRunError{ + Code: int32(errno.CommonInternalErrorCode), + Message: errMsg, + }} + updated, casErr := e.evaluatorRecordRepo.CompareAndSwapEvaluatorRecordResult(ctx, record.ID, record.SpaceID, entity.EvaluatorRunStatusAsyncInvoking, entity.EvaluatorRunStatusFail, output) + if casErr != nil { + return record, errorx.Wrapf(casErr, "mark evaluator async kickoff failed, cause: %v", runErr) + } + if updated { + record.Status = entity.EvaluatorRunStatusFail + record.EvaluatorOutputData = output + } + return record, runErr +} + // AsyncDebugEvaluator Agent evaluator_version 异步调试 func (e *EvaluatorServiceImpl) AsyncDebugEvaluator(ctx context.Context, request *entity.AsyncDebugEvaluatorRequest) (*entity.AsyncDebugEvaluatorResponse, error) { evaluatorDO := request.EvaluatorDO @@ -1101,11 +1153,17 @@ func (e *EvaluatorServiceImpl) AsyncDebugEvaluator(ctx context.Context, request }, nil } -// ReportEvaluatorInvokeResult 上报评估器异步执行结果 +// ReportEvaluatorInvokeResult 上报评估器异步执行结果 using a terminal CAS. func (e *EvaluatorServiceImpl) ReportEvaluatorInvokeResult(ctx context.Context, param *entity.ReportEvaluatorRecordParam) error { + if param == nil { + return errorx.NewByCode(errno.CommonInvalidParamCode, errorx.WithExtraMsg("report evaluator result param is nil")) + } + if param.Status != entity.EvaluatorRunStatusSuccess && param.Status != entity.EvaluatorRunStatusFail { + return errorx.NewByCode(errno.CommonInvalidParamCode, errorx.WithExtraMsg("report evaluator status must be success or fail")) + } logs.CtxInfo(ctx, "[ReportEvaluatorInvokeResult] recordID: %d, spaceID: %d, status: %v", param.RecordID, param.SpaceID, param.Status) - existingRecord, err := e.evaluatorRecordRepo.GetEvaluatorRecord(ctx, param.RecordID, false) + existingRecord, err := e.evaluatorRecordRepo.GetEvaluatorRecord(contexts.WithCtxWriteDB(ctx), param.RecordID, false) if err != nil { logs.CtxError(ctx, "[ReportEvaluatorInvokeResult] GetEvaluatorRecord fail, recordID: %d, err: %v", param.RecordID, err) return err @@ -1113,23 +1171,19 @@ func (e *EvaluatorServiceImpl) ReportEvaluatorInvokeResult(ctx context.Context, if existingRecord == nil { return errorx.NewByCode(errno.EvaluatorRecordNotFoundCode, errorx.WithExtraMsg("evaluator record not found")) } - if existingRecord.SpaceID != param.SpaceID { logs.CtxWarn(ctx, "[ReportEvaluatorInvokeResult] spaceID mismatch, recordID: %d, requestSpaceID: %d, recordSpaceID: %d", param.RecordID, param.SpaceID, existingRecord.SpaceID) return errorx.NewByCode(errno.CommonInvalidParamCode, errorx.WithExtraMsg("spaceID mismatch")) } - if existingRecord.Status != entity.EvaluatorRunStatusAsyncInvoking { - logs.CtxWarn(ctx, "[ReportEvaluatorInvokeResult] skip stale callback, recordID: %d, dbStatus: %v, reportStatus: %v", - param.RecordID, existingRecord.Status, param.Status) - return nil - } - mergedOutputData := param.OutputData if mergedOutputData == nil { mergedOutputData = &entity.EvaluatorOutputData{} } + if mergedOutputData.TimeConsumingMS == 0 && existingRecord.BaseInfo != nil && existingRecord.BaseInfo.CreatedAt != nil { + mergedOutputData.TimeConsumingMS = time.Now().UnixMilli() - gptr.Indirect(existingRecord.BaseInfo.CreatedAt) + } if existingRecord.EvaluatorOutputData != nil && existingRecord.EvaluatorOutputData.Ext != nil { if mergedOutputData.Ext == nil { mergedOutputData.Ext = make(map[string]string) @@ -1141,7 +1195,65 @@ func (e *EvaluatorServiceImpl) ReportEvaluatorInvokeResult(ctx context.Context, } } - return e.evaluatorRecordRepo.UpdateEvaluatorRecordResult(ctx, param.RecordID, param.Status, mergedOutputData) + updated, err := e.evaluatorRecordRepo.CompareAndSwapEvaluatorRecordResult(ctx, param.RecordID, param.SpaceID, entity.EvaluatorRunStatusAsyncInvoking, param.Status, mergedOutputData) + if err != nil { + return err + } + if !updated { + logs.CtxWarn(ctx, "[ReportEvaluatorInvokeResult] skip duplicate/conflicting callback, recordID: %d, reportStatus: %v", param.RecordID, param.Status) + } + return nil +} + +func (e *EvaluatorServiceImpl) ArmEvaluatorResume(ctx context.Context, recordID int64) error { + if e.evalAsyncRepo == nil { + return errorx.New("eval async repo is nil") + } + asyncCtxKey := fmt.Sprintf("evaluator:%d", recordID) + actx, err := e.evalAsyncRepo.MarkEvalAsyncResumeReady(ctx, asyncCtxKey) + if err != nil { + return err + } + if actx == nil || actx.Event == nil { + return nil + } + record, err := e.evaluatorRecordRepo.GetEvaluatorRecord(contexts.WithCtxWriteDB(ctx), recordID, false) + if err != nil { + return err + } + if record == nil { + return errorx.NewByCode(errno.EvaluatorRecordNotFoundCode, errorx.WithExtraMsg("evaluator record not found")) + } + if record.Status == entity.EvaluatorRunStatusAsyncInvoking { + return nil + } + return e.publishEvaluatorResumeEvent(ctx, actx.Event) +} + +func (e *EvaluatorServiceImpl) publishEvaluatorResumeEvent(ctx context.Context, event *entity.ExptItemEvalEvent) error { + if event == nil || e.exptEventPublisher == nil { + return nil + } + delays := []time.Duration{0, 50 * time.Millisecond, 100 * time.Millisecond} + var lastErr error + for _, delay := range delays { + if delay > 0 { + timer := time.NewTimer(delay) + select { + case <-ctx.Done(): + timer.Stop() + return ctx.Err() + case <-timer.C: + } + } + lastErr = e.exptEventPublisher.PublishExptRecordEvalEvent(ctx, event, gptr.Of(time.Second*3), func(event *entity.ExptItemEvalEvent) { + event.AsyncEvaluatorReportTrigger = true + }) + if lastErr == nil { + return nil + } + } + return lastErr } // DebugEvaluator 调试 evaluator_version diff --git a/backend/modules/evaluation/domain/service/evaluator_impl_test.go b/backend/modules/evaluation/domain/service/evaluator_impl_test.go index 8781a1fed0..e1bb976fdf 100644 --- a/backend/modules/evaluation/domain/service/evaluator_impl_test.go +++ b/backend/modules/evaluation/domain/service/evaluator_impl_test.go @@ -21,11 +21,13 @@ import ( idemmocks "github.com/coze-dev/coze-loop/backend/modules/evaluation/domain/component/idem/mocks" componentMocks "github.com/coze-dev/coze-loop/backend/modules/evaluation/domain/component/mocks" "github.com/coze-dev/coze-loop/backend/modules/evaluation/domain/entity" + eventmocks "github.com/coze-dev/coze-loop/backend/modules/evaluation/domain/events/mocks" "github.com/coze-dev/coze-loop/backend/modules/evaluation/domain/service/mocks" "github.com/coze-dev/coze-loop/backend/modules/evaluation/domain/repo" repomocks "github.com/coze-dev/coze-loop/backend/modules/evaluation/domain/repo/mocks" confmocks "github.com/coze-dev/coze-loop/backend/modules/evaluation/pkg/conf/mocks" + "github.com/coze-dev/coze-loop/backend/modules/evaluation/pkg/contexts" "github.com/coze-dev/coze-loop/backend/modules/evaluation/pkg/errno" "github.com/coze-dev/coze-loop/backend/pkg/errorx" "github.com/coze-dev/coze-loop/backend/pkg/lang/ptr" @@ -59,6 +61,8 @@ func TestNewEvaluatorServiceImpl(t *testing.T) { }, mockPlainLimiter, mockErrConfiger, + repomocks.NewMockIEvalAsyncRepo(ctrl), + nil, ) assert.IsType(t, &EvaluatorServiceImpl{}, service) @@ -3262,6 +3266,7 @@ func TestEvaluatorServiceImpl_AsyncRunEvaluator(t *testing.T) { mockEvaluatorRecordRepo := repomocks.NewMockIEvaluatorRecordRepo(ctrl) mockEvaluatorSourceService := mocks.NewMockEvaluatorSourceService(ctrl) mockPlainLimiter := repomocks.NewMockIPlainRateLimiter(ctrl) + mockEvalAsyncRepo := repomocks.NewMockIEvalAsyncRepo(ctrl) mockCConfiger := componentMocks.NewMockIConfiger(ctrl) mockCConfiger.EXPECT().BuildEvalExt(gomock.Any(), gomock.Any(), gomock.Any()).Return(nil).AnyTimes() @@ -3275,6 +3280,7 @@ func TestEvaluatorServiceImpl_AsyncRunEvaluator(t *testing.T) { entity.EvaluatorTypeAgent: mockEvaluatorSourceService, }, plainRateLimiter: mockPlainLimiter, + evalAsyncRepo: mockEvalAsyncRepo, cConfiger: mockCConfiger, } @@ -3323,6 +3329,7 @@ func TestEvaluatorServiceImpl_AsyncRunEvaluator(t *testing.T) { mockLimiter.EXPECT().AllowInvoke(gomock.Any(), req.SpaceID).Return(true) mockPlainLimiter.EXPECT().AllowInvokeWithKeyLimit(gomock.Any(), "async_run_evaluator:100", gomock.Any()).Return(true) mockIDGen.EXPECT().GenID(gomock.Any()).Return(int64(999), nil) + mockEvalAsyncRepo.EXPECT().SetEvalAsyncCtx(gomock.Any(), "evaluator:999", gomock.Any()).Return(nil) mockEvaluatorSourceService.EXPECT().AsyncRun(gomock.Any(), agentEvaluatorDO, req.InputData, req.EvaluatorRunConf, req.SpaceID, int64(999)). Return(map[string]string{"async": "1"}, "trace-1", nil) mockEvaluatorRecordRepo.EXPECT().CreateEvaluatorRecord(gomock.Any(), gomock.Any()).DoAndReturn( @@ -3332,9 +3339,13 @@ func TestEvaluatorServiceImpl_AsyncRunEvaluator(t *testing.T) { assert.Equal(t, req.EvaluatorVersionID, record.EvaluatorVersionID) assert.Equal(t, entity.EvaluatorRunStatusAsyncInvoking, record.Status) assert.Equal(t, req.Ext, record.Ext) - if assert.NotNil(t, record.EvaluatorOutputData) { - assert.Equal(t, map[string]string{"async": "1"}, record.EvaluatorOutputData.Ext) - } + return nil + }, + ) + mockEvaluatorRecordRepo.EXPECT().UpdateEvaluatorRecordAsyncDispatch(gomock.Any(), int64(999), req.SpaceID, "trace-1", gomock.Any()).DoAndReturn( + func(_ context.Context, _, _ int64, traceID string, out *entity.EvaluatorOutputData) error { + assert.Equal(t, "trace-1", traceID) + assert.Equal(t, map[string]string{"async": "1"}, out.Ext) return nil }, ) @@ -3550,13 +3561,13 @@ func TestEvaluatorServiceImpl_ReportEvaluatorInvokeResult(t *testing.T) { }, }, nil, ) - mockEvaluatorRecordRepo.EXPECT().UpdateEvaluatorRecordResult(gomock.Any(), int64(100), entity.EvaluatorRunStatusSuccess, gomock.Any()). - DoAndReturn(func(_ context.Context, _ int64, _ entity.EvaluatorRunStatus, out *entity.EvaluatorOutputData) error { + mockEvaluatorRecordRepo.EXPECT().CompareAndSwapEvaluatorRecordResult(gomock.Any(), int64(100), int64(2), entity.EvaluatorRunStatusAsyncInvoking, entity.EvaluatorRunStatusSuccess, gomock.Any()). + DoAndReturn(func(_ context.Context, _, _ int64, _, _ entity.EvaluatorRunStatus, out *entity.EvaluatorOutputData) (bool, error) { if assert.NotNil(t, out) { assert.Equal(t, "1", out.Ext["new"]) assert.Equal(t, "1", out.Ext["old"]) } - return nil + return true, nil }) }, expectedErrCode: 0, @@ -3599,6 +3610,7 @@ func TestEvaluatorServiceImpl_ReportEvaluatorInvokeResult(t *testing.T) { SpaceID: 2, Status: entity.EvaluatorRunStatusFail, }, nil) + mockEvaluatorRecordRepo.EXPECT().CompareAndSwapEvaluatorRecordResult(gomock.Any(), int64(100), int64(2), entity.EvaluatorRunStatusAsyncInvoking, entity.EvaluatorRunStatusSuccess, gomock.Any()).Return(false, nil) }, expectedErrCode: 0, }, @@ -3666,12 +3678,12 @@ func TestEvaluatorServiceImpl_ReportEvaluatorInvokeResult_OutputDataNilOrExtNil( }, }, nil, ) - mockEvaluatorRecordRepo.EXPECT().UpdateEvaluatorRecordResult(gomock.Any(), int64(100), entity.EvaluatorRunStatusSuccess, gomock.Any()). - DoAndReturn(func(_ context.Context, _ int64, _ entity.EvaluatorRunStatus, out *entity.EvaluatorOutputData) error { + mockEvaluatorRecordRepo.EXPECT().CompareAndSwapEvaluatorRecordResult(gomock.Any(), int64(100), int64(2), entity.EvaluatorRunStatusAsyncInvoking, entity.EvaluatorRunStatusSuccess, gomock.Any()). + DoAndReturn(func(_ context.Context, _, _ int64, _, _ entity.EvaluatorRunStatus, out *entity.EvaluatorOutputData) (bool, error) { if assert.NotNil(t, out) && assert.NotNil(t, out.Ext) { assert.Equal(t, "1", out.Ext["old"]) } - return nil + return true, nil }) err := s.ReportEvaluatorInvokeResult(ctx, param) @@ -4548,3 +4560,256 @@ func TestEvaluatorServiceImpl_CreateEvaluatorRunFailRecord_Errors(t *testing.T) assert.Nil(t, record) }) } + +func TestEvaluatorServiceImpl_AsyncRunEvaluator_RejectsBuiltinCustomRPC(t *testing.T) { + t.Parallel() + + ctrl := gomock.NewController(t) + evaluatorRepo := repomocks.NewMockIEvaluatorRepo(ctrl) + s := &EvaluatorServiceImpl{evaluatorRepo: evaluatorRepo} + evaluatorRepo.EXPECT().BatchGetEvaluatorByVersionID(gomock.Any(), nil, []int64{int64(101)}, false, false).Return([]*entity.Evaluator{{ + Builtin: true, + EvaluatorType: entity.EvaluatorTypeCustomRPC, + CustomRPCEvaluatorVersion: &entity.CustomRPCEvaluatorVersion{ + ID: 101, + IsAsync: true, + }, + }}, nil) + + record, err := s.AsyncRunEvaluator(context.Background(), &entity.AsyncRunEvaluatorRequest{SpaceID: 2, EvaluatorVersionID: 101}) + require.Error(t, err) + assert.Nil(t, record) +} + +func TestEvaluatorServiceImpl_AsyncRunEvaluator_CoordinatorOrderAndFailureCompensation(t *testing.T) { + t.Parallel() + + newFixture := func(t *testing.T) (*EvaluatorServiceImpl, *repomocks.MockIEvaluatorRecordRepo, *repomocks.MockIEvalAsyncRepo, *mocks.MockEvaluatorSourceService) { + t.Helper() + ctrl := gomock.NewController(t) + evaluatorRepo := repomocks.NewMockIEvaluatorRepo(ctrl) + limiter := repomocks.NewMockRateLimiter(ctrl) + plainLimiter := repomocks.NewMockIPlainRateLimiter(ctrl) + idGenerator := idgenmocks.NewMockIIDGenerator(ctrl) + recordRepo := repomocks.NewMockIEvaluatorRecordRepo(ctrl) + asyncRepo := repomocks.NewMockIEvalAsyncRepo(ctrl) + source := mocks.NewMockEvaluatorSourceService(ctrl) + evaluatorDO := &entity.Evaluator{ + ID: 100, + SpaceID: 2, + EvaluatorType: entity.EvaluatorTypeAgent, + AgentEvaluatorVersion: &entity.AgentEvaluatorVersion{ + ID: 101, + }, + } + evaluatorRepo.EXPECT().BatchGetEvaluatorByVersionID(gomock.Any(), nil, []int64{int64(101)}, false, false).Return([]*entity.Evaluator{evaluatorDO}, nil) + limiter.EXPECT().AllowInvoke(gomock.Any(), int64(2)).Return(true) + plainLimiter.EXPECT().AllowInvokeWithKeyLimit(gomock.Any(), "async_run_evaluator:100", gomock.Any()).Return(true) + idGenerator.EXPECT().GenID(gomock.Any()).Return(int64(999), nil) + return &EvaluatorServiceImpl{ + evaluatorRepo: evaluatorRepo, + limiter: limiter, + plainRateLimiter: plainLimiter, + idgen: idGenerator, + evaluatorRecordRepo: recordRepo, + evalAsyncRepo: asyncRepo, + evaluatorSourceServices: map[entity.EvaluatorType]EvaluatorSourceService{ + entity.EvaluatorTypeAgent: source, + }, + }, recordRepo, asyncRepo, source + } + + req := func() *entity.AsyncRunEvaluatorRequest { + return &entity.AsyncRunEvaluatorRequest{ + SpaceID: 2, + EvaluatorVersionID: 101, + InputData: &entity.EvaluatorInputData{}, + AsyncCtx: &entity.EvalAsyncCtx{ + Event: &entity.ExptItemEvalEvent{ExptID: 10}, + ResumeReady: false, + }, + } + } + + t.Run("record then context then provider", func(t *testing.T) { + s, recordRepo, asyncRepo, source := newFixture(t) + order := make([]string, 0, 3) + gomock.InOrder( + recordRepo.EXPECT().CreateEvaluatorRecord(gomock.Any(), gomock.Any()).DoAndReturn(func(_ context.Context, record *entity.EvaluatorRecord) error { + order = append(order, "record") + assert.Equal(t, entity.EvaluatorRunStatusAsyncInvoking, record.Status) + return nil + }), + asyncRepo.EXPECT().SetEvalAsyncCtx(gomock.Any(), "evaluator:999", gomock.Any()).DoAndReturn(func(_ context.Context, _ string, actx *entity.EvalAsyncCtx) error { + order = append(order, "context") + assert.Equal(t, int64(999), actx.RecordID) + assert.Equal(t, int64(101), actx.EvaluatorVersionID) + assert.False(t, actx.ResumeReady) + return nil + }), + source.EXPECT().AsyncRun(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), int64(2), int64(999)).DoAndReturn( + func(context.Context, *entity.Evaluator, *entity.EvaluatorInputData, *entity.EvaluatorRunConfig, int64, int64) (map[string]string, string, error) { + order = append(order, "provider") + return nil, "", nil + }, + ), + ) + + record, err := s.AsyncRunEvaluator(context.Background(), req()) + require.NoError(t, err) + require.NotNil(t, record) + assert.Equal(t, []string{"record", "context", "provider"}, order) + }) + + t.Run("context failure marks same record fail and skips provider", func(t *testing.T) { + s, recordRepo, asyncRepo, source := newFixture(t) + recordRepo.EXPECT().CreateEvaluatorRecord(gomock.Any(), gomock.Any()).Return(nil) + asyncRepo.EXPECT().SetEvalAsyncCtx(gomock.Any(), "evaluator:999", gomock.Any()).Return(errors.New("redis down")) + recordRepo.EXPECT().CompareAndSwapEvaluatorRecordResult(gomock.Any(), int64(999), int64(2), entity.EvaluatorRunStatusAsyncInvoking, entity.EvaluatorRunStatusFail, gomock.Any()).Return(true, nil) + source.EXPECT().AsyncRun(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Times(0) + + record, err := s.AsyncRunEvaluator(context.Background(), req()) + require.Error(t, err) + require.NotNil(t, record) + assert.Equal(t, int64(999), record.ID) + assert.Equal(t, entity.EvaluatorRunStatusFail, record.Status) + }) + + t.Run("dispatch failure marks same record fail", func(t *testing.T) { + s, recordRepo, asyncRepo, source := newFixture(t) + recordRepo.EXPECT().CreateEvaluatorRecord(gomock.Any(), gomock.Any()).Return(nil) + asyncRepo.EXPECT().SetEvalAsyncCtx(gomock.Any(), "evaluator:999", gomock.Any()).Return(nil) + source.EXPECT().AsyncRun(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), int64(2), int64(999)).Return(nil, "", errors.New("dispatch failed")) + recordRepo.EXPECT().CompareAndSwapEvaluatorRecordResult(gomock.Any(), int64(999), int64(2), entity.EvaluatorRunStatusAsyncInvoking, entity.EvaluatorRunStatusFail, gomock.Any()).Return(true, nil) + + record, err := s.AsyncRunEvaluator(context.Background(), req()) + require.Error(t, err) + require.NotNil(t, record) + assert.Equal(t, entity.EvaluatorRunStatusFail, record.Status) + }) + + t.Run("dispatch metadata persistence failure does not fail accepted work", func(t *testing.T) { + s, recordRepo, asyncRepo, source := newFixture(t) + recordRepo.EXPECT().CreateEvaluatorRecord(gomock.Any(), gomock.Any()).Return(nil) + asyncRepo.EXPECT().SetEvalAsyncCtx(gomock.Any(), "evaluator:999", gomock.Any()).Return(nil) + source.EXPECT().AsyncRun(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), int64(2), int64(999)).Return(map[string]string{"session_id": "s1"}, "trace-1", nil) + recordRepo.EXPECT().UpdateEvaluatorRecordAsyncDispatch(gomock.Any(), int64(999), int64(2), "trace-1", gomock.Any()).Return(errors.New("mysql unavailable")) + recordRepo.EXPECT().CompareAndSwapEvaluatorRecordResult(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Times(0) + + record, err := s.AsyncRunEvaluator(context.Background(), req()) + require.NoError(t, err) + require.NotNil(t, record) + assert.Equal(t, entity.EvaluatorRunStatusAsyncInvoking, record.Status) + assert.Equal(t, "trace-1", record.TraceID) + assert.Equal(t, "s1", record.EvaluatorOutputData.Ext["session_id"]) + }) + +} + +func TestEvaluatorServiceImpl_ReportEvaluatorInvokeResult_RejectsNonTerminalStatus(t *testing.T) { + t.Parallel() + + s := &EvaluatorServiceImpl{} + require.Error(t, s.ReportEvaluatorInvokeResult(context.Background(), nil)) + require.Error(t, s.ReportEvaluatorInvokeResult(context.Background(), &entity.ReportEvaluatorRecordParam{ + RecordID: 1, + SpaceID: 2, + Status: entity.EvaluatorRunStatusUnknown, + })) +} + +func TestEvaluatorServiceImpl_ReportEvaluatorInvokeResult_UsesTerminalCAS(t *testing.T) { + t.Parallel() + + ctrl := gomock.NewController(t) + recordRepo := repomocks.NewMockIEvaluatorRecordRepo(ctrl) + s := &EvaluatorServiceImpl{evaluatorRecordRepo: recordRepo} + param := &entity.ReportEvaluatorRecordParam{ + SpaceID: 2, + RecordID: 100, + Status: entity.EvaluatorRunStatusSuccess, + OutputData: &entity.EvaluatorOutputData{Ext: map[string]string{"new": "1"}}, + } + + recordRepo.EXPECT().GetEvaluatorRecord(gomock.Any(), int64(100), false).Return(&entity.EvaluatorRecord{ + ID: 100, + SpaceID: 2, + Status: entity.EvaluatorRunStatusAsyncInvoking, + EvaluatorOutputData: &entity.EvaluatorOutputData{ + Ext: map[string]string{"old": "1"}, + }, + }, nil) + recordRepo.EXPECT().CompareAndSwapEvaluatorRecordResult(gomock.Any(), int64(100), int64(2), entity.EvaluatorRunStatusAsyncInvoking, entity.EvaluatorRunStatusSuccess, gomock.Any()).DoAndReturn( + func(_ context.Context, _, _ int64, _, _ entity.EvaluatorRunStatus, out *entity.EvaluatorOutputData) (bool, error) { + assert.Equal(t, "1", out.Ext["old"]) + assert.Equal(t, "1", out.Ext["new"]) + return false, nil + }, + ) + + err := s.ReportEvaluatorInvokeResult(context.Background(), param) + require.NoError(t, err) +} + +func TestEvaluatorServiceImpl_ArmEvaluatorResume(t *testing.T) { + t.Parallel() + + tests := []struct { + name string + recordStatus entity.EvaluatorRunStatus + withEvent bool + wantPublished bool + }{ + {name: "pending record only arms", recordStatus: entity.EvaluatorRunStatusAsyncInvoking, withEvent: true}, + {name: "terminal record republishes recovery", recordStatus: entity.EvaluatorRunStatusSuccess, withEvent: true, wantPublished: true}, + {name: "direct invocation has no experiment recovery", recordStatus: entity.EvaluatorRunStatusSuccess, withEvent: false}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + ctrl := gomock.NewController(t) + asyncRepo := repomocks.NewMockIEvalAsyncRepo(ctrl) + recordRepo := repomocks.NewMockIEvaluatorRecordRepo(ctrl) + publisher := eventmocks.NewMockExptEventPublisher(ctrl) + var event *entity.ExptItemEvalEvent + if tt.withEvent { + event = &entity.ExptItemEvalEvent{ExptID: 1, ExptRunID: 2, EvalSetItemID: 3} + } + asyncRepo.EXPECT().MarkEvalAsyncResumeReady(gomock.Any(), "evaluator:100").Return(&entity.EvalAsyncCtx{Event: event, ResumeReady: true}, nil) + if tt.withEvent { + recordRepo.EXPECT().GetEvaluatorRecord(gomock.Any(), int64(100), false).DoAndReturn(func(ctx context.Context, _ int64, _ bool, _ ...entity.GetEvaluatorRecordOptionFn) (*entity.EvaluatorRecord, error) { + assert.True(t, contexts.CtxWriteDB(ctx)) + return &entity.EvaluatorRecord{ID: 100, Status: tt.recordStatus}, nil + }) + } + if tt.wantPublished { + publisher.EXPECT().PublishExptRecordEvalEvent(gomock.Any(), event, gomock.Any(), gomock.Any()).DoAndReturn( + func(_ context.Context, ev *entity.ExptItemEvalEvent, _ *time.Duration, modify func(*entity.ExptItemEvalEvent)) error { + modify(ev) + assert.True(t, ev.AsyncEvaluatorReportTrigger) + return nil + }, + ) + } + s := &EvaluatorServiceImpl{evalAsyncRepo: asyncRepo, evaluatorRecordRepo: recordRepo, exptEventPublisher: publisher} + require.NoError(t, s.ArmEvaluatorResume(context.Background(), 100)) + }) + } +} + +func TestEvaluatorServiceImpl_ArmEvaluatorResume_RetriesPublish(t *testing.T) { + t.Parallel() + ctrl := gomock.NewController(t) + asyncRepo := repomocks.NewMockIEvalAsyncRepo(ctrl) + recordRepo := repomocks.NewMockIEvaluatorRecordRepo(ctrl) + publisher := eventmocks.NewMockExptEventPublisher(ctrl) + event := &entity.ExptItemEvalEvent{ExptID: 1} + asyncRepo.EXPECT().MarkEvalAsyncResumeReady(gomock.Any(), "evaluator:100").Return(&entity.EvalAsyncCtx{Event: event, ResumeReady: true}, nil) + recordRepo.EXPECT().GetEvaluatorRecord(gomock.Any(), int64(100), false).Return(&entity.EvaluatorRecord{ID: 100, Status: entity.EvaluatorRunStatusSuccess}, nil) + gomock.InOrder( + publisher.EXPECT().PublishExptRecordEvalEvent(gomock.Any(), event, gomock.Any(), gomock.Any()).Return(errors.New("mq unavailable")), + publisher.EXPECT().PublishExptRecordEvalEvent(gomock.Any(), event, gomock.Any(), gomock.Any()).Return(nil), + ) + s := &EvaluatorServiceImpl{evalAsyncRepo: asyncRepo, evaluatorRecordRepo: recordRepo, exptEventPublisher: publisher} + require.NoError(t, s.ArmEvaluatorResume(context.Background(), 100)) +} diff --git a/backend/modules/evaluation/domain/service/expt_run_item_impl.go b/backend/modules/evaluation/domain/service/expt_run_item_impl.go index 0913c0a16b..1ddb8189f8 100644 --- a/backend/modules/evaluation/domain/service/expt_run_item_impl.go +++ b/backend/modules/evaluation/domain/service/expt_run_item_impl.go @@ -230,6 +230,14 @@ func (e *ExptItemEvalCtxExecutor) storeTurnRunResult(ctx context.Context, etec * if err := e.TurnResultRepo.SaveTurnRunLogs(persistCtx, []*entity.ExptTurnResultRunLog{clone}); err != nil { return err } + for _, record := range result.EvaluatorResults { + if record == nil || record.ID <= 0 || record.Status != entity.EvaluatorRunStatusAsyncInvoking { + continue + } + if err := e.evaluatorService.ArmEvaluatorResume(persistCtx, record.ID); err != nil { + return errorx.Wrapf(err, "arm evaluator async resume fail, record_id: %d", record.ID) + } + } logs.CtxInfo(ctx, "[ExptTurnEval] expt turn eval finished, expt_id: %v, expt_run_id: %v, item_id: %v, turn_id: %v, run_log: %v, err: %v", etec.Expt.ID, etec.Event.ExptRunID, etec.EvalSetItem.ItemID, turn.ID, json.Jsonify(clone), result.EvalErr) diff --git a/backend/modules/evaluation/domain/service/expt_run_item_impl_test.go b/backend/modules/evaluation/domain/service/expt_run_item_impl_test.go index d015431912..02e01a203b 100644 --- a/backend/modules/evaluation/domain/service/expt_run_item_impl_test.go +++ b/backend/modules/evaluation/domain/service/expt_run_item_impl_test.go @@ -1117,3 +1117,57 @@ func Test_buildItemCompleteEvent(t *testing.T) { }) } } + +func TestExptItemEvalCtxExecutor_storeTurnRunResult_ArmsAsyncEvaluatorAfterSave(t *testing.T) { + t.Parallel() + + ctrl := gomock.NewController(t) + turnRepo := repomocks.NewMockIExptTurnResultRepo(ctrl) + evaluatorSvc := servicemocks.NewMockEvaluatorService(ctrl) + turnRepo.EXPECT().SaveTurnRunLogs(gomock.Any(), gomock.Any()).Return(nil) + evaluatorSvc.EXPECT().ArmEvaluatorResume(gomock.Any(), int64(100)).Return(nil) + + executor := &ExptItemEvalCtxExecutor{TurnResultRepo: turnRepo, evaluatorService: evaluatorSvc} + etec := &entity.ExptTurnEvalCtx{ + Turn: &entity.Turn{ID: 1}, + ExptItemEvalCtx: &entity.ExptItemEvalCtx{ + Expt: &entity.Experiment{ID: 1, SpaceID: 2}, + Event: &entity.ExptItemEvalEvent{ExptRunID: 3}, + EvalSetItem: &entity.EvaluationSetItem{ItemID: 4}, + ExistItemEvalResult: &entity.ExptItemEvalResult{TurnResultRunLogs: map[int64]*entity.ExptTurnResultRunLog{ + 1: {ID: 5, TurnID: 1}, + }}, + }, + } + result := &entity.ExptTurnRunResult{EvaluatorResults: []*entity.EvaluatorRecord{{ + ID: 100, EvaluatorVersionID: 101, Status: entity.EvaluatorRunStatusAsyncInvoking, + }}} + require.NoError(t, executor.storeTurnRunResult(context.Background(), etec, result)) +} + +func TestExptItemEvalCtxExecutor_storeTurnRunResult_DoesNotArmWhenSaveFails(t *testing.T) { + t.Parallel() + + ctrl := gomock.NewController(t) + turnRepo := repomocks.NewMockIExptTurnResultRepo(ctrl) + evaluatorSvc := servicemocks.NewMockEvaluatorService(ctrl) + turnRepo.EXPECT().SaveTurnRunLogs(gomock.Any(), gomock.Any()).Return(errors.New("save failed")) + evaluatorSvc.EXPECT().ArmEvaluatorResume(gomock.Any(), gomock.Any()).Times(0) + + executor := &ExptItemEvalCtxExecutor{TurnResultRepo: turnRepo, evaluatorService: evaluatorSvc} + etec := &entity.ExptTurnEvalCtx{ + Turn: &entity.Turn{ID: 1}, + ExptItemEvalCtx: &entity.ExptItemEvalCtx{ + Expt: &entity.Experiment{ID: 1, SpaceID: 2}, + Event: &entity.ExptItemEvalEvent{ExptRunID: 3}, + EvalSetItem: &entity.EvaluationSetItem{ItemID: 4}, + ExistItemEvalResult: &entity.ExptItemEvalResult{TurnResultRunLogs: map[int64]*entity.ExptTurnResultRunLog{ + 1: {ID: 5, TurnID: 1}, + }}, + }, + } + result := &entity.ExptTurnRunResult{EvaluatorResults: []*entity.EvaluatorRecord{{ + ID: 100, EvaluatorVersionID: 101, Status: entity.EvaluatorRunStatusAsyncInvoking, + }}} + require.Error(t, executor.storeTurnRunResult(context.Background(), etec, result)) +} diff --git a/backend/modules/evaluation/domain/service/expt_run_item_turn_impl.go b/backend/modules/evaluation/domain/service/expt_run_item_turn_impl.go index bd8e45a841..45ad15267f 100644 --- a/backend/modules/evaluation/domain/service/expt_run_item_turn_impl.go +++ b/backend/modules/evaluation/domain/service/expt_run_item_turn_impl.go @@ -914,7 +914,6 @@ func (e *DefaultExptTurnEvaluationImpl) asyncCallEvaluatorWithAlias( var err error defer func() { e.metric.EmitTurnExecEvaluatorResult(etec.Event.SpaceID, err != nil) }() - ts := time.Now() evaluatorRecord, err := e.evaluatorService.AsyncRunEvaluator(ctx, &entity.AsyncRunEvaluatorRequest{ SpaceID: etec.Event.SpaceID, EvaluatorVersionID: ev.GetEvaluatorVersionID(), @@ -927,22 +926,19 @@ func (e *DefaultExptTurnEvaluationImpl) asyncCallEvaluatorWithAlias( EvaluatorRunConf: runConf, Alias: alias, SourceType: entity.EvaluatorRecordSourceTypeBuiltin, + AsyncCtx: &entity.EvalAsyncCtx{ + Event: etec.Event, + Session: etec.Event.Session, + EvaluatorVersionID: ev.GetEvaluatorVersionID(), + ResumeReady: false, + }, }) - if err != nil { - return err + if evaluatorRecord != nil { + collector.store(evaluatorRecord) } - - asyncCtxKey := fmt.Sprintf("evaluator:%d", evaluatorRecord.ID) - if err = e.evalAsyncRepo.SetEvalAsyncCtx(ctx, asyncCtxKey, &entity.EvalAsyncCtx{ - Event: etec.Event, - RecordID: evaluatorRecord.ID, - AsyncUnixMS: ts.UnixMilli(), - Session: etec.Event.Session, - EvaluatorVersionID: ev.GetEvaluatorVersionID(), - }); err != nil { + if err != nil { return err } - collector.store(evaluatorRecord) return nil } @@ -957,8 +953,6 @@ func (e *DefaultExptTurnEvaluationImpl) asyncCallEvaluator( var err error defer func() { e.metric.EmitTurnExecEvaluatorResult(etec.Event.SpaceID, err != nil) }() - ts := time.Now() - asyncReq := &entity.AsyncRunEvaluatorRequest{ SpaceID: etec.Event.SpaceID, EvaluatorVersionID: ev.GetEvaluatorVersionID(), @@ -969,40 +963,20 @@ func (e *DefaultExptTurnEvaluationImpl) asyncCallEvaluator( TurnID: etec.Turn.ID, Ext: etec.Ext, EvaluatorRunConf: ec.RunConf, + AsyncCtx: &entity.EvalAsyncCtx{ + Event: etec.Event, + Session: etec.Event.Session, + EvaluatorVersionID: ev.GetEvaluatorVersionID(), + ResumeReady: false, + }, } evaluatorRecord, err := e.evaluatorService.AsyncRunEvaluator(ctx, asyncReq) - if err != nil { - if e.evaluatorService != nil { - failReq := &entity.RunEvaluatorRequest{ - SpaceID: asyncReq.SpaceID, - EvaluatorVersionID: asyncReq.EvaluatorVersionID, - InputData: asyncReq.InputData, - ExperimentID: asyncReq.ExperimentID, - ExperimentRunID: asyncReq.ExperimentRunID, - ItemID: asyncReq.ItemID, - TurnID: asyncReq.TurnID, - Ext: asyncReq.Ext, - EvaluatorRunConf: asyncReq.EvaluatorRunConf, - } - if failedRecord, createErr := e.evaluatorService.CreateEvaluatorRunFailRecord(ctx, failReq, err); createErr == nil && failedRecord != nil { - collector.store(failedRecord) - } - } - return err + if evaluatorRecord != nil { + collector.store(evaluatorRecord) } - - asyncCtxKey := fmt.Sprintf("evaluator:%d", evaluatorRecord.ID) - if err = e.evalAsyncRepo.SetEvalAsyncCtx(ctx, asyncCtxKey, &entity.EvalAsyncCtx{ - Event: etec.Event, - RecordID: evaluatorRecord.ID, - AsyncUnixMS: ts.UnixMilli(), - Session: etec.Event.Session, - EvaluatorVersionID: ev.GetEvaluatorVersionID(), - }); err != nil { + if err != nil { return err } - - collector.store(evaluatorRecord) return nil } diff --git a/backend/modules/evaluation/domain/service/expt_run_item_turn_impl_test.go b/backend/modules/evaluation/domain/service/expt_run_item_turn_impl_test.go index 4ade374a31..2f249b0909 100644 --- a/backend/modules/evaluation/domain/service/expt_run_item_turn_impl_test.go +++ b/backend/modules/evaluation/domain/service/expt_run_item_turn_impl_test.go @@ -391,23 +391,13 @@ func TestDefaultExptTurnEvaluationImpl_asyncCallEvaluator_Agent(t *testing.T) { assert.Equal(t, int64(4), req.ItemID) assert.Equal(t, int64(5), req.TurnID) assert.Equal(t, etec.Ext, req.Ext) + require.NotNil(t, req.AsyncCtx) + assert.Equal(t, etec.Event, req.AsyncCtx.Event) + assert.False(t, req.AsyncCtx.ResumeReady) return mockEvaluatorRecord, nil }, ) - mockEvalAsyncRepo.EXPECT().SetEvalAsyncCtx(gomock.Any(), gomock.Any(), gomock.Any()).DoAndReturn( - func(ctx context.Context, key string, val *entity.EvalAsyncCtx) error { - assert.Equal(t, "evaluator:202", key) - assert.Equal(t, int64(202), val.RecordID) - assert.Equal(t, int64(101), val.EvaluatorVersionID) - assert.Equal(t, etec.Event, val.Event) - // Check timestamp - assert.True(t, val.AsyncUnixMS <= time.Now().UnixMilli()) - assert.True(t, val.AsyncUnixMS > time.Now().Add(-time.Minute).UnixMilli()) - return nil - }, - ) - err := service.asyncCallEvaluator(context.Background(), ev, ec, etec, inputData, &collector) assert.NoError(t, err) @@ -472,12 +462,9 @@ func TestDefaultExptTurnEvaluationImpl_asyncCallEvaluator_Agent_Errors(t *testin mockSetup: func() { mockMetric.EXPECT().EmitTurnExecEvaluatorResult(gomock.Any(), true) runErr := errors.New("async run error") - mockEvaluatorService.EXPECT().AsyncRunEvaluator(gomock.Any(), gomock.Any()).Return(nil, runErr) - mockEvaluatorService.EXPECT().CreateEvaluatorRunFailRecord(gomock.Any(), gomock.Any(), runErr).Return(&entity.EvaluatorRecord{ - ID: 303, - EvaluatorVersionID: 101, - Status: entity.EvaluatorRunStatusFail, - }, nil) + mockEvaluatorService.EXPECT().AsyncRunEvaluator(gomock.Any(), gomock.Any()).Return(&entity.EvaluatorRecord{ + ID: 303, EvaluatorVersionID: 101, Status: entity.EvaluatorRunStatusFail, + }, runErr) }, wantErr: true, }, @@ -487,7 +474,6 @@ func TestDefaultExptTurnEvaluationImpl_asyncCallEvaluator_Agent_Errors(t *testin mockMetric.EXPECT().EmitTurnExecEvaluatorResult(gomock.Any(), true) runErr := errors.New("async run error") mockEvaluatorService.EXPECT().AsyncRunEvaluator(gomock.Any(), gomock.Any()).Return(nil, runErr) - mockEvaluatorService.EXPECT().CreateEvaluatorRunFailRecord(gomock.Any(), gomock.Any(), runErr).Return(nil, errors.New("create failed record error")) }, wantErr: true, }, @@ -496,10 +482,8 @@ func TestDefaultExptTurnEvaluationImpl_asyncCallEvaluator_Agent_Errors(t *testin mockSetup: func() { mockMetric.EXPECT().EmitTurnExecEvaluatorResult(gomock.Any(), true) mockEvaluatorService.EXPECT().AsyncRunEvaluator(gomock.Any(), gomock.Any()).Return(&entity.EvaluatorRecord{ - ID: 202, - EvaluatorVersionID: 101, - }, nil) - mockEvalAsyncRepo.EXPECT().SetEvalAsyncCtx(gomock.Any(), gomock.Any(), gomock.Any()).Return(errors.New("set ctx error")) + ID: 202, EvaluatorVersionID: 101, Status: entity.EvaluatorRunStatusFail, + }, errors.New("set ctx error")) }, wantErr: true, }, @@ -1299,7 +1283,6 @@ func TestDefaultExptTurnEvaluationImpl_CallEvaluators(t *testing.T) { mockEvaluatorService.EXPECT().ShouldInterceptEvaluator(gomock.Any(), gomock.Any()).Return(nil, false, nil) mockEvaluatorService.EXPECT().AsyncRunEvaluator(gomock.Any(), gomock.Any()).Return(&entity.EvaluatorRecord{ID: 101, Status: entity.EvaluatorRunStatusAsyncInvoking}, nil) mockMetric.EXPECT().EmitTurnExecEvaluatorResult(gomock.Any(), false) - mockEvalAsyncRepo.EXPECT().SetEvalAsyncCtx(gomock.Any(), gomock.Any(), gomock.Any()).Return(nil) }, etec: &entity.ExptTurnEvalCtx{ ExptItemEvalCtx: &entity.ExptItemEvalCtx{ @@ -4128,7 +4111,6 @@ func TestDefaultExptTurnEvaluationImpl_CallEvaluators_WithRefresh(t *testing.T) &entity.EvaluatorRecord{ID: 201, EvaluatorVersionID: 101, Status: entity.EvaluatorRunStatusAsyncInvoking}, nil, ) mockMetric.EXPECT().EmitTurnExecEvaluatorResult(gomock.Any(), false) - mockEvalAsyncRepo.EXPECT().SetEvalAsyncCtx(gomock.Any(), gomock.Any(), gomock.Any()).Return(nil) mockEvaluatorRecordService.EXPECT().GetEvaluatorRecord(gomock.Any(), int64(201), false).Return( &entity.EvaluatorRecord{ID: 201, EvaluatorVersionID: 101, Status: entity.EvaluatorRunStatusSuccess}, nil, ) @@ -4167,7 +4149,6 @@ func TestDefaultExptTurnEvaluationImpl_CallEvaluators_WithRefresh(t *testing.T) &entity.EvaluatorRecord{ID: 201, EvaluatorVersionID: 101, Status: entity.EvaluatorRunStatusAsyncInvoking}, nil, ) mockMetric.EXPECT().EmitTurnExecEvaluatorResult(gomock.Any(), false) - mockEvalAsyncRepo.EXPECT().SetEvalAsyncCtx(gomock.Any(), gomock.Any(), gomock.Any()).Return(nil) mockEvaluatorRecordService.EXPECT().GetEvaluatorRecord(gomock.Any(), int64(201), false).Return( &entity.EvaluatorRecord{ID: 201, EvaluatorVersionID: 101, Status: entity.EvaluatorRunStatusAsyncInvoking}, nil, ) @@ -4206,7 +4187,6 @@ func TestDefaultExptTurnEvaluationImpl_CallEvaluators_WithRefresh(t *testing.T) &entity.EvaluatorRecord{ID: 201, EvaluatorVersionID: 101, Status: entity.EvaluatorRunStatusAsyncInvoking}, nil, ) mockMetric.EXPECT().EmitTurnExecEvaluatorResult(gomock.Any(), false) - mockEvalAsyncRepo.EXPECT().SetEvalAsyncCtx(gomock.Any(), gomock.Any(), gomock.Any()).Return(nil) mockEvaluatorRecordService.EXPECT().GetEvaluatorRecord(gomock.Any(), int64(201), false).Return(nil, errors.New("db error")) return &DefaultExptTurnEvaluationImpl{ @@ -5007,3 +4987,50 @@ func TestPickHelpers(t *testing.T) { assert.Equal(t, "sub-key", pickDatasetKey(etec)) }) } + +func TestDefaultExptTurnEvaluationImpl_asyncCallEvaluatorWithAlias_CustomRPC(t *testing.T) { + t.Parallel() + + ctrl := gomock.NewController(t) + evaluatorSvc := svcmocks.NewMockEvaluatorService(ctrl) + metric := metricsmocks.NewMockExptMetric(ctrl) + service := &DefaultExptTurnEvaluationImpl{evaluatorService: evaluatorSvc, metric: metric} + providerCode := "trae_work_long_running" + evaluator := &entity.Evaluator{ + EvaluatorType: entity.EvaluatorTypeCustomRPC, + CustomRPCEvaluatorVersion: &entity.CustomRPCEvaluatorVersion{ + ID: 101, + EvaluatorID: 100, + ProviderEvaluatorCode: &providerCode, + AccessProtocol: entity.EvaluatorAccessProtocolRPC, + IsAsync: true, + }, + } + etec := &entity.ExptTurnEvalCtx{ + ExptItemEvalCtx: &entity.ExptItemEvalCtx{ + Event: &entity.ExptItemEvalEvent{SpaceID: 2, ExptID: 3, ExptRunID: 4, Session: &entity.Session{UserID: "u"}}, + EvalSetItem: &entity.EvaluationSetItem{ItemID: 5}, + }, + Turn: &entity.Turn{ID: 6}, + } + input := &entity.EvaluatorInputData{} + runConf := &entity.EvaluatorRunConfig{Env: gptr.Of("ppe_trae")} + metric.EXPECT().EmitTurnExecEvaluatorResult(int64(2), false) + evaluatorSvc.EXPECT().AsyncRunEvaluator(gomock.Any(), gomock.Any()).DoAndReturn( + func(_ context.Context, req *entity.AsyncRunEvaluatorRequest) (*entity.EvaluatorRecord, error) { + assert.Equal(t, int64(101), req.EvaluatorVersionID) + assert.Equal(t, "judge_a", req.Alias) + assert.Equal(t, entity.EvaluatorRecordSourceTypeBuiltin, req.SourceType) + assert.Same(t, runConf, req.EvaluatorRunConf) + require.NotNil(t, req.AsyncCtx) + assert.Same(t, etec.Event, req.AsyncCtx.Event) + assert.False(t, req.AsyncCtx.ResumeReady) + return &entity.EvaluatorRecord{ID: 7, EvaluatorVersionID: 101, Alias: "judge_a", Status: entity.EvaluatorRunStatusAsyncInvoking}, nil + }, + ) + + collector := &evalRecordCollector{} + require.NoError(t, service.asyncCallEvaluatorWithAlias(context.Background(), evaluator, runConf, "judge_a", etec, input, collector)) + require.Len(t, collector.records, 1) + assert.Equal(t, "judge_a", collector.records[0].Alias) +} diff --git a/backend/modules/evaluation/domain/service/mocks/evaluator_service_mock.go b/backend/modules/evaluation/domain/service/mocks/evaluator_service_mock.go index 34917f8059..5310e42f8b 100644 --- a/backend/modules/evaluation/domain/service/mocks/evaluator_service_mock.go +++ b/backend/modules/evaluation/domain/service/mocks/evaluator_service_mock.go @@ -35,6 +35,20 @@ func (m *MockEvaluatorService) EXPECT() *MockEvaluatorServiceMockRecorder { return m.recorder } +// ArmEvaluatorResume mocks base method. +func (m *MockEvaluatorService) ArmEvaluatorResume(ctx context.Context, recordID int64) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "ArmEvaluatorResume", ctx, recordID) + ret0, _ := ret[0].(error) + return ret0 +} + +// ArmEvaluatorResume indicates an expected call of ArmEvaluatorResume. +func (mr *MockEvaluatorServiceMockRecorder) ArmEvaluatorResume(ctx, recordID any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ArmEvaluatorResume", reflect.TypeOf((*MockEvaluatorService)(nil).ArmEvaluatorResume), ctx, recordID) +} + // AsyncDebugEvaluator mocks base method. func (m *MockEvaluatorService) AsyncDebugEvaluator(arg0 context.Context, arg1 *entity.AsyncDebugEvaluatorRequest) (*entity.AsyncDebugEvaluatorResponse, error) { m.ctrl.T.Helper() diff --git a/backend/modules/evaluation/infra/repo/evaluator/evaluator_record_impl.go b/backend/modules/evaluation/infra/repo/evaluator/evaluator_record_impl.go index 43b68dc48e..443e95ed74 100644 --- a/backend/modules/evaluation/infra/repo/evaluator/evaluator_record_impl.go +++ b/backend/modules/evaluation/infra/repo/evaluator/evaluator_record_impl.go @@ -6,6 +6,8 @@ package evaluator import ( "context" + "gorm.io/gorm" + "github.com/coze-dev/coze-loop/backend/infra/db" "github.com/coze-dev/coze-loop/backend/infra/idgen" "github.com/coze-dev/coze-loop/backend/modules/evaluation/domain/entity" @@ -158,16 +160,92 @@ func (r *EvaluatorRecordRepoImpl) BatchGetEvaluatorRecordForAggr(ctx context.Con return aggrRecords, nil } -func (r *EvaluatorRecordRepoImpl) UpdateEvaluatorRecordResult(ctx context.Context, recordID int64, status entity.EvaluatorRunStatus, outputData *entity.EvaluatorOutputData) error { - var score float64 +func evaluatorRecordResultValues(outputData *entity.EvaluatorOutputData) (score float64, outputDataStr string) { if outputData != nil && outputData.EvaluatorResult != nil && outputData.EvaluatorResult.Score != nil { score = *outputData.EvaluatorResult.Score } - - var outputDataStr string if outputData != nil { outputDataStr = json.Jsonify(outputData) } + return score, outputDataStr +} - return r.evaluatorRecordDao.UpdateEvaluatorRecordResult(ctx, recordID, int8(status), score, outputDataStr) +func mergeEvaluatorOutputExt(dst, src *entity.EvaluatorOutputData) *entity.EvaluatorOutputData { + if dst == nil { + dst = &entity.EvaluatorOutputData{} + } + if src == nil || len(src.Ext) == 0 { + return dst + } + if dst.Ext == nil { + dst.Ext = make(map[string]string, len(src.Ext)) + } + for k, v := range src.Ext { + if _, exists := dst.Ext[k]; !exists { + dst.Ext[k] = v + } + } + return dst +} + +func (r *EvaluatorRecordRepoImpl) UpdateEvaluatorRecordResult(ctx context.Context, recordID int64, status entity.EvaluatorRunStatus, outputData *entity.EvaluatorOutputData) error { + // This legacy entry point is used by async-zombie termination. Resolve space on the primary and + // funnel the terminal write through the same AsyncInvoking CAS so a concurrent callback cannot be overwritten. + po, err := r.evaluatorRecordDao.GetEvaluatorRecord(ctx, recordID, false, db.WithMaster()) + if err != nil || po == nil { + return err + } + _, err = r.CompareAndSwapEvaluatorRecordResult(ctx, recordID, po.SpaceID, entity.EvaluatorRunStatusAsyncInvoking, status, outputData) + return err +} + +func (r *EvaluatorRecordRepoImpl) CompareAndSwapEvaluatorRecordResult(ctx context.Context, recordID, spaceID int64, fromStatus, toStatus entity.EvaluatorRunStatus, outputData *entity.EvaluatorOutputData) (bool, error) { + if r.dbProvider == nil { + score, outputDataStr := evaluatorRecordResultValues(outputData) + rows, err := r.evaluatorRecordDao.CompareAndSwapEvaluatorRecordResult(ctx, recordID, spaceID, int8(fromStatus), int8(toStatus), score, outputDataStr) + return rows > 0, err + } + var updated bool + err := r.dbProvider.Transaction(ctx, func(tx *gorm.DB) error { + opt := db.WithTransaction(tx) + po, err := r.evaluatorRecordDao.GetEvaluatorRecord(ctx, recordID, false, opt, db.WithSelectForUpdate()) + if err != nil || po == nil || po.SpaceID != spaceID || entity.EvaluatorRunStatus(po.Status) != fromStatus { + return err + } + current, err := convertor.ConvertEvaluatorRecordPO2DO(po) + if err != nil { + return err + } + if current != nil { + outputData = mergeEvaluatorOutputExt(outputData, current.EvaluatorOutputData) + } + score, outputDataStr := evaluatorRecordResultValues(outputData) + rows, err := r.evaluatorRecordDao.CompareAndSwapEvaluatorRecordResult(ctx, recordID, spaceID, int8(fromStatus), int8(toStatus), score, outputDataStr, opt) + updated = rows > 0 + return err + }) + return updated, err +} + +func (r *EvaluatorRecordRepoImpl) UpdateEvaluatorRecordAsyncDispatch(ctx context.Context, recordID, spaceID int64, traceID string, outputData *entity.EvaluatorOutputData) error { + if r.dbProvider == nil { + _, outputDataStr := evaluatorRecordResultValues(outputData) + return r.evaluatorRecordDao.UpdateEvaluatorRecordAsyncDispatch(ctx, recordID, spaceID, traceID, outputDataStr) + } + return r.dbProvider.Transaction(ctx, func(tx *gorm.DB) error { + opt := db.WithTransaction(tx) + po, err := r.evaluatorRecordDao.GetEvaluatorRecord(ctx, recordID, false, opt, db.WithSelectForUpdate()) + if err != nil || po == nil || po.SpaceID != spaceID { + return err + } + current, err := convertor.ConvertEvaluatorRecordPO2DO(po) + if err != nil { + return err + } + if current != nil { + outputData = mergeEvaluatorOutputExt(current.EvaluatorOutputData, outputData) + } + _, outputDataStr := evaluatorRecordResultValues(outputData) + return r.evaluatorRecordDao.UpdateEvaluatorRecordAsyncDispatch(ctx, recordID, spaceID, traceID, outputDataStr, opt) + }) } diff --git a/backend/modules/evaluation/infra/repo/evaluator/evaluator_record_impl_test.go b/backend/modules/evaluation/infra/repo/evaluator/evaluator_record_impl_test.go index c3db2b8294..3c81e87e1d 100644 --- a/backend/modules/evaluation/infra/repo/evaluator/evaluator_record_impl_test.go +++ b/backend/modules/evaluation/infra/repo/evaluator/evaluator_record_impl_test.go @@ -13,8 +13,11 @@ import ( "github.com/bytedance/gg/gptr" "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" "go.uber.org/mock/gomock" + "gorm.io/gorm" + "github.com/coze-dev/coze-loop/backend/infra/db" dbmocks "github.com/coze-dev/coze-loop/backend/infra/db/mocks" fsMocks "github.com/coze-dev/coze-loop/backend/infra/fileserver/mocks" idgenmocks "github.com/coze-dev/coze-loop/backend/infra/idgen/mocks" @@ -583,110 +586,24 @@ func TestEvaluatorRecordRepoImpl_GetEvaluatorRecord(t *testing.T) { } func TestEvaluatorRecordRepoImpl_UpdateEvaluatorRecordResult(t *testing.T) { + t.Parallel() ctrl := gomock.NewController(t) - defer ctrl.Finish() - - mockEvaluatorRecordDAO := evaluatormocks.NewMockEvaluatorRecordDAO(ctrl) - - tests := []struct { - name string - recordID int64 - status entity.EvaluatorRunStatus - outputData *entity.EvaluatorOutputData - wantScore float64 - wantOutputDataStr string - daoErr error - }{ - { - name: "outputData为nil", - recordID: 1, - status: entity.EvaluatorRunStatusSuccess, - outputData: nil, - wantScore: 0, - wantOutputDataStr: "", - }, - { - name: "EvaluatorResult为nil", - recordID: 2, - status: entity.EvaluatorRunStatusFail, - outputData: &entity.EvaluatorOutputData{ - EvaluatorResult: nil, - }, - wantScore: 0, - wantOutputDataStr: pkgjson.Jsonify(&entity.EvaluatorOutputData{EvaluatorResult: nil}), - }, - { - name: "Score为nil但Correction有score", - recordID: 3, - status: entity.EvaluatorRunStatusFail, - outputData: &entity.EvaluatorOutputData{ - EvaluatorResult: &entity.EvaluatorResult{ - Score: nil, - Correction: &entity.Correction{ - Score: gptr.Of(float64(2.5)), - }, - }, - }, - wantScore: 0, - wantOutputDataStr: pkgjson.Jsonify(&entity.EvaluatorOutputData{ - EvaluatorResult: &entity.EvaluatorResult{ - Score: nil, - Correction: &entity.Correction{ - Score: gptr.Of(float64(2.5)), - }, - }, - }), - }, - { - name: "Score有值", - recordID: 4, - status: entity.EvaluatorRunStatusSuccess, - outputData: &entity.EvaluatorOutputData{ - EvaluatorResult: &entity.EvaluatorResult{ - Score: gptr.Of(float64(1.25)), - }, - }, - wantScore: 1.25, - wantOutputDataStr: pkgjson.Jsonify(&entity.EvaluatorOutputData{ - EvaluatorResult: &entity.EvaluatorResult{ - Score: gptr.Of(float64(1.25)), - }, - }), - }, - { - name: "DAO返回错误", - recordID: 5, - status: entity.EvaluatorRunStatusSuccess, - outputData: &entity.EvaluatorOutputData{ - EvaluatorResult: &entity.EvaluatorResult{ - Score: gptr.Of(float64(3)), - }, - }, - wantScore: 3, - wantOutputDataStr: pkgjson.Jsonify(&entity.EvaluatorOutputData{ - EvaluatorResult: &entity.EvaluatorResult{ - Score: gptr.Of(float64(3)), - }, - }), - daoErr: assert.AnError, - }, - } - - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - repo := &EvaluatorRecordRepoImpl{ - evaluatorRecordDao: mockEvaluatorRecordDAO, - } + dao := evaluatormocks.NewMockEvaluatorRecordDAO(ctrl) + repo := &EvaluatorRecordRepoImpl{evaluatorRecordDao: dao} + output := &entity.EvaluatorOutputData{EvaluatorRunError: &entity.EvaluatorRunError{Message: "zombie"}} - mockEvaluatorRecordDAO.EXPECT(). - UpdateEvaluatorRecordResult(gomock.Any(), tt.recordID, int8(tt.status), tt.wantScore, tt.wantOutputDataStr). - Return(tt.daoErr). - Times(1) + dao.EXPECT().GetEvaluatorRecord(gomock.Any(), int64(10), false, gomock.Any()).Return(&model.EvaluatorRecord{ID: 10, SpaceID: 20}, nil) + dao.EXPECT().CompareAndSwapEvaluatorRecordResult(gomock.Any(), int64(10), int64(20), int8(entity.EvaluatorRunStatusAsyncInvoking), int8(entity.EvaluatorRunStatusFail), float64(0), pkgjson.Jsonify(output)).Return(int64(1), nil) + require.NoError(t, repo.UpdateEvaluatorRecordResult(context.Background(), 10, entity.EvaluatorRunStatusFail, output)) +} - err := repo.UpdateEvaluatorRecordResult(context.Background(), tt.recordID, tt.status, tt.outputData) - assert.Equal(t, tt.daoErr, err) - }) - } +func TestEvaluatorRecordRepoImpl_UpdateEvaluatorRecordResult_MissingRecordIsNoop(t *testing.T) { + t.Parallel() + ctrl := gomock.NewController(t) + dao := evaluatormocks.NewMockEvaluatorRecordDAO(ctrl) + repo := &EvaluatorRecordRepoImpl{evaluatorRecordDao: dao} + dao.EXPECT().GetEvaluatorRecord(gomock.Any(), int64(10), false, gomock.Any()).Return(nil, nil) + require.NoError(t, repo.UpdateEvaluatorRecordResult(context.Background(), 10, entity.EvaluatorRunStatusFail, nil)) } type nopReader struct{ buf *bytes.Reader } @@ -1138,3 +1055,78 @@ func TestEvaluatorRecordRepoImpl_BatchGetEvaluatorRecordForAggr(t *testing.T) { assert.Nil(t, got) }) } + +func TestEvaluatorRecordRepoImpl_CompareAndSwapEvaluatorRecordResult(t *testing.T) { + t.Parallel() + + ctrl := gomock.NewController(t) + dao := evaluatormocks.NewMockEvaluatorRecordDAO(ctrl) + repo := &EvaluatorRecordRepoImpl{evaluatorRecordDao: dao} + output := &entity.EvaluatorOutputData{EvaluatorResult: &entity.EvaluatorResult{Score: gptr.Of(0.75)}} + dao.EXPECT().CompareAndSwapEvaluatorRecordResult(gomock.Any(), int64(10), int64(20), int8(entity.EvaluatorRunStatusAsyncInvoking), int8(entity.EvaluatorRunStatusSuccess), 0.75, pkgjson.Jsonify(output)).Return(int64(1), nil) + + updated, err := repo.CompareAndSwapEvaluatorRecordResult(context.Background(), 10, 20, entity.EvaluatorRunStatusAsyncInvoking, entity.EvaluatorRunStatusSuccess, output) + require.NoError(t, err) + assert.True(t, updated) +} + +func TestEvaluatorRecordRepoImpl_AsyncCallbackAndDispatchMetadataMerge(t *testing.T) { + t.Parallel() + + newRepo := func(t *testing.T) (*EvaluatorRecordRepoImpl, *evaluatormocks.MockEvaluatorRecordDAO) { + t.Helper() + ctrl := gomock.NewController(t) + dao := evaluatormocks.NewMockEvaluatorRecordDAO(ctrl) + provider := dbmocks.NewMockProvider(ctrl) + provider.EXPECT().Transaction(gomock.Any(), gomock.Any()).DoAndReturn(func(ctx context.Context, fc func(*gorm.DB) error, _ ...db.Option) error { + return fc(nil) + }).AnyTimes() + return &EvaluatorRecordRepoImpl{evaluatorRecordDao: dao, dbProvider: provider}, dao + } + + t.Run("callback after dispatch keeps kickoff ext", func(t *testing.T) { + repo, dao := newRepo(t) + storedOutput := &entity.EvaluatorOutputData{Ext: map[string]string{"session_id": "s1", "lb_key": "k1"}} + storedBytes := []byte(pkgjson.Jsonify(storedOutput)) + dao.EXPECT().GetEvaluatorRecord(gomock.Any(), int64(10), false, gomock.Any(), gomock.Any()).Return(&model.EvaluatorRecord{ + ID: 10, SpaceID: 20, Status: int32(entity.EvaluatorRunStatusAsyncInvoking), OutputData: &storedBytes, + }, nil) + dao.EXPECT().CompareAndSwapEvaluatorRecordResult(gomock.Any(), int64(10), int64(20), int8(entity.EvaluatorRunStatusAsyncInvoking), int8(entity.EvaluatorRunStatusSuccess), 0.8, gomock.Any(), gomock.Any()).DoAndReturn( + func(_ context.Context, _, _ int64, _, _ int8, _ float64, output string, _ ...db.Option) (int64, error) { + var got entity.EvaluatorOutputData + require.NoError(t, pkgjson.Unmarshal([]byte(output), &got)) + assert.Equal(t, "done", got.EvaluatorResult.Reasoning) + assert.Equal(t, "s1", got.Ext["session_id"]) + assert.Equal(t, "k1", got.Ext["lb_key"]) + return 1, nil + }) + + updated, err := repo.CompareAndSwapEvaluatorRecordResult(context.Background(), 10, 20, entity.EvaluatorRunStatusAsyncInvoking, entity.EvaluatorRunStatusSuccess, &entity.EvaluatorOutputData{ + EvaluatorResult: &entity.EvaluatorResult{Score: gptr.Of(0.8), Reasoning: "done"}, + }) + require.NoError(t, err) + assert.True(t, updated) + }) + + t.Run("dispatch after callback keeps terminal output and adds kickoff ext", func(t *testing.T) { + repo, dao := newRepo(t) + storedOutput := &entity.EvaluatorOutputData{EvaluatorResult: &entity.EvaluatorResult{Score: gptr.Of(0.8), Reasoning: "done"}} + storedBytes := []byte(pkgjson.Jsonify(storedOutput)) + dao.EXPECT().GetEvaluatorRecord(gomock.Any(), int64(10), false, gomock.Any(), gomock.Any()).Return(&model.EvaluatorRecord{ + ID: 10, SpaceID: 20, Status: int32(entity.EvaluatorRunStatusSuccess), OutputData: &storedBytes, + }, nil) + dao.EXPECT().UpdateEvaluatorRecordAsyncDispatch(gomock.Any(), int64(10), int64(20), "trace-1", gomock.Any(), gomock.Any()).DoAndReturn( + func(_ context.Context, _, _ int64, _ string, output string, _ ...db.Option) error { + var got entity.EvaluatorOutputData + require.NoError(t, pkgjson.Unmarshal([]byte(output), &got)) + assert.Equal(t, float64(0.8), gptr.Indirect(got.EvaluatorResult.Score)) + assert.Equal(t, "done", got.EvaluatorResult.Reasoning) + assert.Equal(t, "s1", got.Ext["session_id"]) + return nil + }) + + require.NoError(t, repo.UpdateEvaluatorRecordAsyncDispatch(context.Background(), 10, 20, "trace-1", &entity.EvaluatorOutputData{ + Ext: map[string]string{"session_id": "s1"}, + })) + }) +} diff --git a/backend/modules/evaluation/infra/repo/evaluator/mysql/convertor/evaluator_test.go b/backend/modules/evaluation/infra/repo/evaluator/mysql/convertor/evaluator_test.go index 4063cc5fc7..99e6d49d0d 100644 --- a/backend/modules/evaluation/infra/repo/evaluator/mysql/convertor/evaluator_test.go +++ b/backend/modules/evaluation/infra/repo/evaluator/mysql/convertor/evaluator_test.go @@ -442,3 +442,37 @@ func TestConvertEvaluatorVersionPO2DO(t *testing.T) { }) } } + +func TestConvertEvaluatorVersion_CustomRPCAsyncMetainfoRoundTrip(t *testing.T) { + method := evaluatordo.EvaluatorHTTPMethodPost + path := "/async_invoke_evaluator" + serviceName := "trae.work.evaluator" + original := &evaluatordo.Evaluator{ + EvaluatorType: evaluatordo.EvaluatorTypeCustomRPC, + CustomRPCEvaluatorVersion: &evaluatordo.CustomRPCEvaluatorVersion{ + ID: 101, + SpaceID: 202, + EvaluatorID: 303, + EvaluatorType: evaluatordo.EvaluatorTypeCustomRPC, + Version: "1.0.0", + AccessProtocol: evaluatordo.EvaluatorAccessProtocolFaasHTTP, + ServiceName: &serviceName, + AsyncInvokeHTTPInfo: &evaluatordo.EvaluatorHTTPInfo{ + Method: &method, + Path: &path, + }, + IsAsync: true, + }, + } + + po, err := ConvertEvaluatorVersionDO2PO(original) + require.NoError(t, err) + got, err := ConvertEvaluatorVersionPO2DO(po) + require.NoError(t, err) + require.NotNil(t, got) + require.NotNil(t, got.CustomRPCEvaluatorVersion) + assert.True(t, got.CustomRPCEvaluatorVersion.IsAsync) + require.NotNil(t, got.CustomRPCEvaluatorVersion.AsyncInvokeHTTPInfo) + assert.Equal(t, method, gptr.Indirect(got.CustomRPCEvaluatorVersion.AsyncInvokeHTTPInfo.Method)) + assert.Equal(t, path, gptr.Indirect(got.CustomRPCEvaluatorVersion.AsyncInvokeHTTPInfo.Path)) +} diff --git a/backend/modules/evaluation/infra/repo/evaluator/mysql/evaluator_record.go b/backend/modules/evaluation/infra/repo/evaluator/mysql/evaluator_record.go index 653f878dd9..09a382995f 100644 --- a/backend/modules/evaluation/infra/repo/evaluator/mysql/evaluator_record.go +++ b/backend/modules/evaluation/infra/repo/evaluator/mysql/evaluator_record.go @@ -24,6 +24,8 @@ type EvaluatorRecordDAO interface { CreateEvaluatorRecord(ctx context.Context, evaluatorRecord *model.EvaluatorRecord, opts ...db.Option) error UpdateEvaluatorRecord(ctx context.Context, evaluatorRecord *model.EvaluatorRecord, opts ...db.Option) error UpdateEvaluatorRecordResult(ctx context.Context, recordID int64, status int8, score float64, outputData string, opts ...db.Option) error + CompareAndSwapEvaluatorRecordResult(ctx context.Context, recordID, spaceID int64, fromStatus, toStatus int8, score float64, outputData string, opts ...db.Option) (int64, error) + UpdateEvaluatorRecordAsyncDispatch(ctx context.Context, recordID, spaceID int64, traceID, outputData string, opts ...db.Option) error GetEvaluatorRecord(ctx context.Context, evaluatorRecordID int64, includeDeleted bool, opts ...db.Option) (*model.EvaluatorRecord, error) BatchGetEvaluatorRecord(ctx context.Context, evaluatorRecordIDs []int64, includeDeleted bool, opts ...db.Option) ([]*model.EvaluatorRecord, error) // BatchGetEvaluatorRecordForAggr 聚合专用窄查询: 只 SELECT id, score, status, 不取 input_data/output_data/ext @@ -85,6 +87,30 @@ func (dao *EvaluatorRecordDAOImpl) UpdateEvaluatorRecordResult(ctx context.Conte }).Error } +func (dao *EvaluatorRecordDAOImpl) CompareAndSwapEvaluatorRecordResult(ctx context.Context, recordID, spaceID int64, fromStatus, toStatus int8, score float64, outputData string, opts ...db.Option) (int64, error) { + dbsession := dao.provider.NewSession(ctx, opts...) + result := dbsession.WithContext(ctx). + Model(&model.EvaluatorRecord{}). + Where("id = ? AND space_id = ? AND status = ? AND deleted_at IS NULL", recordID, spaceID, fromStatus). + Updates(map[string]interface{}{ + "status": toStatus, + "score": score, + "output_data": outputData, + }) + return result.RowsAffected, result.Error +} + +func (dao *EvaluatorRecordDAOImpl) UpdateEvaluatorRecordAsyncDispatch(ctx context.Context, recordID, spaceID int64, traceID, outputData string, opts ...db.Option) error { + dbsession := dao.provider.NewSession(ctx, opts...) + return dbsession.WithContext(ctx). + Model(&model.EvaluatorRecord{}). + Where("id = ? AND space_id = ? AND deleted_at IS NULL", recordID, spaceID). + Updates(map[string]interface{}{ + "trace_id": traceID, + "output_data": outputData, + }).Error +} + func (dao *EvaluatorRecordDAOImpl) GetEvaluatorRecord(ctx context.Context, evaluatorRecordID int64, includeDeleted bool, opts ...db.Option) (*model.EvaluatorRecord, error) { po := &model.EvaluatorRecord{} @@ -92,6 +118,9 @@ func (dao *EvaluatorRecordDAOImpl) GetEvaluatorRecord(ctx context.Context, evalu dbsession := dao.provider.NewSession(ctx, opts...) query := dbsession.WithContext(ctx).Where("id = ?", evaluatorRecordID) + if contexts.CtxWriteDB(ctx) { + query = query.Clauses(dbresolver.Write) + } if includeDeleted { query = query.Unscoped() // 解除软删除过滤 } diff --git a/backend/modules/evaluation/infra/repo/evaluator/mysql/evaluator_record_aggr_test.go b/backend/modules/evaluation/infra/repo/evaluator/mysql/evaluator_record_aggr_test.go index de08240137..7ea0758206 100644 --- a/backend/modules/evaluation/infra/repo/evaluator/mysql/evaluator_record_aggr_test.go +++ b/backend/modules/evaluation/infra/repo/evaluator/mysql/evaluator_record_aggr_test.go @@ -10,6 +10,7 @@ import ( "github.com/DATA-DOG/go-sqlmock" "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" "go.uber.org/mock/gomock" "gorm.io/driver/mysql" "gorm.io/gorm" @@ -81,3 +82,22 @@ func TestBatchGetEvaluatorRecordForAggr_DAOError(t *testing.T) { assert.Error(t, err) assert.Nil(t, got) } + +func TestCompareAndSwapEvaluatorRecordResult_SQL(t *testing.T) { + t.Parallel() + + ctrl := gomock.NewController(t) + dao, mock, cleanup := newAggrTestDAO(t, ctrl) + defer cleanup() + + mock.ExpectBegin() + mock.ExpectExec(`UPDATE `+"`evaluator_record`"+` SET .+ WHERE \(id = .+ AND space_id = .+ AND status = .+ AND deleted_at IS NULL\) AND `+"`evaluator_record`.`deleted_at`"+` IS NULL`). + WithArgs(sqlmock.AnyArg(), sqlmock.AnyArg(), sqlmock.AnyArg(), sqlmock.AnyArg(), int64(10), int64(20), int8(entity.EvaluatorRunStatusAsyncInvoking)). + WillReturnResult(sqlmock.NewResult(0, 1)) + mock.ExpectCommit() + + rows, err := dao.CompareAndSwapEvaluatorRecordResult(context.Background(), 10, 20, int8(entity.EvaluatorRunStatusAsyncInvoking), int8(entity.EvaluatorRunStatusSuccess), 0.8, `{}`) + require.NoError(t, err) + assert.Equal(t, int64(1), rows) + assert.NoError(t, mock.ExpectationsWereMet()) +} diff --git a/backend/modules/evaluation/infra/repo/evaluator/mysql/mocks/evaluator_record_mock.go b/backend/modules/evaluation/infra/repo/evaluator/mysql/mocks/evaluator_record_mock.go index ccf2020d44..3a6a1be5de 100644 --- a/backend/modules/evaluation/infra/repo/evaluator/mysql/mocks/evaluator_record_mock.go +++ b/backend/modules/evaluation/infra/repo/evaluator/mysql/mocks/evaluator_record_mock.go @@ -81,6 +81,26 @@ func (mr *MockEvaluatorRecordDAOMockRecorder) BatchGetEvaluatorRecordForAggr(arg return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "BatchGetEvaluatorRecordForAggr", reflect.TypeOf((*MockEvaluatorRecordDAO)(nil).BatchGetEvaluatorRecordForAggr), varargs...) } +// CompareAndSwapEvaluatorRecordResult mocks base method. +func (m *MockEvaluatorRecordDAO) CompareAndSwapEvaluatorRecordResult(ctx context.Context, recordID, spaceID int64, fromStatus, toStatus int8, score float64, outputData string, opts ...db.Option) (int64, error) { + m.ctrl.T.Helper() + varargs := []any{ctx, recordID, spaceID, fromStatus, toStatus, score, outputData} + for _, a := range opts { + varargs = append(varargs, a) + } + ret := m.ctrl.Call(m, "CompareAndSwapEvaluatorRecordResult", varargs...) + ret0, _ := ret[0].(int64) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// CompareAndSwapEvaluatorRecordResult indicates an expected call of CompareAndSwapEvaluatorRecordResult. +func (mr *MockEvaluatorRecordDAOMockRecorder) CompareAndSwapEvaluatorRecordResult(ctx, recordID, spaceID, fromStatus, toStatus, score, outputData any, opts ...any) *gomock.Call { + mr.mock.ctrl.T.Helper() + varargs := append([]any{ctx, recordID, spaceID, fromStatus, toStatus, score, outputData}, opts...) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "CompareAndSwapEvaluatorRecordResult", reflect.TypeOf((*MockEvaluatorRecordDAO)(nil).CompareAndSwapEvaluatorRecordResult), varargs...) +} + // CreateEvaluatorRecord mocks base method. func (m *MockEvaluatorRecordDAO) CreateEvaluatorRecord(arg0 context.Context, arg1 *model.EvaluatorRecord, arg2 ...db.Option) error { m.ctrl.T.Helper() @@ -139,6 +159,25 @@ func (mr *MockEvaluatorRecordDAOMockRecorder) UpdateEvaluatorRecord(arg0, arg1 a return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "UpdateEvaluatorRecord", reflect.TypeOf((*MockEvaluatorRecordDAO)(nil).UpdateEvaluatorRecord), varargs...) } +// UpdateEvaluatorRecordAsyncDispatch mocks base method. +func (m *MockEvaluatorRecordDAO) UpdateEvaluatorRecordAsyncDispatch(ctx context.Context, recordID, spaceID int64, traceID, outputData string, opts ...db.Option) error { + m.ctrl.T.Helper() + varargs := []any{ctx, recordID, spaceID, traceID, outputData} + for _, a := range opts { + varargs = append(varargs, a) + } + ret := m.ctrl.Call(m, "UpdateEvaluatorRecordAsyncDispatch", varargs...) + ret0, _ := ret[0].(error) + return ret0 +} + +// UpdateEvaluatorRecordAsyncDispatch indicates an expected call of UpdateEvaluatorRecordAsyncDispatch. +func (mr *MockEvaluatorRecordDAOMockRecorder) UpdateEvaluatorRecordAsyncDispatch(ctx, recordID, spaceID, traceID, outputData any, opts ...any) *gomock.Call { + mr.mock.ctrl.T.Helper() + varargs := append([]any{ctx, recordID, spaceID, traceID, outputData}, opts...) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "UpdateEvaluatorRecordAsyncDispatch", reflect.TypeOf((*MockEvaluatorRecordDAO)(nil).UpdateEvaluatorRecordAsyncDispatch), varargs...) +} + // UpdateEvaluatorRecordResult mocks base method. func (m *MockEvaluatorRecordDAO) UpdateEvaluatorRecordResult(arg0 context.Context, arg1 int64, arg2 int8, arg3 float64, arg4 string, arg5 ...db.Option) error { m.ctrl.T.Helper() diff --git a/backend/modules/evaluation/infra/repo/experiment/redis/convert/item_turn_eval_async_test.go b/backend/modules/evaluation/infra/repo/experiment/redis/convert/item_turn_eval_async_test.go index 8994716189..9c63000096 100644 --- a/backend/modules/evaluation/infra/repo/experiment/redis/convert/item_turn_eval_async_test.go +++ b/backend/modules/evaluation/infra/repo/experiment/redis/convert/item_turn_eval_async_test.go @@ -26,3 +26,14 @@ func TestExptItemTurnEvalAsyncCtx_CallbackURLRoundTrip(t *testing.T) { assert.Equal(t, "https://example.com/hook", out.CallbackURL) assert.Equal(t, int64(123), out.RecordID) } + +func TestExptItemTurnEvalAsyncCtx_ResumeReadyRoundTrip(t *testing.T) { + c := NewExptItemTurnEvalAsyncCtx() + in := &entity.EvalAsyncCtx{RecordID: 123, ResumeReady: true} + b, err := c.FromDO(in) + assert.NoError(t, err) + + out, err := c.ToDO(b) + assert.NoError(t, err) + assert.True(t, out.ResumeReady) +} diff --git a/backend/modules/evaluation/infra/repo/experiment/redis/dao/item_turn_eval_async.go b/backend/modules/evaluation/infra/repo/experiment/redis/dao/item_turn_eval_async.go index 985f88836d..8a0d01a648 100644 --- a/backend/modules/evaluation/infra/repo/experiment/redis/dao/item_turn_eval_async.go +++ b/backend/modules/evaluation/infra/repo/experiment/redis/dao/item_turn_eval_async.go @@ -18,6 +18,8 @@ import ( type IEvalAsyncDAO interface { SetEvalAsyncCtx(ctx context.Context, invokeID string, actx *entity.EvalAsyncCtx) error GetEvalAsyncCtx(ctx context.Context, invokeID string) (*entity.EvalAsyncCtx, error) + GetEvalAsyncCtxStrong(ctx context.Context, invokeID string) (*entity.EvalAsyncCtx, error) + MarkEvalAsyncResumeReady(ctx context.Context, invokeID string) (*entity.EvalAsyncCtx, error) } func NewEvalAsyncDAO(cmdable redis.Cmdable) IEvalAsyncDAO { @@ -47,9 +49,79 @@ func (e *evalAsyncDAOImpl) SetEvalAsyncCtx(ctx context.Context, invokeID string, } func (e *evalAsyncDAOImpl) GetEvalAsyncCtx(ctx context.Context, invokeID string) (*entity.EvalAsyncCtx, error) { + return e.getEvalAsyncCtx(ctx, invokeID) +} + +func (e *evalAsyncDAOImpl) GetEvalAsyncCtxStrong(ctx context.Context, invokeID string) (*entity.EvalAsyncCtx, error) { + var lastErr error + delays := []time.Duration{0, 50 * time.Millisecond, 100 * time.Millisecond, 200 * time.Millisecond} + for _, delay := range delays { + if delay > 0 { + timer := time.NewTimer(delay) + select { + case <-ctx.Done(): + timer.Stop() + return nil, ctx.Err() + case <-timer.C: + } + } + actx, err := e.getEvalAsyncCtx(ctx, invokeID) + if err == nil || !redis.IsNilError(err) { + return actx, err + } + lastErr = err + } + if redis.IsNilError(lastErr) { + return nil, errorx.New("eval async context not found after bounded retry, invoke_id: %s", invokeID) + } + return nil, lastErr +} + +func (e *evalAsyncDAOImpl) MarkEvalAsyncResumeReady(ctx context.Context, invokeID string) (*entity.EvalAsyncCtx, error) { + actx, err := e.GetEvalAsyncCtxStrong(ctx, invokeID) + if err != nil { + return nil, err + } + if actx == nil || actx.ResumeReady { + return actx, nil + } + key := e.makeExptItemTurnEvalAsyncCtxKey(invokeID) + // Keep the raw JSON intact instead of decoding with Redis cjson (Lua numbers cannot safely represent i64 IDs). + // resume_ready is a root-level field emitted by Go's JSON encoder, so an anchored string replacement/insertion is safe. + const markResumeReadyScript = ` +local current = redis.call('GET', KEYS[1]) +if not current then return -1 end +if string.find(current, '"resume_ready":true', 1, true) then return 0 end +local updated, count = string.gsub(current, '"resume_ready":false', '"resume_ready":true', 1) +if count == 0 then + updated, count = string.gsub(current, '^{', '{"resume_ready":true,', 1) +end +if count == 0 then return -2 end +redis.call('SET', KEYS[1], updated, 'KEEPTTL') +return 1 +` + updated, evalErr := e.cmdable.Eval(ctx, markResumeReadyScript, []string{key}).Int64() + if evalErr != nil { + return nil, errorx.Wrapf(evalErr, "redis mark resume ready fail, key: %v", key) + } + if updated < 0 { + return nil, errorx.New("mark eval async resume ready failed, invoke_id: %s, code: %d", invokeID, updated) + } + actx, err = e.GetEvalAsyncCtxStrong(ctx, invokeID) + if err != nil { + return nil, err + } + return actx, nil +} + +func (e *evalAsyncDAOImpl) getEvalAsyncCtx(ctx context.Context, invokeID string) (*entity.EvalAsyncCtx, error) { key := e.makeExptItemTurnEvalAsyncCtxKey(invokeID) got, err := e.cmdable.Get(ctx, key).Result() if err != nil { + // Preserve redis.Nil so the strong-read path can apply bounded retry. + if redis.IsNilError(err) { + return nil, err + } return nil, errorx.Wrapf(err, "redis get fail, key: %v", key) } return convert.NewExptItemTurnEvalAsyncCtx().ToDO(conv.UnsafeStringToBytes(got)) diff --git a/backend/modules/evaluation/infra/repo/experiment/redis/dao/item_turn_eval_async_test.go b/backend/modules/evaluation/infra/repo/experiment/redis/dao/item_turn_eval_async_test.go new file mode 100644 index 0000000000..ffb367f61f --- /dev/null +++ b/backend/modules/evaluation/infra/repo/experiment/redis/dao/item_turn_eval_async_test.go @@ -0,0 +1,84 @@ +// Copyright (c) 2025 coze-dev Authors +// SPDX-License-Identifier: Apache-2.0 + +package dao + +import ( + "context" + "testing" + "time" + + redisv9 "github.com/redis/go-redis/v9" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + infraredis "github.com/coze-dev/coze-loop/backend/infra/redis" + "github.com/coze-dev/coze-loop/backend/modules/evaluation/domain/entity" +) + +func TestEvalAsyncDAO_GetStrongAndMarkResumeReady(t *testing.T) { + t.Parallel() + cmdable := infraredis.NewTestRedis(t) + dao := NewEvalAsyncDAO(cmdable) + ctx := context.Background() + + go func() { + time.Sleep(20 * time.Millisecond) + require.NoError(t, dao.SetEvalAsyncCtx(ctx, "evaluator:1", &entity.EvalAsyncCtx{RecordID: 1})) + }() + + actx, err := dao.GetEvalAsyncCtxStrong(ctx, "evaluator:1") + require.NoError(t, err) + require.NotNil(t, actx) + assert.False(t, actx.ResumeReady) + + actx, err = dao.MarkEvalAsyncResumeReady(ctx, "evaluator:1") + require.NoError(t, err) + assert.True(t, actx.ResumeReady) + + stored, err := dao.GetEvalAsyncCtx(ctx, "evaluator:1") + require.NoError(t, err) + assert.True(t, stored.ResumeReady) +} + +func TestEvalAsyncDAO_GetStrongStopsAfterBoundedRetry(t *testing.T) { + t.Parallel() + dao := NewEvalAsyncDAO(infraredis.NewTestRedis(t)) + start := time.Now() + actx, err := dao.GetEvalAsyncCtxStrong(context.Background(), "missing") + assert.Nil(t, actx) + assert.Error(t, err) + assert.NotErrorIs(t, err, redisv9.Nil) + assert.Contains(t, err.Error(), "eval async context not found") + assert.GreaterOrEqual(t, time.Since(start), 350*time.Millisecond) + assert.Less(t, time.Since(start), time.Second) +} + +func TestEvalAsyncDAO_MarkResumeReadyPreservesLargeIDsAndExistingPayload(t *testing.T) { + t.Parallel() + cmdable := infraredis.NewTestRedis(t) + dao := NewEvalAsyncDAO(cmdable) + ctx := context.Background() + const largeID int64 = 9007199254740993 + original := &entity.EvalAsyncCtx{ + RecordID: largeID, + EvaluatorVersionID: largeID - 1, + Event: &entity.ExptItemEvalEvent{ + ExptID: largeID - 2, + ExptRunID: largeID - 3, + EvalSetItemID: largeID - 4, + }, + } + require.NoError(t, dao.SetEvalAsyncCtx(ctx, "evaluator:large", original)) + + got, err := dao.MarkEvalAsyncResumeReady(ctx, "evaluator:large") + require.NoError(t, err) + require.NotNil(t, got) + assert.True(t, got.ResumeReady) + assert.Equal(t, largeID, got.RecordID) + assert.Equal(t, largeID-1, got.EvaluatorVersionID) + require.NotNil(t, got.Event) + assert.Equal(t, largeID-2, got.Event.ExptID) + assert.Equal(t, largeID-3, got.Event.ExptRunID) + assert.Equal(t, largeID-4, got.Event.EvalSetItemID) +} diff --git a/idl/thrift/coze/loop/evaluation/coze.loop.evaluation.spi.thrift b/idl/thrift/coze/loop/evaluation/coze.loop.evaluation.spi.thrift index 5bf8877c8e..712c928051 100644 --- a/idl/thrift/coze/loop/evaluation/coze.loop.evaluation.spi.thrift +++ b/idl/thrift/coze/loop/evaluation/coze.loop.evaluation.spi.thrift @@ -189,6 +189,20 @@ struct InvokeEvaluatorResponse { 255: base.BaseResp BaseResp } +// asynchronously invoke custom evaluator request +struct AsyncInvokeEvaluatorRequest { + 1: optional i64 workspace_id (api.js_conv="true") + 2: optional i64 invoke_id (api.js_conv="true") // execution id, report result with the same id + 3: optional InvokeCustomEvaluator evaluator + 4: optional InvokeEvaluatorInputData input_data + + 255: optional base.Base Base +} + +struct AsyncInvokeEvaluatorResponse { + 255: base.BaseResp BaseResp +} + service EvaluationSPIService { SearchEvalTargetResponse SearchEvalTarget(1: SearchEvalTargetRequest req) // 搜索评测对象 InvokeEvalTargetResponse InvokeEvalTarget(1: InvokeEvalTargetRequest req) // 执行 @@ -196,4 +210,5 @@ service EvaluationSPIService { // invoke custom evaluator InvokeEvaluatorResponse InvokeEvaluator(1: InvokeEvaluatorRequest req) + AsyncInvokeEvaluatorResponse AsyncInvokeEvaluator(1: AsyncInvokeEvaluatorRequest req) } diff --git a/idl/thrift/coze/loop/evaluation/domain/evaluator.thrift b/idl/thrift/coze/loop/evaluation/domain/evaluator.thrift index 3d05afac07..5bd355e19f 100644 --- a/idl/thrift/coze/loop/evaluation/domain/evaluator.thrift +++ b/idl/thrift/coze/loop/evaluation/domain/evaluator.thrift @@ -123,10 +123,12 @@ struct CustomRPCEvaluator { 3: optional string service_name 4: optional string cluster 5: optional EvaluatorHTTPInfo invoke_http_info // 执行http信息 + 6: optional EvaluatorHTTPInfo async_invoke_http_info // 异步执行http信息 10: optional i64 timeout // ms 11: optional common.RateLimit rate_limit // 自定义评估器的限流配置 12: optional map ext // extra fields + 13: optional bool is_async // 是否异步执行；true 时调用 AsyncInvokeEvaluator } struct EvaluatorVersion { @@ -323,4 +325,4 @@ struct EvaluatorProgressMessage { 2: optional string type // 如 tool_use, tool_result 3: optional string message // 如 Check current user identity and working directory 4: optional i64 created_at_ms -} \ No newline at end of file +} diff --git a/idl/thrift/coze/loop/evaluation/domain_openapi/evaluator.thrift b/idl/thrift/coze/loop/evaluation/domain_openapi/evaluator.thrift index c1b877589a..6c16dc57a9 100644 --- a/idl/thrift/coze/loop/evaluation/domain_openapi/evaluator.thrift +++ b/idl/thrift/coze/loop/evaluation/domain_openapi/evaluator.thrift @@ -56,10 +56,12 @@ struct CustomRPCEvaluator { 3: optional string service_name 4: optional string cluster 5: optional EvaluatorHTTPInfo invoke_http_info + 6: optional EvaluatorHTTPInfo async_invoke_http_info 10: optional i64 timeout // ms 11: optional common.RateLimit rate_limit 12: optional map ext + 13: optional bool is_async } // Agent评估器Prompt配置输出规则 From ca77e9ba4e1617baa1eaa5a6576002af370f3732 Mon Sep 17 00:00:00 2001 From: shenjiayun Date: Thu, 6 Aug 2026 20:22:15 +0800 Subject: [PATCH 02/15] test(evaluation): satisfy backend lint --- .../application/convertor/evaluator/evaluator_test.go | 2 +- .../modules/evaluation/domain/service/evaluator_impl_test.go | 1 - 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/backend/modules/evaluation/application/convertor/evaluator/evaluator_test.go b/backend/modules/evaluation/application/convertor/evaluator/evaluator_test.go index 17b2ba8bc4..e4085b56b8 100755 --- a/backend/modules/evaluation/application/convertor/evaluator/evaluator_test.go +++ b/backend/modules/evaluation/application/convertor/evaluator/evaluator_test.go @@ -1362,7 +1362,7 @@ func TestConvertEvaluatorLangTags_SkipNilInnerMap(t *testing.T) { // TestConvertCustomRPCEvaluatorVersionDTO2DO 测试将 CustomRPC EvaluatorVersion DTO 转换为 DO func TestConvertCustomRPCEvaluatorVersionDTO2DO(t *testing.T) { t.Parallel() - post := evaluatordto.EvaluatorHTTPMethod(evaluatordo.EvaluatorHTTPMethodPost) + post := evaluatordo.EvaluatorHTTPMethodPost asyncPath := "/async_invoke_evaluator" tests := []struct { diff --git a/backend/modules/evaluation/domain/service/evaluator_impl_test.go b/backend/modules/evaluation/domain/service/evaluator_impl_test.go index e1bb976fdf..a7137e9fbc 100644 --- a/backend/modules/evaluation/domain/service/evaluator_impl_test.go +++ b/backend/modules/evaluation/domain/service/evaluator_impl_test.go @@ -4703,7 +4703,6 @@ func TestEvaluatorServiceImpl_AsyncRunEvaluator_CoordinatorOrderAndFailureCompen assert.Equal(t, "trace-1", record.TraceID) assert.Equal(t, "s1", record.EvaluatorOutputData.Ext["session_id"]) }) - } func TestEvaluatorServiceImpl_ReportEvaluatorInvokeResult_RejectsNonTerminalStatus(t *testing.T) { From 014c12ae30c83460d85a084d6acfd338b5939eaf Mon Sep 17 00:00:00 2001 From: shenjiayun Date: Fri, 7 Aug 2026 02:13:55 +0800 Subject: [PATCH 03/15] fix(evaluation): distinguish duplicate async callbacks --- .../application/eval_openapi_app.go | 10 +++- .../application/eval_openapi_app_test.go | 60 +++++++++++++++---- .../modules/evaluation/domain/entity/param.go | 8 +++ .../evaluation/domain/service/evaluator.go | 2 +- .../domain/service/evaluator_impl.go | 37 ++++++++---- .../domain/service/evaluator_impl_test.go | 23 ++++--- .../service/mocks/evaluator_service_mock.go | 7 ++- 7 files changed, 111 insertions(+), 36 deletions(-) diff --git a/backend/modules/evaluation/application/eval_openapi_app.go b/backend/modules/evaluation/application/eval_openapi_app.go index 771c84ecc4..b94612f1d9 100644 --- a/backend/modules/evaluation/application/eval_openapi_app.go +++ b/backend/modules/evaluation/application/eval_openapi_app.go @@ -3569,14 +3569,18 @@ func (e *EvalOpenAPIApplication) ReportEvaluatorInvokeResult_(ctx context.Contex outputData := evaluator_convertor.ToInvokeEvaluatorOutputDataDO(req.GetOutput(), req.GetStatus()) runStatus := evaluator_convertor.ToEvaluatorRunStatusDO(req.GetStatus()) - if err := e.evaluatorService.ReportEvaluatorInvokeResult(ctx, &entity.ReportEvaluatorRecordParam{ + outcome, err := e.evaluatorService.ReportEvaluatorInvokeResult(ctx, &entity.ReportEvaluatorRecordParam{ SpaceID: req.GetWorkspaceID(), RecordID: req.GetInvokeID(), OutputData: outputData, Status: runStatus, - }); err != nil { + }) + if err != nil { return nil, err } + if outcome == entity.ReportEvaluatorResultConflict { + return &openapi.ReportEvaluatorInvokeResultResponse{BaseResp: base.NewBaseResp()}, nil + } if actx.Event != nil { if !actx.ResumeReady { latestCtx, ctxErr := e.asyncRepo.GetEvalAsyncCtxStrong(ctx, asyncCtxKey) @@ -3596,7 +3600,7 @@ func (e *EvalOpenAPIApplication) ReportEvaluatorInvokeResult_(ctx context.Contex } } - if actx.CallbackURL != "" { + if outcome == entity.ReportEvaluatorResultApplied && actx.CallbackURL != "" { payload := &openapi.EvaluatorCallbackPayloadOApi{ InvokeID: gptr.Of(req.GetInvokeID()), WorkspaceID: gptr.Of(req.GetWorkspaceID()), diff --git a/backend/modules/evaluation/application/eval_openapi_app_test.go b/backend/modules/evaluation/application/eval_openapi_app_test.go index 622f721565..2bf09cee53 100755 --- a/backend/modules/evaluation/application/eval_openapi_app_test.go +++ b/backend/modules/evaluation/application/eval_openapi_app_test.go @@ -6657,7 +6657,7 @@ func TestEvalOpenAPIApplication_ReportEvaluatorInvokeResult(t *testing.T) { AsyncUnixMS: time.Now().UnixMilli() - 10, EvaluatorVersionID: 9, }, nil) - evaluatorSvc.EXPECT().ReportEvaluatorInvokeResult(gomock.Any(), gomock.Any()).Return(errors.New("report failed")) + evaluatorSvc.EXPECT().ReportEvaluatorInvokeResult(gomock.Any(), gomock.Any()).Return(entity.ReportEvaluatorResultConflict, errors.New("report failed")) }, wantErr: -1, }, @@ -6680,13 +6680,13 @@ func TestEvalOpenAPIApplication_ReportEvaluatorInvokeResult(t *testing.T) { AsyncUnixMS: time.Now().UnixMilli() - 10, EvaluatorVersionID: 9, }, nil) - evaluatorSvc.EXPECT().ReportEvaluatorInvokeResult(gomock.Any(), gomock.Any()).DoAndReturn(func(_ context.Context, param *entity.ReportEvaluatorRecordParam) error { + evaluatorSvc.EXPECT().ReportEvaluatorInvokeResult(gomock.Any(), gomock.Any()).DoAndReturn(func(_ context.Context, param *entity.ReportEvaluatorRecordParam) (entity.ReportEvaluatorResultOutcome, error) { assert.Equal(t, entity.EvaluatorRunStatusFail, param.Status) if assert.NotNil(t, param.OutputData) && assert.NotNil(t, param.OutputData.EvaluatorUsage) { assert.Equal(t, int64(105119), param.OutputData.EvaluatorUsage.InputTokens) assert.Equal(t, int64(1938), param.OutputData.EvaluatorUsage.OutputTokens) } - return nil + return entity.ReportEvaluatorResultApplied, nil }) publisher.EXPECT().PublishExptRecordEvalEvent(gomock.Any(), gomock.Any(), gomock.Not(gomock.Nil()), gomock.Any()).Return(errors.New("pub failed")) }, @@ -6709,7 +6709,7 @@ func TestEvalOpenAPIApplication_ReportEvaluatorInvokeResult(t *testing.T) { AsyncUnixMS: time.Now().UnixMilli() - 10, EvaluatorVersionID: 9, }, nil) - evaluatorSvc.EXPECT().ReportEvaluatorInvokeResult(gomock.Any(), gomock.Any()).Return(nil) + evaluatorSvc.EXPECT().ReportEvaluatorInvokeResult(gomock.Any(), gomock.Any()).Return(entity.ReportEvaluatorResultApplied, nil) }, }, { @@ -6729,7 +6729,7 @@ func TestEvalOpenAPIApplication_ReportEvaluatorInvokeResult(t *testing.T) { ResumeReady: false, EvaluatorVersionID: 9, }, nil) - evaluatorSvc.EXPECT().ReportEvaluatorInvokeResult(gomock.Any(), gomock.Any()).Return(nil) + evaluatorSvc.EXPECT().ReportEvaluatorInvokeResult(gomock.Any(), gomock.Any()).Return(entity.ReportEvaluatorResultApplied, nil) asyncRepo.EXPECT().GetEvalAsyncCtxStrong(gomock.Any(), "evaluator:2002").Return(&entity.EvalAsyncCtx{ Event: event, ResumeReady: false, @@ -6756,7 +6756,7 @@ func TestEvalOpenAPIApplication_ReportEvaluatorInvokeResult(t *testing.T) { ResumeReady: false, EvaluatorVersionID: 9, }, nil), - evaluatorSvc.EXPECT().ReportEvaluatorInvokeResult(gomock.Any(), gomock.Any()).Return(nil), + evaluatorSvc.EXPECT().ReportEvaluatorInvokeResult(gomock.Any(), gomock.Any()).Return(entity.ReportEvaluatorResultApplied, nil), asyncRepo.EXPECT().GetEvalAsyncCtxStrong(gomock.Any(), "evaluator:2002").Return(&entity.EvalAsyncCtx{ Event: event, ResumeReady: true, @@ -6767,6 +6767,42 @@ func TestEvalOpenAPIApplication_ReportEvaluatorInvokeResult(t *testing.T) { }, }, + { + name: "duplicate terminal callback republishes recovery but does not redispatch external callback", + req: &openapi.ReportEvaluatorInvokeResultRequest{ + WorkspaceID: gptr.Of(workspaceID), InvokeID: gptr.Of(invokeID), Status: gptr.Of(spi.InvokeEvaluatorRunStatus_SUCCESS), + Output: &spi.InvokeEvaluatorOutputData{EvaluatorResult_: &spi.InvokeEvaluatorResult_{Score: gptr.Of(float64(0.9))}}, + }, + setup: func(auth *rpcmocks.MockIAuthProvider, asyncRepo *repomocks.MockIEvalAsyncRepo, evaluatorSvc *servicemocks.MockEvaluatorService, publisher *eventmocks.MockExptEventPublisher, dispatcher *servicemocks.MockIEvaluatorCallbackDispatcher) { + auth.EXPECT().Authorization(gomock.Any(), gomock.Any()).Return(nil) + asyncRepo.EXPECT().GetEvalAsyncCtxStrong(gomock.Any(), "evaluator:2002").Return(&entity.EvalAsyncCtx{ + Event: event, ResumeReady: true, EvaluatorVersionID: 9, CallbackURL: "https://cb.example.com/hook", + }, nil) + evaluatorSvc.EXPECT().ReportEvaluatorInvokeResult(gomock.Any(), gomock.Any()).Return(entity.ReportEvaluatorResultDuplicate, nil) + publisher.EXPECT().PublishExptRecordEvalEvent(gomock.Any(), event, gomock.Any(), gomock.Any()).Return(nil) + dispatcher.EXPECT().Dispatch(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Times(0) + }, + }, + { + name: "conflicting terminal callback does not republish or redispatch", + req: &openapi.ReportEvaluatorInvokeResultRequest{ + WorkspaceID: gptr.Of(workspaceID), + InvokeID: gptr.Of(invokeID), + Status: gptr.Of(spi.InvokeEvaluatorRunStatus_SUCCESS), + Output: &spi.InvokeEvaluatorOutputData{ + EvaluatorResult_: &spi.InvokeEvaluatorResult_{Score: gptr.Of(float64(0.9))}, + }, + }, + setup: func(auth *rpcmocks.MockIAuthProvider, asyncRepo *repomocks.MockIEvalAsyncRepo, evaluatorSvc *servicemocks.MockEvaluatorService, publisher *eventmocks.MockExptEventPublisher, dispatcher *servicemocks.MockIEvaluatorCallbackDispatcher) { + auth.EXPECT().Authorization(gomock.Any(), gomock.Any()).Return(nil) + asyncRepo.EXPECT().GetEvalAsyncCtxStrong(gomock.Any(), "evaluator:2002").Return(&entity.EvalAsyncCtx{ + Event: event, ResumeReady: true, EvaluatorVersionID: 9, CallbackURL: "https://cb.example.com/hook", + }, nil) + evaluatorSvc.EXPECT().ReportEvaluatorInvokeResult(gomock.Any(), gomock.Any()).Return(entity.ReportEvaluatorResultConflict, nil) + publisher.EXPECT().PublishExptRecordEvalEvent(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Times(0) + dispatcher.EXPECT().Dispatch(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Times(0) + }, + }, { name: "success", req: &openapi.ReportEvaluatorInvokeResultRequest{ @@ -6786,13 +6822,13 @@ func TestEvalOpenAPIApplication_ReportEvaluatorInvokeResult(t *testing.T) { AsyncUnixMS: time.Now().UnixMilli() - 50, EvaluatorVersionID: 9, }, nil) - evaluatorSvc.EXPECT().ReportEvaluatorInvokeResult(gomock.Any(), gomock.Any()).DoAndReturn(func(_ context.Context, param *entity.ReportEvaluatorRecordParam) error { + evaluatorSvc.EXPECT().ReportEvaluatorInvokeResult(gomock.Any(), gomock.Any()).DoAndReturn(func(_ context.Context, param *entity.ReportEvaluatorRecordParam) (entity.ReportEvaluatorResultOutcome, error) { assert.Equal(t, workspaceID, param.SpaceID) assert.Equal(t, invokeID, param.RecordID) assert.Equal(t, entity.EvaluatorRunStatusSuccess, param.Status) assert.NotNil(t, param.OutputData) assert.GreaterOrEqual(t, param.OutputData.TimeConsumingMS, int64(0)) - return nil + return entity.ReportEvaluatorResultApplied, nil }) publisher.EXPECT().PublishExptRecordEvalEvent(gomock.Any(), gomock.Any(), gomock.Not(gomock.Nil()), gomock.Any()).DoAndReturn( func(_ context.Context, ev *entity.ExptItemEvalEvent, _ *time.Duration, modifyFunc func(*entity.ExptItemEvalEvent)) error { @@ -6823,7 +6859,7 @@ func TestEvalOpenAPIApplication_ReportEvaluatorInvokeResult(t *testing.T) { EvaluatorVersionID: 9, CallbackURL: "https://cb.example.com/hook", }, nil) - evaluatorSvc.EXPECT().ReportEvaluatorInvokeResult(gomock.Any(), gomock.Any()).Return(nil) + evaluatorSvc.EXPECT().ReportEvaluatorInvokeResult(gomock.Any(), gomock.Any()).Return(entity.ReportEvaluatorResultApplied, nil) dispatcher.EXPECT().Dispatch(gomock.Any(), workspaceID, "https://cb.example.com/hook", gomock.Any()). DoAndReturn(func(_ context.Context, _ int64, _ string, p *openapi.EvaluatorCallbackPayloadOApi) error { assert.Equal(t, invokeID, p.GetInvokeID()) @@ -6852,7 +6888,7 @@ func TestEvalOpenAPIApplication_ReportEvaluatorInvokeResult(t *testing.T) { EvaluatorVersionID: 9, CallbackURL: "https://cb.example.com/hook", }, nil) - evaluatorSvc.EXPECT().ReportEvaluatorInvokeResult(gomock.Any(), gomock.Any()).Return(nil) + evaluatorSvc.EXPECT().ReportEvaluatorInvokeResult(gomock.Any(), gomock.Any()).Return(entity.ReportEvaluatorResultApplied, nil) dispatcher.EXPECT().Dispatch(gomock.Any(), workspaceID, "https://cb.example.com/hook", gomock.Any()). DoAndReturn(func(_ context.Context, _ int64, _ string, p *openapi.EvaluatorCallbackPayloadOApi) error { assert.Equal(t, "fail", p.GetStatus()) @@ -6878,7 +6914,7 @@ func TestEvalOpenAPIApplication_ReportEvaluatorInvokeResult(t *testing.T) { EvaluatorVersionID: 9, CallbackURL: "", }, nil) - evaluatorSvc.EXPECT().ReportEvaluatorInvokeResult(gomock.Any(), gomock.Any()).Return(nil) + evaluatorSvc.EXPECT().ReportEvaluatorInvokeResult(gomock.Any(), gomock.Any()).Return(entity.ReportEvaluatorResultApplied, nil) dispatcher.EXPECT().Dispatch(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Times(0) }, }, @@ -6900,7 +6936,7 @@ func TestEvalOpenAPIApplication_ReportEvaluatorInvokeResult(t *testing.T) { EvaluatorVersionID: 9, CallbackURL: "https://cb.example.com/hook", }, nil) - evaluatorSvc.EXPECT().ReportEvaluatorInvokeResult(gomock.Any(), gomock.Any()).Return(nil) + evaluatorSvc.EXPECT().ReportEvaluatorInvokeResult(gomock.Any(), gomock.Any()).Return(entity.ReportEvaluatorResultApplied, nil) dispatcher.EXPECT().Dispatch(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return(errors.New("boom")) }, // wantErr stays 0 (zero value): report must still succeed despite dispatch error diff --git a/backend/modules/evaluation/domain/entity/param.go b/backend/modules/evaluation/domain/entity/param.go index 4a16676aa4..f672e9846b 100644 --- a/backend/modules/evaluation/domain/entity/param.go +++ b/backend/modules/evaluation/domain/entity/param.go @@ -360,6 +360,14 @@ type GetAsyncDebugEvaluatorInvokeResultResponse struct { InputData *EvaluatorInputData `json:"input_data,omitempty"` } +type ReportEvaluatorResultOutcome int + +const ( + ReportEvaluatorResultApplied ReportEvaluatorResultOutcome = iota + 1 + ReportEvaluatorResultDuplicate + ReportEvaluatorResultConflict +) + type ReportEvaluatorRecordParam struct { SpaceID int64 `json:"space_id"` RecordID int64 `json:"record_id"` diff --git a/backend/modules/evaluation/domain/service/evaluator.go b/backend/modules/evaluation/domain/service/evaluator.go index 2b4fb146b3..87b580639b 100644 --- a/backend/modules/evaluation/domain/service/evaluator.go +++ b/backend/modules/evaluation/domain/service/evaluator.go @@ -66,7 +66,7 @@ type EvaluatorService interface { // ListEvaluatorTags 根据 tagType 聚合标签，并按字母序返回 ListEvaluatorTags(ctx context.Context, tagType entity.EvaluatorTagKeyType) (map[entity.EvaluatorTagKey][]string, error) // ReportEvaluatorInvokeResult 上报评估器异步执行结果. Duplicate/conflicting terminal callbacks are ignored. - ReportEvaluatorInvokeResult(ctx context.Context, param *entity.ReportEvaluatorRecordParam) error + ReportEvaluatorInvokeResult(ctx context.Context, param *entity.ReportEvaluatorRecordParam) (entity.ReportEvaluatorResultOutcome, error) // ArmEvaluatorResume marks ResumeReady after turn refs are durable and republishes recovery when the record is already terminal. ArmEvaluatorResume(ctx context.Context, recordID int64) error } diff --git a/backend/modules/evaluation/domain/service/evaluator_impl.go b/backend/modules/evaluation/domain/service/evaluator_impl.go index 6bc335b7b0..01d4cfa634 100644 --- a/backend/modules/evaluation/domain/service/evaluator_impl.go +++ b/backend/modules/evaluation/domain/service/evaluator_impl.go @@ -1154,27 +1154,34 @@ func (e *EvaluatorServiceImpl) AsyncDebugEvaluator(ctx context.Context, request } // ReportEvaluatorInvokeResult 上报评估器异步执行结果 using a terminal CAS. -func (e *EvaluatorServiceImpl) ReportEvaluatorInvokeResult(ctx context.Context, param *entity.ReportEvaluatorRecordParam) error { +func (e *EvaluatorServiceImpl) ReportEvaluatorInvokeResult(ctx context.Context, param *entity.ReportEvaluatorRecordParam) (entity.ReportEvaluatorResultOutcome, error) { if param == nil { - return errorx.NewByCode(errno.CommonInvalidParamCode, errorx.WithExtraMsg("report evaluator result param is nil")) + return 0, errorx.NewByCode(errno.CommonInvalidParamCode, errorx.WithExtraMsg("report evaluator result param is nil")) } if param.Status != entity.EvaluatorRunStatusSuccess && param.Status != entity.EvaluatorRunStatusFail { - return errorx.NewByCode(errno.CommonInvalidParamCode, errorx.WithExtraMsg("report evaluator status must be success or fail")) + return 0, errorx.NewByCode(errno.CommonInvalidParamCode, errorx.WithExtraMsg("report evaluator status must be success or fail")) } logs.CtxInfo(ctx, "[ReportEvaluatorInvokeResult] recordID: %d, spaceID: %d, status: %v", param.RecordID, param.SpaceID, param.Status) existingRecord, err := e.evaluatorRecordRepo.GetEvaluatorRecord(contexts.WithCtxWriteDB(ctx), param.RecordID, false) if err != nil { logs.CtxError(ctx, "[ReportEvaluatorInvokeResult] GetEvaluatorRecord fail, recordID: %d, err: %v", param.RecordID, err) - return err + return 0, err } if existingRecord == nil { - return errorx.NewByCode(errno.EvaluatorRecordNotFoundCode, errorx.WithExtraMsg("evaluator record not found")) + return 0, errorx.NewByCode(errno.EvaluatorRecordNotFoundCode, errorx.WithExtraMsg("evaluator record not found")) } if existingRecord.SpaceID != param.SpaceID { logs.CtxWarn(ctx, "[ReportEvaluatorInvokeResult] spaceID mismatch, recordID: %d, requestSpaceID: %d, recordSpaceID: %d", param.RecordID, param.SpaceID, existingRecord.SpaceID) - return errorx.NewByCode(errno.CommonInvalidParamCode, errorx.WithExtraMsg("spaceID mismatch")) + return 0, errorx.NewByCode(errno.CommonInvalidParamCode, errorx.WithExtraMsg("spaceID mismatch")) + } + + if existingRecord.Status != entity.EvaluatorRunStatusAsyncInvoking { + if existingRecord.Status == param.Status { + return entity.ReportEvaluatorResultDuplicate, nil + } + return entity.ReportEvaluatorResultConflict, nil } mergedOutputData := param.OutputData @@ -1197,12 +1204,22 @@ func (e *EvaluatorServiceImpl) ReportEvaluatorInvokeResult(ctx context.Context, updated, err := e.evaluatorRecordRepo.CompareAndSwapEvaluatorRecordResult(ctx, param.RecordID, param.SpaceID, entity.EvaluatorRunStatusAsyncInvoking, param.Status, mergedOutputData) if err != nil { - return err + return 0, err } - if !updated { - logs.CtxWarn(ctx, "[ReportEvaluatorInvokeResult] skip duplicate/conflicting callback, recordID: %d, reportStatus: %v", param.RecordID, param.Status) + if updated { + return entity.ReportEvaluatorResultApplied, nil } - return nil + + latestRecord, err := e.evaluatorRecordRepo.GetEvaluatorRecord(contexts.WithCtxWriteDB(ctx), param.RecordID, false) + if err != nil { + return 0, err + } + if latestRecord != nil && latestRecord.Status == param.Status { + logs.CtxWarn(ctx, "[ReportEvaluatorInvokeResult] duplicate terminal callback, recordID: %d, status: %v", param.RecordID, param.Status) + return entity.ReportEvaluatorResultDuplicate, nil + } + logs.CtxWarn(ctx, "[ReportEvaluatorInvokeResult] conflicting terminal callback, recordID: %d, reportStatus: %v", param.RecordID, param.Status) + return entity.ReportEvaluatorResultConflict, nil } func (e *EvaluatorServiceImpl) ArmEvaluatorResume(ctx context.Context, recordID int64) error { diff --git a/backend/modules/evaluation/domain/service/evaluator_impl_test.go b/backend/modules/evaluation/domain/service/evaluator_impl_test.go index a7137e9fbc..5258a2767b 100644 --- a/backend/modules/evaluation/domain/service/evaluator_impl_test.go +++ b/backend/modules/evaluation/domain/service/evaluator_impl_test.go @@ -1141,6 +1141,7 @@ func TestEvaluatorServiceImpl_CreateEvaluator(t *testing.T) { expectedID int64 expectedErr error expectedErrCode int32 + expectedOutcome entity.ReportEvaluatorResultOutcome }{ { name: "失败 - validateCreateEvaluatorRequest - CheckNameExist 返回错误", @@ -3321,6 +3322,7 @@ func TestEvaluatorServiceImpl_AsyncRunEvaluator(t *testing.T) { name string setupMocks func() expectedErrCode int32 + expectedOutcome entity.ReportEvaluatorResultOutcome }{ { name: "成功 - 异步运行 Agent 评估器", @@ -3539,6 +3541,7 @@ func TestEvaluatorServiceImpl_ReportEvaluatorInvokeResult(t *testing.T) { param *entity.ReportEvaluatorRecordParam setupMocks func() expectedErrCode int32 + expectedOutcome entity.ReportEvaluatorResultOutcome }{ { name: "成功 - 合并 Ext 并更新记录", @@ -3571,6 +3574,7 @@ func TestEvaluatorServiceImpl_ReportEvaluatorInvokeResult(t *testing.T) { }) }, expectedErrCode: 0, + expectedOutcome: entity.ReportEvaluatorResultApplied, }, { name: "失败 - record 不存在", @@ -3610,19 +3614,20 @@ func TestEvaluatorServiceImpl_ReportEvaluatorInvokeResult(t *testing.T) { SpaceID: 2, Status: entity.EvaluatorRunStatusFail, }, nil) - mockEvaluatorRecordRepo.EXPECT().CompareAndSwapEvaluatorRecordResult(gomock.Any(), int64(100), int64(2), entity.EvaluatorRunStatusAsyncInvoking, entity.EvaluatorRunStatusSuccess, gomock.Any()).Return(false, nil) }, expectedErrCode: 0, + expectedOutcome: entity.ReportEvaluatorResultConflict, }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { tt.setupMocks() - err := s.ReportEvaluatorInvokeResult(ctx, tt.param) + outcome, err := s.ReportEvaluatorInvokeResult(ctx, tt.param) if tt.expectedErrCode == 0 { assert.NoError(t, err) + assert.Equal(t, tt.expectedOutcome, outcome) return } assert.Error(t, err) @@ -3686,7 +3691,7 @@ func TestEvaluatorServiceImpl_ReportEvaluatorInvokeResult_OutputDataNilOrExtNil( return true, nil }) - err := s.ReportEvaluatorInvokeResult(ctx, param) + _, err := s.ReportEvaluatorInvokeResult(ctx, param) assert.NoError(t, err) }) } @@ -4709,12 +4714,14 @@ func TestEvaluatorServiceImpl_ReportEvaluatorInvokeResult_RejectsNonTerminalStat t.Parallel() s := &EvaluatorServiceImpl{} - require.Error(t, s.ReportEvaluatorInvokeResult(context.Background(), nil)) - require.Error(t, s.ReportEvaluatorInvokeResult(context.Background(), &entity.ReportEvaluatorRecordParam{ + _, err := s.ReportEvaluatorInvokeResult(context.Background(), nil) + require.Error(t, err) + _, err = s.ReportEvaluatorInvokeResult(context.Background(), &entity.ReportEvaluatorRecordParam{ RecordID: 1, SpaceID: 2, Status: entity.EvaluatorRunStatusUnknown, - })) + }) + require.Error(t, err) } func TestEvaluatorServiceImpl_ReportEvaluatorInvokeResult_UsesTerminalCAS(t *testing.T) { @@ -4745,9 +4752,11 @@ func TestEvaluatorServiceImpl_ReportEvaluatorInvokeResult_UsesTerminalCAS(t *tes return false, nil }, ) + recordRepo.EXPECT().GetEvaluatorRecord(gomock.Any(), int64(100), false).Return(&entity.EvaluatorRecord{ID: 100, SpaceID: 2, Status: entity.EvaluatorRunStatusFail}, nil) - err := s.ReportEvaluatorInvokeResult(context.Background(), param) + outcome, err := s.ReportEvaluatorInvokeResult(context.Background(), param) require.NoError(t, err) + assert.Equal(t, entity.ReportEvaluatorResultConflict, outcome) } func TestEvaluatorServiceImpl_ArmEvaluatorResume(t *testing.T) { diff --git a/backend/modules/evaluation/domain/service/mocks/evaluator_service_mock.go b/backend/modules/evaluation/domain/service/mocks/evaluator_service_mock.go index 5310e42f8b..f9db9902f8 100644 --- a/backend/modules/evaluation/domain/service/mocks/evaluator_service_mock.go +++ b/backend/modules/evaluation/domain/service/mocks/evaluator_service_mock.go @@ -337,11 +337,12 @@ func (mr *MockEvaluatorServiceMockRecorder) ListEvaluatorVersion(arg0, arg1 inte } // ReportEvaluatorInvokeResult mocks base method. -func (m *MockEvaluatorService) ReportEvaluatorInvokeResult(arg0 context.Context, arg1 *entity.ReportEvaluatorRecordParam) error { +func (m *MockEvaluatorService) ReportEvaluatorInvokeResult(arg0 context.Context, arg1 *entity.ReportEvaluatorRecordParam) (entity.ReportEvaluatorResultOutcome, error) { m.ctrl.T.Helper() ret := m.ctrl.Call(m, "ReportEvaluatorInvokeResult", arg0, arg1) - ret0, _ := ret[0].(error) - return ret0 + ret0, _ := ret[0].(entity.ReportEvaluatorResultOutcome) + ret1, _ := ret[1].(error) + return ret0, ret1 } // ReportEvaluatorInvokeResult indicates an expected call of ReportEvaluatorInvokeResult. From 06bbe8ab563720b7cba52c7d11cc949ac0103823 Mon Sep 17 00:00:00 2001 From: shenjiayun Date: Fri, 7 Aug 2026 12:44:25 +0800 Subject: [PATCH 04/15] fix(evaluation): preserve async context ttl compatibly --- .../repo/experiment/redis/dao/item_turn_eval_async.go | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/backend/modules/evaluation/infra/repo/experiment/redis/dao/item_turn_eval_async.go b/backend/modules/evaluation/infra/repo/experiment/redis/dao/item_turn_eval_async.go index 8a0d01a648..9572c80382 100644 --- a/backend/modules/evaluation/infra/repo/experiment/redis/dao/item_turn_eval_async.go +++ b/backend/modules/evaluation/infra/repo/experiment/redis/dao/item_turn_eval_async.go @@ -97,7 +97,12 @@ if count == 0 then updated, count = string.gsub(current, '^{', '{"resume_ready":true,', 1) end if count == 0 then return -2 end -redis.call('SET', KEYS[1], updated, 'KEEPTTL') +local ttl = redis.call('PTTL', KEYS[1]) +if ttl > 0 then + redis.call('SET', KEYS[1], updated, 'PX', ttl) +else + redis.call('SET', KEYS[1], updated) +end return 1 ` updated, evalErr := e.cmdable.Eval(ctx, markResumeReadyScript, []string{key}).Int64() From c3080a4b5d13df60696e7b791e3ea4c8242356e4 Mon Sep 17 00:00:00 2001 From: shenjiayun Date: Fri, 7 Aug 2026 13:05:58 +0800 Subject: [PATCH 05/15] fix(evaluation): avoid nondeterministic redis ttl script --- .../redis/dao/item_turn_eval_async.go | 33 +++++++++---------- .../redis/dao/item_turn_eval_async_test.go | 7 ++++ 2 files changed, 22 insertions(+), 18 deletions(-) diff --git a/backend/modules/evaluation/infra/repo/experiment/redis/dao/item_turn_eval_async.go b/backend/modules/evaluation/infra/repo/experiment/redis/dao/item_turn_eval_async.go index 9572c80382..e49d7ffebc 100644 --- a/backend/modules/evaluation/infra/repo/experiment/redis/dao/item_turn_eval_async.go +++ b/backend/modules/evaluation/infra/repo/experiment/redis/dao/item_turn_eval_async.go @@ -77,6 +77,19 @@ func (e *evalAsyncDAOImpl) GetEvalAsyncCtxStrong(ctx context.Context, invokeID s return nil, lastErr } +const markResumeReadyScript = ` +local current = redis.call('GET', KEYS[1]) +if not current then return -1 end +if string.find(current, '"resume_ready":true', 1, true) then return 0 end +local updated, count = string.gsub(current, '"resume_ready":false', '"resume_ready":true', 1) +if count == 0 then + updated, count = string.gsub(current, '^{', '{"resume_ready":true,', 1) +end +if count == 0 then return -2 end +redis.call('SET', KEYS[1], updated, 'EX', ARGV[1]) +return 1 +` + func (e *evalAsyncDAOImpl) MarkEvalAsyncResumeReady(ctx context.Context, invokeID string) (*entity.EvalAsyncCtx, error) { actx, err := e.GetEvalAsyncCtxStrong(ctx, invokeID) if err != nil { @@ -88,24 +101,8 @@ func (e *evalAsyncDAOImpl) MarkEvalAsyncResumeReady(ctx context.Context, invokeI key := e.makeExptItemTurnEvalAsyncCtxKey(invokeID) // Keep the raw JSON intact instead of decoding with Redis cjson (Lua numbers cannot safely represent i64 IDs). // resume_ready is a root-level field emitted by Go's JSON encoder, so an anchored string replacement/insertion is safe. - const markResumeReadyScript = ` -local current = redis.call('GET', KEYS[1]) -if not current then return -1 end -if string.find(current, '"resume_ready":true', 1, true) then return 0 end -local updated, count = string.gsub(current, '"resume_ready":false', '"resume_ready":true', 1) -if count == 0 then - updated, count = string.gsub(current, '^{', '{"resume_ready":true,', 1) -end -if count == 0 then return -2 end -local ttl = redis.call('PTTL', KEYS[1]) -if ttl > 0 then - redis.call('SET', KEYS[1], updated, 'PX', ttl) -else - redis.call('SET', KEYS[1], updated) -end -return 1 -` - updated, evalErr := e.cmdable.Eval(ctx, markResumeReadyScript, []string{key}).Int64() + + updated, evalErr := e.cmdable.Eval(ctx, markResumeReadyScript, []string{key}, int64((12*time.Hour)/time.Second)).Int64() if evalErr != nil { return nil, errorx.Wrapf(evalErr, "redis mark resume ready fail, key: %v", key) } diff --git a/backend/modules/evaluation/infra/repo/experiment/redis/dao/item_turn_eval_async_test.go b/backend/modules/evaluation/infra/repo/experiment/redis/dao/item_turn_eval_async_test.go index ffb367f61f..8c8389c8cc 100644 --- a/backend/modules/evaluation/infra/repo/experiment/redis/dao/item_turn_eval_async_test.go +++ b/backend/modules/evaluation/infra/repo/experiment/redis/dao/item_turn_eval_async_test.go @@ -82,3 +82,10 @@ func TestEvalAsyncDAO_MarkResumeReadyPreservesLargeIDsAndExistingPayload(t *test assert.Equal(t, largeID-3, got.Event.ExptRunID) assert.Equal(t, largeID-4, got.Event.EvalSetItemID) } + +func TestMarkResumeReadyScriptAvoidsVersionSensitiveTTLCommands(t *testing.T) { + t.Parallel() + assert.NotContains(t, markResumeReadyScript, "KEEPTTL") + assert.NotContains(t, markResumeReadyScript, "PTTL") + assert.Contains(t, markResumeReadyScript, "'EX', ARGV[1]") +} From dac41d39210225992317ac3f2a22259c47bb79f7 Mon Sep 17 00:00:00 2001 From: shenjiayun Date: Fri, 7 Aug 2026 14:13:06 +0800 Subject: [PATCH 06/15] fix(evaluation): preserve concurrent async callback result --- .../domain/service/evaluator_impl.go | 13 +++++++++ .../domain/service/evaluator_impl_test.go | 27 +++++++++++++++++++ 2 files changed, 40 insertions(+) diff --git a/backend/modules/evaluation/domain/service/evaluator_impl.go b/backend/modules/evaluation/domain/service/evaluator_impl.go index 01d4cfa634..f7351a6583 100644 --- a/backend/modules/evaluation/domain/service/evaluator_impl.go +++ b/backend/modules/evaluation/domain/service/evaluator_impl.go @@ -1085,6 +1085,19 @@ func (e *EvaluatorServiceImpl) failAsyncEvaluatorRecord(ctx context.Context, rec if updated { record.Status = entity.EvaluatorRunStatusFail record.EvaluatorOutputData = output + return record, runErr + } + + // The provider call can return an ACK error after it has already accepted the task and + // synchronously reported a terminal result. In that race the terminal CAS above loses by + // design; re-read the primary and let the callback outcome win instead of failing the turn + // with a stale dispatch error. + latest, getErr := e.evaluatorRecordRepo.GetEvaluatorRecord(contexts.WithCtxWriteDB(ctx), record.ID, false, entity.WithoutLoadStorageData()) + if getErr != nil { + return record, errorx.Wrapf(getErr, "reload evaluator record after async kickoff failure, cause: %v", runErr) + } + if latest != nil && (latest.Status == entity.EvaluatorRunStatusSuccess || latest.Status == entity.EvaluatorRunStatusFail) { + return latest, nil } return record, runErr } diff --git a/backend/modules/evaluation/domain/service/evaluator_impl_test.go b/backend/modules/evaluation/domain/service/evaluator_impl_test.go index 5258a2767b..201dfcf550 100644 --- a/backend/modules/evaluation/domain/service/evaluator_impl_test.go +++ b/backend/modules/evaluation/domain/service/evaluator_impl_test.go @@ -6,6 +6,7 @@ package service import ( "context" "errors" + "fmt" "testing" "time" @@ -4693,6 +4694,32 @@ func TestEvaluatorServiceImpl_AsyncRunEvaluator_CoordinatorOrderAndFailureCompen assert.Equal(t, entity.EvaluatorRunStatusFail, record.Status) }) + t.Run("dispatch ack failure preserves concurrent terminal callback", func(t *testing.T) { + for _, terminalStatus := range []entity.EvaluatorRunStatus{ + entity.EvaluatorRunStatusSuccess, + entity.EvaluatorRunStatusFail, + } { + t.Run(fmt.Sprintf("status_%d", terminalStatus), func(t *testing.T) { + s, recordRepo, asyncRepo, source := newFixture(t) + recordRepo.EXPECT().CreateEvaluatorRecord(gomock.Any(), gomock.Any()).Return(nil) + asyncRepo.EXPECT().SetEvalAsyncCtx(gomock.Any(), "evaluator:999", gomock.Any()).Return(nil) + source.EXPECT().AsyncRun(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), int64(2), int64(999)).Return(nil, "", errors.New("dispatch ack lost")) + recordRepo.EXPECT().CompareAndSwapEvaluatorRecordResult(gomock.Any(), int64(999), int64(2), entity.EvaluatorRunStatusAsyncInvoking, entity.EvaluatorRunStatusFail, gomock.Any()).Return(false, nil) + recordRepo.EXPECT().GetEvaluatorRecord(gomock.Any(), int64(999), false, gomock.Any()).Return(&entity.EvaluatorRecord{ + ID: 999, + SpaceID: 2, + Status: terminalStatus, + EvaluatorOutputData: &entity.EvaluatorOutputData{}, + }, nil) + + record, err := s.AsyncRunEvaluator(context.Background(), req()) + require.NoError(t, err) + require.NotNil(t, record) + assert.Equal(t, terminalStatus, record.Status) + }) + } + }) + t.Run("dispatch metadata persistence failure does not fail accepted work", func(t *testing.T) { s, recordRepo, asyncRepo, source := newFixture(t) recordRepo.EXPECT().CreateEvaluatorRecord(gomock.Any(), gomock.Any()).Return(nil) From 87b5eaa9f0fbea0ad444cfda7bdd3032e379da8d Mon Sep 17 00:00:00 2001 From: shenjiayun Date: Fri, 7 Aug 2026 17:31:41 +0800 Subject: [PATCH 07/15] test(evaluation): cover async evaluator failure paths --- .../application/eval_openapi_app_test.go | 34 ++++ .../domain/entity/evaluator_test.go | 1 + .../evaluator_version_custom_rpc_test.go | 9 ++ .../domain/service/evaluator_impl_test.go | 146 ++++++++++++++++++ .../domain/service/expt_run_item_impl_test.go | 29 ++++ .../evaluator/evaluator_record_impl_test.go | 58 +++++++ 6 files changed, 277 insertions(+) diff --git a/backend/modules/evaluation/application/eval_openapi_app_test.go b/backend/modules/evaluation/application/eval_openapi_app_test.go index 2bf09cee53..6acc29f786 100755 --- a/backend/modules/evaluation/application/eval_openapi_app_test.go +++ b/backend/modules/evaluation/application/eval_openapi_app_test.go @@ -6573,6 +6573,16 @@ func TestEvalOpenAPIApplication_ReportEvaluatorInvokeResult(t *testing.T) { }, wantErr: errno.CommonInvalidParamCode, }, + { + name: "missing workspace id rejected before auth", + req: &openapi.ReportEvaluatorInvokeResultRequest{ + InvokeID: gptr.Of(invokeID), + Status: gptr.Of(spi.InvokeEvaluatorRunStatus_SUCCESS), + }, + setup: func(*rpcmocks.MockIAuthProvider, *repomocks.MockIEvalAsyncRepo, *servicemocks.MockEvaluatorService, *eventmocks.MockExptEventPublisher, *servicemocks.MockIEvaluatorCallbackDispatcher) { + }, + wantErr: errno.CommonInvalidParamCode, + }, { name: "success without score rejected", @@ -6766,6 +6776,30 @@ func TestEvalOpenAPIApplication_ReportEvaluatorInvokeResult(t *testing.T) { publisher.EXPECT().PublishExptRecordEvalEvent(gomock.Any(), event, gomock.Not(gomock.Nil()), gomock.Any()).Return(nil) }, }, + { + name: "resume readiness reread failure is returned", + req: &openapi.ReportEvaluatorInvokeResultRequest{ + WorkspaceID: gptr.Of(workspaceID), + InvokeID: gptr.Of(invokeID), + Status: gptr.Of(spi.InvokeEvaluatorRunStatus_SUCCESS), + Output: &spi.InvokeEvaluatorOutputData{ + EvaluatorResult_: &spi.InvokeEvaluatorResult_{Score: gptr.Of(float64(0.9))}, + }, + }, + setup: func(auth *rpcmocks.MockIAuthProvider, asyncRepo *repomocks.MockIEvalAsyncRepo, evaluatorSvc *servicemocks.MockEvaluatorService, _ *eventmocks.MockExptEventPublisher, _ *servicemocks.MockIEvaluatorCallbackDispatcher) { + auth.EXPECT().Authorization(gomock.Any(), gomock.Any()).Return(nil) + gomock.InOrder( + asyncRepo.EXPECT().GetEvalAsyncCtxStrong(gomock.Any(), "evaluator:2002").Return(&entity.EvalAsyncCtx{ + Event: event, + ResumeReady: false, + EvaluatorVersionID: 9, + }, nil), + evaluatorSvc.EXPECT().ReportEvaluatorInvokeResult(gomock.Any(), gomock.Any()).Return(entity.ReportEvaluatorResultApplied, nil), + asyncRepo.EXPECT().GetEvalAsyncCtxStrong(gomock.Any(), "evaluator:2002").Return(nil, errors.New("redis reread failed")), + ) + }, + wantErr: -1, + }, { name: "duplicate terminal callback republishes recovery but does not redispatch external callback", diff --git a/backend/modules/evaluation/domain/entity/evaluator_test.go b/backend/modules/evaluation/domain/entity/evaluator_test.go index fa859bd2d7..aee7513e25 100644 --- a/backend/modules/evaluation/domain/entity/evaluator_test.go +++ b/backend/modules/evaluation/domain/entity/evaluator_test.go @@ -2387,6 +2387,7 @@ func TestEvaluator_IsAsync_CustomRPC(t *testing.T) { eval *Evaluator want bool }{ + {name: "nil evaluator remains sync", eval: nil, want: false}, {name: "agent remains async", eval: &Evaluator{EvaluatorType: EvaluatorTypeAgent}, want: true}, {name: "custom rpc async version", eval: &Evaluator{EvaluatorType: EvaluatorTypeCustomRPC, CustomRPCEvaluatorVersion: &CustomRPCEvaluatorVersion{IsAsync: true}}, want: true}, {name: "custom rpc sync version", eval: &Evaluator{EvaluatorType: EvaluatorTypeCustomRPC, CustomRPCEvaluatorVersion: &CustomRPCEvaluatorVersion{}}, want: false}, diff --git a/backend/modules/evaluation/domain/entity/evaluator_version_custom_rpc_test.go b/backend/modules/evaluation/domain/entity/evaluator_version_custom_rpc_test.go index 72203fbdc5..03c34eee37 100644 --- a/backend/modules/evaluation/domain/entity/evaluator_version_custom_rpc_test.go +++ b/backend/modules/evaluation/domain/entity/evaluator_version_custom_rpc_test.go @@ -444,6 +444,15 @@ func TestCustomRPCEvaluatorVersion_ValidateBaseInfo_Async(t *testing.T) { }, wantErr: true, }, + { + name: "unknown async protocol is rejected", + version: &CustomRPCEvaluatorVersion{ + AccessProtocol: "unsupported", + ServiceName: gptr.Of("trae.work.evaluator"), + IsAsync: true, + }, + wantErr: true, + }, { name: "sync faas http keeps existing behavior", version: &CustomRPCEvaluatorVersion{ diff --git a/backend/modules/evaluation/domain/service/evaluator_impl_test.go b/backend/modules/evaluation/domain/service/evaluator_impl_test.go index 201dfcf550..afbd346f34 100644 --- a/backend/modules/evaluation/domain/service/evaluator_impl_test.go +++ b/backend/modules/evaluation/domain/service/evaluator_impl_test.go @@ -4694,6 +4694,19 @@ func TestEvaluatorServiceImpl_AsyncRunEvaluator_CoordinatorOrderAndFailureCompen assert.Equal(t, entity.EvaluatorRunStatusFail, record.Status) }) + t.Run("missing async repo marks same record fail", func(t *testing.T) { + s, recordRepo, _, source := newFixture(t) + s.evalAsyncRepo = nil + recordRepo.EXPECT().CreateEvaluatorRecord(gomock.Any(), gomock.Any()).Return(nil) + recordRepo.EXPECT().CompareAndSwapEvaluatorRecordResult(gomock.Any(), int64(999), int64(2), entity.EvaluatorRunStatusAsyncInvoking, entity.EvaluatorRunStatusFail, gomock.Any()).Return(true, nil) + source.EXPECT().AsyncRun(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Times(0) + + record, err := s.AsyncRunEvaluator(context.Background(), req()) + require.Error(t, err) + require.NotNil(t, record) + assert.Equal(t, entity.EvaluatorRunStatusFail, record.Status) + }) + t.Run("dispatch ack failure preserves concurrent terminal callback", func(t *testing.T) { for _, terminalStatus := range []entity.EvaluatorRunStatus{ entity.EvaluatorRunStatusSuccess, @@ -4735,6 +4748,56 @@ func TestEvaluatorServiceImpl_AsyncRunEvaluator_CoordinatorOrderAndFailureCompen assert.Equal(t, "trace-1", record.TraceID) assert.Equal(t, "s1", record.EvaluatorOutputData.Ext["session_id"]) }) + + t.Run("dispatch failure returns CAS error", func(t *testing.T) { + s, recordRepo, asyncRepo, source := newFixture(t) + recordRepo.EXPECT().CreateEvaluatorRecord(gomock.Any(), gomock.Any()).Return(nil) + asyncRepo.EXPECT().SetEvalAsyncCtx(gomock.Any(), "evaluator:999", gomock.Any()).Return(nil) + source.EXPECT().AsyncRun(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), int64(2), int64(999)).Return(nil, "", errors.New("dispatch failed")) + recordRepo.EXPECT().CompareAndSwapEvaluatorRecordResult(gomock.Any(), int64(999), int64(2), entity.EvaluatorRunStatusAsyncInvoking, entity.EvaluatorRunStatusFail, gomock.Any()).Return(false, errors.New("cas failed")) + + record, err := s.AsyncRunEvaluator(context.Background(), req()) + require.Error(t, err) + require.NotNil(t, record) + assert.Contains(t, err.Error(), "mark evaluator async kickoff failed") + }) + + t.Run("dispatch failure returns reload error after lost CAS", func(t *testing.T) { + s, recordRepo, asyncRepo, source := newFixture(t) + recordRepo.EXPECT().CreateEvaluatorRecord(gomock.Any(), gomock.Any()).Return(nil) + asyncRepo.EXPECT().SetEvalAsyncCtx(gomock.Any(), "evaluator:999", gomock.Any()).Return(nil) + source.EXPECT().AsyncRun(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), int64(2), int64(999)).Return(nil, "", errors.New("dispatch failed")) + recordRepo.EXPECT().CompareAndSwapEvaluatorRecordResult(gomock.Any(), int64(999), int64(2), entity.EvaluatorRunStatusAsyncInvoking, entity.EvaluatorRunStatusFail, gomock.Any()).Return(false, nil) + recordRepo.EXPECT().GetEvaluatorRecord(gomock.Any(), int64(999), false, gomock.Any()).Return(nil, errors.New("reload failed")) + + record, err := s.AsyncRunEvaluator(context.Background(), req()) + require.Error(t, err) + require.NotNil(t, record) + assert.Contains(t, err.Error(), "reload evaluator record") + }) + + t.Run("dispatch failure remains error when lost CAS has no terminal record", func(t *testing.T) { + s, recordRepo, asyncRepo, source := newFixture(t) + recordRepo.EXPECT().CreateEvaluatorRecord(gomock.Any(), gomock.Any()).Return(nil) + asyncRepo.EXPECT().SetEvalAsyncCtx(gomock.Any(), "evaluator:999", gomock.Any()).Return(nil) + source.EXPECT().AsyncRun(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), int64(2), int64(999)).Return(nil, "", errors.New("dispatch failed")) + recordRepo.EXPECT().CompareAndSwapEvaluatorRecordResult(gomock.Any(), int64(999), int64(2), entity.EvaluatorRunStatusAsyncInvoking, entity.EvaluatorRunStatusFail, gomock.Any()).Return(false, nil) + recordRepo.EXPECT().GetEvaluatorRecord(gomock.Any(), int64(999), false, gomock.Any()).Return(nil, nil) + + record, err := s.AsyncRunEvaluator(context.Background(), req()) + require.EqualError(t, err, "dispatch failed") + require.NotNil(t, record) + assert.Equal(t, entity.EvaluatorRunStatusAsyncInvoking, record.Status) + }) +} + +func TestEvaluatorServiceImpl_failAsyncEvaluatorRecord_NilRecord(t *testing.T) { + t.Parallel() + s := &EvaluatorServiceImpl{} + want := errors.New("run failed") + record, err := s.failAsyncEvaluatorRecord(context.Background(), nil, want) + assert.Nil(t, record) + assert.ErrorIs(t, err, want) } func TestEvaluatorServiceImpl_ReportEvaluatorInvokeResult_RejectsNonTerminalStatus(t *testing.T) { @@ -4848,3 +4911,86 @@ func TestEvaluatorServiceImpl_ArmEvaluatorResume_RetriesPublish(t *testing.T) { s := &EvaluatorServiceImpl{evalAsyncRepo: asyncRepo, evaluatorRecordRepo: recordRepo, exptEventPublisher: publisher} require.NoError(t, s.ArmEvaluatorResume(context.Background(), 100)) } + +func TestEvaluatorServiceImpl_ArmEvaluatorResume_Errors(t *testing.T) { + t.Parallel() + + t.Run("missing async repo", func(t *testing.T) { + t.Parallel() + s := &EvaluatorServiceImpl{} + require.Error(t, s.ArmEvaluatorResume(context.Background(), 100)) + }) + + t.Run("mark resume ready error", func(t *testing.T) { + t.Parallel() + ctrl := gomock.NewController(t) + asyncRepo := repomocks.NewMockIEvalAsyncRepo(ctrl) + asyncRepo.EXPECT().MarkEvalAsyncResumeReady(gomock.Any(), "evaluator:100").Return(nil, errors.New("redis failed")) + s := &EvaluatorServiceImpl{evalAsyncRepo: asyncRepo} + require.Error(t, s.ArmEvaluatorResume(context.Background(), 100)) + }) + + t.Run("record reload error", func(t *testing.T) { + t.Parallel() + ctrl := gomock.NewController(t) + asyncRepo := repomocks.NewMockIEvalAsyncRepo(ctrl) + recordRepo := repomocks.NewMockIEvaluatorRecordRepo(ctrl) + asyncRepo.EXPECT().MarkEvalAsyncResumeReady(gomock.Any(), "evaluator:100").Return(&entity.EvalAsyncCtx{Event: &entity.ExptItemEvalEvent{ExptID: 1}}, nil) + recordRepo.EXPECT().GetEvaluatorRecord(gomock.Any(), int64(100), false).Return(nil, errors.New("mysql failed")) + s := &EvaluatorServiceImpl{evalAsyncRepo: asyncRepo, evaluatorRecordRepo: recordRepo} + require.Error(t, s.ArmEvaluatorResume(context.Background(), 100)) + }) + + t.Run("missing record", func(t *testing.T) { + t.Parallel() + ctrl := gomock.NewController(t) + asyncRepo := repomocks.NewMockIEvalAsyncRepo(ctrl) + recordRepo := repomocks.NewMockIEvaluatorRecordRepo(ctrl) + asyncRepo.EXPECT().MarkEvalAsyncResumeReady(gomock.Any(), "evaluator:100").Return(&entity.EvalAsyncCtx{Event: &entity.ExptItemEvalEvent{ExptID: 1}}, nil) + recordRepo.EXPECT().GetEvaluatorRecord(gomock.Any(), int64(100), false).Return(nil, nil) + s := &EvaluatorServiceImpl{evalAsyncRepo: asyncRepo, evaluatorRecordRepo: recordRepo} + err := s.ArmEvaluatorResume(context.Background(), 100) + statusErr, ok := errorx.FromStatusError(err) + require.True(t, ok) + assert.Equal(t, int32(errno.EvaluatorRecordNotFoundCode), statusErr.Code()) + }) +} + +func TestEvaluatorServiceImpl_publishEvaluatorResumeEvent(t *testing.T) { + t.Parallel() + + t.Run("nil dependencies are no-op", func(t *testing.T) { + t.Parallel() + s := &EvaluatorServiceImpl{} + require.NoError(t, s.publishEvaluatorResumeEvent(context.Background(), nil)) + require.NoError(t, s.publishEvaluatorResumeEvent(context.Background(), &entity.ExptItemEvalEvent{})) + }) + + t.Run("returns last error after all retries", func(t *testing.T) { + t.Parallel() + ctrl := gomock.NewController(t) + publisher := eventmocks.NewMockExptEventPublisher(ctrl) + event := &entity.ExptItemEvalEvent{ExptID: 1} + publisher.EXPECT().PublishExptRecordEvalEvent(gomock.Any(), event, gomock.Any(), gomock.Any()).Return(errors.New("mq unavailable")).Times(3) + s := &EvaluatorServiceImpl{exptEventPublisher: publisher} + err := s.publishEvaluatorResumeEvent(context.Background(), event) + require.EqualError(t, err, "mq unavailable") + }) + + t.Run("context cancellation stops retry delay", func(t *testing.T) { + t.Parallel() + ctrl := gomock.NewController(t) + publisher := eventmocks.NewMockExptEventPublisher(ctrl) + event := &entity.ExptItemEvalEvent{ExptID: 1} + ctx, cancel := context.WithCancel(context.Background()) + publisher.EXPECT().PublishExptRecordEvalEvent(gomock.Any(), event, gomock.Any(), gomock.Any()).DoAndReturn( + func(context.Context, *entity.ExptItemEvalEvent, *time.Duration, func(*entity.ExptItemEvalEvent)) error { + cancel() + return errors.New("mq unavailable") + }, + ).Times(1) + s := &EvaluatorServiceImpl{exptEventPublisher: publisher} + err := s.publishEvaluatorResumeEvent(ctx, event) + require.ErrorIs(t, err, context.Canceled) + }) +} diff --git a/backend/modules/evaluation/domain/service/expt_run_item_impl_test.go b/backend/modules/evaluation/domain/service/expt_run_item_impl_test.go index 02e01a203b..607c0609dd 100644 --- a/backend/modules/evaluation/domain/service/expt_run_item_impl_test.go +++ b/backend/modules/evaluation/domain/service/expt_run_item_impl_test.go @@ -1171,3 +1171,32 @@ func TestExptItemEvalCtxExecutor_storeTurnRunResult_DoesNotArmWhenSaveFails(t *t }}} require.Error(t, executor.storeTurnRunResult(context.Background(), etec, result)) } + +func TestExptItemEvalCtxExecutor_storeTurnRunResult_ReturnsArmError(t *testing.T) { + t.Parallel() + + ctrl := gomock.NewController(t) + turnRepo := repomocks.NewMockIExptTurnResultRepo(ctrl) + evaluatorSvc := servicemocks.NewMockEvaluatorService(ctrl) + turnRepo.EXPECT().SaveTurnRunLogs(gomock.Any(), gomock.Any()).Return(nil) + evaluatorSvc.EXPECT().ArmEvaluatorResume(gomock.Any(), int64(100)).Return(errors.New("arm failed")) + + executor := &ExptItemEvalCtxExecutor{TurnResultRepo: turnRepo, evaluatorService: evaluatorSvc} + etec := &entity.ExptTurnEvalCtx{ + Turn: &entity.Turn{ID: 1}, + ExptItemEvalCtx: &entity.ExptItemEvalCtx{ + Expt: &entity.Experiment{ID: 1, SpaceID: 2}, + Event: &entity.ExptItemEvalEvent{ExptRunID: 3}, + EvalSetItem: &entity.EvaluationSetItem{ItemID: 4}, + ExistItemEvalResult: &entity.ExptItemEvalResult{TurnResultRunLogs: map[int64]*entity.ExptTurnResultRunLog{ + 1: {ID: 5, TurnID: 1}, + }}, + }, + } + result := &entity.ExptTurnRunResult{EvaluatorResults: []*entity.EvaluatorRecord{{ + ID: 100, EvaluatorVersionID: 101, Status: entity.EvaluatorRunStatusAsyncInvoking, + }}} + err := executor.storeTurnRunResult(context.Background(), etec, result) + require.Error(t, err) + assert.Contains(t, err.Error(), "arm evaluator async resume fail") +} diff --git a/backend/modules/evaluation/infra/repo/evaluator/evaluator_record_impl_test.go b/backend/modules/evaluation/infra/repo/evaluator/evaluator_record_impl_test.go index 3c81e87e1d..59a225e245 100644 --- a/backend/modules/evaluation/infra/repo/evaluator/evaluator_record_impl_test.go +++ b/backend/modules/evaluation/infra/repo/evaluator/evaluator_record_impl_test.go @@ -1070,6 +1070,64 @@ func TestEvaluatorRecordRepoImpl_CompareAndSwapEvaluatorRecordResult(t *testing. assert.True(t, updated) } +func TestMergeEvaluatorOutputExt(t *testing.T) { + t.Parallel() + + got := mergeEvaluatorOutputExt(nil, &entity.EvaluatorOutputData{Ext: map[string]string{"source": "1"}}) + require.NotNil(t, got) + assert.Equal(t, "1", got.Ext["source"]) + assert.Same(t, got, mergeEvaluatorOutputExt(got, nil)) +} + +func TestEvaluatorRecordRepoImpl_AsyncTransactionErrors(t *testing.T) { + t.Parallel() + + newRepo := func(t *testing.T) (*EvaluatorRecordRepoImpl, *evaluatormocks.MockEvaluatorRecordDAO) { + t.Helper() + ctrl := gomock.NewController(t) + dao := evaluatormocks.NewMockEvaluatorRecordDAO(ctrl) + provider := dbmocks.NewMockProvider(ctrl) + provider.EXPECT().Transaction(gomock.Any(), gomock.Any()).DoAndReturn(func(ctx context.Context, fc func(*gorm.DB) error, _ ...db.Option) error { + return fc(nil) + }).AnyTimes() + return &EvaluatorRecordRepoImpl{evaluatorRecordDao: dao, dbProvider: provider}, dao + } + + t.Run("callback get record error", func(t *testing.T) { + repo, dao := newRepo(t) + dao.EXPECT().GetEvaluatorRecord(gomock.Any(), int64(10), false, gomock.Any(), gomock.Any()).Return(nil, errors.New("get failed")) + updated, err := repo.CompareAndSwapEvaluatorRecordResult(context.Background(), 10, 20, entity.EvaluatorRunStatusAsyncInvoking, entity.EvaluatorRunStatusSuccess, nil) + assert.False(t, updated) + require.EqualError(t, err, "get failed") + }) + + t.Run("callback invalid stored output", func(t *testing.T) { + repo, dao := newRepo(t) + bad := []byte("{invalid") + dao.EXPECT().GetEvaluatorRecord(gomock.Any(), int64(10), false, gomock.Any(), gomock.Any()).Return(&model.EvaluatorRecord{ + ID: 10, SpaceID: 20, Status: int32(entity.EvaluatorRunStatusAsyncInvoking), OutputData: &bad, + }, nil) + updated, err := repo.CompareAndSwapEvaluatorRecordResult(context.Background(), 10, 20, entity.EvaluatorRunStatusAsyncInvoking, entity.EvaluatorRunStatusSuccess, nil) + assert.False(t, updated) + assert.Error(t, err) + }) + + t.Run("dispatch get record error", func(t *testing.T) { + repo, dao := newRepo(t) + dao.EXPECT().GetEvaluatorRecord(gomock.Any(), int64(10), false, gomock.Any(), gomock.Any()).Return(nil, errors.New("get failed")) + require.EqualError(t, repo.UpdateEvaluatorRecordAsyncDispatch(context.Background(), 10, 20, "trace", nil), "get failed") + }) + + t.Run("dispatch invalid stored output", func(t *testing.T) { + repo, dao := newRepo(t) + bad := []byte("{invalid") + dao.EXPECT().GetEvaluatorRecord(gomock.Any(), int64(10), false, gomock.Any(), gomock.Any()).Return(&model.EvaluatorRecord{ + ID: 10, SpaceID: 20, Status: int32(entity.EvaluatorRunStatusSuccess), OutputData: &bad, + }, nil) + assert.Error(t, repo.UpdateEvaluatorRecordAsyncDispatch(context.Background(), 10, 20, "trace", nil)) + }) +} + func TestEvaluatorRecordRepoImpl_AsyncCallbackAndDispatchMetadataMerge(t *testing.T) { t.Parallel() From 6427e276c1d12bea276ad0172fabdb42e05ab17e Mon Sep 17 00:00:00 2001 From: shenjiayun Date: Fri, 7 Aug 2026 21:47:07 +0800 Subject: [PATCH 08/15] test(evaluation): cover async target mixed evaluator recovery --- .../domain/service/expt_run_item_impl_test.go | 94 ++++++ .../service/expt_run_item_turn_impl_test.go | 306 ++++++++++++++++++ 2 files changed, 400 insertions(+) diff --git a/backend/modules/evaluation/domain/service/expt_run_item_impl_test.go b/backend/modules/evaluation/domain/service/expt_run_item_impl_test.go index 607c0609dd..cec2f8a35e 100644 --- a/backend/modules/evaluation/domain/service/expt_run_item_impl_test.go +++ b/backend/modules/evaluation/domain/service/expt_run_item_impl_test.go @@ -1200,3 +1200,97 @@ func TestExptItemEvalCtxExecutor_storeTurnRunResult_ReturnsArmError(t *testing.T require.Error(t, err) assert.Contains(t, err.Error(), "arm evaluator async resume fail") } + +func TestExptItemEvalCtxExecutor_storeTurnRunResult_ArmsEveryPendingEvaluator(t *testing.T) { + t.Parallel() + + ctrl := gomock.NewController(t) + turnRepo := repomocks.NewMockIExptTurnResultRepo(ctrl) + evaluatorSvc := servicemocks.NewMockEvaluatorService(ctrl) + + turnRepo.EXPECT().SaveTurnRunLogs(gomock.Any(), gomock.Any()).DoAndReturn( + func(_ context.Context, logs []*entity.ExptTurnResultRunLog) error { + require.Len(t, logs, 1) + require.NotNil(t, logs[0].EvaluatorResultIds) + require.Len(t, logs[0].EvaluatorResultIds.Registered, 5) + seen := make(map[int64]int) + for _, ref := range logs[0].EvaluatorResultIds.Registered { + require.NotNil(t, ref) + seen[ref.VersionID]++ + } + for _, versionID := range []int64{101, 102, 201, 202, 203} { + assert.Equal(t, 1, seen[versionID]) + } + return nil + }, + ) + for _, recordID := range []int64{1201, 1202, 1203} { + evaluatorSvc.EXPECT().ArmEvaluatorResume(gomock.Any(), recordID).Return(nil) + } + + executor := &ExptItemEvalCtxExecutor{TurnResultRepo: turnRepo, evaluatorService: evaluatorSvc} + etec := &entity.ExptTurnEvalCtx{ + Turn: &entity.Turn{ID: 1}, + ExptItemEvalCtx: &entity.ExptItemEvalCtx{ + Expt: &entity.Experiment{ID: 1, SpaceID: 2}, + Event: &entity.ExptItemEvalEvent{ExptRunID: 3}, + EvalSetItem: &entity.EvaluationSetItem{ItemID: 4}, + ExistItemEvalResult: &entity.ExptItemEvalResult{TurnResultRunLogs: map[int64]*entity.ExptTurnResultRunLog{ + 1: {ID: 5, TurnID: 1}, + }}, + }, + } + result := &entity.ExptTurnRunResult{ + AsyncAbort: true, + EvaluatorResults: []*entity.EvaluatorRecord{ + {ID: 1101, EvaluatorVersionID: 101, Status: entity.EvaluatorRunStatusSuccess}, + {ID: 1102, EvaluatorVersionID: 102, Status: entity.EvaluatorRunStatusSuccess}, + {ID: 1201, EvaluatorVersionID: 201, Status: entity.EvaluatorRunStatusAsyncInvoking}, + {ID: 1202, EvaluatorVersionID: 202, Status: entity.EvaluatorRunStatusAsyncInvoking}, + {ID: 1203, EvaluatorVersionID: 203, Status: entity.EvaluatorRunStatusAsyncInvoking}, + }, + } + require.NoError(t, executor.storeTurnRunResult(context.Background(), etec, result)) +} + +func TestExptItemEvalCtxExecutor_storeTurnRunResult_AllEvaluatorsTerminalCompletesWithoutArming(t *testing.T) { + t.Parallel() + + ctrl := gomock.NewController(t) + turnRepo := repomocks.NewMockIExptTurnResultRepo(ctrl) + evaluatorSvc := servicemocks.NewMockEvaluatorService(ctrl) + + turnRepo.EXPECT().SaveTurnRunLogs(gomock.Any(), gomock.Any()).DoAndReturn( + func(_ context.Context, logs []*entity.ExptTurnResultRunLog) error { + require.Len(t, logs, 1) + assert.Equal(t, entity.TurnRunState_Success, logs[0].Status) + require.NotNil(t, logs[0].EvaluatorResultIds) + require.Len(t, logs[0].EvaluatorResultIds.Registered, 5) + return nil + }, + ) + evaluatorSvc.EXPECT().ArmEvaluatorResume(gomock.Any(), gomock.Any()).Times(0) + + executor := &ExptItemEvalCtxExecutor{TurnResultRepo: turnRepo, evaluatorService: evaluatorSvc} + etec := &entity.ExptTurnEvalCtx{ + Turn: &entity.Turn{ID: 1}, + ExptItemEvalCtx: &entity.ExptItemEvalCtx{ + Expt: &entity.Experiment{ID: 1, SpaceID: 2}, + Event: &entity.ExptItemEvalEvent{ExptRunID: 3}, + EvalSetItem: &entity.EvaluationSetItem{ItemID: 4}, + ExistItemEvalResult: &entity.ExptItemEvalResult{TurnResultRunLogs: map[int64]*entity.ExptTurnResultRunLog{ + 1: {ID: 5, TurnID: 1, Status: entity.TurnRunState_Processing}, + }}, + }, + } + result := &entity.ExptTurnRunResult{ + EvaluatorResults: []*entity.EvaluatorRecord{ + {ID: 1101, EvaluatorVersionID: 101, Status: entity.EvaluatorRunStatusSuccess}, + {ID: 1102, EvaluatorVersionID: 102, Status: entity.EvaluatorRunStatusSuccess}, + {ID: 1201, EvaluatorVersionID: 201, Status: entity.EvaluatorRunStatusSuccess}, + {ID: 1202, EvaluatorVersionID: 202, Status: entity.EvaluatorRunStatusSuccess}, + {ID: 1203, EvaluatorVersionID: 203, Status: entity.EvaluatorRunStatusSuccess}, + }, + } + require.NoError(t, executor.storeTurnRunResult(context.Background(), etec, result)) +} diff --git a/backend/modules/evaluation/domain/service/expt_run_item_turn_impl_test.go b/backend/modules/evaluation/domain/service/expt_run_item_turn_impl_test.go index 2f249b0909..335aa06da8 100644 --- a/backend/modules/evaluation/domain/service/expt_run_item_turn_impl_test.go +++ b/backend/modules/evaluation/domain/service/expt_run_item_turn_impl_test.go @@ -5,6 +5,7 @@ package service import ( "context" "errors" + "fmt" "runtime" "strconv" "sync" @@ -5034,3 +5035,308 @@ func TestDefaultExptTurnEvaluationImpl_asyncCallEvaluatorWithAlias_CustomRPC(t * require.Len(t, collector.records, 1) assert.Equal(t, "judge_a", collector.records[0].Alias) } + +// TestDefaultExptTurnEvaluationImpl_AsyncTargetCallbackRunsMixedEvaluators is the +// regression guard for the full target/evaluator state machine. In particular, +// an AsyncReportTrigger means "the target result is ready"; it must not be +// mistaken for AsyncEvaluatorReportTrigger and skip evaluator execution. +func TestDefaultExptTurnEvaluationImpl_AsyncTargetCallbackRunsMixedEvaluators(t *testing.T) { + t.Parallel() + + ctrl := gomock.NewController(t) + metric := metricsmocks.NewMockExptMetric(ctrl) + evaluatorSvc := svcmocks.NewMockEvaluatorService(ctrl) + benefitSvc := benefitmocks.NewMockIBenefitService(ctrl) + recordSvc := svcmocks.NewMockEvaluatorRecordService(ctrl) + + service := &DefaultExptTurnEvaluationImpl{ + metric: metric, + evaluatorService: evaluatorSvc, + benefitService: benefitSvc, + evaluatorRecordService: recordSvc, + } + + const ( + spaceID = int64(2) + targetID = int64(900) + ) + targetStatus := entity.EvalTargetRunStatusSuccess + targetRecord := &entity.EvalTargetRecord{ + ID: targetID, + Status: &targetStatus, + EvalTargetOutputData: &entity.EvalTargetOutputData{OutputFields: map[string]*entity.Content{ + "actual_output": {Text: gptr.Of("target done")}, + }}, + } + + newEvaluator := func(versionID int64, async bool) *entity.Evaluator { + if async { + code := fmt.Sprintf("async-%d", versionID) + return &entity.Evaluator{ + ID: versionID, EvaluatorType: entity.EvaluatorTypeCustomRPC, + CustomRPCEvaluatorVersion: &entity.CustomRPCEvaluatorVersion{ + ID: versionID, EvaluatorID: versionID, ProviderEvaluatorCode: &code, + AccessProtocol: entity.EvaluatorAccessProtocolRPC, IsAsync: true, + }, + } + } + return &entity.Evaluator{ + ID: versionID, EvaluatorType: entity.EvaluatorTypePrompt, + PromptEvaluatorVersion: &entity.PromptEvaluatorVersion{ID: versionID}, + } + } + newConf := func(versionID int64) *entity.EvaluatorConf { + return &entity.EvaluatorConf{ + EvaluatorVersionID: versionID, + IngressConf: &entity.EvaluatorIngressConf{ + EvalSetAdapter: &entity.FieldAdapter{FieldConfs: []*entity.FieldConf{}}, + TargetAdapter: &entity.FieldAdapter{FieldConfs: []*entity.FieldConf{}}, + }, + RunConf: &entity.EvaluatorRunConfig{Env: gptr.Of("ppe_trae_work_async_evaluator")}, + } + } + + syncVersions := map[int64]struct{}{101: {}, 102: {}} + asyncVersions := map[int64]struct{}{201: {}, 202: {}, 203: {}} + allEvaluators := []*entity.Evaluator{ + newEvaluator(101, false), newEvaluator(102, false), + newEvaluator(201, true), newEvaluator(202, true), newEvaluator(203, true), + } + allConfs := []*entity.EvaluatorConf{ + newConf(101), newConf(102), newConf(201), newConf(202), newConf(203), + } + + event := &entity.ExptItemEvalEvent{ + SpaceID: spaceID, ExptID: 3, ExptRunID: 4, EvalSetItemID: 5, + Session: &entity.Session{UserID: "u"}, AsyncReportTrigger: true, + } + etec := &entity.ExptTurnEvalCtx{ + ExptItemEvalCtx: &entity.ExptItemEvalCtx{ + Event: event, + EvalSetItem: &entity.EvaluationSetItem{ItemID: 5}, + Expt: &entity.Experiment{ + ID: 3, SpaceID: spaceID, TargetVersionID: 99, + Target: &entity.EvalTarget{ID: 88, EvalTargetType: entity.EvalTargetTypeCustomRPCServer, + EvalTargetVersion: &entity.EvalTargetVersion{ID: 99, EvalTargetType: entity.EvalTargetTypeCustomRPCServer, + CustomRPCServer: &entity.CustomRPCServer{IsAsync: gptr.Of(true)}}}, + Evaluators: allEvaluators, + EvalConf: &entity.EvaluationConfiguration{ConnectorConf: entity.Connector{ + TargetConf: &entity.TargetConf{TargetVersionID: 99}, + EvaluatorsConf: &entity.EvaluatorsConf{EvaluatorConcurNum: gptr.Of(5), EvaluatorConf: allConfs}, + }}, + }, + }, + Turn: &entity.Turn{ID: 6, FieldDataList: []*entity.FieldData{}}, + ExptTurnRunResult: &entity.ExptTurnRunResult{TargetResult: targetRecord}, + } + + metric.EXPECT().EmitTurnExecEval(spaceID, gomock.Any()) + metric.EXPECT().EmitTurnExecResult(spaceID, gomock.Any(), true, gomock.Any(), gomock.Any(), gomock.Any()) + benefitSvc.EXPECT().CheckAndDeductEvalBenefit(gomock.Any(), gomock.Any()).Return(&benefit.CheckAndDeductEvalBenefitResult{}, nil) + evaluatorSvc.EXPECT().ShouldInterceptEvaluator(gomock.Any(), gomock.Any()).Return(nil, false, nil).Times(5) + metric.EXPECT().EmitTurnExecEvaluatorResult(spaceID, false).Times(5) + + var mu sync.Mutex + runCounts := make(map[int64]int) + evaluatorSvc.EXPECT().RunEvaluator(gomock.Any(), gomock.Any()).DoAndReturn( + func(_ context.Context, req *entity.RunEvaluatorRequest) (*entity.EvaluatorRecord, error) { + mu.Lock() + runCounts[req.EvaluatorVersionID]++ + mu.Unlock() + _, ok := syncVersions[req.EvaluatorVersionID] + require.True(t, ok, "unexpected sync evaluator %d", req.EvaluatorVersionID) + require.Equal(t, "ppe_trae_work_async_evaluator", gptr.Indirect(req.EvaluatorRunConf.Env)) + return &entity.EvaluatorRecord{ID: 1000 + req.EvaluatorVersionID, EvaluatorVersionID: req.EvaluatorVersionID, Status: entity.EvaluatorRunStatusSuccess}, nil + }, + ).Times(2) + evaluatorSvc.EXPECT().AsyncRunEvaluator(gomock.Any(), gomock.Any()).DoAndReturn( + func(_ context.Context, req *entity.AsyncRunEvaluatorRequest) (*entity.EvaluatorRecord, error) { + mu.Lock() + runCounts[req.EvaluatorVersionID]++ + mu.Unlock() + _, ok := asyncVersions[req.EvaluatorVersionID] + require.True(t, ok, "unexpected async evaluator %d", req.EvaluatorVersionID) + require.Equal(t, "ppe_trae_work_async_evaluator", gptr.Indirect(req.EvaluatorRunConf.Env)) + require.NotNil(t, req.AsyncCtx) + require.Same(t, event, req.AsyncCtx.Event) + require.False(t, req.AsyncCtx.ResumeReady) + return &entity.EvaluatorRecord{ID: 1000 + req.EvaluatorVersionID, EvaluatorVersionID: req.EvaluatorVersionID, Status: entity.EvaluatorRunStatusAsyncInvoking}, nil + }, + ).Times(3) + recordSvc.EXPECT().GetEvaluatorRecord(gomock.Any(), gomock.Any(), false).DoAndReturn( + func(_ context.Context, recordID int64, _ bool) (*entity.EvaluatorRecord, error) { + versionID := recordID - 1000 + return &entity.EvaluatorRecord{ID: recordID, EvaluatorVersionID: versionID, Status: entity.EvaluatorRunStatusAsyncInvoking}, nil + }, + ).Times(3) + + result := service.Eval(context.Background(), etec) + require.NoError(t, result.EvalErr) + require.Same(t, targetRecord, result.TargetResult) + require.True(t, result.AsyncAbort) + require.Len(t, result.EvaluatorResults, 5) + for versionID := range syncVersions { + record := result.GetEvaluatorRecord(versionID) + require.NotNil(t, record) + assert.Equal(t, entity.EvaluatorRunStatusSuccess, record.Status) + } + for versionID := range asyncVersions { + record := result.GetEvaluatorRecord(versionID) + require.NotNil(t, record) + assert.Equal(t, entity.EvaluatorRunStatusAsyncInvoking, record.Status) + } + for versionID, count := range runCounts { + assert.Equal(t, 1, count, "evaluator %d must run exactly once", versionID) + } + assert.Len(t, runCounts, 5) +} + +// TestDefaultExptTurnEvaluationImpl_AsyncEvaluatorCallbacksPreserveAllRecords +// models staggered callbacks after the initial mixed run. Every callback event +// must reuse the complete persisted record set and must never re-run target or +// evaluators. The turn only converges after the final async record is terminal. +func TestDefaultExptTurnEvaluationImpl_AsyncEvaluatorCallbacksPreserveAllRecords(t *testing.T) { + t.Parallel() + + targetStatus := entity.EvalTargetRunStatusSuccess + target := &entity.EvalTargetRecord{ID: 900, Status: &targetStatus, EvalTargetOutputData: &entity.EvalTargetOutputData{OutputFields: map[string]*entity.Content{}}} + newExpt := func() *entity.Experiment { + return &entity.Experiment{ + TargetVersionID: 99, + Target: &entity.EvalTarget{EvalTargetVersion: &entity.EvalTargetVersion{ID: 99, CustomRPCServer: &entity.CustomRPCServer{IsAsync: gptr.Of(true)}}}, + Evaluators: []*entity.Evaluator{{ID: 101}, {ID: 102}, {ID: 201}, {ID: 202}, {ID: 203}}, + EvalConf: &entity.EvaluationConfiguration{ConnectorConf: entity.Connector{EvaluatorsConf: &entity.EvaluatorsConf{}}}, + } + } + newRecords := func(statuses map[int64]entity.EvaluatorRunStatus) []*entity.EvaluatorRecord { + versions := []int64{101, 102, 201, 202, 203} + records := make([]*entity.EvaluatorRecord, 0, len(versions)) + for _, versionID := range versions { + records = append(records, &entity.EvaluatorRecord{ID: 1000 + versionID, EvaluatorVersionID: versionID, Status: statuses[versionID]}) + } + return records + } + + phases := []struct { + name string + statuses map[int64]entity.EvaluatorRunStatus + wantAbort bool + }{ + { + name: "first async callback keeps the other two pending", + statuses: map[int64]entity.EvaluatorRunStatus{101: entity.EvaluatorRunStatusSuccess, 102: entity.EvaluatorRunStatusSuccess, + 201: entity.EvaluatorRunStatusSuccess, 202: entity.EvaluatorRunStatusAsyncInvoking, 203: entity.EvaluatorRunStatusAsyncInvoking}, + wantAbort: true, + }, + { + name: "second async callback keeps the last one pending", + statuses: map[int64]entity.EvaluatorRunStatus{101: entity.EvaluatorRunStatusSuccess, 102: entity.EvaluatorRunStatusSuccess, + 201: entity.EvaluatorRunStatusSuccess, 202: entity.EvaluatorRunStatusSuccess, 203: entity.EvaluatorRunStatusAsyncInvoking}, + wantAbort: true, + }, + { + name: "last async callback completes the turn", + statuses: map[int64]entity.EvaluatorRunStatus{101: entity.EvaluatorRunStatusSuccess, 102: entity.EvaluatorRunStatusSuccess, + 201: entity.EvaluatorRunStatusSuccess, 202: entity.EvaluatorRunStatusSuccess, 203: entity.EvaluatorRunStatusSuccess}, + wantAbort: false, + }, + } + + for _, phase := range phases { + phase := phase + t.Run(phase.name, func(t *testing.T) { + t.Parallel() + ctrl := gomock.NewController(t) + metric := metricsmocks.NewMockExptMetric(ctrl) + service := &DefaultExptTurnEvaluationImpl{metric: metric} + event := &entity.ExptItemEvalEvent{SpaceID: 2, AsyncEvaluatorReportTrigger: true, Session: &entity.Session{UserID: "u"}} + records := newRecords(phase.statuses) + etec := &entity.ExptTurnEvalCtx{ + ExptItemEvalCtx: &entity.ExptItemEvalCtx{Event: event, Expt: newExpt()}, + ExptTurnRunResult: &entity.ExptTurnRunResult{TargetResult: target, EvaluatorResults: records}, + } + metric.EXPECT().EmitTurnExecEval(int64(2), gomock.Any()) + metric.EXPECT().EmitTurnExecResult(int64(2), gomock.Any(), true, gomock.Any(), gomock.Any(), gomock.Any()) + + result := service.Eval(context.Background(), etec) + require.NoError(t, result.EvalErr) + require.Same(t, target, result.TargetResult) + assert.Equal(t, phase.wantAbort, result.AsyncAbort) + require.Len(t, result.EvaluatorResults, 5) + seen := make(map[int64]int) + for _, record := range result.EvaluatorResults { + require.NotNil(t, record) + seen[record.EvaluatorVersionID]++ + assert.Equal(t, phase.statuses[record.EvaluatorVersionID], record.Status) + } + for _, versionID := range []int64{101, 102, 201, 202, 203} { + assert.Equal(t, 1, seen[versionID], "record for evaluator %d must be preserved exactly once", versionID) + } + }) + } +} + +// TestDefaultExptTurnEvaluationImpl_MixedEvaluatorCompletionOrder covers both +// orderings around refreshAsyncEvaluatorRecords: the async callback can arrive +// before a slow synchronous evaluator returns, or remain pending after the +// synchronous evaluator has completed. +func TestDefaultExptTurnEvaluationImpl_MixedEvaluatorCompletionOrder(t *testing.T) { + t.Parallel() + + newCase := func(t *testing.T, refreshedStatus entity.EvaluatorRunStatus, wantAbort bool) { + ctrl := gomock.NewController(t) + metric := metricsmocks.NewMockExptMetric(ctrl) + evaluatorSvc := svcmocks.NewMockEvaluatorService(ctrl) + benefitSvc := benefitmocks.NewMockIBenefitService(ctrl) + recordSvc := svcmocks.NewMockEvaluatorRecordService(ctrl) + service := &DefaultExptTurnEvaluationImpl{metric: metric, evaluatorService: evaluatorSvc, benefitService: benefitSvc, evaluatorRecordService: recordSvc} + + code := "async-fast" + expt := &entity.Experiment{ + SpaceID: 2, + Evaluators: []*entity.Evaluator{ + {ID: 101, EvaluatorType: entity.EvaluatorTypePrompt, PromptEvaluatorVersion: &entity.PromptEvaluatorVersion{ID: 101}}, + {ID: 201, EvaluatorType: entity.EvaluatorTypeCustomRPC, CustomRPCEvaluatorVersion: &entity.CustomRPCEvaluatorVersion{ID: 201, EvaluatorID: 201, ProviderEvaluatorCode: &code, IsAsync: true}}, + }, + EvalConf: &entity.EvaluationConfiguration{ConnectorConf: entity.Connector{EvaluatorsConf: &entity.EvaluatorsConf{ + EvaluatorConcurNum: gptr.Of(2), EvaluatorConf: []*entity.EvaluatorConf{ + {EvaluatorVersionID: 101, IngressConf: &entity.EvaluatorIngressConf{EvalSetAdapter: &entity.FieldAdapter{}, TargetAdapter: &entity.FieldAdapter{}}}, + {EvaluatorVersionID: 201, IngressConf: &entity.EvaluatorIngressConf{EvalSetAdapter: &entity.FieldAdapter{}, TargetAdapter: &entity.FieldAdapter{}}}, + }, + }}}, + } + etec := &entity.ExptTurnEvalCtx{ + ExptItemEvalCtx: &entity.ExptItemEvalCtx{Expt: expt, Event: &entity.ExptItemEvalEvent{SpaceID: 2, ExptID: 3, ExptRunID: 4, Session: &entity.Session{UserID: "u"}}, EvalSetItem: &entity.EvaluationSetItem{ItemID: 5}}, + ExptTurnRunResult: &entity.ExptTurnRunResult{}, Turn: &entity.Turn{ID: 6}, + } + target := &entity.EvalTargetRecord{EvalTargetOutputData: &entity.EvalTargetOutputData{OutputFields: map[string]*entity.Content{}}} + + benefitSvc.EXPECT().CheckAndDeductEvalBenefit(gomock.Any(), gomock.Any()).Return(&benefit.CheckAndDeductEvalBenefitResult{}, nil) + evaluatorSvc.EXPECT().ShouldInterceptEvaluator(gomock.Any(), gomock.Any()).Return(nil, false, nil).Times(2) + metric.EXPECT().EmitTurnExecEvaluatorResult(int64(2), false).Times(2) + evaluatorSvc.EXPECT().RunEvaluator(gomock.Any(), gomock.Any()).DoAndReturn(func(context.Context, *entity.RunEvaluatorRequest) (*entity.EvaluatorRecord, error) { + // This makes the sync branch observably slower than an immediate provider callback. + time.Sleep(20 * time.Millisecond) + return &entity.EvaluatorRecord{ID: 1101, EvaluatorVersionID: 101, Status: entity.EvaluatorRunStatusSuccess}, nil + }) + evaluatorSvc.EXPECT().AsyncRunEvaluator(gomock.Any(), gomock.Any()).Return(&entity.EvaluatorRecord{ID: 1201, EvaluatorVersionID: 201, Status: entity.EvaluatorRunStatusAsyncInvoking}, nil) + recordSvc.EXPECT().GetEvaluatorRecord(gomock.Any(), int64(1201), false).Return(&entity.EvaluatorRecord{ID: 1201, EvaluatorVersionID: 201, Status: refreshedStatus}, nil) + + records, err := service.CallEvaluators(context.Background(), etec, target) + require.NoError(t, err) + require.Len(t, records, 2) + result := (&entity.ExptTurnRunResult{}).SetEvaluatorResults(records) + assert.Equal(t, wantAbort, result.AbortWithEvaluatorResults(context.Background(), etec.Event)) + assert.Equal(t, entity.EvaluatorRunStatusSuccess, result.GetEvaluatorRecord(101).Status) + assert.Equal(t, refreshedStatus, result.GetEvaluatorRecord(201).Status) + } + + t.Run("sync slow async callback fast", func(t *testing.T) { + t.Parallel() + newCase(t, entity.EvaluatorRunStatusSuccess, false) + }) + t.Run("sync fast relative to async callback", func(t *testing.T) { + t.Parallel() + newCase(t, entity.EvaluatorRunStatusAsyncInvoking, true) + }) +} From 46eec5c773891837ac6b550c59ffb682975a8841 Mon Sep 17 00:00:00 2001 From: shenjiayun Date: Fri, 7 Aug 2026 23:40:12 +0800 Subject: [PATCH 09/15] style(evaluation): format async recovery tests --- .../service/expt_run_item_turn_impl_test.go | 28 +++++++++++++------ 1 file changed, 19 insertions(+), 9 deletions(-) diff --git a/backend/modules/evaluation/domain/service/expt_run_item_turn_impl_test.go b/backend/modules/evaluation/domain/service/expt_run_item_turn_impl_test.go index 335aa06da8..2f9d0e55da 100644 --- a/backend/modules/evaluation/domain/service/expt_run_item_turn_impl_test.go +++ b/backend/modules/evaluation/domain/service/expt_run_item_turn_impl_test.go @@ -5116,9 +5116,13 @@ func TestDefaultExptTurnEvaluationImpl_AsyncTargetCallbackRunsMixedEvaluators(t EvalSetItem: &entity.EvaluationSetItem{ItemID: 5}, Expt: &entity.Experiment{ ID: 3, SpaceID: spaceID, TargetVersionID: 99, - Target: &entity.EvalTarget{ID: 88, EvalTargetType: entity.EvalTargetTypeCustomRPCServer, - EvalTargetVersion: &entity.EvalTargetVersion{ID: 99, EvalTargetType: entity.EvalTargetTypeCustomRPCServer, - CustomRPCServer: &entity.CustomRPCServer{IsAsync: gptr.Of(true)}}}, + Target: &entity.EvalTarget{ + ID: 88, EvalTargetType: entity.EvalTargetTypeCustomRPCServer, + EvalTargetVersion: &entity.EvalTargetVersion{ + ID: 99, EvalTargetType: entity.EvalTargetTypeCustomRPCServer, + CustomRPCServer: &entity.CustomRPCServer{IsAsync: gptr.Of(true)}, + }, + }, Evaluators: allEvaluators, EvalConf: &entity.EvaluationConfiguration{ConnectorConf: entity.Connector{ TargetConf: &entity.TargetConf{TargetVersionID: 99}, @@ -5224,20 +5228,26 @@ func TestDefaultExptTurnEvaluationImpl_AsyncEvaluatorCallbacksPreserveAllRecords }{ { name: "first async callback keeps the other two pending", - statuses: map[int64]entity.EvaluatorRunStatus{101: entity.EvaluatorRunStatusSuccess, 102: entity.EvaluatorRunStatusSuccess, - 201: entity.EvaluatorRunStatusSuccess, 202: entity.EvaluatorRunStatusAsyncInvoking, 203: entity.EvaluatorRunStatusAsyncInvoking}, + statuses: map[int64]entity.EvaluatorRunStatus{ + 101: entity.EvaluatorRunStatusSuccess, 102: entity.EvaluatorRunStatusSuccess, + 201: entity.EvaluatorRunStatusSuccess, 202: entity.EvaluatorRunStatusAsyncInvoking, 203: entity.EvaluatorRunStatusAsyncInvoking, + }, wantAbort: true, }, { name: "second async callback keeps the last one pending", - statuses: map[int64]entity.EvaluatorRunStatus{101: entity.EvaluatorRunStatusSuccess, 102: entity.EvaluatorRunStatusSuccess, - 201: entity.EvaluatorRunStatusSuccess, 202: entity.EvaluatorRunStatusSuccess, 203: entity.EvaluatorRunStatusAsyncInvoking}, + statuses: map[int64]entity.EvaluatorRunStatus{ + 101: entity.EvaluatorRunStatusSuccess, 102: entity.EvaluatorRunStatusSuccess, + 201: entity.EvaluatorRunStatusSuccess, 202: entity.EvaluatorRunStatusSuccess, 203: entity.EvaluatorRunStatusAsyncInvoking, + }, wantAbort: true, }, { name: "last async callback completes the turn", - statuses: map[int64]entity.EvaluatorRunStatus{101: entity.EvaluatorRunStatusSuccess, 102: entity.EvaluatorRunStatusSuccess, - 201: entity.EvaluatorRunStatusSuccess, 202: entity.EvaluatorRunStatusSuccess, 203: entity.EvaluatorRunStatusSuccess}, + statuses: map[int64]entity.EvaluatorRunStatus{ + 101: entity.EvaluatorRunStatusSuccess, 102: entity.EvaluatorRunStatusSuccess, + 201: entity.EvaluatorRunStatusSuccess, 202: entity.EvaluatorRunStatusSuccess, 203: entity.EvaluatorRunStatusSuccess, + }, wantAbort: false, }, } From 99c44eefb9ae68f0228143da335e4f4f1cb8fd0c Mon Sep 17 00:00:00 2001 From: shenjiayun Date: Sat, 8 Aug 2026 09:00:16 +0800 Subject: [PATCH 10/15] fix(evaluation): harden async evaluator recovery M-7362275687 --- .../domain/service/evaluator_impl.go | 9 +++- .../domain/service/evaluator_impl_test.go | 51 +++++++++++++++++++ .../domain/service/expt_run_item_impl.go | 6 ++- .../domain/service/expt_run_item_impl_test.go | 8 ++- 4 files changed, 67 insertions(+), 7 deletions(-) diff --git a/backend/modules/evaluation/domain/service/evaluator_impl.go b/backend/modules/evaluation/domain/service/evaluator_impl.go index f7351a6583..a0aa04d128 100644 --- a/backend/modules/evaluation/domain/service/evaluator_impl.go +++ b/backend/modules/evaluation/domain/service/evaluator_impl.go @@ -1022,7 +1022,14 @@ func (e *EvaluatorServiceImpl) AsyncRunEvaluator(ctx context.Context, request *e UpdatedAt: gptr.Of(now), }, } - if err := e.evaluatorRecordRepo.CreateEvaluatorRecord(ctx, recordDO); err != nil { + // CreateEvaluatorRecord may truncate oversized Content fields in-place before persisting them. + // Persist a deep copy so the provider still receives the original, complete input. + persistedRecord := *recordDO + persistedRecord.EvaluatorInputData = deepCopyEvaluatorInputData(request.InputData) + if persistedRecord.EvaluatorInputData == request.InputData && request.InputData != nil { + return nil, errorx.New("deep copy evaluator input data failed") + } + if err := e.evaluatorRecordRepo.CreateEvaluatorRecord(ctx, &persistedRecord); err != nil { logs.CtxError(ctx, "[AsyncRunEvaluator] CreateEvaluatorRecord fail, invokeID: %d, err: %v", invokeID, err) return nil, err } diff --git a/backend/modules/evaluation/domain/service/evaluator_impl_test.go b/backend/modules/evaluation/domain/service/evaluator_impl_test.go index afbd346f34..fa72dfcd0a 100644 --- a/backend/modules/evaluation/domain/service/evaluator_impl_test.go +++ b/backend/modules/evaluation/domain/service/evaluator_impl_test.go @@ -4587,6 +4587,57 @@ func TestEvaluatorServiceImpl_AsyncRunEvaluator_RejectsBuiltinCustomRPC(t *testi assert.Nil(t, record) } +func TestEvaluatorServiceImpl_AsyncRunEvaluator_PersistsCopyWithoutMutatingProviderInput(t *testing.T) { + t.Parallel() + + ctrl := gomock.NewController(t) + evaluatorRepo := repomocks.NewMockIEvaluatorRepo(ctrl) + limiter := repomocks.NewMockRateLimiter(ctrl) + plainLimiter := repomocks.NewMockIPlainRateLimiter(ctrl) + idGenerator := idgenmocks.NewMockIIDGenerator(ctrl) + recordRepo := repomocks.NewMockIEvaluatorRecordRepo(ctrl) + asyncRepo := repomocks.NewMockIEvalAsyncRepo(ctrl) + source := mocks.NewMockEvaluatorSourceService(ctrl) + + originalText := "full evaluator input that must reach provider" + input := &entity.EvaluatorInputData{InputFields: map[string]*entity.Content{ + "actual_output": {ContentType: gptr.Of(entity.ContentTypeText), Text: gptr.Of(originalText)}, + }} + evaluatorDO := &entity.Evaluator{ + ID: 100, SpaceID: 2, EvaluatorType: entity.EvaluatorTypeAgent, + AgentEvaluatorVersion: &entity.AgentEvaluatorVersion{ID: 101}, + } + evaluatorRepo.EXPECT().BatchGetEvaluatorByVersionID(gomock.Any(), nil, []int64{int64(101)}, false, false).Return([]*entity.Evaluator{evaluatorDO}, nil) + limiter.EXPECT().AllowInvoke(gomock.Any(), int64(2)).Return(true) + plainLimiter.EXPECT().AllowInvokeWithKeyLimit(gomock.Any(), "async_run_evaluator:100", gomock.Any()).Return(true) + idGenerator.EXPECT().GenID(gomock.Any()).Return(int64(999), nil) + recordRepo.EXPECT().CreateEvaluatorRecord(gomock.Any(), gomock.Any()).DoAndReturn(func(_ context.Context, record *entity.EvaluatorRecord) error { + require.NotSame(t, input, record.EvaluatorInputData) + record.EvaluatorInputData.InputFields["actual_output"].SetText("persisted preview") + return nil + }) + asyncRepo.EXPECT().SetEvalAsyncCtx(gomock.Any(), "evaluator:999", gomock.Any()).Return(nil) + source.EXPECT().AsyncRun(gomock.Any(), evaluatorDO, gomock.Any(), gomock.Any(), int64(2), int64(999)).DoAndReturn( + func(_ context.Context, _ *entity.Evaluator, got *entity.EvaluatorInputData, _ *entity.EvaluatorRunConfig, _, _ int64) (map[string]string, string, error) { + require.Same(t, input, got) + assert.Equal(t, originalText, got.InputFields["actual_output"].GetText()) + return nil, "", nil + }, + ) + + record, err := (&EvaluatorServiceImpl{ + evaluatorRepo: evaluatorRepo, limiter: limiter, plainRateLimiter: plainLimiter, + idgen: idGenerator, evaluatorRecordRepo: recordRepo, evalAsyncRepo: asyncRepo, + evaluatorSourceServices: map[entity.EvaluatorType]EvaluatorSourceService{entity.EvaluatorTypeAgent: source}, + }).AsyncRunEvaluator(context.Background(), &entity.AsyncRunEvaluatorRequest{ + SpaceID: 2, EvaluatorVersionID: 101, InputData: input, + }) + require.NoError(t, err) + require.NotNil(t, record) + assert.Equal(t, originalText, input.InputFields["actual_output"].GetText()) + assert.Same(t, input, record.EvaluatorInputData) +} + func TestEvaluatorServiceImpl_AsyncRunEvaluator_CoordinatorOrderAndFailureCompensation(t *testing.T) { t.Parallel() diff --git a/backend/modules/evaluation/domain/service/expt_run_item_impl.go b/backend/modules/evaluation/domain/service/expt_run_item_impl.go index 1ddb8189f8..d341ee50b8 100644 --- a/backend/modules/evaluation/domain/service/expt_run_item_impl.go +++ b/backend/modules/evaluation/domain/service/expt_run_item_impl.go @@ -235,7 +235,11 @@ func (e *ExptItemEvalCtxExecutor) storeTurnRunResult(ctx context.Context, etec * continue } if err := e.evaluatorService.ArmEvaluatorResume(persistCtx, record.ID); err != nil { - return errorx.Wrapf(err, "arm evaluator async resume fail, record_id: %d", record.ID) + // The turn references are already durable and the provider has accepted the work. + // Failing the item here would be unretriable (AsyncAbort sets CtxForceNoRetry) and + // could overwrite a valid terminal callback. Keep the item processing; callbacks can + // retry publication, and the existing zombie policy remains the final fallback. + logs.CtxError(ctx, "[ExptTurnEval] arm evaluator async resume failed after refs persisted, keep item processing, record_id: %d, err: %v", record.ID, err) } } diff --git a/backend/modules/evaluation/domain/service/expt_run_item_impl_test.go b/backend/modules/evaluation/domain/service/expt_run_item_impl_test.go index cec2f8a35e..20fea51a6e 100644 --- a/backend/modules/evaluation/domain/service/expt_run_item_impl_test.go +++ b/backend/modules/evaluation/domain/service/expt_run_item_impl_test.go @@ -1172,7 +1172,7 @@ func TestExptItemEvalCtxExecutor_storeTurnRunResult_DoesNotArmWhenSaveFails(t *t require.Error(t, executor.storeTurnRunResult(context.Background(), etec, result)) } -func TestExptItemEvalCtxExecutor_storeTurnRunResult_ReturnsArmError(t *testing.T) { +func TestExptItemEvalCtxExecutor_storeTurnRunResult_ArmErrorKeepsAsyncProcessing(t *testing.T) { t.Parallel() ctrl := gomock.NewController(t) @@ -1193,12 +1193,10 @@ func TestExptItemEvalCtxExecutor_storeTurnRunResult_ReturnsArmError(t *testing.T }}, }, } - result := &entity.ExptTurnRunResult{EvaluatorResults: []*entity.EvaluatorRecord{{ + result := &entity.ExptTurnRunResult{AsyncAbort: true, EvaluatorResults: []*entity.EvaluatorRecord{{ ID: 100, EvaluatorVersionID: 101, Status: entity.EvaluatorRunStatusAsyncInvoking, }}} - err := executor.storeTurnRunResult(context.Background(), etec, result) - require.Error(t, err) - assert.Contains(t, err.Error(), "arm evaluator async resume fail") + require.NoError(t, executor.storeTurnRunResult(context.Background(), etec, result)) } func TestExptItemEvalCtxExecutor_storeTurnRunResult_ArmsEveryPendingEvaluator(t *testing.T) { From 7f962800ed11a1353690fdf1563ffcd90cd4daa7 Mon Sep 17 00:00:00 2001 From: shenjiayun Date: Sat, 8 Aug 2026 22:49:33 +0800 Subject: [PATCH 11/15] fix(evaluation): make async resume recovery durable M-7362275687 --- .../application/eval_openapi_app.go | 13 +- .../application/eval_openapi_app_test.go | 56 +++-- .../evaluation/application/evaluator_app.go | 5 +- .../evaluation/domain/entity/expt_run.go | 14 +- .../modules/evaluation/domain/entity/param.go | 2 +- .../modules/evaluation/domain/repo/expt.go | 2 +- .../domain/service/evaluator_impl.go | 2 +- .../domain/service/evaluator_impl_test.go | 6 +- .../domain/service/expt_run_item_impl.go | 36 +++- .../domain/service/expt_run_item_impl_test.go | 33 +++ .../domain/service/expt_run_item_turn_impl.go | 18 +- .../service/expt_run_item_turn_impl_test.go | 2 + .../service/expt_run_scheduler_event_impl.go | 143 +++++++++++++ .../expt_run_scheduler_event_impl_test.go | 191 ++++++++++++++++++ .../convert/item_turn_eval_async_test.go | 22 +- .../redis/dao/item_turn_eval_async.go | 23 ++- .../dao/item_turn_eval_async_primary_test.go | 62 ++++++ .../redis/dao/item_turn_eval_async_test.go | 20 +- 18 files changed, 599 insertions(+), 51 deletions(-) create mode 100644 backend/modules/evaluation/infra/repo/experiment/redis/dao/item_turn_eval_async_primary_test.go diff --git a/backend/modules/evaluation/application/eval_openapi_app.go b/backend/modules/evaluation/application/eval_openapi_app.go index b94612f1d9..842babe571 100644 --- a/backend/modules/evaluation/application/eval_openapi_app.go +++ b/backend/modules/evaluation/application/eval_openapi_app.go @@ -3002,10 +3002,11 @@ func (e *EvalOpenAPIApplication) AsyncRunEvaluatorOApi(ctx context.Context, req EvaluatorRunConf: runConf, Ext: req.Ext, AsyncCtx: &entity.EvalAsyncCtx{ - Session: &entity.Session{UserID: usersession.UserIDInCtxOrEmpty(ctx)}, - CallbackURL: req.GetCallbackURL(), - ResumeReady: true, - AsyncUnixMS: startTime.UnixMilli(), + Session: &entity.Session{UserID: usersession.UserIDInCtxOrEmpty(ctx)}, + CallbackURL: req.GetCallbackURL(), + ResumeBarrierEnabled: true, + ResumeReady: true, + AsyncUnixMS: startTime.UnixMilli(), }, }) if err != nil { @@ -3582,7 +3583,7 @@ func (e *EvalOpenAPIApplication) ReportEvaluatorInvokeResult_(ctx context.Contex return &openapi.ReportEvaluatorInvokeResultResponse{BaseResp: base.NewBaseResp()}, nil } if actx.Event != nil { - if !actx.ResumeReady { + if !actx.CanResumeExperiment() { latestCtx, ctxErr := e.asyncRepo.GetEvalAsyncCtxStrong(ctx, asyncCtxKey) if ctxErr != nil { return nil, ctxErr @@ -3591,7 +3592,7 @@ func (e *EvalOpenAPIApplication) ReportEvaluatorInvokeResult_(ctx context.Contex actx = latestCtx } } - if actx.ResumeReady { + if actx.CanResumeExperiment() { if err := e.publisher.PublishExptRecordEvalEvent(ctx, actx.Event, gptr.Of(time.Second*3), func(event *entity.ExptItemEvalEvent) { event.AsyncEvaluatorReportTrigger = true }); err != nil { diff --git a/backend/modules/evaluation/application/eval_openapi_app_test.go b/backend/modules/evaluation/application/eval_openapi_app_test.go index 6acc29f786..1fdc4f2d62 100755 --- a/backend/modules/evaluation/application/eval_openapi_app_test.go +++ b/backend/modules/evaluation/application/eval_openapi_app_test.go @@ -6735,15 +6735,17 @@ func TestEvalOpenAPIApplication_ReportEvaluatorInvokeResult(t *testing.T) { setup: func(auth *rpcmocks.MockIAuthProvider, asyncRepo *repomocks.MockIEvalAsyncRepo, evaluatorSvc *servicemocks.MockEvaluatorService, publisher *eventmocks.MockExptEventPublisher, _ *servicemocks.MockIEvaluatorCallbackDispatcher) { auth.EXPECT().Authorization(gomock.Any(), gomock.Any()).Return(nil) asyncRepo.EXPECT().GetEvalAsyncCtxStrong(gomock.Any(), "evaluator:2002").Return(&entity.EvalAsyncCtx{ - Event: event, - ResumeReady: false, - EvaluatorVersionID: 9, + Event: event, + ResumeBarrierEnabled: true, + ResumeReady: false, + EvaluatorVersionID: 9, }, nil) evaluatorSvc.EXPECT().ReportEvaluatorInvokeResult(gomock.Any(), gomock.Any()).Return(entity.ReportEvaluatorResultApplied, nil) asyncRepo.EXPECT().GetEvalAsyncCtxStrong(gomock.Any(), "evaluator:2002").Return(&entity.EvalAsyncCtx{ - Event: event, - ResumeReady: false, - EvaluatorVersionID: 9, + Event: event, + ResumeBarrierEnabled: true, + ResumeReady: false, + EvaluatorVersionID: 9, }, nil) publisher.EXPECT().PublishExptRecordEvalEvent(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Times(0) }, @@ -6762,9 +6764,10 @@ func TestEvalOpenAPIApplication_ReportEvaluatorInvokeResult(t *testing.T) { auth.EXPECT().Authorization(gomock.Any(), gomock.Any()).Return(nil) gomock.InOrder( asyncRepo.EXPECT().GetEvalAsyncCtxStrong(gomock.Any(), "evaluator:2002").Return(&entity.EvalAsyncCtx{ - Event: event, - ResumeReady: false, - EvaluatorVersionID: 9, + Event: event, + ResumeBarrierEnabled: true, + ResumeReady: false, + EvaluatorVersionID: 9, }, nil), evaluatorSvc.EXPECT().ReportEvaluatorInvokeResult(gomock.Any(), gomock.Any()).Return(entity.ReportEvaluatorResultApplied, nil), asyncRepo.EXPECT().GetEvalAsyncCtxStrong(gomock.Any(), "evaluator:2002").Return(&entity.EvalAsyncCtx{ @@ -6790,9 +6793,10 @@ func TestEvalOpenAPIApplication_ReportEvaluatorInvokeResult(t *testing.T) { auth.EXPECT().Authorization(gomock.Any(), gomock.Any()).Return(nil) gomock.InOrder( asyncRepo.EXPECT().GetEvalAsyncCtxStrong(gomock.Any(), "evaluator:2002").Return(&entity.EvalAsyncCtx{ - Event: event, - ResumeReady: false, - EvaluatorVersionID: 9, + Event: event, + ResumeBarrierEnabled: true, + ResumeReady: false, + EvaluatorVersionID: 9, }, nil), evaluatorSvc.EXPECT().ReportEvaluatorInvokeResult(gomock.Any(), gomock.Any()).Return(entity.ReportEvaluatorResultApplied, nil), asyncRepo.EXPECT().GetEvalAsyncCtxStrong(gomock.Any(), "evaluator:2002").Return(nil, errors.New("redis reread failed")), @@ -8622,6 +8626,7 @@ func TestEvalOpenAPIApplication_AsyncRunEvaluatorOApi(t *testing.T) { auth.EXPECT().AuthorizationWithoutSPI(gomock.Any(), gomock.Any()).Return(nil) evaluatorSvc.EXPECT().AsyncRunEvaluator(gomock.Any(), gomock.Any()).DoAndReturn(func(_ context.Context, req *entity.AsyncRunEvaluatorRequest) (*entity.EvaluatorRecord, error) { require.NotNil(t, req.AsyncCtx) + assert.True(t, req.AsyncCtx.ResumeBarrierEnabled) assert.True(t, req.AsyncCtx.ResumeReady) assert.Nil(t, req.AsyncCtx.Event) return record, nil @@ -8640,6 +8645,7 @@ func TestEvalOpenAPIApplication_AsyncRunEvaluatorOApi(t *testing.T) { evaluatorSvc.EXPECT().GetEvaluatorVersion(gomock.Any(), gomock.Any(), evaluatorVersionID, false, false).Return(evaluator, nil) evaluatorSvc.EXPECT().AsyncRunEvaluator(gomock.Any(), gomock.Any()).DoAndReturn(func(_ context.Context, req *entity.AsyncRunEvaluatorRequest) (*entity.EvaluatorRecord, error) { require.NotNil(t, req.AsyncCtx) + assert.True(t, req.AsyncCtx.ResumeBarrierEnabled) assert.True(t, req.AsyncCtx.ResumeReady) assert.Nil(t, req.AsyncCtx.Event) return record, nil @@ -9006,3 +9012,29 @@ func TestEvalOpenAPIApplication_ListEvaluationSetVersionsOApi_SharedExecuteHides assert.Nil(t, resp.Data.Versions[0].EvaluationSetSchema) assert.NotNil(t, version.EvaluationSetSchema) } + +func TestEvalOpenAPIApplication_ReportEvaluatorInvokeResult_LegacyContextResumesWithoutReread(t *testing.T) { + t.Parallel() + + ctrl := gomock.NewController(t) + auth := rpcmocks.NewMockIAuthProvider(ctrl) + asyncRepo := repomocks.NewMockIEvalAsyncRepo(ctrl) + evaluatorSvc := servicemocks.NewMockEvaluatorService(ctrl) + publisher := eventmocks.NewMockExptEventPublisher(ctrl) + dispatcher := servicemocks.NewMockIEvaluatorCallbackDispatcher(ctrl) + event := &entity.ExptItemEvalEvent{ExptID: 1, ExptRunID: 2, EvalSetItemID: 3} + + auth.EXPECT().Authorization(gomock.Any(), gomock.Any()).Return(nil) + asyncRepo.EXPECT().GetEvalAsyncCtxStrong(gomock.Any(), "evaluator:2002").Return(&entity.EvalAsyncCtx{ + Event: event, EvaluatorVersionID: 9, + }, nil).Times(1) + evaluatorSvc.EXPECT().ReportEvaluatorInvokeResult(gomock.Any(), gomock.Any()).Return(entity.ReportEvaluatorResultApplied, nil) + publisher.EXPECT().PublishExptRecordEvalEvent(gomock.Any(), event, gomock.Any(), gomock.Any()).Return(nil) + + app := &EvalOpenAPIApplication{auth: auth, asyncRepo: asyncRepo, evaluatorService: evaluatorSvc, publisher: publisher, callbackDispatcher: dispatcher} + _, err := app.ReportEvaluatorInvokeResult_(context.Background(), &openapi.ReportEvaluatorInvokeResultRequest{ + WorkspaceID: gptr.Of(int64(1001)), InvokeID: gptr.Of(int64(2002)), Status: gptr.Of(spi.InvokeEvaluatorRunStatus_SUCCESS), + Output: &spi.InvokeEvaluatorOutputData{EvaluatorResult_: &spi.InvokeEvaluatorResult_{Score: gptr.Of(float64(1))}}, + }) + require.NoError(t, err) +} diff --git a/backend/modules/evaluation/application/evaluator_app.go b/backend/modules/evaluation/application/evaluator_app.go index a636635026..a39522ced1 100644 --- a/backend/modules/evaluation/application/evaluator_app.go +++ b/backend/modules/evaluation/application/evaluator_app.go @@ -2104,8 +2104,9 @@ func (e *EvaluatorHandlerImpl) AsyncRunEvaluator(ctx context.Context, req *evalu } asyncReq := buildAsyncRunEvaluatorRequest(evaluatorDO.Name, req) asyncReq.AsyncCtx = &entity.EvalAsyncCtx{ - Session: &entity.Session{UserID: session.UserIDInCtxOrEmpty(ctx)}, - ResumeReady: true, + Session: &entity.Session{UserID: session.UserIDInCtxOrEmpty(ctx)}, + ResumeBarrierEnabled: true, + ResumeReady: true, } resp, err := e.evaluatorService.AsyncRunEvaluator(ctx, asyncReq) if err != nil { diff --git a/backend/modules/evaluation/domain/entity/expt_run.go b/backend/modules/evaluation/domain/entity/expt_run.go index a5bbb2f558..c9e4a4b890 100644 --- a/backend/modules/evaluation/domain/entity/expt_run.go +++ b/backend/modules/evaluation/domain/entity/expt_run.go @@ -636,6 +636,15 @@ type ExptSchedulerMode interface { PublishResult(ctx context.Context, turnEvaluatorRefs []*ExptTurnEvaluatorResultRef, event *ExptScheduleEvent) error } +func (e *EvalAsyncCtx) CanResumeExperiment() bool { + if e == nil { + return false + } + // Contexts written before the ResumeReady barrier existed have neither field. + // Preserve their historical behavior during rolling upgrades. + return !e.ResumeBarrierEnabled || e.ResumeReady +} + type CKDBConfig struct { ExptTurnResultFilterDBName string `json:"expt_turn_result_filter_db_name" mapstructure:"expt_turn_result_filter_db_name"` DatasetItemsSnapshotDBName string `json:"dataset_items_snapshot_db_name" mapstructure:"dataset_items_snapshot_db_name"` @@ -649,8 +658,9 @@ type EvalAsyncCtx struct { Callee string EvaluatorVersionID int64 // evaluator version id, used for evaluator async scenario EnableExtractTrajectory *bool - ResumeReady bool `json:"resume_ready,omitempty"` // experiment turn refs are durable and callback may resume scheduling - CallbackURL string `json:"callback_url,omitempty"` // 异步执行完成后回调通知的 URL，为空则不回调 + ResumeBarrierEnabled bool `json:"resume_barrier_enabled,omitempty"` // new experiment evaluator contexts require refs to be durable before resume + ResumeReady bool `json:"resume_ready,omitempty"` // experiment turn refs are durable and callback may resume scheduling + CallbackURL string `json:"callback_url,omitempty"` // 异步执行完成后回调通知的 URL，为空则不回调 // 下述字段用于沙箱内部 step 上报的 tag 反查, 由 target async 写入位点从 etec 填充, // 调试场景 (无实验上下文) 保留零值, 由上报侧回退为占位符. TargetID int64 diff --git a/backend/modules/evaluation/domain/entity/param.go b/backend/modules/evaluation/domain/entity/param.go index f672e9846b..93e2c5d139 100644 --- a/backend/modules/evaluation/domain/entity/param.go +++ b/backend/modules/evaluation/domain/entity/param.go @@ -327,7 +327,7 @@ type AsyncRunEvaluatorRequest struct { Alias string `json:"alias,omitempty"` SourceType EvaluatorRecordSourceType `json:"source_type,omitempty"` // AsyncCtx is persisted after the AsyncInvoking record is created and before the provider is dispatched. - // Experiment calls set ResumeReady=false; direct calls set it true. + // New experiment calls enable the resume barrier and set ResumeReady=false; direct calls set both enabled and ready. AsyncCtx *EvalAsyncCtx `json:"-"` } diff --git a/backend/modules/evaluation/domain/repo/expt.go b/backend/modules/evaluation/domain/repo/expt.go index a3bc06c95c..392843b35f 100644 --- a/backend/modules/evaluation/domain/repo/expt.go +++ b/backend/modules/evaluation/domain/repo/expt.go @@ -146,7 +146,7 @@ type IExptResultExportRecordRepo interface { type IEvalAsyncRepo interface { GetEvalAsyncCtx(ctx context.Context, invokeID string) (*entity.EvalAsyncCtx, error) - // GetEvalAsyncCtxStrong uses the shared direct Redis client and retries a transient missing key at 50/100/200ms. + // GetEvalAsyncCtxStrong reads through a Redis script (master-routed command) and retries a transient missing key at 50/100/200ms. GetEvalAsyncCtxStrong(ctx context.Context, invokeID string) (*entity.EvalAsyncCtx, error) SetEvalAsyncCtx(ctx context.Context, invokeID string, actx *entity.EvalAsyncCtx) error // MarkEvalAsyncResumeReady atomically persists ResumeReady=true and returns the latest context. diff --git a/backend/modules/evaluation/domain/service/evaluator_impl.go b/backend/modules/evaluation/domain/service/evaluator_impl.go index a0aa04d128..fa6b3abbdf 100644 --- a/backend/modules/evaluation/domain/service/evaluator_impl.go +++ b/backend/modules/evaluation/domain/service/evaluator_impl.go @@ -1036,7 +1036,7 @@ func (e *EvaluatorServiceImpl) AsyncRunEvaluator(ctx context.Context, request *e asyncCtx := request.AsyncCtx if asyncCtx == nil { - asyncCtx = &entity.EvalAsyncCtx{ResumeReady: true} + asyncCtx = &entity.EvalAsyncCtx{ResumeBarrierEnabled: true, ResumeReady: true} } asyncCtx.RecordID = invokeID asyncCtx.EvaluatorVersionID = request.EvaluatorVersionID diff --git a/backend/modules/evaluation/domain/service/evaluator_impl_test.go b/backend/modules/evaluation/domain/service/evaluator_impl_test.go index fa72dfcd0a..0eed592ccb 100644 --- a/backend/modules/evaluation/domain/service/evaluator_impl_test.go +++ b/backend/modules/evaluation/domain/service/evaluator_impl_test.go @@ -4682,8 +4682,9 @@ func TestEvaluatorServiceImpl_AsyncRunEvaluator_CoordinatorOrderAndFailureCompen EvaluatorVersionID: 101, InputData: &entity.EvaluatorInputData{}, AsyncCtx: &entity.EvalAsyncCtx{ - Event: &entity.ExptItemEvalEvent{ExptID: 10}, - ResumeReady: false, + Event: &entity.ExptItemEvalEvent{ExptID: 10}, + ResumeBarrierEnabled: true, + ResumeReady: false, }, } } @@ -4701,6 +4702,7 @@ func TestEvaluatorServiceImpl_AsyncRunEvaluator_CoordinatorOrderAndFailureCompen order = append(order, "context") assert.Equal(t, int64(999), actx.RecordID) assert.Equal(t, int64(101), actx.EvaluatorVersionID) + assert.True(t, actx.ResumeBarrierEnabled) assert.False(t, actx.ResumeReady) return nil }), diff --git a/backend/modules/evaluation/domain/service/expt_run_item_impl.go b/backend/modules/evaluation/domain/service/expt_run_item_impl.go index d341ee50b8..aadbc8e90b 100644 --- a/backend/modules/evaluation/domain/service/expt_run_item_impl.go +++ b/backend/modules/evaluation/domain/service/expt_run_item_impl.go @@ -10,6 +10,7 @@ import ( "strings" "time" + "github.com/bytedance/gg/gmap" "github.com/bytedance/gg/gptr" "github.com/jinzhu/copier" @@ -77,7 +78,10 @@ type ExptItemEvalCtxExecutor struct { sandboxAgentNotifier ISandboxAgentNotifier // 沙箱 agent 实验单行失败飞书通知; 可空 } -const exptRunLogPersistTimeout = 5 * time.Second +const ( + exptRunLogPersistTimeout = 5 * time.Second + asyncEvaluatorResumeRepairExtKey = "__async_evaluator_resume_repair__" +) func (e *ExptItemEvalCtxExecutor) Eval(ctx context.Context, eiec *entity.ExptItemEvalCtx) error { // if err := e.SetItemRunProcessing(ctx, event.ExptID, event.ExptRunID, event.EvalSetItemID, event.SpaceID, event.Session); err != nil { @@ -153,7 +157,7 @@ func (e *ExptItemEvalCtxExecutor) storeTurnRunResult(ctx context.Context, etec * return errorx.Wrapf(err, "ExptTurnResultRunLog copy fail") } - clone.Ext = etec.Ext + clone.Ext = gmap.Clone(etec.Ext) var evalErr error @@ -227,19 +231,31 @@ func (e *ExptItemEvalCtxExecutor) storeTurnRunResult(ctx context.Context, etec * result.SetEvalErr(evalErr) + pendingEvaluatorRecordIDs := make([]int64, 0) + for _, record := range result.EvaluatorResults { + if record != nil && record.ID > 0 && record.Status == entity.EvaluatorRunStatusAsyncInvoking { + pendingEvaluatorRecordIDs = append(pendingEvaluatorRecordIDs, record.ID) + } + } + if len(pendingEvaluatorRecordIDs) > 0 { + if clone.Ext == nil { + clone.Ext = make(map[string]string) + } + // Durable repair intent: if arm or MQ publication fails after this save, the scheduler + // can recover from turn refs plus terminal evaluator records without another provider callback. + clone.Ext[asyncEvaluatorResumeRepairExtKey] = "true" + } + if err := e.TurnResultRepo.SaveTurnRunLogs(persistCtx, []*entity.ExptTurnResultRunLog{clone}); err != nil { return err } - for _, record := range result.EvaluatorResults { - if record == nil || record.ID <= 0 || record.Status != entity.EvaluatorRunStatusAsyncInvoking { - continue - } - if err := e.evaluatorService.ArmEvaluatorResume(persistCtx, record.ID); err != nil { + for _, recordID := range pendingEvaluatorRecordIDs { + if err := e.evaluatorService.ArmEvaluatorResume(persistCtx, recordID); err != nil { // The turn references are already durable and the provider has accepted the work. // Failing the item here would be unretriable (AsyncAbort sets CtxForceNoRetry) and - // could overwrite a valid terminal callback. Keep the item processing; callbacks can - // retry publication, and the existing zombie policy remains the final fallback. - logs.CtxError(ctx, "[ExptTurnEval] arm evaluator async resume failed after refs persisted, keep item processing, record_id: %d, err: %v", record.ID, err) + // could overwrite a valid terminal callback. Keep the item processing; the scheduler + // repairs from the durable turn refs/record state, with Zombie only as final timeout. + logs.CtxError(ctx, "[ExptTurnEval] arm evaluator async resume failed after refs persisted, keep item processing, record_id: %d, err: %v", recordID, err) } } diff --git a/backend/modules/evaluation/domain/service/expt_run_item_impl_test.go b/backend/modules/evaluation/domain/service/expt_run_item_impl_test.go index 20fea51a6e..a58f3705b3 100644 --- a/backend/modules/evaluation/domain/service/expt_run_item_impl_test.go +++ b/backend/modules/evaluation/domain/service/expt_run_item_impl_test.go @@ -1292,3 +1292,36 @@ func TestExptItemEvalCtxExecutor_storeTurnRunResult_AllEvaluatorsTerminalComplet } require.NoError(t, executor.storeTurnRunResult(context.Background(), etec, result)) } + +func TestExptItemEvalCtxExecutor_storeTurnRunResult_ArmErrorPersistsResumeRepairMarker(t *testing.T) { + t.Parallel() + + ctrl := gomock.NewController(t) + turnRepo := repomocks.NewMockIExptTurnResultRepo(ctrl) + evaluatorSvc := servicemocks.NewMockEvaluatorService(ctrl) + turnRepo.EXPECT().SaveTurnRunLogs(gomock.Any(), gomock.Any()).DoAndReturn( + func(_ context.Context, logs []*entity.ExptTurnResultRunLog) error { + require.Len(t, logs, 1) + assert.Equal(t, "true", logs[0].Ext[asyncEvaluatorResumeRepairExtKey]) + return nil + }, + ) + evaluatorSvc.EXPECT().ArmEvaluatorResume(gomock.Any(), int64(100)).Return(errors.New("arm failed")) + + executor := &ExptItemEvalCtxExecutor{TurnResultRepo: turnRepo, evaluatorService: evaluatorSvc} + etec := &entity.ExptTurnEvalCtx{ + Turn: &entity.Turn{ID: 1}, + ExptItemEvalCtx: &entity.ExptItemEvalCtx{ + Expt: &entity.Experiment{ID: 1, SpaceID: 2}, + Event: &entity.ExptItemEvalEvent{ExptRunID: 3}, + EvalSetItem: &entity.EvaluationSetItem{ItemID: 4}, + ExistItemEvalResult: &entity.ExptItemEvalResult{TurnResultRunLogs: map[int64]*entity.ExptTurnResultRunLog{ + 1: {ID: 5, TurnID: 1}, + }}, + }, + } + result := &entity.ExptTurnRunResult{AsyncAbort: true, EvaluatorResults: []*entity.EvaluatorRecord{{ + ID: 100, EvaluatorVersionID: 101, Status: entity.EvaluatorRunStatusAsyncInvoking, + }}} + require.NoError(t, executor.storeTurnRunResult(context.Background(), etec, result)) +} diff --git a/backend/modules/evaluation/domain/service/expt_run_item_turn_impl.go b/backend/modules/evaluation/domain/service/expt_run_item_turn_impl.go index 45ad15267f..4fc79b25b0 100644 --- a/backend/modules/evaluation/domain/service/expt_run_item_turn_impl.go +++ b/backend/modules/evaluation/domain/service/expt_run_item_turn_impl.go @@ -927,10 +927,11 @@ func (e *DefaultExptTurnEvaluationImpl) asyncCallEvaluatorWithAlias( Alias: alias, SourceType: entity.EvaluatorRecordSourceTypeBuiltin, AsyncCtx: &entity.EvalAsyncCtx{ - Event: etec.Event, - Session: etec.Event.Session, - EvaluatorVersionID: ev.GetEvaluatorVersionID(), - ResumeReady: false, + Event: etec.Event, + Session: etec.Event.Session, + EvaluatorVersionID: ev.GetEvaluatorVersionID(), + ResumeBarrierEnabled: true, + ResumeReady: false, }, }) if evaluatorRecord != nil { @@ -964,10 +965,11 @@ func (e *DefaultExptTurnEvaluationImpl) asyncCallEvaluator( Ext: etec.Ext, EvaluatorRunConf: ec.RunConf, AsyncCtx: &entity.EvalAsyncCtx{ - Event: etec.Event, - Session: etec.Event.Session, - EvaluatorVersionID: ev.GetEvaluatorVersionID(), - ResumeReady: false, + Event: etec.Event, + Session: etec.Event.Session, + EvaluatorVersionID: ev.GetEvaluatorVersionID(), + ResumeBarrierEnabled: true, + ResumeReady: false, }, } evaluatorRecord, err := e.evaluatorService.AsyncRunEvaluator(ctx, asyncReq) diff --git a/backend/modules/evaluation/domain/service/expt_run_item_turn_impl_test.go b/backend/modules/evaluation/domain/service/expt_run_item_turn_impl_test.go index 2f9d0e55da..21089ef016 100644 --- a/backend/modules/evaluation/domain/service/expt_run_item_turn_impl_test.go +++ b/backend/modules/evaluation/domain/service/expt_run_item_turn_impl_test.go @@ -394,6 +394,7 @@ func TestDefaultExptTurnEvaluationImpl_asyncCallEvaluator_Agent(t *testing.T) { assert.Equal(t, etec.Ext, req.Ext) require.NotNil(t, req.AsyncCtx) assert.Equal(t, etec.Event, req.AsyncCtx.Event) + assert.True(t, req.AsyncCtx.ResumeBarrierEnabled) assert.False(t, req.AsyncCtx.ResumeReady) return mockEvaluatorRecord, nil }, @@ -5025,6 +5026,7 @@ func TestDefaultExptTurnEvaluationImpl_asyncCallEvaluatorWithAlias_CustomRPC(t * assert.Same(t, runConf, req.EvaluatorRunConf) require.NotNil(t, req.AsyncCtx) assert.Same(t, etec.Event, req.AsyncCtx.Event) + assert.True(t, req.AsyncCtx.ResumeBarrierEnabled) assert.False(t, req.AsyncCtx.ResumeReady) return &entity.EvaluatorRecord{ID: 7, EvaluatorVersionID: 101, Alias: "judge_a", Status: entity.EvaluatorRunStatusAsyncInvoking}, nil }, diff --git a/backend/modules/evaluation/domain/service/expt_run_scheduler_event_impl.go b/backend/modules/evaluation/domain/service/expt_run_scheduler_event_impl.go index d43b9eb07e..7ca8f197b3 100644 --- a/backend/modules/evaluation/domain/service/expt_run_scheduler_event_impl.go +++ b/backend/modules/evaluation/domain/service/expt_run_scheduler_event_impl.go @@ -392,6 +392,10 @@ func (e *ExptSchedulerImpl) schedule(ctx context.Context, event *entity.ExptSche return err } + if err := e.reconcileTerminalAsyncEvaluatorItems(ctx, event, exptDetail, incomplete); err != nil { + return err + } + incomplete, zombies, err := e.handleZombies(ctx, event, incomplete, exptDetail) if err != nil { return err @@ -553,6 +557,145 @@ func (e *ExptSchedulerImpl) handleToSubmits(ctx context.Context, event *entity.E return nil } +// reconcileTerminalAsyncEvaluatorItems repairs a lost evaluator callback resume event. +// Turn evaluator refs and evaluator record terminal states are durable DB facts, so the +// scheduler can safely republish recovery without relying on the provider to callback again. +func (e *ExptSchedulerImpl) reconcileTerminalAsyncEvaluatorItems(ctx context.Context, event *entity.ExptScheduleEvent, expt *entity.Experiment, items []*entity.ExptEvalItem) error { + if event == nil || expt == nil || len(items) == 0 || e.ExptTurnResultRepo == nil || e.EvaluatorRecordRepo == nil || e.Publisher == nil { + return nil + } + + if !expt.AsyncCallEvaluators() { + return nil + } + + processingItemIDs := make([]int64, 0, len(items)) + for _, item := range items { + if item != nil && item.State == entity.ItemRunState_Processing && item.ItemID > 0 { + processingItemIDs = append(processingItemIDs, item.ItemID) + } + } + if len(processingItemIDs) == 0 { + return nil + } + + turnRunLogs, err := e.ExptTurnResultRepo.MGetItemTurnRunLogs(ctx, event.ExptID, event.ExptRunID, processingItemIDs, event.SpaceID) + if err != nil { + logs.CtxError(ctx, "[ExptEval] read turn refs for async evaluator repair failed, keep repair pending, expt_id: %d, expt_run_id: %d, err: %v", + event.ExptID, event.ExptRunID, err) + return nil + } + + itemRecordIDs := make(map[int64]map[int64]struct{}) + allRecordIDs := make([]int64, 0) + seenRecordIDs := make(map[int64]struct{}) + for _, runLog := range turnRunLogs { + if runLog == nil || runLog.ItemID <= 0 || runLog.Status != entity.TurnRunState_Processing || + runLog.Ext[asyncEvaluatorResumeRepairExtKey] != "true" || runLog.EvaluatorResultIds == nil { + continue + } + ids := make([]int64, 0) + seen := make(map[int64]struct{}) + appendID := func(id int64) { + if id <= 0 { + return + } + if _, exists := seen[id]; exists { + return + } + seen[id] = struct{}{} + ids = append(ids, id) + } + if runLog.EvaluatorResultIds.IsNewFormat() { + for _, result := range runLog.EvaluatorResultIds.Registered { + if result != nil { + appendID(result.RecordID) + } + } + for _, result := range runLog.EvaluatorResultIds.Inline { + if result != nil { + appendID(result.RecordID) + } + } + } else { + for _, id := range runLog.EvaluatorResultIds.EvalVerIDToResID { + appendID(id) + } + } + if len(ids) == 0 { + continue + } + set := itemRecordIDs[runLog.ItemID] + if set == nil { + set = make(map[int64]struct{}) + itemRecordIDs[runLog.ItemID] = set + } + for _, id := range ids { + set[id] = struct{}{} + if _, exists := seenRecordIDs[id]; !exists { + seenRecordIDs[id] = struct{}{} + allRecordIDs = append(allRecordIDs, id) + } + } + } + if len(allRecordIDs) == 0 { + return nil + } + + records, err := e.EvaluatorRecordRepo.BatchGetEvaluatorRecord(contexts.WithCtxWriteDB(ctx), allRecordIDs, false, false, entity.WithoutLoadStorageData()) + if err != nil { + logs.CtxError(ctx, "[ExptEval] read evaluator records for async repair failed, keep repair pending, expt_id: %d, expt_run_id: %d, err: %v", + event.ExptID, event.ExptRunID, err) + return nil + } + recordStatus := make(map[int64]entity.EvaluatorRunStatus, len(records)) + for _, record := range records { + if record != nil { + recordStatus[record.ID] = record.Status + } + } + + for _, item := range items { + if item == nil || item.State != entity.ItemRunState_Processing { + continue + } + ids := itemRecordIDs[item.ItemID] + if len(ids) == 0 { + continue + } + allTerminal := true + for id := range ids { + status, exists := recordStatus[id] + if !exists || status == entity.EvaluatorRunStatusAsyncInvoking || status == entity.EvaluatorRunStatusUnknown { + allTerminal = false + break + } + } + if !allTerminal { + continue + } + itemEvent := &entity.ExptItemEvalEvent{ + SpaceID: event.SpaceID, + ExptID: event.ExptID, + ExptRunID: event.ExptRunID, + ExptRunMode: event.ExptRunMode, + EvalSetItemID: item.ItemID, + CreateAt: time.Now().Unix(), + Ext: event.Ext, + Session: event.Session, + } + if err := e.Publisher.PublishExptRecordEvalEvent(ctx, itemEvent, gptr.Of(time.Second*3), func(resume *entity.ExptItemEvalEvent) { + resume.AsyncEvaluatorReportTrigger = true + }); err != nil { + // The repair marker and terminal DB records remain durable. A later scheduler tick + // will retry, so an MQ outage here must not fail the entire experiment. + logs.CtxError(ctx, "[ExptEval] republish async evaluator resume event failed, keep repair pending, expt_id: %d, expt_run_id: %d, item_id: %d, err: %v", + event.ExptID, event.ExptRunID, item.ItemID, err) + } + } + return nil +} + func (e *ExptSchedulerImpl) handleZombies(ctx context.Context, event *entity.ExptScheduleEvent, items []*entity.ExptEvalItem, expt *entity.Experiment) (alives, zombies []*entity.ExptEvalItem, err error) { asyncExec := false if expt != nil { diff --git a/backend/modules/evaluation/domain/service/expt_run_scheduler_event_impl_test.go b/backend/modules/evaluation/domain/service/expt_run_scheduler_event_impl_test.go index d7064c74b2..b53b8cd0dc 100644 --- a/backend/modules/evaluation/domain/service/expt_run_scheduler_event_impl_test.go +++ b/backend/modules/evaluation/domain/service/expt_run_scheduler_event_impl_test.go @@ -12,6 +12,7 @@ import ( "time" "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" "go.uber.org/mock/gomock" auditmocks "github.com/coze-dev/coze-loop/backend/infra/external/audit/mocks" @@ -2118,3 +2119,193 @@ func TestIsSandboxAgentExpt(t *testing.T) { EvalTargetVersion: &entity.EvalTargetVersion{EvalTargetType: entity.EvalTargetTypeSandboxAgent}, }})) } + +func TestExptSchedulerImpl_ReconcileTerminalAsyncEvaluatorItems(t *testing.T) { + t.Parallel() + + ctrl := gomock.NewController(t) + turnRepo := mock_repo.NewMockIExptTurnResultRepo(ctrl) + recordRepo := mock_repo.NewMockIEvaluatorRecordRepo(ctrl) + publisher := eventmocks.NewMockExptEventPublisher(ctrl) + + items := []*entity.ExptEvalItem{{ExptID: 1, ItemID: 10, State: entity.ItemRunState_Processing}} + event := &entity.ExptScheduleEvent{SpaceID: 3, ExptID: 1, ExptRunID: 2, ExptRunMode: entity.EvaluationModeSubmit, Session: &entity.Session{UserID: "u"}} + turnRepo.EXPECT().MGetItemTurnRunLogs(gomock.Any(), int64(1), int64(2), []int64{10}, int64(3)).Return([]*entity.ExptTurnResultRunLog{{ + ItemID: 10, + Status: entity.TurnRunState_Processing, + Ext: map[string]string{asyncEvaluatorResumeRepairExtKey: "true"}, + EvaluatorResultIds: &entity.EvaluatorResults{Registered: []*entity.RegisteredEvalResult{{ + VersionID: 101, RecordID: 1001, + }}}, + }}, nil) + recordRepo.EXPECT().BatchGetEvaluatorRecord(gomock.Any(), []int64{1001}, false, false, gomock.Any()).Return([]*entity.EvaluatorRecord{{ + ID: 1001, Status: entity.EvaluatorRunStatusSuccess, + }}, nil) + publisher.EXPECT().PublishExptRecordEvalEvent(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).DoAndReturn( + func(_ context.Context, itemEvent *entity.ExptItemEvalEvent, _ *time.Duration, modify func(*entity.ExptItemEvalEvent)) error { + modify(itemEvent) + assert.Equal(t, int64(10), itemEvent.EvalSetItemID) + assert.True(t, itemEvent.AsyncEvaluatorReportTrigger) + return nil + }, + ) + + svc := &ExptSchedulerImpl{ExptTurnResultRepo: turnRepo, EvaluatorRecordRepo: recordRepo, Publisher: publisher} + require.NoError(t, svc.reconcileTerminalAsyncEvaluatorItems(context.Background(), event, asyncEvaluatorExperiment(), items)) +} + +func TestExptSchedulerImpl_ReconcileTerminalAsyncEvaluatorItems_WaitsForAllRecords(t *testing.T) { + t.Parallel() + + ctrl := gomock.NewController(t) + turnRepo := mock_repo.NewMockIExptTurnResultRepo(ctrl) + recordRepo := mock_repo.NewMockIEvaluatorRecordRepo(ctrl) + publisher := eventmocks.NewMockExptEventPublisher(ctrl) + items := []*entity.ExptEvalItem{{ExptID: 1, ItemID: 10, State: entity.ItemRunState_Processing}} + event := &entity.ExptScheduleEvent{SpaceID: 3, ExptID: 1, ExptRunID: 2} + turnRepo.EXPECT().MGetItemTurnRunLogs(gomock.Any(), int64(1), int64(2), []int64{10}, int64(3)).Return([]*entity.ExptTurnResultRunLog{{ + ItemID: 10, Status: entity.TurnRunState_Processing, + Ext: map[string]string{asyncEvaluatorResumeRepairExtKey: "true"}, + EvaluatorResultIds: &entity.EvaluatorResults{Registered: []*entity.RegisteredEvalResult{ + {VersionID: 101, RecordID: 1001}, {VersionID: 102, RecordID: 1002}, + }}, + }}, nil) + recordRepo.EXPECT().BatchGetEvaluatorRecord(gomock.Any(), gomock.InAnyOrder([]int64{1001, 1002}), false, false, gomock.Any()).Return([]*entity.EvaluatorRecord{ + {ID: 1001, Status: entity.EvaluatorRunStatusSuccess}, + {ID: 1002, Status: entity.EvaluatorRunStatusAsyncInvoking}, + }, nil) + publisher.EXPECT().PublishExptRecordEvalEvent(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Times(0) + + svc := &ExptSchedulerImpl{ExptTurnResultRepo: turnRepo, EvaluatorRecordRepo: recordRepo, Publisher: publisher} + require.NoError(t, svc.reconcileTerminalAsyncEvaluatorItems(context.Background(), event, asyncEvaluatorExperiment(), items)) +} + +func TestExptSchedulerImpl_ReconcileTerminalAsyncEvaluatorItems_RetriesOnNextTick(t *testing.T) { + t.Parallel() + + ctrl := gomock.NewController(t) + turnRepo := mock_repo.NewMockIExptTurnResultRepo(ctrl) + recordRepo := mock_repo.NewMockIEvaluatorRecordRepo(ctrl) + publisher := eventmocks.NewMockExptEventPublisher(ctrl) + items := []*entity.ExptEvalItem{{ExptID: 1, ItemID: 10, State: entity.ItemRunState_Processing}} + event := &entity.ExptScheduleEvent{SpaceID: 3, ExptID: 1, ExptRunID: 2} + turnLogs := []*entity.ExptTurnResultRunLog{{ + ItemID: 10, Status: entity.TurnRunState_Processing, + Ext: map[string]string{asyncEvaluatorResumeRepairExtKey: "true"}, + EvaluatorResultIds: &entity.EvaluatorResults{Registered: []*entity.RegisteredEvalResult{{VersionID: 101, RecordID: 1001}}}, + }} + turnRepo.EXPECT().MGetItemTurnRunLogs(gomock.Any(), int64(1), int64(2), []int64{10}, int64(3)).Return(turnLogs, nil).Times(2) + recordRepo.EXPECT().BatchGetEvaluatorRecord(gomock.Any(), []int64{1001}, false, false, gomock.Any()).Return([]*entity.EvaluatorRecord{{ID: 1001, Status: entity.EvaluatorRunStatusSuccess}}, nil).Times(2) + gomock.InOrder( + publisher.EXPECT().PublishExptRecordEvalEvent(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return(errors.New("mq down")), + publisher.EXPECT().PublishExptRecordEvalEvent(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return(nil), + ) + + svc := &ExptSchedulerImpl{ExptTurnResultRepo: turnRepo, EvaluatorRecordRepo: recordRepo, Publisher: publisher} + require.NoError(t, svc.reconcileTerminalAsyncEvaluatorItems(context.Background(), event, asyncEvaluatorExperiment(), items)) + require.NoError(t, svc.reconcileTerminalAsyncEvaluatorItems(context.Background(), event, asyncEvaluatorExperiment(), items)) +} + +func TestExptSchedulerImpl_ReconcileTerminalAsyncEvaluatorItems_IgnoresUnmarkedTurns(t *testing.T) { + t.Parallel() + + ctrl := gomock.NewController(t) + turnRepo := mock_repo.NewMockIExptTurnResultRepo(ctrl) + recordRepo := mock_repo.NewMockIEvaluatorRecordRepo(ctrl) + publisher := eventmocks.NewMockExptEventPublisher(ctrl) + items := []*entity.ExptEvalItem{{ExptID: 1, ItemID: 10, State: entity.ItemRunState_Processing}} + event := &entity.ExptScheduleEvent{SpaceID: 3, ExptID: 1, ExptRunID: 2} + turnRepo.EXPECT().MGetItemTurnRunLogs(gomock.Any(), int64(1), int64(2), []int64{10}, int64(3)).Return([]*entity.ExptTurnResultRunLog{{ + ItemID: 10, Status: entity.TurnRunState_Processing, + EvaluatorResultIds: &entity.EvaluatorResults{Registered: []*entity.RegisteredEvalResult{{VersionID: 101, RecordID: 1001}}}, + }}, nil) + recordRepo.EXPECT().BatchGetEvaluatorRecord(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Times(0) + publisher.EXPECT().PublishExptRecordEvalEvent(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Times(0) + + svc := &ExptSchedulerImpl{ExptTurnResultRepo: turnRepo, EvaluatorRecordRepo: recordRepo, Publisher: publisher} + require.NoError(t, svc.reconcileTerminalAsyncEvaluatorItems(context.Background(), event, asyncEvaluatorExperiment(), items)) +} + +func asyncEvaluatorExperiment() *entity.Experiment { + return &entity.Experiment{Evaluators: []*entity.Evaluator{{ + EvaluatorType: entity.EvaluatorTypeAgent, + }}} +} + +func TestExptSchedulerImpl_ReconcileTerminalAsyncEvaluatorItems_SkipsSyncOnlyExperiments(t *testing.T) { + t.Parallel() + + ctrl := gomock.NewController(t) + turnRepo := mock_repo.NewMockIExptTurnResultRepo(ctrl) + recordRepo := mock_repo.NewMockIEvaluatorRecordRepo(ctrl) + publisher := eventmocks.NewMockExptEventPublisher(ctrl) + turnRepo.EXPECT().MGetItemTurnRunLogs(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Times(0) + recordRepo.EXPECT().BatchGetEvaluatorRecord(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Times(0) + publisher.EXPECT().PublishExptRecordEvalEvent(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Times(0) + + svc := &ExptSchedulerImpl{ExptTurnResultRepo: turnRepo, EvaluatorRecordRepo: recordRepo, Publisher: publisher} + expt := &entity.Experiment{Evaluators: []*entity.Evaluator{{EvaluatorType: entity.EvaluatorTypePrompt}}} + items := []*entity.ExptEvalItem{{ItemID: 10, State: entity.ItemRunState_Processing}} + require.NoError(t, svc.reconcileTerminalAsyncEvaluatorItems(context.Background(), &entity.ExptScheduleEvent{}, expt, items)) +} + +func TestExptSchedulerImpl_ReconcileTerminalAsyncEvaluatorItems_ReadErrorsAreBestEffort(t *testing.T) { + t.Parallel() + + t.Run("turn log read error", func(t *testing.T) { + t.Parallel() + ctrl := gomock.NewController(t) + turnRepo := mock_repo.NewMockIExptTurnResultRepo(ctrl) + recordRepo := mock_repo.NewMockIEvaluatorRecordRepo(ctrl) + publisher := eventmocks.NewMockExptEventPublisher(ctrl) + turnRepo.EXPECT().MGetItemTurnRunLogs(gomock.Any(), int64(1), int64(2), []int64{10}, int64(3)).Return(nil, errors.New("db down")) + recordRepo.EXPECT().BatchGetEvaluatorRecord(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Times(0) + publisher.EXPECT().PublishExptRecordEvalEvent(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Times(0) + + svc := &ExptSchedulerImpl{ExptTurnResultRepo: turnRepo, EvaluatorRecordRepo: recordRepo, Publisher: publisher} + event := &entity.ExptScheduleEvent{SpaceID: 3, ExptID: 1, ExptRunID: 2} + items := []*entity.ExptEvalItem{{ItemID: 10, State: entity.ItemRunState_Processing}} + require.NoError(t, svc.reconcileTerminalAsyncEvaluatorItems(context.Background(), event, asyncEvaluatorExperiment(), items)) + }) + + t.Run("record read error", func(t *testing.T) { + t.Parallel() + ctrl := gomock.NewController(t) + turnRepo := mock_repo.NewMockIExptTurnResultRepo(ctrl) + recordRepo := mock_repo.NewMockIEvaluatorRecordRepo(ctrl) + publisher := eventmocks.NewMockExptEventPublisher(ctrl) + turnRepo.EXPECT().MGetItemTurnRunLogs(gomock.Any(), int64(1), int64(2), []int64{10}, int64(3)).Return([]*entity.ExptTurnResultRunLog{{ + ItemID: 10, Status: entity.TurnRunState_Processing, + Ext: map[string]string{asyncEvaluatorResumeRepairExtKey: "true"}, + EvaluatorResultIds: &entity.EvaluatorResults{Registered: []*entity.RegisteredEvalResult{{RecordID: 1001}}}, + }}, nil) + recordRepo.EXPECT().BatchGetEvaluatorRecord(gomock.Any(), []int64{1001}, false, false, gomock.Any()).Return(nil, errors.New("db down")) + publisher.EXPECT().PublishExptRecordEvalEvent(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Times(0) + + svc := &ExptSchedulerImpl{ExptTurnResultRepo: turnRepo, EvaluatorRecordRepo: recordRepo, Publisher: publisher} + event := &entity.ExptScheduleEvent{SpaceID: 3, ExptID: 1, ExptRunID: 2} + items := []*entity.ExptEvalItem{{ItemID: 10, State: entity.ItemRunState_Processing}} + require.NoError(t, svc.reconcileTerminalAsyncEvaluatorItems(context.Background(), event, asyncEvaluatorExperiment(), items)) + }) +} + +func TestExptSchedulerImpl_ReconcileTerminalAsyncEvaluatorItems_DoesNotRepublishWhenRecordMissing(t *testing.T) { + t.Parallel() + + ctrl := gomock.NewController(t) + turnRepo := mock_repo.NewMockIExptTurnResultRepo(ctrl) + recordRepo := mock_repo.NewMockIEvaluatorRecordRepo(ctrl) + publisher := eventmocks.NewMockExptEventPublisher(ctrl) + turnRepo.EXPECT().MGetItemTurnRunLogs(gomock.Any(), int64(1), int64(2), []int64{10}, int64(3)).Return([]*entity.ExptTurnResultRunLog{{ + ItemID: 10, Status: entity.TurnRunState_Processing, + Ext: map[string]string{asyncEvaluatorResumeRepairExtKey: "true"}, + EvaluatorResultIds: &entity.EvaluatorResults{Registered: []*entity.RegisteredEvalResult{{RecordID: 1001}, {RecordID: 1002}}}, + }}, nil) + recordRepo.EXPECT().BatchGetEvaluatorRecord(gomock.Any(), gomock.InAnyOrder([]int64{1001, 1002}), false, false, gomock.Any()).Return([]*entity.EvaluatorRecord{{ID: 1001, Status: entity.EvaluatorRunStatusSuccess}}, nil) + publisher.EXPECT().PublishExptRecordEvalEvent(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Times(0) + + svc := &ExptSchedulerImpl{ExptTurnResultRepo: turnRepo, EvaluatorRecordRepo: recordRepo, Publisher: publisher} + event := &entity.ExptScheduleEvent{SpaceID: 3, ExptID: 1, ExptRunID: 2} + items := []*entity.ExptEvalItem{{ItemID: 10, State: entity.ItemRunState_Processing}} + require.NoError(t, svc.reconcileTerminalAsyncEvaluatorItems(context.Background(), event, asyncEvaluatorExperiment(), items)) +} diff --git a/backend/modules/evaluation/infra/repo/experiment/redis/convert/item_turn_eval_async_test.go b/backend/modules/evaluation/infra/repo/experiment/redis/convert/item_turn_eval_async_test.go index 9c63000096..39dfd0e471 100644 --- a/backend/modules/evaluation/infra/repo/experiment/redis/convert/item_turn_eval_async_test.go +++ b/backend/modules/evaluation/infra/repo/experiment/redis/convert/item_turn_eval_async_test.go @@ -7,6 +7,7 @@ import ( "testing" "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" "github.com/coze-dev/coze-loop/backend/modules/evaluation/domain/entity" ) @@ -29,11 +30,30 @@ func TestExptItemTurnEvalAsyncCtx_CallbackURLRoundTrip(t *testing.T) { func TestExptItemTurnEvalAsyncCtx_ResumeReadyRoundTrip(t *testing.T) { c := NewExptItemTurnEvalAsyncCtx() - in := &entity.EvalAsyncCtx{RecordID: 123, ResumeReady: true} + in := &entity.EvalAsyncCtx{RecordID: 123, ResumeBarrierEnabled: true, ResumeReady: true} b, err := c.FromDO(in) assert.NoError(t, err) out, err := c.ToDO(b) assert.NoError(t, err) + assert.True(t, out.ResumeBarrierEnabled) assert.True(t, out.ResumeReady) } + +func TestExptItemTurnEvalAsyncCtx_LegacyPayloadDefaultsToResumeAllowed(t *testing.T) { + c := NewExptItemTurnEvalAsyncCtx() + out, err := c.ToDO([]byte(`{"RecordID":123}`)) + require.NoError(t, err) + assert.True(t, out.CanResumeExperiment()) +} + +func TestExptItemTurnEvalAsyncCtx_NewBarrierRequiresReady(t *testing.T) { + c := NewExptItemTurnEvalAsyncCtx() + out, err := c.ToDO([]byte(`{"resume_barrier_enabled":true}`)) + require.NoError(t, err) + assert.False(t, out.CanResumeExperiment()) + + out, err = c.ToDO([]byte(`{"resume_barrier_enabled":true,"resume_ready":true}`)) + require.NoError(t, err) + assert.True(t, out.CanResumeExperiment()) +} diff --git a/backend/modules/evaluation/infra/repo/experiment/redis/dao/item_turn_eval_async.go b/backend/modules/evaluation/infra/repo/experiment/redis/dao/item_turn_eval_async.go index e49d7ffebc..a48194f3eb 100644 --- a/backend/modules/evaluation/infra/repo/experiment/redis/dao/item_turn_eval_async.go +++ b/backend/modules/evaluation/infra/repo/experiment/redis/dao/item_turn_eval_async.go @@ -13,6 +13,7 @@ import ( "github.com/coze-dev/coze-loop/backend/modules/evaluation/infra/repo/experiment/redis/convert" "github.com/coze-dev/coze-loop/backend/pkg/errorx" "github.com/coze-dev/coze-loop/backend/pkg/lang/conv" + redisv9 "github.com/redis/go-redis/v9" ) type IEvalAsyncDAO interface { @@ -65,7 +66,7 @@ func (e *evalAsyncDAOImpl) GetEvalAsyncCtxStrong(ctx context.Context, invokeID s case <-timer.C: } } - actx, err := e.getEvalAsyncCtx(ctx, invokeID) + actx, err := e.getEvalAsyncCtxFromPrimary(ctx, invokeID) if err == nil || !redis.IsNilError(err) { return actx, err } @@ -95,7 +96,7 @@ func (e *evalAsyncDAOImpl) MarkEvalAsyncResumeReady(ctx context.Context, invokeI if err != nil { return nil, err } - if actx == nil || actx.ResumeReady { + if actx == nil || !actx.ResumeBarrierEnabled || actx.ResumeReady { return actx, nil } key := e.makeExptItemTurnEvalAsyncCtxKey(invokeID) @@ -116,6 +117,24 @@ func (e *evalAsyncDAOImpl) MarkEvalAsyncResumeReady(ctx context.Context, invokeI return actx, nil } +// Redis-v6 routes EVAL/EVALSHA by the script key and only routes commands whose +// command metadata is ReadOnly to replicas. EVAL is therefore master-routed even +// though this script only performs GET. This gives callback/arm reads the latest +// context without changing the shared client's read priority for unrelated traffic. +const getEvalAsyncCtxFromPrimaryScript = `return redis.call('GET', KEYS[1])` + +func (e *evalAsyncDAOImpl) getEvalAsyncCtxFromPrimary(ctx context.Context, invokeID string) (*entity.EvalAsyncCtx, error) { + key := e.makeExptItemTurnEvalAsyncCtxKey(invokeID) + got, err := e.cmdable.Eval(ctx, getEvalAsyncCtxFromPrimaryScript, []string{key}).Text() + if err != nil { + if redis.IsNilError(err) || err == redisv9.Nil { + return nil, redisv9.Nil + } + return nil, errorx.Wrapf(err, "redis primary get fail, key: %v", key) + } + return convert.NewExptItemTurnEvalAsyncCtx().ToDO(conv.UnsafeStringToBytes(got)) +} + func (e *evalAsyncDAOImpl) getEvalAsyncCtx(ctx context.Context, invokeID string) (*entity.EvalAsyncCtx, error) { key := e.makeExptItemTurnEvalAsyncCtxKey(invokeID) got, err := e.cmdable.Get(ctx, key).Result() diff --git a/backend/modules/evaluation/infra/repo/experiment/redis/dao/item_turn_eval_async_primary_test.go b/backend/modules/evaluation/infra/repo/experiment/redis/dao/item_turn_eval_async_primary_test.go new file mode 100644 index 0000000000..943ea90706 --- /dev/null +++ b/backend/modules/evaluation/infra/repo/experiment/redis/dao/item_turn_eval_async_primary_test.go @@ -0,0 +1,62 @@ +// Copyright (c) 2025 coze-dev Authors +// SPDX-License-Identifier: Apache-2.0 + +package dao + +import ( + "context" + "testing" + + redismocks "github.com/coze-dev/coze-loop/backend/infra/redis/mocks" + "github.com/redis/go-redis/v9" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "go.uber.org/mock/gomock" +) + +func TestEvalAsyncDAO_GetEvalAsyncCtxStrongReadsThroughMasterRoutedScript(t *testing.T) { + t.Parallel() + + ctrl := gomock.NewController(t) + cmdable := redismocks.NewMockPersistentCmdable(ctrl) + dao := NewEvalAsyncDAO(cmdable) + key := "[experiment]item_turn_eval_async_ctx:evaluator:1" + payload := `{"RecordID":1,"resume_ready":true}` + + cmdable.EXPECT().Eval(gomock.Any(), gomock.Any(), []string{key}).DoAndReturn( + func(ctx context.Context, _ string, _ []string, _ ...any) *redis.Cmd { + return redis.NewCmdResult(payload, nil) + }, + ) + + got, err := dao.GetEvalAsyncCtxStrong(context.Background(), "evaluator:1") + require.NoError(t, err) + require.NotNil(t, got) + assert.True(t, got.ResumeReady) +} + +func TestEvalAsyncDAO_GetEvalAsyncCtxStrongRetriesPrimaryScriptOnMissing(t *testing.T) { + t.Parallel() + + ctrl := gomock.NewController(t) + cmdable := redismocks.NewMockPersistentCmdable(ctrl) + dao := NewEvalAsyncDAO(cmdable) + key := "[experiment]item_turn_eval_async_ctx:evaluator:2" + payload := `{"RecordID":2,"resume_barrier_enabled":true,"resume_ready":true}` + calls := 0 + cmdable.EXPECT().Eval(gomock.Any(), gomock.Any(), []string{key}).DoAndReturn( + func(ctx context.Context, _ string, _ []string, _ ...any) *redis.Cmd { + calls++ + if calls < 3 { + return redis.NewCmdResult(nil, redis.Nil) + } + return redis.NewCmdResult(payload, nil) + }, + ).Times(3) + + got, err := dao.GetEvalAsyncCtxStrong(context.Background(), "evaluator:2") + require.NoError(t, err) + require.NotNil(t, got) + assert.Equal(t, 3, calls) + assert.True(t, got.CanResumeExperiment()) +} diff --git a/backend/modules/evaluation/infra/repo/experiment/redis/dao/item_turn_eval_async_test.go b/backend/modules/evaluation/infra/repo/experiment/redis/dao/item_turn_eval_async_test.go index 8c8389c8cc..f0a27a0474 100644 --- a/backend/modules/evaluation/infra/repo/experiment/redis/dao/item_turn_eval_async_test.go +++ b/backend/modules/evaluation/infra/repo/experiment/redis/dao/item_turn_eval_async_test.go @@ -24,7 +24,7 @@ func TestEvalAsyncDAO_GetStrongAndMarkResumeReady(t *testing.T) { go func() { time.Sleep(20 * time.Millisecond) - require.NoError(t, dao.SetEvalAsyncCtx(ctx, "evaluator:1", &entity.EvalAsyncCtx{RecordID: 1})) + require.NoError(t, dao.SetEvalAsyncCtx(ctx, "evaluator:1", &entity.EvalAsyncCtx{RecordID: 1, ResumeBarrierEnabled: true})) }() actx, err := dao.GetEvalAsyncCtxStrong(ctx, "evaluator:1") @@ -61,8 +61,9 @@ func TestEvalAsyncDAO_MarkResumeReadyPreservesLargeIDsAndExistingPayload(t *test ctx := context.Background() const largeID int64 = 9007199254740993 original := &entity.EvalAsyncCtx{ - RecordID: largeID, - EvaluatorVersionID: largeID - 1, + ResumeBarrierEnabled: true, + RecordID: largeID, + EvaluatorVersionID: largeID - 1, Event: &entity.ExptItemEvalEvent{ ExptID: largeID - 2, ExptRunID: largeID - 3, @@ -89,3 +90,16 @@ func TestMarkResumeReadyScriptAvoidsVersionSensitiveTTLCommands(t *testing.T) { assert.NotContains(t, markResumeReadyScript, "PTTL") assert.Contains(t, markResumeReadyScript, "'EX', ARGV[1]") } + +func TestEvalAsyncDAO_MarkResumeReadyDoesNotArmLegacyContext(t *testing.T) { + cmdable := infraredis.NewTestRedis(t) + dao := NewEvalAsyncDAO(cmdable) + ctx := context.Background() + + require.NoError(t, dao.SetEvalAsyncCtx(ctx, "legacy", &entity.EvalAsyncCtx{RecordID: 1})) + got, err := dao.MarkEvalAsyncResumeReady(ctx, "legacy") + require.NoError(t, err) + assert.False(t, got.ResumeBarrierEnabled) + assert.False(t, got.ResumeReady) + assert.True(t, got.CanResumeExperiment()) +} From 3759f6a821c756c46fae25e398f056b8f13dcd95 Mon Sep 17 00:00:00 2001 From: shenjiayun Date: Sat, 8 Aug 2026 22:52:14 +0800 Subject: [PATCH 12/15] Revert "fix(evaluation): make async resume recovery durable M-7362275687" This reverts commit 7c4c6fcd055be1726ec1bedecbaa501ed311b125. --- .../application/eval_openapi_app.go | 13 +- .../application/eval_openapi_app_test.go | 56 ++--- .../evaluation/application/evaluator_app.go | 5 +- .../evaluation/domain/entity/expt_run.go | 14 +- .../modules/evaluation/domain/entity/param.go | 2 +- .../modules/evaluation/domain/repo/expt.go | 2 +- .../domain/service/evaluator_impl.go | 2 +- .../domain/service/evaluator_impl_test.go | 6 +- .../domain/service/expt_run_item_impl.go | 36 +--- .../domain/service/expt_run_item_impl_test.go | 33 --- .../domain/service/expt_run_item_turn_impl.go | 18 +- .../service/expt_run_item_turn_impl_test.go | 2 - .../service/expt_run_scheduler_event_impl.go | 143 ------------- .../expt_run_scheduler_event_impl_test.go | 191 ------------------ .../convert/item_turn_eval_async_test.go | 22 +- .../redis/dao/item_turn_eval_async.go | 23 +-- .../dao/item_turn_eval_async_primary_test.go | 62 ------ .../redis/dao/item_turn_eval_async_test.go | 20 +- 18 files changed, 51 insertions(+), 599 deletions(-) delete mode 100644 backend/modules/evaluation/infra/repo/experiment/redis/dao/item_turn_eval_async_primary_test.go diff --git a/backend/modules/evaluation/application/eval_openapi_app.go b/backend/modules/evaluation/application/eval_openapi_app.go index 842babe571..b94612f1d9 100644 --- a/backend/modules/evaluation/application/eval_openapi_app.go +++ b/backend/modules/evaluation/application/eval_openapi_app.go @@ -3002,11 +3002,10 @@ func (e *EvalOpenAPIApplication) AsyncRunEvaluatorOApi(ctx context.Context, req EvaluatorRunConf: runConf, Ext: req.Ext, AsyncCtx: &entity.EvalAsyncCtx{ - Session: &entity.Session{UserID: usersession.UserIDInCtxOrEmpty(ctx)}, - CallbackURL: req.GetCallbackURL(), - ResumeBarrierEnabled: true, - ResumeReady: true, - AsyncUnixMS: startTime.UnixMilli(), + Session: &entity.Session{UserID: usersession.UserIDInCtxOrEmpty(ctx)}, + CallbackURL: req.GetCallbackURL(), + ResumeReady: true, + AsyncUnixMS: startTime.UnixMilli(), }, }) if err != nil { @@ -3583,7 +3582,7 @@ func (e *EvalOpenAPIApplication) ReportEvaluatorInvokeResult_(ctx context.Contex return &openapi.ReportEvaluatorInvokeResultResponse{BaseResp: base.NewBaseResp()}, nil } if actx.Event != nil { - if !actx.CanResumeExperiment() { + if !actx.ResumeReady { latestCtx, ctxErr := e.asyncRepo.GetEvalAsyncCtxStrong(ctx, asyncCtxKey) if ctxErr != nil { return nil, ctxErr @@ -3592,7 +3591,7 @@ func (e *EvalOpenAPIApplication) ReportEvaluatorInvokeResult_(ctx context.Contex actx = latestCtx } } - if actx.CanResumeExperiment() { + if actx.ResumeReady { if err := e.publisher.PublishExptRecordEvalEvent(ctx, actx.Event, gptr.Of(time.Second*3), func(event *entity.ExptItemEvalEvent) { event.AsyncEvaluatorReportTrigger = true }); err != nil { diff --git a/backend/modules/evaluation/application/eval_openapi_app_test.go b/backend/modules/evaluation/application/eval_openapi_app_test.go index 1fdc4f2d62..6acc29f786 100755 --- a/backend/modules/evaluation/application/eval_openapi_app_test.go +++ b/backend/modules/evaluation/application/eval_openapi_app_test.go @@ -6735,17 +6735,15 @@ func TestEvalOpenAPIApplication_ReportEvaluatorInvokeResult(t *testing.T) { setup: func(auth *rpcmocks.MockIAuthProvider, asyncRepo *repomocks.MockIEvalAsyncRepo, evaluatorSvc *servicemocks.MockEvaluatorService, publisher *eventmocks.MockExptEventPublisher, _ *servicemocks.MockIEvaluatorCallbackDispatcher) { auth.EXPECT().Authorization(gomock.Any(), gomock.Any()).Return(nil) asyncRepo.EXPECT().GetEvalAsyncCtxStrong(gomock.Any(), "evaluator:2002").Return(&entity.EvalAsyncCtx{ - Event: event, - ResumeBarrierEnabled: true, - ResumeReady: false, - EvaluatorVersionID: 9, + Event: event, + ResumeReady: false, + EvaluatorVersionID: 9, }, nil) evaluatorSvc.EXPECT().ReportEvaluatorInvokeResult(gomock.Any(), gomock.Any()).Return(entity.ReportEvaluatorResultApplied, nil) asyncRepo.EXPECT().GetEvalAsyncCtxStrong(gomock.Any(), "evaluator:2002").Return(&entity.EvalAsyncCtx{ - Event: event, - ResumeBarrierEnabled: true, - ResumeReady: false, - EvaluatorVersionID: 9, + Event: event, + ResumeReady: false, + EvaluatorVersionID: 9, }, nil) publisher.EXPECT().PublishExptRecordEvalEvent(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Times(0) }, @@ -6764,10 +6762,9 @@ func TestEvalOpenAPIApplication_ReportEvaluatorInvokeResult(t *testing.T) { auth.EXPECT().Authorization(gomock.Any(), gomock.Any()).Return(nil) gomock.InOrder( asyncRepo.EXPECT().GetEvalAsyncCtxStrong(gomock.Any(), "evaluator:2002").Return(&entity.EvalAsyncCtx{ - Event: event, - ResumeBarrierEnabled: true, - ResumeReady: false, - EvaluatorVersionID: 9, + Event: event, + ResumeReady: false, + EvaluatorVersionID: 9, }, nil), evaluatorSvc.EXPECT().ReportEvaluatorInvokeResult(gomock.Any(), gomock.Any()).Return(entity.ReportEvaluatorResultApplied, nil), asyncRepo.EXPECT().GetEvalAsyncCtxStrong(gomock.Any(), "evaluator:2002").Return(&entity.EvalAsyncCtx{ @@ -6793,10 +6790,9 @@ func TestEvalOpenAPIApplication_ReportEvaluatorInvokeResult(t *testing.T) { auth.EXPECT().Authorization(gomock.Any(), gomock.Any()).Return(nil) gomock.InOrder( asyncRepo.EXPECT().GetEvalAsyncCtxStrong(gomock.Any(), "evaluator:2002").Return(&entity.EvalAsyncCtx{ - Event: event, - ResumeBarrierEnabled: true, - ResumeReady: false, - EvaluatorVersionID: 9, + Event: event, + ResumeReady: false, + EvaluatorVersionID: 9, }, nil), evaluatorSvc.EXPECT().ReportEvaluatorInvokeResult(gomock.Any(), gomock.Any()).Return(entity.ReportEvaluatorResultApplied, nil), asyncRepo.EXPECT().GetEvalAsyncCtxStrong(gomock.Any(), "evaluator:2002").Return(nil, errors.New("redis reread failed")), @@ -8626,7 +8622,6 @@ func TestEvalOpenAPIApplication_AsyncRunEvaluatorOApi(t *testing.T) { auth.EXPECT().AuthorizationWithoutSPI(gomock.Any(), gomock.Any()).Return(nil) evaluatorSvc.EXPECT().AsyncRunEvaluator(gomock.Any(), gomock.Any()).DoAndReturn(func(_ context.Context, req *entity.AsyncRunEvaluatorRequest) (*entity.EvaluatorRecord, error) { require.NotNil(t, req.AsyncCtx) - assert.True(t, req.AsyncCtx.ResumeBarrierEnabled) assert.True(t, req.AsyncCtx.ResumeReady) assert.Nil(t, req.AsyncCtx.Event) return record, nil @@ -8645,7 +8640,6 @@ func TestEvalOpenAPIApplication_AsyncRunEvaluatorOApi(t *testing.T) { evaluatorSvc.EXPECT().GetEvaluatorVersion(gomock.Any(), gomock.Any(), evaluatorVersionID, false, false).Return(evaluator, nil) evaluatorSvc.EXPECT().AsyncRunEvaluator(gomock.Any(), gomock.Any()).DoAndReturn(func(_ context.Context, req *entity.AsyncRunEvaluatorRequest) (*entity.EvaluatorRecord, error) { require.NotNil(t, req.AsyncCtx) - assert.True(t, req.AsyncCtx.ResumeBarrierEnabled) assert.True(t, req.AsyncCtx.ResumeReady) assert.Nil(t, req.AsyncCtx.Event) return record, nil @@ -9012,29 +9006,3 @@ func TestEvalOpenAPIApplication_ListEvaluationSetVersionsOApi_SharedExecuteHides assert.Nil(t, resp.Data.Versions[0].EvaluationSetSchema) assert.NotNil(t, version.EvaluationSetSchema) } - -func TestEvalOpenAPIApplication_ReportEvaluatorInvokeResult_LegacyContextResumesWithoutReread(t *testing.T) { - t.Parallel() - - ctrl := gomock.NewController(t) - auth := rpcmocks.NewMockIAuthProvider(ctrl) - asyncRepo := repomocks.NewMockIEvalAsyncRepo(ctrl) - evaluatorSvc := servicemocks.NewMockEvaluatorService(ctrl) - publisher := eventmocks.NewMockExptEventPublisher(ctrl) - dispatcher := servicemocks.NewMockIEvaluatorCallbackDispatcher(ctrl) - event := &entity.ExptItemEvalEvent{ExptID: 1, ExptRunID: 2, EvalSetItemID: 3} - - auth.EXPECT().Authorization(gomock.Any(), gomock.Any()).Return(nil) - asyncRepo.EXPECT().GetEvalAsyncCtxStrong(gomock.Any(), "evaluator:2002").Return(&entity.EvalAsyncCtx{ - Event: event, EvaluatorVersionID: 9, - }, nil).Times(1) - evaluatorSvc.EXPECT().ReportEvaluatorInvokeResult(gomock.Any(), gomock.Any()).Return(entity.ReportEvaluatorResultApplied, nil) - publisher.EXPECT().PublishExptRecordEvalEvent(gomock.Any(), event, gomock.Any(), gomock.Any()).Return(nil) - - app := &EvalOpenAPIApplication{auth: auth, asyncRepo: asyncRepo, evaluatorService: evaluatorSvc, publisher: publisher, callbackDispatcher: dispatcher} - _, err := app.ReportEvaluatorInvokeResult_(context.Background(), &openapi.ReportEvaluatorInvokeResultRequest{ - WorkspaceID: gptr.Of(int64(1001)), InvokeID: gptr.Of(int64(2002)), Status: gptr.Of(spi.InvokeEvaluatorRunStatus_SUCCESS), - Output: &spi.InvokeEvaluatorOutputData{EvaluatorResult_: &spi.InvokeEvaluatorResult_{Score: gptr.Of(float64(1))}}, - }) - require.NoError(t, err) -} diff --git a/backend/modules/evaluation/application/evaluator_app.go b/backend/modules/evaluation/application/evaluator_app.go index a39522ced1..a636635026 100644 --- a/backend/modules/evaluation/application/evaluator_app.go +++ b/backend/modules/evaluation/application/evaluator_app.go @@ -2104,9 +2104,8 @@ func (e *EvaluatorHandlerImpl) AsyncRunEvaluator(ctx context.Context, req *evalu } asyncReq := buildAsyncRunEvaluatorRequest(evaluatorDO.Name, req) asyncReq.AsyncCtx = &entity.EvalAsyncCtx{ - Session: &entity.Session{UserID: session.UserIDInCtxOrEmpty(ctx)}, - ResumeBarrierEnabled: true, - ResumeReady: true, + Session: &entity.Session{UserID: session.UserIDInCtxOrEmpty(ctx)}, + ResumeReady: true, } resp, err := e.evaluatorService.AsyncRunEvaluator(ctx, asyncReq) if err != nil { diff --git a/backend/modules/evaluation/domain/entity/expt_run.go b/backend/modules/evaluation/domain/entity/expt_run.go index c9e4a4b890..a5bbb2f558 100644 --- a/backend/modules/evaluation/domain/entity/expt_run.go +++ b/backend/modules/evaluation/domain/entity/expt_run.go @@ -636,15 +636,6 @@ type ExptSchedulerMode interface { PublishResult(ctx context.Context, turnEvaluatorRefs []*ExptTurnEvaluatorResultRef, event *ExptScheduleEvent) error } -func (e *EvalAsyncCtx) CanResumeExperiment() bool { - if e == nil { - return false - } - // Contexts written before the ResumeReady barrier existed have neither field. - // Preserve their historical behavior during rolling upgrades. - return !e.ResumeBarrierEnabled || e.ResumeReady -} - type CKDBConfig struct { ExptTurnResultFilterDBName string `json:"expt_turn_result_filter_db_name" mapstructure:"expt_turn_result_filter_db_name"` DatasetItemsSnapshotDBName string `json:"dataset_items_snapshot_db_name" mapstructure:"dataset_items_snapshot_db_name"` @@ -658,9 +649,8 @@ type EvalAsyncCtx struct { Callee string EvaluatorVersionID int64 // evaluator version id, used for evaluator async scenario EnableExtractTrajectory *bool - ResumeBarrierEnabled bool `json:"resume_barrier_enabled,omitempty"` // new experiment evaluator contexts require refs to be durable before resume - ResumeReady bool `json:"resume_ready,omitempty"` // experiment turn refs are durable and callback may resume scheduling - CallbackURL string `json:"callback_url,omitempty"` // 异步执行完成后回调通知的 URL，为空则不回调 + ResumeReady bool `json:"resume_ready,omitempty"` // experiment turn refs are durable and callback may resume scheduling + CallbackURL string `json:"callback_url,omitempty"` // 异步执行完成后回调通知的 URL，为空则不回调 // 下述字段用于沙箱内部 step 上报的 tag 反查, 由 target async 写入位点从 etec 填充, // 调试场景 (无实验上下文) 保留零值, 由上报侧回退为占位符. TargetID int64 diff --git a/backend/modules/evaluation/domain/entity/param.go b/backend/modules/evaluation/domain/entity/param.go index 93e2c5d139..f672e9846b 100644 --- a/backend/modules/evaluation/domain/entity/param.go +++ b/backend/modules/evaluation/domain/entity/param.go @@ -327,7 +327,7 @@ type AsyncRunEvaluatorRequest struct { Alias string `json:"alias,omitempty"` SourceType EvaluatorRecordSourceType `json:"source_type,omitempty"` // AsyncCtx is persisted after the AsyncInvoking record is created and before the provider is dispatched. - // New experiment calls enable the resume barrier and set ResumeReady=false; direct calls set both enabled and ready. + // Experiment calls set ResumeReady=false; direct calls set it true. AsyncCtx *EvalAsyncCtx `json:"-"` } diff --git a/backend/modules/evaluation/domain/repo/expt.go b/backend/modules/evaluation/domain/repo/expt.go index 392843b35f..a3bc06c95c 100644 --- a/backend/modules/evaluation/domain/repo/expt.go +++ b/backend/modules/evaluation/domain/repo/expt.go @@ -146,7 +146,7 @@ type IExptResultExportRecordRepo interface { type IEvalAsyncRepo interface { GetEvalAsyncCtx(ctx context.Context, invokeID string) (*entity.EvalAsyncCtx, error) - // GetEvalAsyncCtxStrong reads through a Redis script (master-routed command) and retries a transient missing key at 50/100/200ms. + // GetEvalAsyncCtxStrong uses the shared direct Redis client and retries a transient missing key at 50/100/200ms. GetEvalAsyncCtxStrong(ctx context.Context, invokeID string) (*entity.EvalAsyncCtx, error) SetEvalAsyncCtx(ctx context.Context, invokeID string, actx *entity.EvalAsyncCtx) error // MarkEvalAsyncResumeReady atomically persists ResumeReady=true and returns the latest context. diff --git a/backend/modules/evaluation/domain/service/evaluator_impl.go b/backend/modules/evaluation/domain/service/evaluator_impl.go index fa6b3abbdf..a0aa04d128 100644 --- a/backend/modules/evaluation/domain/service/evaluator_impl.go +++ b/backend/modules/evaluation/domain/service/evaluator_impl.go @@ -1036,7 +1036,7 @@ func (e *EvaluatorServiceImpl) AsyncRunEvaluator(ctx context.Context, request *e asyncCtx := request.AsyncCtx if asyncCtx == nil { - asyncCtx = &entity.EvalAsyncCtx{ResumeBarrierEnabled: true, ResumeReady: true} + asyncCtx = &entity.EvalAsyncCtx{ResumeReady: true} } asyncCtx.RecordID = invokeID asyncCtx.EvaluatorVersionID = request.EvaluatorVersionID diff --git a/backend/modules/evaluation/domain/service/evaluator_impl_test.go b/backend/modules/evaluation/domain/service/evaluator_impl_test.go index 0eed592ccb..fa72dfcd0a 100644 --- a/backend/modules/evaluation/domain/service/evaluator_impl_test.go +++ b/backend/modules/evaluation/domain/service/evaluator_impl_test.go @@ -4682,9 +4682,8 @@ func TestEvaluatorServiceImpl_AsyncRunEvaluator_CoordinatorOrderAndFailureCompen EvaluatorVersionID: 101, InputData: &entity.EvaluatorInputData{}, AsyncCtx: &entity.EvalAsyncCtx{ - Event: &entity.ExptItemEvalEvent{ExptID: 10}, - ResumeBarrierEnabled: true, - ResumeReady: false, + Event: &entity.ExptItemEvalEvent{ExptID: 10}, + ResumeReady: false, }, } } @@ -4702,7 +4701,6 @@ func TestEvaluatorServiceImpl_AsyncRunEvaluator_CoordinatorOrderAndFailureCompen order = append(order, "context") assert.Equal(t, int64(999), actx.RecordID) assert.Equal(t, int64(101), actx.EvaluatorVersionID) - assert.True(t, actx.ResumeBarrierEnabled) assert.False(t, actx.ResumeReady) return nil }), diff --git a/backend/modules/evaluation/domain/service/expt_run_item_impl.go b/backend/modules/evaluation/domain/service/expt_run_item_impl.go index aadbc8e90b..d341ee50b8 100644 --- a/backend/modules/evaluation/domain/service/expt_run_item_impl.go +++ b/backend/modules/evaluation/domain/service/expt_run_item_impl.go @@ -10,7 +10,6 @@ import ( "strings" "time" - "github.com/bytedance/gg/gmap" "github.com/bytedance/gg/gptr" "github.com/jinzhu/copier" @@ -78,10 +77,7 @@ type ExptItemEvalCtxExecutor struct { sandboxAgentNotifier ISandboxAgentNotifier // 沙箱 agent 实验单行失败飞书通知; 可空 } -const ( - exptRunLogPersistTimeout = 5 * time.Second - asyncEvaluatorResumeRepairExtKey = "__async_evaluator_resume_repair__" -) +const exptRunLogPersistTimeout = 5 * time.Second func (e *ExptItemEvalCtxExecutor) Eval(ctx context.Context, eiec *entity.ExptItemEvalCtx) error { // if err := e.SetItemRunProcessing(ctx, event.ExptID, event.ExptRunID, event.EvalSetItemID, event.SpaceID, event.Session); err != nil { @@ -157,7 +153,7 @@ func (e *ExptItemEvalCtxExecutor) storeTurnRunResult(ctx context.Context, etec * return errorx.Wrapf(err, "ExptTurnResultRunLog copy fail") } - clone.Ext = gmap.Clone(etec.Ext) + clone.Ext = etec.Ext var evalErr error @@ -231,31 +227,19 @@ func (e *ExptItemEvalCtxExecutor) storeTurnRunResult(ctx context.Context, etec * result.SetEvalErr(evalErr) - pendingEvaluatorRecordIDs := make([]int64, 0) - for _, record := range result.EvaluatorResults { - if record != nil && record.ID > 0 && record.Status == entity.EvaluatorRunStatusAsyncInvoking { - pendingEvaluatorRecordIDs = append(pendingEvaluatorRecordIDs, record.ID) - } - } - if len(pendingEvaluatorRecordIDs) > 0 { - if clone.Ext == nil { - clone.Ext = make(map[string]string) - } - // Durable repair intent: if arm or MQ publication fails after this save, the scheduler - // can recover from turn refs plus terminal evaluator records without another provider callback. - clone.Ext[asyncEvaluatorResumeRepairExtKey] = "true" - } - if err := e.TurnResultRepo.SaveTurnRunLogs(persistCtx, []*entity.ExptTurnResultRunLog{clone}); err != nil { return err } - for _, recordID := range pendingEvaluatorRecordIDs { - if err := e.evaluatorService.ArmEvaluatorResume(persistCtx, recordID); err != nil { + for _, record := range result.EvaluatorResults { + if record == nil || record.ID <= 0 || record.Status != entity.EvaluatorRunStatusAsyncInvoking { + continue + } + if err := e.evaluatorService.ArmEvaluatorResume(persistCtx, record.ID); err != nil { // The turn references are already durable and the provider has accepted the work. // Failing the item here would be unretriable (AsyncAbort sets CtxForceNoRetry) and - // could overwrite a valid terminal callback. Keep the item processing; the scheduler - // repairs from the durable turn refs/record state, with Zombie only as final timeout. - logs.CtxError(ctx, "[ExptTurnEval] arm evaluator async resume failed after refs persisted, keep item processing, record_id: %d, err: %v", recordID, err) + // could overwrite a valid terminal callback. Keep the item processing; callbacks can + // retry publication, and the existing zombie policy remains the final fallback. + logs.CtxError(ctx, "[ExptTurnEval] arm evaluator async resume failed after refs persisted, keep item processing, record_id: %d, err: %v", record.ID, err) } } diff --git a/backend/modules/evaluation/domain/service/expt_run_item_impl_test.go b/backend/modules/evaluation/domain/service/expt_run_item_impl_test.go index a58f3705b3..20fea51a6e 100644 --- a/backend/modules/evaluation/domain/service/expt_run_item_impl_test.go +++ b/backend/modules/evaluation/domain/service/expt_run_item_impl_test.go @@ -1292,36 +1292,3 @@ func TestExptItemEvalCtxExecutor_storeTurnRunResult_AllEvaluatorsTerminalComplet } require.NoError(t, executor.storeTurnRunResult(context.Background(), etec, result)) } - -func TestExptItemEvalCtxExecutor_storeTurnRunResult_ArmErrorPersistsResumeRepairMarker(t *testing.T) { - t.Parallel() - - ctrl := gomock.NewController(t) - turnRepo := repomocks.NewMockIExptTurnResultRepo(ctrl) - evaluatorSvc := servicemocks.NewMockEvaluatorService(ctrl) - turnRepo.EXPECT().SaveTurnRunLogs(gomock.Any(), gomock.Any()).DoAndReturn( - func(_ context.Context, logs []*entity.ExptTurnResultRunLog) error { - require.Len(t, logs, 1) - assert.Equal(t, "true", logs[0].Ext[asyncEvaluatorResumeRepairExtKey]) - return nil - }, - ) - evaluatorSvc.EXPECT().ArmEvaluatorResume(gomock.Any(), int64(100)).Return(errors.New("arm failed")) - - executor := &ExptItemEvalCtxExecutor{TurnResultRepo: turnRepo, evaluatorService: evaluatorSvc} - etec := &entity.ExptTurnEvalCtx{ - Turn: &entity.Turn{ID: 1}, - ExptItemEvalCtx: &entity.ExptItemEvalCtx{ - Expt: &entity.Experiment{ID: 1, SpaceID: 2}, - Event: &entity.ExptItemEvalEvent{ExptRunID: 3}, - EvalSetItem: &entity.EvaluationSetItem{ItemID: 4}, - ExistItemEvalResult: &entity.ExptItemEvalResult{TurnResultRunLogs: map[int64]*entity.ExptTurnResultRunLog{ - 1: {ID: 5, TurnID: 1}, - }}, - }, - } - result := &entity.ExptTurnRunResult{AsyncAbort: true, EvaluatorResults: []*entity.EvaluatorRecord{{ - ID: 100, EvaluatorVersionID: 101, Status: entity.EvaluatorRunStatusAsyncInvoking, - }}} - require.NoError(t, executor.storeTurnRunResult(context.Background(), etec, result)) -} diff --git a/backend/modules/evaluation/domain/service/expt_run_item_turn_impl.go b/backend/modules/evaluation/domain/service/expt_run_item_turn_impl.go index 4fc79b25b0..45ad15267f 100644 --- a/backend/modules/evaluation/domain/service/expt_run_item_turn_impl.go +++ b/backend/modules/evaluation/domain/service/expt_run_item_turn_impl.go @@ -927,11 +927,10 @@ func (e *DefaultExptTurnEvaluationImpl) asyncCallEvaluatorWithAlias( Alias: alias, SourceType: entity.EvaluatorRecordSourceTypeBuiltin, AsyncCtx: &entity.EvalAsyncCtx{ - Event: etec.Event, - Session: etec.Event.Session, - EvaluatorVersionID: ev.GetEvaluatorVersionID(), - ResumeBarrierEnabled: true, - ResumeReady: false, + Event: etec.Event, + Session: etec.Event.Session, + EvaluatorVersionID: ev.GetEvaluatorVersionID(), + ResumeReady: false, }, }) if evaluatorRecord != nil { @@ -965,11 +964,10 @@ func (e *DefaultExptTurnEvaluationImpl) asyncCallEvaluator( Ext: etec.Ext, EvaluatorRunConf: ec.RunConf, AsyncCtx: &entity.EvalAsyncCtx{ - Event: etec.Event, - Session: etec.Event.Session, - EvaluatorVersionID: ev.GetEvaluatorVersionID(), - ResumeBarrierEnabled: true, - ResumeReady: false, + Event: etec.Event, + Session: etec.Event.Session, + EvaluatorVersionID: ev.GetEvaluatorVersionID(), + ResumeReady: false, }, } evaluatorRecord, err := e.evaluatorService.AsyncRunEvaluator(ctx, asyncReq) diff --git a/backend/modules/evaluation/domain/service/expt_run_item_turn_impl_test.go b/backend/modules/evaluation/domain/service/expt_run_item_turn_impl_test.go index 21089ef016..2f9d0e55da 100644 --- a/backend/modules/evaluation/domain/service/expt_run_item_turn_impl_test.go +++ b/backend/modules/evaluation/domain/service/expt_run_item_turn_impl_test.go @@ -394,7 +394,6 @@ func TestDefaultExptTurnEvaluationImpl_asyncCallEvaluator_Agent(t *testing.T) { assert.Equal(t, etec.Ext, req.Ext) require.NotNil(t, req.AsyncCtx) assert.Equal(t, etec.Event, req.AsyncCtx.Event) - assert.True(t, req.AsyncCtx.ResumeBarrierEnabled) assert.False(t, req.AsyncCtx.ResumeReady) return mockEvaluatorRecord, nil }, @@ -5026,7 +5025,6 @@ func TestDefaultExptTurnEvaluationImpl_asyncCallEvaluatorWithAlias_CustomRPC(t * assert.Same(t, runConf, req.EvaluatorRunConf) require.NotNil(t, req.AsyncCtx) assert.Same(t, etec.Event, req.AsyncCtx.Event) - assert.True(t, req.AsyncCtx.ResumeBarrierEnabled) assert.False(t, req.AsyncCtx.ResumeReady) return &entity.EvaluatorRecord{ID: 7, EvaluatorVersionID: 101, Alias: "judge_a", Status: entity.EvaluatorRunStatusAsyncInvoking}, nil }, diff --git a/backend/modules/evaluation/domain/service/expt_run_scheduler_event_impl.go b/backend/modules/evaluation/domain/service/expt_run_scheduler_event_impl.go index 7ca8f197b3..d43b9eb07e 100644 --- a/backend/modules/evaluation/domain/service/expt_run_scheduler_event_impl.go +++ b/backend/modules/evaluation/domain/service/expt_run_scheduler_event_impl.go @@ -392,10 +392,6 @@ func (e *ExptSchedulerImpl) schedule(ctx context.Context, event *entity.ExptSche return err } - if err := e.reconcileTerminalAsyncEvaluatorItems(ctx, event, exptDetail, incomplete); err != nil { - return err - } - incomplete, zombies, err := e.handleZombies(ctx, event, incomplete, exptDetail) if err != nil { return err @@ -557,145 +553,6 @@ func (e *ExptSchedulerImpl) handleToSubmits(ctx context.Context, event *entity.E return nil } -// reconcileTerminalAsyncEvaluatorItems repairs a lost evaluator callback resume event. -// Turn evaluator refs and evaluator record terminal states are durable DB facts, so the -// scheduler can safely republish recovery without relying on the provider to callback again. -func (e *ExptSchedulerImpl) reconcileTerminalAsyncEvaluatorItems(ctx context.Context, event *entity.ExptScheduleEvent, expt *entity.Experiment, items []*entity.ExptEvalItem) error { - if event == nil || expt == nil || len(items) == 0 || e.ExptTurnResultRepo == nil || e.EvaluatorRecordRepo == nil || e.Publisher == nil { - return nil - } - - if !expt.AsyncCallEvaluators() { - return nil - } - - processingItemIDs := make([]int64, 0, len(items)) - for _, item := range items { - if item != nil && item.State == entity.ItemRunState_Processing && item.ItemID > 0 { - processingItemIDs = append(processingItemIDs, item.ItemID) - } - } - if len(processingItemIDs) == 0 { - return nil - } - - turnRunLogs, err := e.ExptTurnResultRepo.MGetItemTurnRunLogs(ctx, event.ExptID, event.ExptRunID, processingItemIDs, event.SpaceID) - if err != nil { - logs.CtxError(ctx, "[ExptEval] read turn refs for async evaluator repair failed, keep repair pending, expt_id: %d, expt_run_id: %d, err: %v", - event.ExptID, event.ExptRunID, err) - return nil - } - - itemRecordIDs := make(map[int64]map[int64]struct{}) - allRecordIDs := make([]int64, 0) - seenRecordIDs := make(map[int64]struct{}) - for _, runLog := range turnRunLogs { - if runLog == nil || runLog.ItemID <= 0 || runLog.Status != entity.TurnRunState_Processing || - runLog.Ext[asyncEvaluatorResumeRepairExtKey] != "true" || runLog.EvaluatorResultIds == nil { - continue - } - ids := make([]int64, 0) - seen := make(map[int64]struct{}) - appendID := func(id int64) { - if id <= 0 { - return - } - if _, exists := seen[id]; exists { - return - } - seen[id] = struct{}{} - ids = append(ids, id) - } - if runLog.EvaluatorResultIds.IsNewFormat() { - for _, result := range runLog.EvaluatorResultIds.Registered { - if result != nil { - appendID(result.RecordID) - } - } - for _, result := range runLog.EvaluatorResultIds.Inline { - if result != nil { - appendID(result.RecordID) - } - } - } else { - for _, id := range runLog.EvaluatorResultIds.EvalVerIDToResID { - appendID(id) - } - } - if len(ids) == 0 { - continue - } - set := itemRecordIDs[runLog.ItemID] - if set == nil { - set = make(map[int64]struct{}) - itemRecordIDs[runLog.ItemID] = set - } - for _, id := range ids { - set[id] = struct{}{} - if _, exists := seenRecordIDs[id]; !exists { - seenRecordIDs[id] = struct{}{} - allRecordIDs = append(allRecordIDs, id) - } - } - } - if len(allRecordIDs) == 0 { - return nil - } - - records, err := e.EvaluatorRecordRepo.BatchGetEvaluatorRecord(contexts.WithCtxWriteDB(ctx), allRecordIDs, false, false, entity.WithoutLoadStorageData()) - if err != nil { - logs.CtxError(ctx, "[ExptEval] read evaluator records for async repair failed, keep repair pending, expt_id: %d, expt_run_id: %d, err: %v", - event.ExptID, event.ExptRunID, err) - return nil - } - recordStatus := make(map[int64]entity.EvaluatorRunStatus, len(records)) - for _, record := range records { - if record != nil { - recordStatus[record.ID] = record.Status - } - } - - for _, item := range items { - if item == nil || item.State != entity.ItemRunState_Processing { - continue - } - ids := itemRecordIDs[item.ItemID] - if len(ids) == 0 { - continue - } - allTerminal := true - for id := range ids { - status, exists := recordStatus[id] - if !exists || status == entity.EvaluatorRunStatusAsyncInvoking || status == entity.EvaluatorRunStatusUnknown { - allTerminal = false - break - } - } - if !allTerminal { - continue - } - itemEvent := &entity.ExptItemEvalEvent{ - SpaceID: event.SpaceID, - ExptID: event.ExptID, - ExptRunID: event.ExptRunID, - ExptRunMode: event.ExptRunMode, - EvalSetItemID: item.ItemID, - CreateAt: time.Now().Unix(), - Ext: event.Ext, - Session: event.Session, - } - if err := e.Publisher.PublishExptRecordEvalEvent(ctx, itemEvent, gptr.Of(time.Second*3), func(resume *entity.ExptItemEvalEvent) { - resume.AsyncEvaluatorReportTrigger = true - }); err != nil { - // The repair marker and terminal DB records remain durable. A later scheduler tick - // will retry, so an MQ outage here must not fail the entire experiment. - logs.CtxError(ctx, "[ExptEval] republish async evaluator resume event failed, keep repair pending, expt_id: %d, expt_run_id: %d, item_id: %d, err: %v", - event.ExptID, event.ExptRunID, item.ItemID, err) - } - } - return nil -} - func (e *ExptSchedulerImpl) handleZombies(ctx context.Context, event *entity.ExptScheduleEvent, items []*entity.ExptEvalItem, expt *entity.Experiment) (alives, zombies []*entity.ExptEvalItem, err error) { asyncExec := false if expt != nil { diff --git a/backend/modules/evaluation/domain/service/expt_run_scheduler_event_impl_test.go b/backend/modules/evaluation/domain/service/expt_run_scheduler_event_impl_test.go index b53b8cd0dc..d7064c74b2 100644 --- a/backend/modules/evaluation/domain/service/expt_run_scheduler_event_impl_test.go +++ b/backend/modules/evaluation/domain/service/expt_run_scheduler_event_impl_test.go @@ -12,7 +12,6 @@ import ( "time" "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" "go.uber.org/mock/gomock" auditmocks "github.com/coze-dev/coze-loop/backend/infra/external/audit/mocks" @@ -2119,193 +2118,3 @@ func TestIsSandboxAgentExpt(t *testing.T) { EvalTargetVersion: &entity.EvalTargetVersion{EvalTargetType: entity.EvalTargetTypeSandboxAgent}, }})) } - -func TestExptSchedulerImpl_ReconcileTerminalAsyncEvaluatorItems(t *testing.T) { - t.Parallel() - - ctrl := gomock.NewController(t) - turnRepo := mock_repo.NewMockIExptTurnResultRepo(ctrl) - recordRepo := mock_repo.NewMockIEvaluatorRecordRepo(ctrl) - publisher := eventmocks.NewMockExptEventPublisher(ctrl) - - items := []*entity.ExptEvalItem{{ExptID: 1, ItemID: 10, State: entity.ItemRunState_Processing}} - event := &entity.ExptScheduleEvent{SpaceID: 3, ExptID: 1, ExptRunID: 2, ExptRunMode: entity.EvaluationModeSubmit, Session: &entity.Session{UserID: "u"}} - turnRepo.EXPECT().MGetItemTurnRunLogs(gomock.Any(), int64(1), int64(2), []int64{10}, int64(3)).Return([]*entity.ExptTurnResultRunLog{{ - ItemID: 10, - Status: entity.TurnRunState_Processing, - Ext: map[string]string{asyncEvaluatorResumeRepairExtKey: "true"}, - EvaluatorResultIds: &entity.EvaluatorResults{Registered: []*entity.RegisteredEvalResult{{ - VersionID: 101, RecordID: 1001, - }}}, - }}, nil) - recordRepo.EXPECT().BatchGetEvaluatorRecord(gomock.Any(), []int64{1001}, false, false, gomock.Any()).Return([]*entity.EvaluatorRecord{{ - ID: 1001, Status: entity.EvaluatorRunStatusSuccess, - }}, nil) - publisher.EXPECT().PublishExptRecordEvalEvent(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).DoAndReturn( - func(_ context.Context, itemEvent *entity.ExptItemEvalEvent, _ *time.Duration, modify func(*entity.ExptItemEvalEvent)) error { - modify(itemEvent) - assert.Equal(t, int64(10), itemEvent.EvalSetItemID) - assert.True(t, itemEvent.AsyncEvaluatorReportTrigger) - return nil - }, - ) - - svc := &ExptSchedulerImpl{ExptTurnResultRepo: turnRepo, EvaluatorRecordRepo: recordRepo, Publisher: publisher} - require.NoError(t, svc.reconcileTerminalAsyncEvaluatorItems(context.Background(), event, asyncEvaluatorExperiment(), items)) -} - -func TestExptSchedulerImpl_ReconcileTerminalAsyncEvaluatorItems_WaitsForAllRecords(t *testing.T) { - t.Parallel() - - ctrl := gomock.NewController(t) - turnRepo := mock_repo.NewMockIExptTurnResultRepo(ctrl) - recordRepo := mock_repo.NewMockIEvaluatorRecordRepo(ctrl) - publisher := eventmocks.NewMockExptEventPublisher(ctrl) - items := []*entity.ExptEvalItem{{ExptID: 1, ItemID: 10, State: entity.ItemRunState_Processing}} - event := &entity.ExptScheduleEvent{SpaceID: 3, ExptID: 1, ExptRunID: 2} - turnRepo.EXPECT().MGetItemTurnRunLogs(gomock.Any(), int64(1), int64(2), []int64{10}, int64(3)).Return([]*entity.ExptTurnResultRunLog{{ - ItemID: 10, Status: entity.TurnRunState_Processing, - Ext: map[string]string{asyncEvaluatorResumeRepairExtKey: "true"}, - EvaluatorResultIds: &entity.EvaluatorResults{Registered: []*entity.RegisteredEvalResult{ - {VersionID: 101, RecordID: 1001}, {VersionID: 102, RecordID: 1002}, - }}, - }}, nil) - recordRepo.EXPECT().BatchGetEvaluatorRecord(gomock.Any(), gomock.InAnyOrder([]int64{1001, 1002}), false, false, gomock.Any()).Return([]*entity.EvaluatorRecord{ - {ID: 1001, Status: entity.EvaluatorRunStatusSuccess}, - {ID: 1002, Status: entity.EvaluatorRunStatusAsyncInvoking}, - }, nil) - publisher.EXPECT().PublishExptRecordEvalEvent(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Times(0) - - svc := &ExptSchedulerImpl{ExptTurnResultRepo: turnRepo, EvaluatorRecordRepo: recordRepo, Publisher: publisher} - require.NoError(t, svc.reconcileTerminalAsyncEvaluatorItems(context.Background(), event, asyncEvaluatorExperiment(), items)) -} - -func TestExptSchedulerImpl_ReconcileTerminalAsyncEvaluatorItems_RetriesOnNextTick(t *testing.T) { - t.Parallel() - - ctrl := gomock.NewController(t) - turnRepo := mock_repo.NewMockIExptTurnResultRepo(ctrl) - recordRepo := mock_repo.NewMockIEvaluatorRecordRepo(ctrl) - publisher := eventmocks.NewMockExptEventPublisher(ctrl) - items := []*entity.ExptEvalItem{{ExptID: 1, ItemID: 10, State: entity.ItemRunState_Processing}} - event := &entity.ExptScheduleEvent{SpaceID: 3, ExptID: 1, ExptRunID: 2} - turnLogs := []*entity.ExptTurnResultRunLog{{ - ItemID: 10, Status: entity.TurnRunState_Processing, - Ext: map[string]string{asyncEvaluatorResumeRepairExtKey: "true"}, - EvaluatorResultIds: &entity.EvaluatorResults{Registered: []*entity.RegisteredEvalResult{{VersionID: 101, RecordID: 1001}}}, - }} - turnRepo.EXPECT().MGetItemTurnRunLogs(gomock.Any(), int64(1), int64(2), []int64{10}, int64(3)).Return(turnLogs, nil).Times(2) - recordRepo.EXPECT().BatchGetEvaluatorRecord(gomock.Any(), []int64{1001}, false, false, gomock.Any()).Return([]*entity.EvaluatorRecord{{ID: 1001, Status: entity.EvaluatorRunStatusSuccess}}, nil).Times(2) - gomock.InOrder( - publisher.EXPECT().PublishExptRecordEvalEvent(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return(errors.New("mq down")), - publisher.EXPECT().PublishExptRecordEvalEvent(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return(nil), - ) - - svc := &ExptSchedulerImpl{ExptTurnResultRepo: turnRepo, EvaluatorRecordRepo: recordRepo, Publisher: publisher} - require.NoError(t, svc.reconcileTerminalAsyncEvaluatorItems(context.Background(), event, asyncEvaluatorExperiment(), items)) - require.NoError(t, svc.reconcileTerminalAsyncEvaluatorItems(context.Background(), event, asyncEvaluatorExperiment(), items)) -} - -func TestExptSchedulerImpl_ReconcileTerminalAsyncEvaluatorItems_IgnoresUnmarkedTurns(t *testing.T) { - t.Parallel() - - ctrl := gomock.NewController(t) - turnRepo := mock_repo.NewMockIExptTurnResultRepo(ctrl) - recordRepo := mock_repo.NewMockIEvaluatorRecordRepo(ctrl) - publisher := eventmocks.NewMockExptEventPublisher(ctrl) - items := []*entity.ExptEvalItem{{ExptID: 1, ItemID: 10, State: entity.ItemRunState_Processing}} - event := &entity.ExptScheduleEvent{SpaceID: 3, ExptID: 1, ExptRunID: 2} - turnRepo.EXPECT().MGetItemTurnRunLogs(gomock.Any(), int64(1), int64(2), []int64{10}, int64(3)).Return([]*entity.ExptTurnResultRunLog{{ - ItemID: 10, Status: entity.TurnRunState_Processing, - EvaluatorResultIds: &entity.EvaluatorResults{Registered: []*entity.RegisteredEvalResult{{VersionID: 101, RecordID: 1001}}}, - }}, nil) - recordRepo.EXPECT().BatchGetEvaluatorRecord(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Times(0) - publisher.EXPECT().PublishExptRecordEvalEvent(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Times(0) - - svc := &ExptSchedulerImpl{ExptTurnResultRepo: turnRepo, EvaluatorRecordRepo: recordRepo, Publisher: publisher} - require.NoError(t, svc.reconcileTerminalAsyncEvaluatorItems(context.Background(), event, asyncEvaluatorExperiment(), items)) -} - -func asyncEvaluatorExperiment() *entity.Experiment { - return &entity.Experiment{Evaluators: []*entity.Evaluator{{ - EvaluatorType: entity.EvaluatorTypeAgent, - }}} -} - -func TestExptSchedulerImpl_ReconcileTerminalAsyncEvaluatorItems_SkipsSyncOnlyExperiments(t *testing.T) { - t.Parallel() - - ctrl := gomock.NewController(t) - turnRepo := mock_repo.NewMockIExptTurnResultRepo(ctrl) - recordRepo := mock_repo.NewMockIEvaluatorRecordRepo(ctrl) - publisher := eventmocks.NewMockExptEventPublisher(ctrl) - turnRepo.EXPECT().MGetItemTurnRunLogs(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Times(0) - recordRepo.EXPECT().BatchGetEvaluatorRecord(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Times(0) - publisher.EXPECT().PublishExptRecordEvalEvent(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Times(0) - - svc := &ExptSchedulerImpl{ExptTurnResultRepo: turnRepo, EvaluatorRecordRepo: recordRepo, Publisher: publisher} - expt := &entity.Experiment{Evaluators: []*entity.Evaluator{{EvaluatorType: entity.EvaluatorTypePrompt}}} - items := []*entity.ExptEvalItem{{ItemID: 10, State: entity.ItemRunState_Processing}} - require.NoError(t, svc.reconcileTerminalAsyncEvaluatorItems(context.Background(), &entity.ExptScheduleEvent{}, expt, items)) -} - -func TestExptSchedulerImpl_ReconcileTerminalAsyncEvaluatorItems_ReadErrorsAreBestEffort(t *testing.T) { - t.Parallel() - - t.Run("turn log read error", func(t *testing.T) { - t.Parallel() - ctrl := gomock.NewController(t) - turnRepo := mock_repo.NewMockIExptTurnResultRepo(ctrl) - recordRepo := mock_repo.NewMockIEvaluatorRecordRepo(ctrl) - publisher := eventmocks.NewMockExptEventPublisher(ctrl) - turnRepo.EXPECT().MGetItemTurnRunLogs(gomock.Any(), int64(1), int64(2), []int64{10}, int64(3)).Return(nil, errors.New("db down")) - recordRepo.EXPECT().BatchGetEvaluatorRecord(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Times(0) - publisher.EXPECT().PublishExptRecordEvalEvent(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Times(0) - - svc := &ExptSchedulerImpl{ExptTurnResultRepo: turnRepo, EvaluatorRecordRepo: recordRepo, Publisher: publisher} - event := &entity.ExptScheduleEvent{SpaceID: 3, ExptID: 1, ExptRunID: 2} - items := []*entity.ExptEvalItem{{ItemID: 10, State: entity.ItemRunState_Processing}} - require.NoError(t, svc.reconcileTerminalAsyncEvaluatorItems(context.Background(), event, asyncEvaluatorExperiment(), items)) - }) - - t.Run("record read error", func(t *testing.T) { - t.Parallel() - ctrl := gomock.NewController(t) - turnRepo := mock_repo.NewMockIExptTurnResultRepo(ctrl) - recordRepo := mock_repo.NewMockIEvaluatorRecordRepo(ctrl) - publisher := eventmocks.NewMockExptEventPublisher(ctrl) - turnRepo.EXPECT().MGetItemTurnRunLogs(gomock.Any(), int64(1), int64(2), []int64{10}, int64(3)).Return([]*entity.ExptTurnResultRunLog{{ - ItemID: 10, Status: entity.TurnRunState_Processing, - Ext: map[string]string{asyncEvaluatorResumeRepairExtKey: "true"}, - EvaluatorResultIds: &entity.EvaluatorResults{Registered: []*entity.RegisteredEvalResult{{RecordID: 1001}}}, - }}, nil) - recordRepo.EXPECT().BatchGetEvaluatorRecord(gomock.Any(), []int64{1001}, false, false, gomock.Any()).Return(nil, errors.New("db down")) - publisher.EXPECT().PublishExptRecordEvalEvent(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Times(0) - - svc := &ExptSchedulerImpl{ExptTurnResultRepo: turnRepo, EvaluatorRecordRepo: recordRepo, Publisher: publisher} - event := &entity.ExptScheduleEvent{SpaceID: 3, ExptID: 1, ExptRunID: 2} - items := []*entity.ExptEvalItem{{ItemID: 10, State: entity.ItemRunState_Processing}} - require.NoError(t, svc.reconcileTerminalAsyncEvaluatorItems(context.Background(), event, asyncEvaluatorExperiment(), items)) - }) -} - -func TestExptSchedulerImpl_ReconcileTerminalAsyncEvaluatorItems_DoesNotRepublishWhenRecordMissing(t *testing.T) { - t.Parallel() - - ctrl := gomock.NewController(t) - turnRepo := mock_repo.NewMockIExptTurnResultRepo(ctrl) - recordRepo := mock_repo.NewMockIEvaluatorRecordRepo(ctrl) - publisher := eventmocks.NewMockExptEventPublisher(ctrl) - turnRepo.EXPECT().MGetItemTurnRunLogs(gomock.Any(), int64(1), int64(2), []int64{10}, int64(3)).Return([]*entity.ExptTurnResultRunLog{{ - ItemID: 10, Status: entity.TurnRunState_Processing, - Ext: map[string]string{asyncEvaluatorResumeRepairExtKey: "true"}, - EvaluatorResultIds: &entity.EvaluatorResults{Registered: []*entity.RegisteredEvalResult{{RecordID: 1001}, {RecordID: 1002}}}, - }}, nil) - recordRepo.EXPECT().BatchGetEvaluatorRecord(gomock.Any(), gomock.InAnyOrder([]int64{1001, 1002}), false, false, gomock.Any()).Return([]*entity.EvaluatorRecord{{ID: 1001, Status: entity.EvaluatorRunStatusSuccess}}, nil) - publisher.EXPECT().PublishExptRecordEvalEvent(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Times(0) - - svc := &ExptSchedulerImpl{ExptTurnResultRepo: turnRepo, EvaluatorRecordRepo: recordRepo, Publisher: publisher} - event := &entity.ExptScheduleEvent{SpaceID: 3, ExptID: 1, ExptRunID: 2} - items := []*entity.ExptEvalItem{{ItemID: 10, State: entity.ItemRunState_Processing}} - require.NoError(t, svc.reconcileTerminalAsyncEvaluatorItems(context.Background(), event, asyncEvaluatorExperiment(), items)) -} diff --git a/backend/modules/evaluation/infra/repo/experiment/redis/convert/item_turn_eval_async_test.go b/backend/modules/evaluation/infra/repo/experiment/redis/convert/item_turn_eval_async_test.go index 39dfd0e471..9c63000096 100644 --- a/backend/modules/evaluation/infra/repo/experiment/redis/convert/item_turn_eval_async_test.go +++ b/backend/modules/evaluation/infra/repo/experiment/redis/convert/item_turn_eval_async_test.go @@ -7,7 +7,6 @@ import ( "testing" "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" "github.com/coze-dev/coze-loop/backend/modules/evaluation/domain/entity" ) @@ -30,30 +29,11 @@ func TestExptItemTurnEvalAsyncCtx_CallbackURLRoundTrip(t *testing.T) { func TestExptItemTurnEvalAsyncCtx_ResumeReadyRoundTrip(t *testing.T) { c := NewExptItemTurnEvalAsyncCtx() - in := &entity.EvalAsyncCtx{RecordID: 123, ResumeBarrierEnabled: true, ResumeReady: true} + in := &entity.EvalAsyncCtx{RecordID: 123, ResumeReady: true} b, err := c.FromDO(in) assert.NoError(t, err) out, err := c.ToDO(b) assert.NoError(t, err) - assert.True(t, out.ResumeBarrierEnabled) assert.True(t, out.ResumeReady) } - -func TestExptItemTurnEvalAsyncCtx_LegacyPayloadDefaultsToResumeAllowed(t *testing.T) { - c := NewExptItemTurnEvalAsyncCtx() - out, err := c.ToDO([]byte(`{"RecordID":123}`)) - require.NoError(t, err) - assert.True(t, out.CanResumeExperiment()) -} - -func TestExptItemTurnEvalAsyncCtx_NewBarrierRequiresReady(t *testing.T) { - c := NewExptItemTurnEvalAsyncCtx() - out, err := c.ToDO([]byte(`{"resume_barrier_enabled":true}`)) - require.NoError(t, err) - assert.False(t, out.CanResumeExperiment()) - - out, err = c.ToDO([]byte(`{"resume_barrier_enabled":true,"resume_ready":true}`)) - require.NoError(t, err) - assert.True(t, out.CanResumeExperiment()) -} diff --git a/backend/modules/evaluation/infra/repo/experiment/redis/dao/item_turn_eval_async.go b/backend/modules/evaluation/infra/repo/experiment/redis/dao/item_turn_eval_async.go index a48194f3eb..e49d7ffebc 100644 --- a/backend/modules/evaluation/infra/repo/experiment/redis/dao/item_turn_eval_async.go +++ b/backend/modules/evaluation/infra/repo/experiment/redis/dao/item_turn_eval_async.go @@ -13,7 +13,6 @@ import ( "github.com/coze-dev/coze-loop/backend/modules/evaluation/infra/repo/experiment/redis/convert" "github.com/coze-dev/coze-loop/backend/pkg/errorx" "github.com/coze-dev/coze-loop/backend/pkg/lang/conv" - redisv9 "github.com/redis/go-redis/v9" ) type IEvalAsyncDAO interface { @@ -66,7 +65,7 @@ func (e *evalAsyncDAOImpl) GetEvalAsyncCtxStrong(ctx context.Context, invokeID s case <-timer.C: } } - actx, err := e.getEvalAsyncCtxFromPrimary(ctx, invokeID) + actx, err := e.getEvalAsyncCtx(ctx, invokeID) if err == nil || !redis.IsNilError(err) { return actx, err } @@ -96,7 +95,7 @@ func (e *evalAsyncDAOImpl) MarkEvalAsyncResumeReady(ctx context.Context, invokeI if err != nil { return nil, err } - if actx == nil || !actx.ResumeBarrierEnabled || actx.ResumeReady { + if actx == nil || actx.ResumeReady { return actx, nil } key := e.makeExptItemTurnEvalAsyncCtxKey(invokeID) @@ -117,24 +116,6 @@ func (e *evalAsyncDAOImpl) MarkEvalAsyncResumeReady(ctx context.Context, invokeI return actx, nil } -// Redis-v6 routes EVAL/EVALSHA by the script key and only routes commands whose -// command metadata is ReadOnly to replicas. EVAL is therefore master-routed even -// though this script only performs GET. This gives callback/arm reads the latest -// context without changing the shared client's read priority for unrelated traffic. -const getEvalAsyncCtxFromPrimaryScript = `return redis.call('GET', KEYS[1])` - -func (e *evalAsyncDAOImpl) getEvalAsyncCtxFromPrimary(ctx context.Context, invokeID string) (*entity.EvalAsyncCtx, error) { - key := e.makeExptItemTurnEvalAsyncCtxKey(invokeID) - got, err := e.cmdable.Eval(ctx, getEvalAsyncCtxFromPrimaryScript, []string{key}).Text() - if err != nil { - if redis.IsNilError(err) || err == redisv9.Nil { - return nil, redisv9.Nil - } - return nil, errorx.Wrapf(err, "redis primary get fail, key: %v", key) - } - return convert.NewExptItemTurnEvalAsyncCtx().ToDO(conv.UnsafeStringToBytes(got)) -} - func (e *evalAsyncDAOImpl) getEvalAsyncCtx(ctx context.Context, invokeID string) (*entity.EvalAsyncCtx, error) { key := e.makeExptItemTurnEvalAsyncCtxKey(invokeID) got, err := e.cmdable.Get(ctx, key).Result() diff --git a/backend/modules/evaluation/infra/repo/experiment/redis/dao/item_turn_eval_async_primary_test.go b/backend/modules/evaluation/infra/repo/experiment/redis/dao/item_turn_eval_async_primary_test.go deleted file mode 100644 index 943ea90706..0000000000 --- a/backend/modules/evaluation/infra/repo/experiment/redis/dao/item_turn_eval_async_primary_test.go +++ /dev/null @@ -1,62 +0,0 @@ -// Copyright (c) 2025 coze-dev Authors -// SPDX-License-Identifier: Apache-2.0 - -package dao - -import ( - "context" - "testing" - - redismocks "github.com/coze-dev/coze-loop/backend/infra/redis/mocks" - "github.com/redis/go-redis/v9" - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" - "go.uber.org/mock/gomock" -) - -func TestEvalAsyncDAO_GetEvalAsyncCtxStrongReadsThroughMasterRoutedScript(t *testing.T) { - t.Parallel() - - ctrl := gomock.NewController(t) - cmdable := redismocks.NewMockPersistentCmdable(ctrl) - dao := NewEvalAsyncDAO(cmdable) - key := "[experiment]item_turn_eval_async_ctx:evaluator:1" - payload := `{"RecordID":1,"resume_ready":true}` - - cmdable.EXPECT().Eval(gomock.Any(), gomock.Any(), []string{key}).DoAndReturn( - func(ctx context.Context, _ string, _ []string, _ ...any) *redis.Cmd { - return redis.NewCmdResult(payload, nil) - }, - ) - - got, err := dao.GetEvalAsyncCtxStrong(context.Background(), "evaluator:1") - require.NoError(t, err) - require.NotNil(t, got) - assert.True(t, got.ResumeReady) -} - -func TestEvalAsyncDAO_GetEvalAsyncCtxStrongRetriesPrimaryScriptOnMissing(t *testing.T) { - t.Parallel() - - ctrl := gomock.NewController(t) - cmdable := redismocks.NewMockPersistentCmdable(ctrl) - dao := NewEvalAsyncDAO(cmdable) - key := "[experiment]item_turn_eval_async_ctx:evaluator:2" - payload := `{"RecordID":2,"resume_barrier_enabled":true,"resume_ready":true}` - calls := 0 - cmdable.EXPECT().Eval(gomock.Any(), gomock.Any(), []string{key}).DoAndReturn( - func(ctx context.Context, _ string, _ []string, _ ...any) *redis.Cmd { - calls++ - if calls < 3 { - return redis.NewCmdResult(nil, redis.Nil) - } - return redis.NewCmdResult(payload, nil) - }, - ).Times(3) - - got, err := dao.GetEvalAsyncCtxStrong(context.Background(), "evaluator:2") - require.NoError(t, err) - require.NotNil(t, got) - assert.Equal(t, 3, calls) - assert.True(t, got.CanResumeExperiment()) -} diff --git a/backend/modules/evaluation/infra/repo/experiment/redis/dao/item_turn_eval_async_test.go b/backend/modules/evaluation/infra/repo/experiment/redis/dao/item_turn_eval_async_test.go index f0a27a0474..8c8389c8cc 100644 --- a/backend/modules/evaluation/infra/repo/experiment/redis/dao/item_turn_eval_async_test.go +++ b/backend/modules/evaluation/infra/repo/experiment/redis/dao/item_turn_eval_async_test.go @@ -24,7 +24,7 @@ func TestEvalAsyncDAO_GetStrongAndMarkResumeReady(t *testing.T) { go func() { time.Sleep(20 * time.Millisecond) - require.NoError(t, dao.SetEvalAsyncCtx(ctx, "evaluator:1", &entity.EvalAsyncCtx{RecordID: 1, ResumeBarrierEnabled: true})) + require.NoError(t, dao.SetEvalAsyncCtx(ctx, "evaluator:1", &entity.EvalAsyncCtx{RecordID: 1})) }() actx, err := dao.GetEvalAsyncCtxStrong(ctx, "evaluator:1") @@ -61,9 +61,8 @@ func TestEvalAsyncDAO_MarkResumeReadyPreservesLargeIDsAndExistingPayload(t *test ctx := context.Background() const largeID int64 = 9007199254740993 original := &entity.EvalAsyncCtx{ - ResumeBarrierEnabled: true, - RecordID: largeID, - EvaluatorVersionID: largeID - 1, + RecordID: largeID, + EvaluatorVersionID: largeID - 1, Event: &entity.ExptItemEvalEvent{ ExptID: largeID - 2, ExptRunID: largeID - 3, @@ -90,16 +89,3 @@ func TestMarkResumeReadyScriptAvoidsVersionSensitiveTTLCommands(t *testing.T) { assert.NotContains(t, markResumeReadyScript, "PTTL") assert.Contains(t, markResumeReadyScript, "'EX', ARGV[1]") } - -func TestEvalAsyncDAO_MarkResumeReadyDoesNotArmLegacyContext(t *testing.T) { - cmdable := infraredis.NewTestRedis(t) - dao := NewEvalAsyncDAO(cmdable) - ctx := context.Background() - - require.NoError(t, dao.SetEvalAsyncCtx(ctx, "legacy", &entity.EvalAsyncCtx{RecordID: 1})) - got, err := dao.MarkEvalAsyncResumeReady(ctx, "legacy") - require.NoError(t, err) - assert.False(t, got.ResumeBarrierEnabled) - assert.False(t, got.ResumeReady) - assert.True(t, got.CanResumeExperiment()) -} From b3b6fa44158252c91754c8ba2d379799be2f8719 Mon Sep 17 00:00:00 2001 From: shenjiayun Date: Wed, 12 Aug 2026 16:36:42 +0800 Subject: [PATCH 13/15] fix(evaluation): close async evaluator recovery gaps M-7362275687 --- .../application/eval_openapi_app.go | 8 +- .../application/eval_openapi_app_test.go | 54 ++++++++- .../domain/service/evaluator_impl.go | 20 +++- .../domain/service/evaluator_impl_test.go | 19 +++- .../domain/service/expt_run_item_impl.go | 24 ++-- .../domain/service/expt_run_item_impl_test.go | 94 ++++++++++++++++ .../domain/service/expt_run_item_turn_impl.go | 4 +- .../service/expt_run_item_turn_impl_test.go | 105 +++++++++++++++--- 8 files changed, 302 insertions(+), 26 deletions(-) diff --git a/backend/modules/evaluation/application/eval_openapi_app.go b/backend/modules/evaluation/application/eval_openapi_app.go index b94612f1d9..84a5adb473 100644 --- a/backend/modules/evaluation/application/eval_openapi_app.go +++ b/backend/modules/evaluation/application/eval_openapi_app.go @@ -3581,6 +3581,7 @@ func (e *EvalOpenAPIApplication) ReportEvaluatorInvokeResult_(ctx context.Contex if outcome == entity.ReportEvaluatorResultConflict { return &openapi.ReportEvaluatorInvokeResultResponse{BaseResp: base.NewBaseResp()}, nil } + var resumeErr error if actx.Event != nil { if !actx.ResumeReady { latestCtx, ctxErr := e.asyncRepo.GetEvalAsyncCtxStrong(ctx, asyncCtxKey) @@ -3591,7 +3592,9 @@ func (e *EvalOpenAPIApplication) ReportEvaluatorInvokeResult_(ctx context.Contex actx = latestCtx } } - if actx.ResumeReady { + if !actx.ResumeReady { + resumeErr = errorx.New("evaluator resume is not ready, invoke_id: %v", req.GetInvokeID()) + } else { if err := e.publisher.PublishExptRecordEvalEvent(ctx, actx.Event, gptr.Of(time.Second*3), func(event *entity.ExptItemEvalEvent) { event.AsyncEvaluatorReportTrigger = true }); err != nil { @@ -3617,6 +3620,9 @@ func (e *EvalOpenAPIApplication) ReportEvaluatorInvokeResult_(ctx context.Contex // 不返回错误：回调失败不影响运行时回报接口成功 } } + if resumeErr != nil { + return nil, resumeErr + } return &openapi.ReportEvaluatorInvokeResultResponse{BaseResp: base.NewBaseResp()}, nil } diff --git a/backend/modules/evaluation/application/eval_openapi_app_test.go b/backend/modules/evaluation/application/eval_openapi_app_test.go index 6acc29f786..c6ae8970a0 100755 --- a/backend/modules/evaluation/application/eval_openapi_app_test.go +++ b/backend/modules/evaluation/application/eval_openapi_app_test.go @@ -6723,7 +6723,7 @@ func TestEvalOpenAPIApplication_ReportEvaluatorInvokeResult(t *testing.T) { }, }, { - name: "success before resume ready skips experiment publish", + name: "success before resume ready asks provider to retry", req: &openapi.ReportEvaluatorInvokeResultRequest{ WorkspaceID: gptr.Of(workspaceID), InvokeID: gptr.Of(invokeID), @@ -6747,6 +6747,58 @@ func TestEvalOpenAPIApplication_ReportEvaluatorInvokeResult(t *testing.T) { }, nil) publisher.EXPECT().PublishExptRecordEvalEvent(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Times(0) }, + wantErr: -1, + }, + { + name: "duplicate terminal callback before resume ready asks provider to retry", + req: &openapi.ReportEvaluatorInvokeResultRequest{ + WorkspaceID: gptr.Of(workspaceID), + InvokeID: gptr.Of(invokeID), + Status: gptr.Of(spi.InvokeEvaluatorRunStatus_SUCCESS), + Output: &spi.InvokeEvaluatorOutputData{ + EvaluatorResult_: &spi.InvokeEvaluatorResult_{Score: gptr.Of(float64(0.9))}, + }, + }, + setup: func(auth *rpcmocks.MockIAuthProvider, asyncRepo *repomocks.MockIEvalAsyncRepo, evaluatorSvc *servicemocks.MockEvaluatorService, publisher *eventmocks.MockExptEventPublisher, _ *servicemocks.MockIEvaluatorCallbackDispatcher) { + auth.EXPECT().Authorization(gomock.Any(), gomock.Any()).Return(nil) + asyncRepo.EXPECT().GetEvalAsyncCtxStrong(gomock.Any(), "evaluator:2002").Return(&entity.EvalAsyncCtx{ + Event: event, + ResumeReady: false, + EvaluatorVersionID: 9, + }, nil) + evaluatorSvc.EXPECT().ReportEvaluatorInvokeResult(gomock.Any(), gomock.Any()).Return(entity.ReportEvaluatorResultDuplicate, nil) + asyncRepo.EXPECT().GetEvalAsyncCtxStrong(gomock.Any(), "evaluator:2002").Return(&entity.EvalAsyncCtx{ + Event: event, + ResumeReady: false, + EvaluatorVersionID: 9, + }, nil) + publisher.EXPECT().PublishExptRecordEvalEvent(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Times(0) + }, + wantErr: -1, + }, + { + name: "applied callback before resume ready dispatches external callback once then asks provider to retry", + req: &openapi.ReportEvaluatorInvokeResultRequest{ + WorkspaceID: gptr.Of(workspaceID), + InvokeID: gptr.Of(invokeID), + Status: gptr.Of(spi.InvokeEvaluatorRunStatus_SUCCESS), + Output: &spi.InvokeEvaluatorOutputData{ + EvaluatorResult_: &spi.InvokeEvaluatorResult_{Score: gptr.Of(float64(0.9))}, + }, + }, + setup: func(auth *rpcmocks.MockIAuthProvider, asyncRepo *repomocks.MockIEvalAsyncRepo, evaluatorSvc *servicemocks.MockEvaluatorService, publisher *eventmocks.MockExptEventPublisher, dispatcher *servicemocks.MockIEvaluatorCallbackDispatcher) { + auth.EXPECT().Authorization(gomock.Any(), gomock.Any()).Return(nil) + asyncRepo.EXPECT().GetEvalAsyncCtxStrong(gomock.Any(), "evaluator:2002").Return(&entity.EvalAsyncCtx{ + Event: event, ResumeReady: false, EvaluatorVersionID: 9, CallbackURL: "https://cb.example.com/hook", + }, nil) + evaluatorSvc.EXPECT().ReportEvaluatorInvokeResult(gomock.Any(), gomock.Any()).Return(entity.ReportEvaluatorResultApplied, nil) + asyncRepo.EXPECT().GetEvalAsyncCtxStrong(gomock.Any(), "evaluator:2002").Return(&entity.EvalAsyncCtx{ + Event: event, ResumeReady: false, EvaluatorVersionID: 9, CallbackURL: "https://cb.example.com/hook", + }, nil) + publisher.EXPECT().PublishExptRecordEvalEvent(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Times(0) + dispatcher.EXPECT().Dispatch(gomock.Any(), workspaceID, "https://cb.example.com/hook", gomock.Any()).Times(1) + }, + wantErr: -1, }, { name: "resume becomes ready between initial read and terminal CAS", diff --git a/backend/modules/evaluation/domain/service/evaluator_impl.go b/backend/modules/evaluation/domain/service/evaluator_impl.go index a0aa04d128..e405fec7ad 100644 --- a/backend/modules/evaluation/domain/service/evaluator_impl.go +++ b/backend/modules/evaluation/domain/service/evaluator_impl.go @@ -1247,7 +1247,25 @@ func (e *EvaluatorServiceImpl) ArmEvaluatorResume(ctx context.Context, recordID return errorx.New("eval async repo is nil") } asyncCtxKey := fmt.Sprintf("evaluator:%d", recordID) - actx, err := e.evalAsyncRepo.MarkEvalAsyncResumeReady(ctx, asyncCtxKey) + var ( + actx *entity.EvalAsyncCtx + err error + ) + for _, delay := range []time.Duration{0, 50 * time.Millisecond, 100 * time.Millisecond} { + if delay > 0 { + timer := time.NewTimer(delay) + select { + case <-ctx.Done(): + timer.Stop() + return ctx.Err() + case <-timer.C: + } + } + actx, err = e.evalAsyncRepo.MarkEvalAsyncResumeReady(ctx, asyncCtxKey) + if err == nil { + break + } + } if err != nil { return err } diff --git a/backend/modules/evaluation/domain/service/evaluator_impl_test.go b/backend/modules/evaluation/domain/service/evaluator_impl_test.go index fa72dfcd0a..7b302ea2d1 100644 --- a/backend/modules/evaluation/domain/service/evaluator_impl_test.go +++ b/backend/modules/evaluation/domain/service/evaluator_impl_test.go @@ -4963,6 +4963,23 @@ func TestEvaluatorServiceImpl_ArmEvaluatorResume_RetriesPublish(t *testing.T) { require.NoError(t, s.ArmEvaluatorResume(context.Background(), 100)) } +func TestEvaluatorServiceImpl_ArmEvaluatorResume_RetriesResumeReadyMark(t *testing.T) { + t.Parallel() + + ctrl := gomock.NewController(t) + asyncRepo := repomocks.NewMockIEvalAsyncRepo(ctrl) + recordRepo := repomocks.NewMockIEvaluatorRecordRepo(ctrl) + event := &entity.ExptItemEvalEvent{ExptID: 1} + gomock.InOrder( + asyncRepo.EXPECT().MarkEvalAsyncResumeReady(gomock.Any(), "evaluator:100").Return(nil, errors.New("redis unavailable")), + asyncRepo.EXPECT().MarkEvalAsyncResumeReady(gomock.Any(), "evaluator:100").Return(&entity.EvalAsyncCtx{Event: event, ResumeReady: true}, nil), + ) + recordRepo.EXPECT().GetEvaluatorRecord(gomock.Any(), int64(100), false).Return(&entity.EvaluatorRecord{ID: 100, Status: entity.EvaluatorRunStatusAsyncInvoking}, nil) + + s := &EvaluatorServiceImpl{evalAsyncRepo: asyncRepo, evaluatorRecordRepo: recordRepo} + require.NoError(t, s.ArmEvaluatorResume(context.Background(), 100)) +} + func TestEvaluatorServiceImpl_ArmEvaluatorResume_Errors(t *testing.T) { t.Parallel() @@ -4976,7 +4993,7 @@ func TestEvaluatorServiceImpl_ArmEvaluatorResume_Errors(t *testing.T) { t.Parallel() ctrl := gomock.NewController(t) asyncRepo := repomocks.NewMockIEvalAsyncRepo(ctrl) - asyncRepo.EXPECT().MarkEvalAsyncResumeReady(gomock.Any(), "evaluator:100").Return(nil, errors.New("redis failed")) + asyncRepo.EXPECT().MarkEvalAsyncResumeReady(gomock.Any(), "evaluator:100").Return(nil, errors.New("redis failed")).Times(3) s := &EvaluatorServiceImpl{evalAsyncRepo: asyncRepo} require.Error(t, s.ArmEvaluatorResume(context.Background(), 100)) }) diff --git a/backend/modules/evaluation/domain/service/expt_run_item_impl.go b/backend/modules/evaluation/domain/service/expt_run_item_impl.go index d341ee50b8..2752310f67 100644 --- a/backend/modules/evaluation/domain/service/expt_run_item_impl.go +++ b/backend/modules/evaluation/domain/service/expt_run_item_impl.go @@ -8,6 +8,7 @@ import ( "fmt" "strconv" "strings" + "sync" "time" "github.com/bytedance/gg/gptr" @@ -230,18 +231,27 @@ func (e *ExptItemEvalCtxExecutor) storeTurnRunResult(ctx context.Context, etec * if err := e.TurnResultRepo.SaveTurnRunLogs(persistCtx, []*entity.ExptTurnResultRunLog{clone}); err != nil { return err } + resumeCtx, resumeCancel := context.WithTimeout(context.WithoutCancel(ctx), exptRunLogPersistTimeout) + defer resumeCancel() + var resumeWG sync.WaitGroup for _, record := range result.EvaluatorResults { if record == nil || record.ID <= 0 || record.Status != entity.EvaluatorRunStatusAsyncInvoking { continue } - if err := e.evaluatorService.ArmEvaluatorResume(persistCtx, record.ID); err != nil { - // The turn references are already durable and the provider has accepted the work. - // Failing the item here would be unretriable (AsyncAbort sets CtxForceNoRetry) and - // could overwrite a valid terminal callback. Keep the item processing; callbacks can - // retry publication, and the existing zombie policy remains the final fallback. - logs.CtxError(ctx, "[ExptTurnEval] arm evaluator async resume failed after refs persisted, keep item processing, record_id: %d, err: %v", record.ID, err) - } + recordID := record.ID + resumeWG.Add(1) + go func() { + defer resumeWG.Done() + if err := e.evaluatorService.ArmEvaluatorResume(resumeCtx, recordID); err != nil { + // The turn references are already durable and the provider has accepted the work. + // Failing the item here would be unretriable (AsyncAbort sets CtxForceNoRetry) and + // could overwrite a valid terminal callback. Keep the item processing while the + // provider retries the callback; the existing zombie policy remains the final fallback. + logs.CtxError(ctx, "[ExptTurnEval] arm evaluator async resume failed after refs persisted, keep item processing, record_id: %d, err: %v", recordID, err) + } + }() } + resumeWG.Wait() logs.CtxInfo(ctx, "[ExptTurnEval] expt turn eval finished, expt_id: %v, expt_run_id: %v, item_id: %v, turn_id: %v, run_log: %v, err: %v", etec.Expt.ID, etec.Event.ExptRunID, etec.EvalSetItem.ItemID, turn.ID, json.Jsonify(clone), result.EvalErr) diff --git a/backend/modules/evaluation/domain/service/expt_run_item_impl_test.go b/backend/modules/evaluation/domain/service/expt_run_item_impl_test.go index 20fea51a6e..3d815e71fc 100644 --- a/backend/modules/evaluation/domain/service/expt_run_item_impl_test.go +++ b/backend/modules/evaluation/domain/service/expt_run_item_impl_test.go @@ -8,6 +8,7 @@ import ( "errors" "fmt" "testing" + "time" "github.com/bytedance/gg/gptr" "github.com/stretchr/testify/assert" @@ -1145,6 +1146,99 @@ func TestExptItemEvalCtxExecutor_storeTurnRunResult_ArmsAsyncEvaluatorAfterSave( require.NoError(t, executor.storeTurnRunResult(context.Background(), etec, result)) } +func TestExptItemEvalCtxExecutor_storeTurnRunResult_ArmingHasIndependentTimeoutBudget(t *testing.T) { + t.Parallel() + + ctrl := gomock.NewController(t) + turnRepo := repomocks.NewMockIExptTurnResultRepo(ctrl) + evaluatorSvc := servicemocks.NewMockEvaluatorService(ctrl) + + var persistDeadline time.Time + turnRepo.EXPECT().SaveTurnRunLogs(gomock.Any(), gomock.Any()).DoAndReturn( + func(ctx context.Context, _ []*entity.ExptTurnResultRunLog) error { + var ok bool + persistDeadline, ok = ctx.Deadline() + require.True(t, ok) + time.Sleep(20 * time.Millisecond) + return nil + }, + ) + evaluatorSvc.EXPECT().ArmEvaluatorResume(gomock.Any(), int64(100)).DoAndReturn( + func(ctx context.Context, _ int64) error { + resumeDeadline, ok := ctx.Deadline() + require.True(t, ok) + assert.True(t, resumeDeadline.After(persistDeadline), "arming must not consume the turn-log persistence timeout budget") + return nil + }, + ) + + executor := &ExptItemEvalCtxExecutor{TurnResultRepo: turnRepo, evaluatorService: evaluatorSvc} + etec := &entity.ExptTurnEvalCtx{ + Turn: &entity.Turn{ID: 1}, + ExptItemEvalCtx: &entity.ExptItemEvalCtx{ + Expt: &entity.Experiment{ID: 1, SpaceID: 2}, + Event: &entity.ExptItemEvalEvent{ExptRunID: 3}, + EvalSetItem: &entity.EvaluationSetItem{ItemID: 4}, + ExistItemEvalResult: &entity.ExptItemEvalResult{TurnResultRunLogs: map[int64]*entity.ExptTurnResultRunLog{ + 1: {ID: 5, TurnID: 1}, + }}, + }, + } + result := &entity.ExptTurnRunResult{EvaluatorResults: []*entity.EvaluatorRecord{{ + ID: 100, EvaluatorVersionID: 101, Status: entity.EvaluatorRunStatusAsyncInvoking, + }}} + require.NoError(t, executor.storeTurnRunResult(context.Background(), etec, result)) +} + +func TestExptItemEvalCtxExecutor_storeTurnRunResult_ArmsPendingEvaluatorsConcurrently(t *testing.T) { + t.Parallel() + + ctrl := gomock.NewController(t) + turnRepo := repomocks.NewMockIExptTurnResultRepo(ctrl) + evaluatorSvc := servicemocks.NewMockEvaluatorService(ctrl) + turnRepo.EXPECT().SaveTurnRunLogs(gomock.Any(), gomock.Any()).Return(nil) + + secondStarted := make(chan struct{}) + firstObservedSecond := make(chan bool, 1) + evaluatorSvc.EXPECT().ArmEvaluatorResume(gomock.Any(), int64(100)).DoAndReturn( + func(context.Context, int64) error { + select { + case <-secondStarted: + firstObservedSecond <- true + case <-time.After(time.Second): + firstObservedSecond <- false + } + return nil + }, + ) + evaluatorSvc.EXPECT().ArmEvaluatorResume(gomock.Any(), int64(200)).DoAndReturn( + func(context.Context, int64) error { + close(secondStarted) + return nil + }, + ) + + executor := &ExptItemEvalCtxExecutor{TurnResultRepo: turnRepo, evaluatorService: evaluatorSvc} + etec := &entity.ExptTurnEvalCtx{ + Turn: &entity.Turn{ID: 1}, + ExptItemEvalCtx: &entity.ExptItemEvalCtx{ + Expt: &entity.Experiment{ID: 1, SpaceID: 2}, + Event: &entity.ExptItemEvalEvent{ExptRunID: 3}, + EvalSetItem: &entity.EvaluationSetItem{ItemID: 4}, + ExistItemEvalResult: &entity.ExptItemEvalResult{TurnResultRunLogs: map[int64]*entity.ExptTurnResultRunLog{ + 1: {ID: 5, TurnID: 1}, + }}, + }, + } + result := &entity.ExptTurnRunResult{EvaluatorResults: []*entity.EvaluatorRecord{ + {ID: 100, EvaluatorVersionID: 101, Status: entity.EvaluatorRunStatusAsyncInvoking}, + {ID: 200, EvaluatorVersionID: 201, Status: entity.EvaluatorRunStatusAsyncInvoking}, + }} + + require.NoError(t, executor.storeTurnRunResult(context.Background(), etec, result)) + require.True(t, <-firstObservedSecond, "one slow arming operation must not delay the other pending evaluators") +} + func TestExptItemEvalCtxExecutor_storeTurnRunResult_DoesNotArmWhenSaveFails(t *testing.T) { t.Parallel() diff --git a/backend/modules/evaluation/domain/service/expt_run_item_turn_impl.go b/backend/modules/evaluation/domain/service/expt_run_item_turn_impl.go index 45ad15267f..4c706e174c 100644 --- a/backend/modules/evaluation/domain/service/expt_run_item_turn_impl.go +++ b/backend/modules/evaluation/domain/service/expt_run_item_turn_impl.go @@ -113,7 +113,7 @@ func (e *DefaultExptTurnEvaluationImpl) CallTarget(ctx context.Context, etec *en tr := etec.ExptTurnRunResult.GetTargetResult() - if etec.Event.AsyncReportTrigger { + if etec.Event.AsyncReportTrigger || etec.Event.AsyncEvaluatorReportTrigger { etec.Event.WithCtxTargetCalled(ctx) return tr, nil } @@ -137,7 +137,7 @@ func (e *DefaultExptTurnEvaluationImpl) CallTarget(ctx context.Context, etec *en } func (e *DefaultExptTurnEvaluationImpl) validateEvalTargetCtx(etec *entity.ExptTurnEvalCtx) error { - if etec.Event.AsyncReportTrigger && etec.ExptTurnRunResult.GetTargetResult() == nil { + if (etec.Event.AsyncReportTrigger || etec.Event.AsyncEvaluatorReportTrigger) && etec.ExptTurnRunResult.GetTargetResult() == nil { return errorx.NewByCode(errno.CommonInternalErrorCode, errorx.WithExtraMsg("target result must not be nil in async reported event")) } return nil diff --git a/backend/modules/evaluation/domain/service/expt_run_item_turn_impl_test.go b/backend/modules/evaluation/domain/service/expt_run_item_turn_impl_test.go index 2f9d0e55da..3bc28b9303 100644 --- a/backend/modules/evaluation/domain/service/expt_run_item_turn_impl_test.go +++ b/backend/modules/evaluation/domain/service/expt_run_item_turn_impl_test.go @@ -893,6 +893,74 @@ func TestDefaultExptTurnEvaluationImpl_CallTarget_AsyncReport(t *testing.T) { } } +func TestDefaultExptTurnEvaluationImpl_CallTarget_AsyncEvaluatorReportReusesTarget(t *testing.T) { + t.Parallel() + + for _, runMode := range []entity.ExptRunMode{ + entity.EvaluationModeSubmit, + entity.EvaluationModeRetryAll, + entity.EvaluationModeRetryItems, + } { + runMode := runMode + t.Run(fmt.Sprintf("run_mode_%d", runMode), func(t *testing.T) { + t.Parallel() + + ctrl := gomock.NewController(t) + benefitSvc := benefitmocks.NewMockIBenefitService(ctrl) + targetSvc := svcmocks.NewMockIEvalTargetService(ctrl) + benefitSvc.EXPECT().CheckAndDeductEvalBenefit(gomock.Any(), gomock.Any()).Times(0) + targetSvc.EXPECT().ExecuteTarget(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Times(0) + targetSvc.EXPECT().AsyncExecuteTarget(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Times(0) + + service := &DefaultExptTurnEvaluationImpl{benefitService: benefitSvc, evalTargetService: targetSvc} + status := entity.EvalTargetRunStatusSuccess + target := &entity.EvalTargetRecord{ID: 101, Status: &status} + etec := &entity.ExptTurnEvalCtx{ + ExptItemEvalCtx: &entity.ExptItemEvalCtx{ + Expt: &entity.Experiment{ID: 1, TargetVersionID: 1}, + Event: &entity.ExptItemEvalEvent{ + ExptRunMode: runMode, + RetryTimes: 0, + AsyncEvaluatorReportTrigger: true, + Session: &entity.Session{UserID: "u"}, + }, + }, + ExptTurnRunResult: &entity.ExptTurnRunResult{TargetResult: target}, + } + + got, err := service.CallTarget(context.Background(), etec) + + require.NoError(t, err) + require.Same(t, target, got) + }) + } +} + +func TestDefaultExptTurnEvaluationImpl_CallTarget_AsyncEvaluatorReportRequiresTarget(t *testing.T) { + t.Parallel() + + ctrl := gomock.NewController(t) + benefitSvc := benefitmocks.NewMockIBenefitService(ctrl) + targetSvc := svcmocks.NewMockIEvalTargetService(ctrl) + benefitSvc.EXPECT().CheckAndDeductEvalBenefit(gomock.Any(), gomock.Any()).Times(0) + targetSvc.EXPECT().ExecuteTarget(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Times(0) + targetSvc.EXPECT().AsyncExecuteTarget(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Times(0) + + service := &DefaultExptTurnEvaluationImpl{benefitService: benefitSvc, evalTargetService: targetSvc} + etec := &entity.ExptTurnEvalCtx{ + ExptItemEvalCtx: &entity.ExptItemEvalCtx{ + Expt: &entity.Experiment{ID: 1, TargetVersionID: 1}, + Event: &entity.ExptItemEvalEvent{AsyncEvaluatorReportTrigger: true, Session: &entity.Session{UserID: "u"}}, + }, + ExptTurnRunResult: &entity.ExptTurnRunResult{}, + } + + got, err := service.CallTarget(context.Background(), etec) + + require.Error(t, err) + require.Nil(t, got) +} + func TestDefaultExptTurnEvaluationImpl_CallTarget_ExistedRecord_Status(t *testing.T) { ctrl := gomock.NewController(t) defer ctrl.Finish() @@ -5069,22 +5137,28 @@ func TestDefaultExptTurnEvaluationImpl_AsyncTargetCallbackRunsMixedEvaluators(t }}, } - newEvaluator := func(versionID int64, async bool) *entity.Evaluator { - if async { - code := fmt.Sprintf("async-%d", versionID) - return &entity.Evaluator{ - ID: versionID, EvaluatorType: entity.EvaluatorTypeCustomRPC, - CustomRPCEvaluatorVersion: &entity.CustomRPCEvaluatorVersion{ - ID: versionID, EvaluatorID: versionID, ProviderEvaluatorCode: &code, - AccessProtocol: entity.EvaluatorAccessProtocolRPC, IsAsync: true, - }, - } - } + newSyncEvaluator := func(versionID int64) *entity.Evaluator { return &entity.Evaluator{ ID: versionID, EvaluatorType: entity.EvaluatorTypePrompt, PromptEvaluatorVersion: &entity.PromptEvaluatorVersion{ID: versionID}, } } + newAsyncCustomRPCEvaluator := func(versionID int64) *entity.Evaluator { + code := fmt.Sprintf("async-%d", versionID) + return &entity.Evaluator{ + ID: versionID, EvaluatorType: entity.EvaluatorTypeCustomRPC, + CustomRPCEvaluatorVersion: &entity.CustomRPCEvaluatorVersion{ + ID: versionID, EvaluatorID: versionID, ProviderEvaluatorCode: &code, + AccessProtocol: entity.EvaluatorAccessProtocolRPC, IsAsync: true, + }, + } + } + newAgentEvaluator := func(versionID int64) *entity.Evaluator { + return &entity.Evaluator{ + ID: versionID, EvaluatorType: entity.EvaluatorTypeAgent, + AgentEvaluatorVersion: &entity.AgentEvaluatorVersion{ID: versionID, EvaluatorID: versionID}, + } + } newConf := func(versionID int64) *entity.EvaluatorConf { return &entity.EvaluatorConf{ EvaluatorVersionID: versionID, @@ -5097,10 +5171,12 @@ func TestDefaultExptTurnEvaluationImpl_AsyncTargetCallbackRunsMixedEvaluators(t } syncVersions := map[int64]struct{}{101: {}, 102: {}} + agentVersions := map[int64]struct{}{201: {}} + asyncCustomRPCVersions := map[int64]struct{}{202: {}, 203: {}} asyncVersions := map[int64]struct{}{201: {}, 202: {}, 203: {}} allEvaluators := []*entity.Evaluator{ - newEvaluator(101, false), newEvaluator(102, false), - newEvaluator(201, true), newEvaluator(202, true), newEvaluator(203, true), + newSyncEvaluator(101), newSyncEvaluator(102), + newAgentEvaluator(201), newAsyncCustomRPCEvaluator(202), newAsyncCustomRPCEvaluator(203), } allConfs := []*entity.EvaluatorConf{ newConf(101), newConf(102), newConf(201), newConf(202), newConf(203), @@ -5160,6 +5236,9 @@ func TestDefaultExptTurnEvaluationImpl_AsyncTargetCallbackRunsMixedEvaluators(t mu.Unlock() _, ok := asyncVersions[req.EvaluatorVersionID] require.True(t, ok, "unexpected async evaluator %d", req.EvaluatorVersionID) + _, isAgent := agentVersions[req.EvaluatorVersionID] + _, isCustomRPC := asyncCustomRPCVersions[req.EvaluatorVersionID] + require.True(t, isAgent || isCustomRPC, "async evaluator %d must be Agent or CustomRPC", req.EvaluatorVersionID) require.Equal(t, "ppe_trae_work_async_evaluator", gptr.Indirect(req.EvaluatorRunConf.Env)) require.NotNil(t, req.AsyncCtx) require.Same(t, event, req.AsyncCtx.Event) From f48a6bd17abe8f8d9ce153bf48172bc46d8cd22e Mon Sep 17 00:00:00 2001 From: shenjiayun Date: Wed, 12 Aug 2026 19:24:26 +0800 Subject: [PATCH 14/15] test(evaluation): cover result-set async evaluator resume M-7362275687 --- .../service/expt_run_item_turn_impl_test.go | 45 +++++++++++++++++++ 1 file changed, 45 insertions(+) diff --git a/backend/modules/evaluation/domain/service/expt_run_item_turn_impl_test.go b/backend/modules/evaluation/domain/service/expt_run_item_turn_impl_test.go index 3bc28b9303..be23c1c641 100644 --- a/backend/modules/evaluation/domain/service/expt_run_item_turn_impl_test.go +++ b/backend/modules/evaluation/domain/service/expt_run_item_turn_impl_test.go @@ -961,6 +961,51 @@ func TestDefaultExptTurnEvaluationImpl_CallTarget_AsyncEvaluatorReportRequiresTa require.Nil(t, got) } +func TestDefaultExptTurnEvaluationImpl_ResultSetAsyncEvaluatorReportSkipsTargetValidation(t *testing.T) { + t.Parallel() + + ctrl := gomock.NewController(t) + metric := metricsmocks.NewMockExptMetric(ctrl) + metric.EXPECT().EmitTurnExecEval(int64(2), gomock.Any()) + metric.EXPECT().EmitTurnExecResult(int64(2), gomock.Any(), true, gomock.Any(), gomock.Any(), gomock.Any()) + + existingRecord := &entity.EvaluatorRecord{ + ID: 1001, + EvaluatorVersionID: 2001, + Status: entity.EvaluatorRunStatusSuccess, + } + service := &DefaultExptTurnEvaluationImpl{metric: metric} + etec := &entity.ExptTurnEvalCtx{ + ExptItemEvalCtx: &entity.ExptItemEvalCtx{ + Expt: &entity.Experiment{ + TargetVersionID: 0, + ExptType: entity.ExptType_Offline, + EvalConf: &entity.EvaluationConfiguration{ConnectorConf: entity.Connector{ + EvaluatorsConf: &entity.EvaluatorsConf{}, + }}, + }, + Event: &entity.ExptItemEvalEvent{ + SpaceID: 2, + AsyncEvaluatorReportTrigger: true, + Session: &entity.Session{UserID: "u"}, + }, + }, + ExptTurnRunResult: &entity.ExptTurnRunResult{ + TargetResult: nil, + EvaluatorResults: []*entity.EvaluatorRecord{existingRecord}, + }, + } + + result := service.Eval(context.Background(), etec) + + require.NoError(t, result.EvalErr) + require.NotNil(t, result.TargetResult) + require.NotNil(t, result.TargetResult.EvalTargetOutputData) + assert.Empty(t, result.TargetResult.EvalTargetOutputData.OutputFields) + require.Len(t, result.EvaluatorResults, 1) + assert.Same(t, existingRecord, result.EvaluatorResults[0]) +} + func TestDefaultExptTurnEvaluationImpl_CallTarget_ExistedRecord_Status(t *testing.T) { ctrl := gomock.NewController(t) defer ctrl.Finish() From 8741a167970c52527e84038fa7faa8d91cba286e Mon Sep 17 00:00:00 2001 From: shenjiayun Date: Wed, 12 Aug 2026 22:05:23 +0800 Subject: [PATCH 15/15] fix(evaluation): clamp async evaluator duration M-7362275687 --- .../domain/service/evaluator_impl.go | 4 ++- .../domain/service/evaluator_impl_test.go | 34 +++++++++++++++++++ 2 files changed, 37 insertions(+), 1 deletion(-) diff --git a/backend/modules/evaluation/domain/service/evaluator_impl.go b/backend/modules/evaluation/domain/service/evaluator_impl.go index e405fec7ad..e8ee9a0a59 100644 --- a/backend/modules/evaluation/domain/service/evaluator_impl.go +++ b/backend/modules/evaluation/domain/service/evaluator_impl.go @@ -1209,7 +1209,9 @@ func (e *EvaluatorServiceImpl) ReportEvaluatorInvokeResult(ctx context.Context, mergedOutputData = &entity.EvaluatorOutputData{} } if mergedOutputData.TimeConsumingMS == 0 && existingRecord.BaseInfo != nil && existingRecord.BaseInfo.CreatedAt != nil { - mergedOutputData.TimeConsumingMS = time.Now().UnixMilli() - gptr.Indirect(existingRecord.BaseInfo.CreatedAt) + if elapsedMS := time.Now().UnixMilli() - gptr.Indirect(existingRecord.BaseInfo.CreatedAt); elapsedMS > 0 { + mergedOutputData.TimeConsumingMS = elapsedMS + } } if existingRecord.EvaluatorOutputData != nil && existingRecord.EvaluatorOutputData.Ext != nil { if mergedOutputData.Ext == nil { diff --git a/backend/modules/evaluation/domain/service/evaluator_impl_test.go b/backend/modules/evaluation/domain/service/evaluator_impl_test.go index 7b302ea2d1..e11a30fb78 100644 --- a/backend/modules/evaluation/domain/service/evaluator_impl_test.go +++ b/backend/modules/evaluation/domain/service/evaluator_impl_test.go @@ -4900,6 +4900,40 @@ func TestEvaluatorServiceImpl_ReportEvaluatorInvokeResult_UsesTerminalCAS(t *tes assert.Equal(t, entity.ReportEvaluatorResultConflict, outcome) } +func TestEvaluatorServiceImpl_ReportEvaluatorInvokeResult_ClampsFallbackDurationToZero(t *testing.T) { + t.Parallel() + + ctrl := gomock.NewController(t) + recordRepo := repomocks.NewMockIEvaluatorRecordRepo(ctrl) + s := &EvaluatorServiceImpl{evaluatorRecordRepo: recordRepo} + futureCreatedAt := time.Now().Add(time.Second).UnixMilli() + + recordRepo.EXPECT().GetEvaluatorRecord(gomock.Any(), int64(100), false).Return(&entity.EvaluatorRecord{ + ID: 100, + SpaceID: 2, + Status: entity.EvaluatorRunStatusAsyncInvoking, + BaseInfo: &entity.BaseInfo{ + CreatedAt: gptr.Of(futureCreatedAt), + }, + }, nil) + recordRepo.EXPECT().CompareAndSwapEvaluatorRecordResult(gomock.Any(), int64(100), int64(2), entity.EvaluatorRunStatusAsyncInvoking, entity.EvaluatorRunStatusSuccess, gomock.Any()).DoAndReturn( + func(_ context.Context, _, _ int64, _, _ entity.EvaluatorRunStatus, out *entity.EvaluatorOutputData) (bool, error) { + require.NotNil(t, out) + assert.Zero(t, out.TimeConsumingMS) + return true, nil + }, + ) + + outcome, err := s.ReportEvaluatorInvokeResult(context.Background(), &entity.ReportEvaluatorRecordParam{ + SpaceID: 2, + RecordID: 100, + Status: entity.EvaluatorRunStatusSuccess, + OutputData: &entity.EvaluatorOutputData{}, + }) + require.NoError(t, err) + assert.Equal(t, entity.ReportEvaluatorResultApplied, outcome) +} + func TestEvaluatorServiceImpl_ArmEvaluatorResume(t *testing.T) { t.Parallel()