Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
15 changes: 11 additions & 4 deletions api/dms/service/v1/data_export_task.go
Original file line number Diff line number Diff line change
Expand Up @@ -85,7 +85,10 @@ type GetDataExportTask struct {
AuditResult AuditTaskResult `json:"audit_result"`
ExportType string `json:"export_type"` // Export Type example: SQL Meta
ExportFileType string `json:"export_file_type"` // Export Content example: CSV SQL EXCEL

// 失败阶段 wire:task_schedule / connect / prepare / sql_execute / file_generate;非失败可省略
ExportFailStage string `json:"export_fail_stage,omitempty"`
// 失败人类可读原因;非失败可省略
ExportFailReason string `json:"export_fail_reason,omitempty"`
}

// SQL审核结果
Expand Down Expand Up @@ -122,9 +125,13 @@ type ListDataExportTaskSQLsReply struct {
}

type ListDataExportTaskSQL struct {
ID uint `json:"uid"`
ExportSQL string `json:"sql"`
ExportResult string `json:"export_result"` // 导出结果
ID uint `json:"uid"`
ExportSQL string `json:"sql"`
// 导出结果(人类可读;成败以 export_status 为准)
ExportResult string `json:"export_result"`
// 导出执行状态:success / failed / not_executed;未开始可为空
// Enum: success,failed,not_executed
ExportStatus string `json:"export_status"`
ExportSQLType string `json:"export_sql_type"`
AuditLevel string `json:"audit_level"`
AuditSQLResult []AuditSQLResult `json:"audit_sql_result"`
Expand Down
4 changes: 3 additions & 1 deletion api/dms/service/v1/data_export_workflow.go
Original file line number Diff line number Diff line change
Expand Up @@ -298,7 +298,9 @@ type WorkflowRecord struct {
Tasks []*Task `json:"tasks"`
CurrentStepNumber uint `json:"current_step_number,omitempty"`
Status DataExportWorkflowStatus `json:"status"`
Steps []*WorkflowStep `json:"workflow_step_list,omitempty"`
// 导出失败摘要(人类可读);失败类状态时有值,成功/非导出失败为空
ExportFailSummary string `json:"export_fail_summary,omitempty"`
Steps []*WorkflowStep `json:"workflow_step_list,omitempty"`
}

type Task struct {
Expand Down
26 changes: 26 additions & 0 deletions api/swagger.json
Original file line number Diff line number Diff line change
Expand Up @@ -11614,6 +11614,16 @@
"format": "date-time",
"x-go-name": "ExportEndTime"
},
"export_fail_reason": {
"description": "失败人类可读原因;非失败可省略",
"type": "string",
"x-go-name": "ExportFailReason"
},
"export_fail_stage": {
"description": "失败阶段 wire:task_schedule / connect / prepare / sql_execute / file_generate;非失败可省略",
"type": "string",
"x-go-name": "ExportFailStage"
},
"export_file_type": {
"type": "string",
"x-go-name": "ExportFileType"
Expand Down Expand Up @@ -14109,13 +14119,24 @@
"x-go-name": "AuditSQLResult"
},
"export_result": {
"description": "导出结果(人类可读;成败以 export_status 为准)",
"type": "string",
"x-go-name": "ExportResult"
},
"export_sql_type": {
"type": "string",
"x-go-name": "ExportSQLType"
},
"export_status": {
"description": "导出执行状态 success/failed/not_executed;未开始可为空",
"type": "string",
"enum": [
"success",
"failed",
"not_executed"
],
"x-go-name": "ExportStatus"
},
"lineage_analysis_snapshot": {
"$ref": "#/definitions/AnalyzeResult"
},
Expand Down Expand Up @@ -20640,6 +20661,11 @@
"format": "uint64",
"x-go-name": "CurrentStepNumber"
},
"export_fail_summary": {
"description": "导出失败摘要(人类可读);失败类状态时有值,成功/非导出失败为空",
"type": "string",
"x-go-name": "ExportFailSummary"
},
"status": {
"type": "string",
"enum": [
Expand Down
21 changes: 21 additions & 0 deletions api/swagger.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -2334,6 +2334,14 @@ definitions:
format: date-time
type: string
x-go-name: ExportEndTime
export_fail_reason:
description: 失败人类可读原因;非失败可省略
type: string
x-go-name: ExportFailReason
export_fail_stage:
description: '失败阶段 wire:task_schedule / connect / prepare / sql_execute / file_generate;非失败可省略'
type: string
x-go-name: ExportFailStage
export_file_type:
type: string
x-go-name: ExportFileType
Expand Down Expand Up @@ -4306,11 +4314,20 @@ definitions:
type: array
x-go-name: AuditSQLResult
export_result:
description: 导出结果(人类可读;成败以 export_status 为准)
type: string
x-go-name: ExportResult
export_sql_type:
type: string
x-go-name: ExportSQLType
export_status:
description: 导出执行状态 success/failed/not_executed;未开始可为空
enum:
- success
- failed
- not_executed
type: string
x-go-name: ExportStatus
lineage_analysis_snapshot:
$ref: '#/definitions/AnalyzeResult'
masking_config_snapshot:
Expand Down Expand Up @@ -9731,6 +9748,10 @@ definitions:
format: uint64
type: integer
x-go-name: CurrentStepNumber
export_fail_summary:
description: 导出失败摘要(人类可读);失败类状态时有值,成功/非导出失败为空
type: string
x-go-name: ExportFailSummary
status:
enum:
- wait_for_approve
Expand Down
39 changes: 35 additions & 4 deletions internal/dms/biz/data_export_task.go
Original file line number Diff line number Diff line change
Expand Up @@ -39,27 +39,58 @@ type DataExportTask struct {
AuditScore int32
AuditLevel string

ExportStatus DataExportTaskStatus
ExportStartTime *time.Time
ExportEndTime *time.Time
DbService *DBService
ExportStatus DataExportTaskStatus
ExportStartTime *time.Time
ExportEndTime *time.Time
ExportFailStage string
ExportFailReason string
DbService *DBService

DataExportTaskRecords []*DataExportTaskRecord
}

// 任务级导出失败阶段(与 S2 §8.2 / swagger / UI 文案表一致)
const (
DataExportFailStageTaskSchedule = "task_schedule"
DataExportFailStageConnect = "connect"
DataExportFailStagePrepare = "prepare"
DataExportFailStageSQLExecute = "sql_execute"
DataExportFailStageFileGenerate = "file_generate"
)

func (t *DataExportTask) InstanceName() string {
if t.DbService != nil {
return t.DbService.Name
}
return ""
}

// SQL 级导出执行状态(与 S3 §8.2 wire 一致;成败以本字段为准,不以 export_result 判)
type DataExportSQLExportStatus string

const (
DataExportSQLStatusSuccess DataExportSQLExportStatus = "success"
DataExportSQLStatusFailed DataExportSQLExportStatus = "failed"
DataExportSQLStatusNotExecuted DataExportSQLExportStatus = "not_executed"
)

func (s DataExportSQLExportStatus) String() string {
return string(s)
}

// 未执行说明文案(产品锁定)
const DataExportSQLNotExecutedResult = "导出任务已失败,本条 SQL 未执行"

// 确无原因时任务/工单统一兜底(产品锁定);不得覆盖已有业务错误
const DataExportFailReasonFallback = "导出失败,暂未获取到具体原因,请联系管理员查看服务日志"

type DataExportTaskRecord struct {
Number uint
DataExportTaskId string
ExportSQL string
AuditLevel string
ExportResult string
ExportStatus DataExportSQLExportStatus
ExportSQLType string
AuditSQLResults model.AuditResults
}
Expand Down
4 changes: 4 additions & 0 deletions internal/dms/biz/data_export_workflow.go
Original file line number Diff line number Diff line change
Expand Up @@ -96,6 +96,8 @@ type WorkflowRecord struct {
CurrentStep *WorkflowStep
WorkflowSteps []*WorkflowStep
UpdateTime time.Time
// ExportFailSummary 导出失败摘要(持久化);成功/非失败为空
ExportFailSummary string
}

type WorkflowStep struct {
Expand All @@ -114,6 +116,8 @@ type WorkflowRepo interface {
ListDataExportWorkflows(ctx context.Context, opt *ListWorkflowsOption) ([]*Workflow, int64, error)
GetDataExportWorkflow(ctx context.Context, dataExportWorkflowUid string) (*Workflow, error)
UpdateWorkflowStatusById(ctx context.Context, dataExportWorkflowUid string, status DataExportWorkflowStatus) error
// UpdateWorkflowExportStatusById 更新工单导出状态与失败摘要(进入 exporting 时 summary 传空以清空上一轮)
UpdateWorkflowExportStatusById(ctx context.Context, dataExportWorkflowUid string, status DataExportWorkflowStatus, exportFailSummary string) error
GetDataExportWorkflowsByIds(ctx context.Context, dataExportWorkflowUid []string) ([]*Workflow, error)
CancelWorkflow(ctx context.Context, workflowRecordIds []string, workflowSteps []*WorkflowStep, operateId string) error
AuditWorkflow(ctx context.Context, dataExportWorkflowUid string, status DataExportWorkflowStatus, step *WorkflowStep, operateId, reason string) error
Expand Down
20 changes: 12 additions & 8 deletions internal/dms/service/data_export_workflow.go
Original file line number Diff line number Diff line change
Expand Up @@ -226,6 +226,7 @@ func (d *DMSService) GetDataExportWorkflow(ctx context.Context, req *dmsV1.GetDa
WorkflowRecord: dmsV1.WorkflowRecord{
CurrentStepNumber: uint(w.WorkflowRecord.CurrentWorkflowStepId),
Status: dmsV1.DataExportWorkflowStatus(w.WorkflowRecord.Status),
ExportFailSummary: w.WorkflowRecord.ExportFailSummary,
},
}

Expand Down Expand Up @@ -297,14 +298,16 @@ func (d *DMSService) BatchGetDataExportTask(ctx context.Context, req *dmsV1.Batc
data := make([]*dmsV1.GetDataExportTask, 0)
for _, task := range tasks {
data = append(data, &dmsV1.GetDataExportTask{
TaskUid: task.UID,
DBInfo: dmsV1.TaskDBInfo{UidWithName: convertBizUidWithName(d.DBServiceUsecase.GetBizDBWithNameByUids(ctx, []string{task.DBServiceUid}))[0], DBType: "", DatabaseName: task.DatabaseName},
Status: dmsV1.DataExportTaskStatus(task.ExportStatus),
ExportStartTime: task.ExportStartTime,
ExportEndTime: task.ExportEndTime,
FileName: task.ExportFileName,
ExportType: task.ExportType,
ExportFileType: task.ExportFileType,
TaskUid: task.UID,
DBInfo: dmsV1.TaskDBInfo{UidWithName: convertBizUidWithName(d.DBServiceUsecase.GetBizDBWithNameByUids(ctx, []string{task.DBServiceUid}))[0], DBType: "", DatabaseName: task.DatabaseName},
Status: dmsV1.DataExportTaskStatus(task.ExportStatus),
ExportStartTime: task.ExportStartTime,
ExportEndTime: task.ExportEndTime,
FileName: task.ExportFileName,
ExportType: task.ExportType,
ExportFileType: task.ExportFileType,
ExportFailStage: task.ExportFailStage,
ExportFailReason: task.ExportFailReason,
AuditResult: dmsV1.AuditTaskResult{
AuditLevel: task.AuditLevel,
Score: task.AuditScore,
Expand Down Expand Up @@ -352,6 +355,7 @@ func (d *DMSService) ListDataExportTaskSQLs(ctx context.Context, req *dmsV1.List
ExportSQL: w.ExportSQL,
AuditLevel: w.AuditLevel,
ExportResult: w.ExportResult,
ExportStatus: w.ExportStatus.String(),
ExportSQLType: w.ExportSQLType,
}
if d.UnmaskingWorkflowUsecase != nil {
Expand Down
8 changes: 8 additions & 0 deletions internal/dms/storage/convert.go
Original file line number Diff line number Diff line change
Expand Up @@ -1100,6 +1100,7 @@ func convertBizWorkflowRecord(b *biz.WorkflowRecord) *model.WorkflowRecord {
CurrentWorkflowStepId: b.CurrentWorkflowStepId,
Status: b.Status.String(),
TaskIds: taskIds,
ExportFailSummary: b.ExportFailSummary,
}
if b.WorkflowSteps != nil {
for _, step := range b.WorkflowSteps {
Expand All @@ -1116,6 +1117,7 @@ func convertModelWorkflowRecord(m *model.WorkflowRecord) (wr *biz.WorkflowRecord
Tasks: make([]biz.Task, 0),
CurrentWorkflowStepId: m.CurrentWorkflowStepId,
CurrentStep: convertModelWorkflowStep(m, m.CurrentWorkflowStepId),
ExportFailSummary: m.ExportFailSummary,
}

if len(m.Steps) != 0 {
Expand Down Expand Up @@ -1183,6 +1185,8 @@ func convertBizDataExportTask(b *biz.DataExportTask) *model.DataExportTask {
ExportStatus: b.ExportStatus.String(),
ExportStartTime: b.ExportStartTime,
ExportEndTime: b.ExportEndTime,
ExportFailStage: b.ExportFailStage,
ExportFailReason: b.ExportFailReason,
AuditPassRate: b.AuditPassRate,
AuditScore: b.AuditScore,
AuditLevel: b.AuditLevel,
Expand Down Expand Up @@ -1213,6 +1217,8 @@ func convertModelDataExportTask(m *model.DataExportTask) *biz.DataExportTask {
ExportStatus: biz.DataExportTaskStatus(m.ExportStatus),
ExportStartTime: m.ExportStartTime,
ExportEndTime: m.ExportEndTime,
ExportFailStage: m.ExportFailStage,
ExportFailReason: m.ExportFailReason,
}
if m.DataExportTaskRecords != nil {
for _, r := range m.DataExportTaskRecords {
Expand All @@ -1230,6 +1236,7 @@ func convertBizDataExportTaskRecords(b *biz.DataExportTaskRecord) *model.DataExp
ExportSQL: b.ExportSQL,
ExportSQLType: b.ExportSQLType,
ExportResult: b.ExportResult,
ExportStatus: b.ExportStatus.String(),
AuditLevel: b.AuditLevel,
AuditResults: b.AuditSQLResults,
}
Expand All @@ -1244,6 +1251,7 @@ func convertModelDataExportTaskRecords(m *model.DataExportTaskRecord) *biz.DataE
ExportSQLType: m.ExportSQLType,
AuditLevel: m.AuditLevel,
ExportResult: m.ExportResult,
ExportStatus: biz.DataExportSQLExportStatus(m.ExportStatus),
AuditSQLResults: m.AuditResults,
}
return b
Expand Down
8 changes: 7 additions & 1 deletion internal/dms/storage/model/model.go
Original file line number Diff line number Diff line change
Expand Up @@ -509,6 +509,8 @@ type WorkflowRecord struct {
CurrentWorkflowStepId uint64 `json:"current_workflow_step_id"`
Status string `gorm:"default:\"wait_for_export\""`
TaskIds Strings `json:"task_ids" gorm:"type:json"`
// ExportFailSummary 导出执行失败时的工单级人类可读摘要;成功或非导出失败为空
ExportFailSummary string `json:"export_fail_summary" gorm:"type:text"`

Steps []*WorkflowStep `gorm:"foreignkey:WorkflowRecordUid"`
}
Expand Down Expand Up @@ -572,7 +574,11 @@ type DataExportTask struct {
ExportStatus string `json:"export_status" gorm:"column:export_status;size:32"`
ExportStartTime *time.Time `json:"export_start_time" gorm:"column:export_start_time"`
ExportEndTime *time.Time `json:"export_end_time" gorm:"column:export_end_time"`
CreateUserUID string `json:"create_user_uid" gorm:"size:32;column:create_user_uid"`
// ExportFailStage 任务失败阶段 wire(connect/prepare/sql_execute/file_generate/...);成功为空
ExportFailStage string `json:"export_fail_stage" gorm:"column:export_fail_stage;size:32"`
// ExportFailReason 任务失败人类可读原因;成功为空
ExportFailReason string `json:"export_fail_reason" gorm:"column:export_fail_reason;type:text"`
CreateUserUID string `json:"create_user_uid" gorm:"size:32;column:create_user_uid"`
// Audit Result
AuditPassRate float64 `json:"audit_pass_rate"`
AuditScore int32 `json:"audit_score"`
Expand Down
13 changes: 13 additions & 0 deletions internal/dms/storage/workflow.go
Original file line number Diff line number Diff line change
Expand Up @@ -318,6 +318,19 @@ func (d *WorkflowRepo) UpdateWorkflowStatusById(ctx context.Context, dataExportW
})
}

func (d *WorkflowRepo) UpdateWorkflowExportStatusById(ctx context.Context, dataExportWorkflowUid string, status biz.DataExportWorkflowStatus, exportFailSummary string) error {
return transaction(d.log, ctx, d.db, func(tx *gorm.DB) error {
fields := map[string]interface{}{
"status": status,
"export_fail_summary": exportFailSummary,
}
if err := tx.WithContext(ctx).Model(&model.WorkflowRecord{}).Where("uid = ?", dataExportWorkflowUid).Updates(fields).Error; err != nil {
return fmt.Errorf("failed to update workflow export status, err: %v", err)
}
return nil
})
}

func (d *WorkflowRepo) CancelWorkflow(ctx context.Context, workflowRecordIds []string, workflowSteps []*biz.WorkflowStep, operateId string) error {
return transaction(d.log, ctx, d.db, func(tx *gorm.DB) error {
if err := tx.WithContext(ctx).Model(&model.WorkflowRecord{}).Where("uid in (?)", workflowRecordIds).Update("status", biz.DataExportWorkflowStatusCancel).Error; err != nil {
Expand Down
1 change: 1 addition & 0 deletions internal/pkg/locale/active.en.toml
Original file line number Diff line number Diff line change
Expand Up @@ -137,6 +137,7 @@ NotifyDataWorkflowBodyHead = "\n📋 Data Export Workflow Topic: %v\n📍 Projec
NotifyDataWorkflowBodyInstanceAndSchema = "🗄️ Data Source: %v\n📊 Schema: %v"
NotifyDataWorkflowBodyLink = "🔗 Data Export Workflow Link: %v"
NotifyDataWorkflowBodyReason = "❌ Rejection Reason: %v"
NotifyDataWorkflowBodyExportFailReason = "❌ Failure Reason: %v"
NotifyDataWorkflowBodyReport = "⭐ Data Export Workflow Audit Score: %v"
NotifyDataWorkflowBodyStartEnd = "▶️ Execute Start Time: %v\n◀️ Execute End Time: %v"
NotifyDataWorkflowBodyWorkFlowErr = "⚠️ Failed to read data export workflow task content, please check the workflow status through the SQLE interface"
Expand Down
1 change: 1 addition & 0 deletions internal/pkg/locale/active.zh.toml
Original file line number Diff line number Diff line change
Expand Up @@ -137,6 +137,7 @@ NotifyDataWorkflowBodyHead = "\n📋 数据导出工单主题: %v\n📍 所属
NotifyDataWorkflowBodyInstanceAndSchema = "🗄️ 数据源: %v\n📊 schema: %v\n"
NotifyDataWorkflowBodyLink = "🔗 数据导出工单链接: %v"
NotifyDataWorkflowBodyReason = "❌ 驳回原因: %v"
NotifyDataWorkflowBodyExportFailReason = "❌ 失败原因: %v"
NotifyDataWorkflowBodyReport = "⭐ 数据导出工单审核得分: %v"
NotifyDataWorkflowBodyStartEnd = "▶️ 数据导出开始时间: %v\n◀️ 数据导出结束时间: %v"
NotifyDataWorkflowBodyWorkFlowErr = "❌ 读取工单任务内容失败,请通过SQLE界面确认工单状态"
Expand Down
1 change: 1 addition & 0 deletions internal/pkg/locale/message_zh.go
Original file line number Diff line number Diff line change
Expand Up @@ -259,6 +259,7 @@ var (
NotifyDataWorkflowBodyInstanceAndSchema = &i18n.Message{ID: "NotifyDataWorkflowBodyInstanceAndSchema", Other: "🗄️ 数据源: %v\n📊 schema: %v\n"}
NotifyDataWorkflowBodyLink = &i18n.Message{ID: "NotifyDataWorkflowBodyLink", Other: "🔗 数据导出工单链接: %v"}
NotifyDataWorkflowBodyReason = &i18n.Message{ID: "NotifyDataWorkflowBodyReason", Other: "❌ 驳回原因: %v"}
NotifyDataWorkflowBodyExportFailReason = &i18n.Message{ID: "NotifyDataWorkflowBodyExportFailReason", Other: "❌ 失败原因: %v"}
NotifyDataWorkflowBodyReport = &i18n.Message{ID: "NotifyDataWorkflowBodyReport", Other: "⭐ 数据导出工单审核得分: %v"}
NotifyDataWorkflowBodyStartEnd = &i18n.Message{ID: "NotifyDataWorkflowBodyStartEnd", Other: "▶️ 数据导出开始时间: %v\n◀️ 数据导出结束时间: %v"}
NotifyDataWorkflowBodyWorkFlowErr = &i18n.Message{ID: "NotifyDataWorkflowBodyWorkFlowErr", Other: "❌ 读取工单任务内容失败,请通过SQLE界面确认工单状态"}
Expand Down