diff --git a/cmd/simrun/main.go b/cmd/simrun/main.go index ad65a1e..d03626c 100644 --- a/cmd/simrun/main.go +++ b/cmd/simrun/main.go @@ -39,7 +39,7 @@ func main() { // Create stores runStore := db.NewRunStore(database.Pool) - scenarioStore := db.NewScenarioStore(database.Pool) + scenarioStore := db.NewAssessmentStore(database.Pool) packStore := db.NewPackStore(database.Pool) configStore := db.NewConfigStore(database.Pool) secretStore := db.NewSecretStore(database.Pool) @@ -130,8 +130,8 @@ func main() { log.Warnf("Retention sweep: failed to load config: %v", err) return } - web.SweepRunLogs(bootstrap.DataDir, cfg.AssessmentLogRetentionEnabled, cfg.AssessmentLogRetentionDays) - web.SweepAssessments(ctx, runStore, bootstrap.DataDir, cfg.AssessmentRetentionEnabled, cfg.AssessmentRetentionDays) + web.SweepRunLogs(bootstrap.DataDir, cfg.RunLogRetentionEnabled, cfg.RunLogRetentionDays) + web.SweepRuns(ctx, runStore, bootstrap.DataDir, cfg.RunRetentionEnabled, cfg.RunRetentionDays) } sweep() for { diff --git a/internal/collectors/collector.go b/internal/collectors/collector.go index db941d7..cd98168 100644 --- a/internal/collectors/collector.go +++ b/internal/collectors/collector.go @@ -9,7 +9,7 @@ import ( type Collector interface { // Collect searches for logs matching the configured query and indicators, // and writes them to the output file. This is called once at the end of - // scenario execution (when assertions pass or timeout is reached). + // scenario execution (when expectations pass or timeout is reached). // It returns the number of documents collected and any error that occurred. Collect(ctx context.Context, indicators map[string]string) (int, error) diff --git a/internal/config/appconfig.go b/internal/config/appconfig.go index 2eef584..3dc4d30 100644 --- a/internal/config/appconfig.go +++ b/internal/config/appconfig.go @@ -7,27 +7,27 @@ package config // belongs in connectors, anything secret belongs in secret_groups, anything // set at deploy belongs in Bootstrap. type AppConfig struct { - Parallelism int `json:"parallelism"` - TerraformVersion string `json:"terraform_version"` - PackLogsEnabled bool `json:"pack_logs_enabled"` - SSHLoggingEnabled bool `json:"ssh_logging_enabled"` - AssessmentLogRetentionEnabled bool `json:"assessment_log_retention_enabled"` - AssessmentLogRetentionDays int `json:"assessment_log_retention_days"` - AssessmentRetentionEnabled bool `json:"assessment_retention_enabled"` - AssessmentRetentionDays int `json:"assessment_retention_days"` + Parallelism int `json:"parallelism"` + TerraformVersion string `json:"terraform_version"` + PackLogsEnabled bool `json:"pack_logs_enabled"` + SSHLoggingEnabled bool `json:"ssh_logging_enabled"` + RunLogRetentionEnabled bool `json:"run_log_retention_enabled"` + RunLogRetentionDays int `json:"run_log_retention_days"` + RunRetentionEnabled bool `json:"run_retention_enabled"` + RunRetentionDays int `json:"run_retention_days"` } // DefaultAppConfig returns the default values used when no row exists for // a key. Keep these aligned with the migration that backfills app_config. func DefaultAppConfig() AppConfig { return AppConfig{ - Parallelism: 5, - TerraformVersion: "", - PackLogsEnabled: true, - SSHLoggingEnabled: false, - AssessmentLogRetentionEnabled: true, - AssessmentLogRetentionDays: 7, - AssessmentRetentionEnabled: false, - AssessmentRetentionDays: 30, + Parallelism: 5, + TerraformVersion: "", + PackLogsEnabled: true, + SSHLoggingEnabled: false, + RunLogRetentionEnabled: true, + RunLogRetentionDays: 7, + RunRetentionEnabled: false, + RunRetentionDays: 30, } } diff --git a/internal/config/appconfig_test.go b/internal/config/appconfig_test.go index a177161..fadc06d 100644 --- a/internal/config/appconfig_test.go +++ b/internal/config/appconfig_test.go @@ -8,13 +8,13 @@ import ( func TestDefaultAppConfig(t *testing.T) { assert.Equal(t, AppConfig{ - Parallelism: 5, - TerraformVersion: "", - PackLogsEnabled: true, - SSHLoggingEnabled: false, - AssessmentLogRetentionEnabled: true, - AssessmentLogRetentionDays: 7, - AssessmentRetentionEnabled: false, - AssessmentRetentionDays: 30, + Parallelism: 5, + TerraformVersion: "", + PackLogsEnabled: true, + SSHLoggingEnabled: false, + RunLogRetentionEnabled: true, + RunLogRetentionDays: 7, + RunRetentionEnabled: false, + RunRetentionDays: 30, }, DefaultAppConfig()) } diff --git a/internal/db/assessments.go b/internal/db/assessments.go new file mode 100644 index 0000000..ba1c537 --- /dev/null +++ b/internal/db/assessments.go @@ -0,0 +1,202 @@ +package db + +import ( + "context" + "strconv" + "strings" + "time" + + "github.com/google/uuid" + "github.com/jackc/pgx/v5/pgxpool" +) + +// AssessmentStore manages saved assessment (definition) YAML persistence. +type AssessmentStore interface { + Save(ctx context.Context, name, assessmentType, yaml, createdBy string) (*Assessment, error) + Get(ctx context.Context, id uuid.UUID) (*Assessment, error) + // GetByName returns the assessment with the given unique name. + GetByName(ctx context.Context, name string) (*Assessment, error) + // List returns a filtered, paginated slice of assessments for the UI. + List(ctx context.Context, filters ListAssessmentsFilters, limit, offset int) (AssessmentPage, error) + // ListAll returns every assessment in updated_at DESC order. For internal + // callers (e.g. coverage maps) that need the full set in one shot. + ListAll(ctx context.Context) ([]Assessment, error) + Update(ctx context.Context, id uuid.UUID, name, assessmentType, yaml, updatedBy string) error + Delete(ctx context.Context, id uuid.UUID) error +} + +// Assessment represents a saved assessment definition (the saved YAML). +type Assessment struct { + ID uuid.UUID `json:"id"` + Name string `json:"name"` + Type string `json:"type"` + YAML string `json:"yaml"` + CreatedBy string `json:"createdBy"` + UpdatedBy string `json:"updatedBy"` + CreatedAt time.Time `json:"createdAt"` + UpdatedAt time.Time `json:"updatedAt"` +} + +// AssessmentPage is a paginated slice of assessments with the total row count. +type AssessmentPage struct { + Assessments []Assessment `json:"assessments"` + Total int `json:"total"` +} + +// ListAssessmentsFilters narrows the result set for AssessmentStore.List. Zero +// values mean "no constraint on this dimension". +type ListAssessmentsFilters struct { + // Name is an ILIKE %name% match against assessments.name. + Name string + // Types restricts assessments.type to the listed values. + Types []string + // Since restricts assessments to updated_at >= Since. + Since *time.Time +} + +type assessmentStore struct { + pool *pgxpool.Pool +} + +// NewAssessmentStore creates a new AssessmentStore backed by PostgreSQL. +func NewAssessmentStore(pool *pgxpool.Pool) AssessmentStore { + return &assessmentStore{pool: pool} +} + +func (s *assessmentStore) Save(ctx context.Context, name, assessmentType, yaml, createdBy string) (*Assessment, error) { + var sc Assessment + err := s.pool.QueryRow(ctx, + `INSERT INTO assessments (name, type, yaml, created_by, updated_by) VALUES ($1, $2, $3, $4, $4) + RETURNING id, name, type, yaml, created_by, updated_by, created_at, updated_at`, + name, assessmentType, yaml, createdBy, + ).Scan(&sc.ID, &sc.Name, &sc.Type, &sc.YAML, &sc.CreatedBy, &sc.UpdatedBy, &sc.CreatedAt, &sc.UpdatedAt) + if err != nil { + return nil, err + } + return &sc, nil +} + +func (s *assessmentStore) Get(ctx context.Context, id uuid.UUID) (*Assessment, error) { + var sc Assessment + err := s.pool.QueryRow(ctx, + `SELECT id, name, type, yaml, created_by, updated_by, created_at, updated_at FROM assessments WHERE id = $1`, id, + ).Scan(&sc.ID, &sc.Name, &sc.Type, &sc.YAML, &sc.CreatedBy, &sc.UpdatedBy, &sc.CreatedAt, &sc.UpdatedAt) + if err != nil { + return nil, err + } + return &sc, nil +} + +func (s *assessmentStore) GetByName(ctx context.Context, name string) (*Assessment, error) { + var sc Assessment + err := s.pool.QueryRow(ctx, + `SELECT id, name, type, yaml, created_by, updated_by, created_at, updated_at FROM assessments WHERE name = $1`, name, + ).Scan(&sc.ID, &sc.Name, &sc.Type, &sc.YAML, &sc.CreatedBy, &sc.UpdatedBy, &sc.CreatedAt, &sc.UpdatedAt) + if err != nil { + return nil, err + } + return &sc, nil +} + +func (s *assessmentStore) List(ctx context.Context, filters ListAssessmentsFilters, limit, offset int) (AssessmentPage, error) { + where, args := buildAssessmentsWhere(filters) + rows, err := s.pool.Query(ctx, + `SELECT id, name, type, yaml, created_by, updated_by, created_at, updated_at, + COUNT(*) OVER() AS total_count + FROM assessments + `+where+` + ORDER BY updated_at DESC + LIMIT $`+strconv.Itoa(len(args)+1)+` OFFSET $`+strconv.Itoa(len(args)+2), + append(args, limit, offset)..., + ) + if err != nil { + return AssessmentPage{}, err + } + defer rows.Close() + + page := AssessmentPage{Assessments: []Assessment{}} + for rows.Next() { + var sc Assessment + var total int + if err := rows.Scan(&sc.ID, &sc.Name, &sc.Type, &sc.YAML, &sc.CreatedBy, &sc.UpdatedBy, &sc.CreatedAt, &sc.UpdatedAt, &total); err != nil { + return AssessmentPage{}, err + } + page.Assessments = append(page.Assessments, sc) + page.Total = total + } + if err := rows.Err(); err != nil { + return AssessmentPage{}, err + } + if len(page.Assessments) == 0 { + // COUNT(*) OVER() collapses to no rows when LIMIT/OFFSET yields nothing. + // Re-run the same filter as a plain COUNT so the UI can show "of N". + countSQL := `SELECT COUNT(*) FROM assessments ` + where + if err := s.pool.QueryRow(ctx, countSQL, args...).Scan(&page.Total); err != nil { + return AssessmentPage{}, err + } + } + return page, nil +} + +func (s *assessmentStore) ListAll(ctx context.Context) ([]Assessment, error) { + rows, err := s.pool.Query(ctx, + `SELECT id, name, type, yaml, created_by, updated_by, created_at, updated_at + FROM assessments ORDER BY updated_at DESC`, + ) + if err != nil { + return nil, err + } + defer rows.Close() + + assessments := []Assessment{} + for rows.Next() { + var sc Assessment + if err := rows.Scan(&sc.ID, &sc.Name, &sc.Type, &sc.YAML, &sc.CreatedBy, &sc.UpdatedBy, &sc.CreatedAt, &sc.UpdatedAt); err != nil { + return nil, err + } + assessments = append(assessments, sc) + } + return assessments, rows.Err() +} + +// buildAssessmentsWhere returns a WHERE clause (or "") and its positional args +// for assessments. Placeholders are $1..$N in argument order. +func buildAssessmentsWhere(f ListAssessmentsFilters) (string, []any) { + var clauses []string + var args []any + if f.Name != "" { + args = append(args, "%"+f.Name+"%") + clauses = append(clauses, "name ILIKE $"+strconv.Itoa(len(args))) + } + if len(f.Types) > 0 { + placeholders := make([]string, len(f.Types)) + for i, t := range f.Types { + args = append(args, t) + placeholders[i] = "$" + strconv.Itoa(len(args)) + } + clauses = append(clauses, "type IN ("+strings.Join(placeholders, ",")+")") + } + if f.Since != nil { + args = append(args, *f.Since) + clauses = append(clauses, "updated_at >= $"+strconv.Itoa(len(args))) + } + if len(clauses) == 0 { + return "", args + } + return "WHERE " + strings.Join(clauses, " AND "), args +} + +func (s *assessmentStore) Update(ctx context.Context, id uuid.UUID, name, assessmentType, yaml, updatedBy string) error { + _, err := s.pool.Exec(ctx, + `UPDATE assessments SET name = $2, type = $3, yaml = $4, updated_by = $5, updated_at = NOW() WHERE id = $1`, + id, name, assessmentType, yaml, updatedBy, + ) + return err +} + +func (s *assessmentStore) Delete(ctx context.Context, id uuid.UUID) error { + _, err := s.pool.Exec(ctx, + `DELETE FROM assessments WHERE id = $1`, id, + ) + return err +} diff --git a/internal/db/config.go b/internal/db/config.go index 912a993..c29599c 100644 --- a/internal/db/config.go +++ b/internal/db/config.go @@ -96,28 +96,28 @@ func parseAppConfig(all map[string]json.RawMessage) config.AppConfig { cfg.SSHLoggingEnabled = b } } - if v, ok := all["assessment_log_retention_enabled"]; ok { + if v, ok := all["run_log_retention_enabled"]; ok { var b bool if err := json.Unmarshal(v, &b); err == nil { - cfg.AssessmentLogRetentionEnabled = b + cfg.RunLogRetentionEnabled = b } } - if v, ok := all["assessment_log_retention_days"]; ok { + if v, ok := all["run_log_retention_days"]; ok { var n int if err := json.Unmarshal(v, &n); err == nil && n > 0 { - cfg.AssessmentLogRetentionDays = n + cfg.RunLogRetentionDays = n } } - if v, ok := all["assessment_retention_enabled"]; ok { + if v, ok := all["run_retention_enabled"]; ok { var b bool if err := json.Unmarshal(v, &b); err == nil { - cfg.AssessmentRetentionEnabled = b + cfg.RunRetentionEnabled = b } } - if v, ok := all["assessment_retention_days"]; ok { + if v, ok := all["run_retention_days"]; ok { var n int if err := json.Unmarshal(v, &n); err == nil && n > 0 { - cfg.AssessmentRetentionDays = n + cfg.RunRetentionDays = n } } @@ -138,10 +138,10 @@ func appConfigKVs(c config.AppConfig) []appConfigKV { {"terraform_version", c.TerraformVersion}, {"pack_logs_enabled", c.PackLogsEnabled}, {"ssh_logging_enabled", c.SSHLoggingEnabled}, - {"assessment_log_retention_enabled", c.AssessmentLogRetentionEnabled}, - {"assessment_log_retention_days", c.AssessmentLogRetentionDays}, - {"assessment_retention_enabled", c.AssessmentRetentionEnabled}, - {"assessment_retention_days", c.AssessmentRetentionDays}, + {"run_log_retention_enabled", c.RunLogRetentionEnabled}, + {"run_log_retention_days", c.RunLogRetentionDays}, + {"run_retention_enabled", c.RunRetentionEnabled}, + {"run_retention_days", c.RunRetentionDays}, } } diff --git a/internal/db/config_test.go b/internal/db/config_test.go index 749ef9e..a63a490 100644 --- a/internal/db/config_test.go +++ b/internal/db/config_test.go @@ -47,24 +47,24 @@ func TestParseAppConfig_NonPositiveParallelismKeepsDefault(t *testing.T) { func TestParseAppConfig_AllSet(t *testing.T) { got := parseAppConfig(map[string]json.RawMessage{ - "parallelism": json.RawMessage(`12`), - "terraform_version": json.RawMessage(`"1.6.0"`), - "pack_logs_enabled": json.RawMessage(`false`), - "ssh_logging_enabled": json.RawMessage(`true`), - "assessment_log_retention_enabled": json.RawMessage(`false`), - "assessment_log_retention_days": json.RawMessage(`14`), - "assessment_retention_enabled": json.RawMessage(`true`), - "assessment_retention_days": json.RawMessage(`90`), + "parallelism": json.RawMessage(`12`), + "terraform_version": json.RawMessage(`"1.6.0"`), + "pack_logs_enabled": json.RawMessage(`false`), + "ssh_logging_enabled": json.RawMessage(`true`), + "run_log_retention_enabled": json.RawMessage(`false`), + "run_log_retention_days": json.RawMessage(`14`), + "run_retention_enabled": json.RawMessage(`true`), + "run_retention_days": json.RawMessage(`90`), }) assert.Equal(t, config.AppConfig{ - Parallelism: 12, - TerraformVersion: "1.6.0", - PackLogsEnabled: false, - SSHLoggingEnabled: true, - AssessmentLogRetentionEnabled: false, - AssessmentLogRetentionDays: 14, - AssessmentRetentionEnabled: true, - AssessmentRetentionDays: 90, + Parallelism: 12, + TerraformVersion: "1.6.0", + PackLogsEnabled: false, + SSHLoggingEnabled: true, + RunLogRetentionEnabled: false, + RunLogRetentionDays: 14, + RunRetentionEnabled: true, + RunRetentionDays: 90, }, got) } @@ -75,36 +75,36 @@ func TestParseAppConfig_NonPositiveRetentionDaysKeepsDefault(t *testing.T) { for _, v := range []string{`0`, `-1`} { t.Run(v, func(t *testing.T) { got := parseAppConfig(map[string]json.RawMessage{ - "assessment_log_retention_days": json.RawMessage(v), - "assessment_retention_days": json.RawMessage(v), + "run_log_retention_days": json.RawMessage(v), + "run_retention_days": json.RawMessage(v), }) - assert.Equal(t, def.AssessmentLogRetentionDays, got.AssessmentLogRetentionDays) - assert.Equal(t, def.AssessmentRetentionDays, got.AssessmentRetentionDays) + assert.Equal(t, def.RunLogRetentionDays, got.RunLogRetentionDays) + assert.Equal(t, def.RunRetentionDays, got.RunRetentionDays) }) } } func TestAppConfigKVs_MarshalsToExpectedJSON(t *testing.T) { c := config.AppConfig{ - Parallelism: 7, - TerraformVersion: "1.5.7", - PackLogsEnabled: true, - SSHLoggingEnabled: false, - AssessmentLogRetentionEnabled: true, - AssessmentLogRetentionDays: 7, - AssessmentRetentionEnabled: false, - AssessmentRetentionDays: 30, + Parallelism: 7, + TerraformVersion: "1.5.7", + PackLogsEnabled: true, + SSHLoggingEnabled: false, + RunLogRetentionEnabled: true, + RunLogRetentionDays: 7, + RunRetentionEnabled: false, + RunRetentionDays: 30, } want := map[string]string{ - "parallelism": `7`, - "terraform_version": `"1.5.7"`, - "pack_logs_enabled": `true`, - "ssh_logging_enabled": `false`, - "assessment_log_retention_enabled": `true`, - "assessment_log_retention_days": `7`, - "assessment_retention_enabled": `false`, - "assessment_retention_days": `30`, + "parallelism": `7`, + "terraform_version": `"1.5.7"`, + "pack_logs_enabled": `true`, + "ssh_logging_enabled": `false`, + "run_log_retention_enabled": `true`, + "run_log_retention_days": `7`, + "run_retention_enabled": `false`, + "run_retention_days": `30`, } kvs := appConfigKVs(c) @@ -121,14 +121,14 @@ func TestFakeConfigStore_UpdateGetAppConfigRoundtrip(t *testing.T) { ctx := context.Background() want := config.AppConfig{ - Parallelism: 12, - TerraformVersion: "1.6.0", - PackLogsEnabled: false, - SSHLoggingEnabled: true, - AssessmentLogRetentionEnabled: false, - AssessmentLogRetentionDays: 14, - AssessmentRetentionEnabled: true, - AssessmentRetentionDays: 90, + Parallelism: 12, + TerraformVersion: "1.6.0", + PackLogsEnabled: false, + SSHLoggingEnabled: true, + RunLogRetentionEnabled: false, + RunLogRetentionDays: 14, + RunRetentionEnabled: true, + RunRetentionDays: 90, } require.NoError(t, f.UpdateAppConfig(ctx, want)) @@ -138,8 +138,8 @@ func TestFakeConfigStore_UpdateGetAppConfigRoundtrip(t *testing.T) { assert.Equal(t, []string{ "parallelism", "terraform_version", "pack_logs_enabled", "ssh_logging_enabled", - "assessment_log_retention_enabled", "assessment_log_retention_days", - "assessment_retention_enabled", "assessment_retention_days", + "run_log_retention_enabled", "run_log_retention_days", + "run_retention_enabled", "run_retention_days", }, f.sets) } diff --git a/internal/db/migrations/012_rename_saved_scenarios_to_assessments.down.sql b/internal/db/migrations/012_rename_saved_scenarios_to_assessments.down.sql new file mode 100644 index 0000000..69abda9 --- /dev/null +++ b/internal/db/migrations/012_rename_saved_scenarios_to_assessments.down.sql @@ -0,0 +1,6 @@ +-- Reverse 012: drop the unique slug constraint and restore the table/index names. +-- Disambiguating suffixes added on the up path are not removed (the original +-- duplicate names cannot be recovered), which is acceptable for a rollback. +ALTER TABLE assessments DROP CONSTRAINT assessments_name_key; +ALTER INDEX idx_assessments_updated_at RENAME TO idx_saved_scenarios_updated_at; +ALTER TABLE assessments RENAME TO saved_scenarios; diff --git a/internal/db/migrations/012_rename_saved_scenarios_to_assessments.up.sql b/internal/db/migrations/012_rename_saved_scenarios_to_assessments.up.sql new file mode 100644 index 0000000..03ac267 --- /dev/null +++ b/internal/db/migrations/012_rename_saved_scenarios_to_assessments.up.sql @@ -0,0 +1,29 @@ +-- Rename the saved-definition table to "assessments" (GitHub-Actions "workflow"). +-- The per-case "scenario" vocabulary (scenario_results, YAML scenarios:) is unchanged. +ALTER TABLE saved_scenarios RENAME TO assessments; +ALTER INDEX idx_saved_scenarios_updated_at RENAME TO idx_assessments_updated_at; + +-- name becomes a human-addressable unique slug (each assessment serializes to +-- .yaml). Disambiguate any pre-existing duplicate names before adding the +-- UNIQUE constraint: keep the oldest row's name, suffix the rest, and log each +-- rename so the operation is not silent. +DO $$ +DECLARE + r RECORD; +BEGIN + FOR r IN + SELECT id, name, + row_number() OVER (PARTITION BY name ORDER BY created_at, id) AS rn + FROM assessments + LOOP + IF r.rn > 1 THEN + UPDATE assessments + SET name = r.name || '-' || substr(r.id::text, 1, 8) + WHERE id = r.id; + RAISE NOTICE 'assessment-vocabulary-refactor: renamed duplicate assessment "%" to "%-%"', + r.name, r.name, substr(r.id::text, 1, 8); + END IF; + END LOOP; +END $$; + +ALTER TABLE assessments ADD CONSTRAINT assessments_name_key UNIQUE (name); diff --git a/internal/db/migrations/013_rename_runs_scenario_id_to_assessment_id.down.sql b/internal/db/migrations/013_rename_runs_scenario_id_to_assessment_id.down.sql new file mode 100644 index 0000000..c826814 --- /dev/null +++ b/internal/db/migrations/013_rename_runs_scenario_id_to_assessment_id.down.sql @@ -0,0 +1,2 @@ +ALTER INDEX idx_runs_assessment_id RENAME TO idx_runs_scenario_id; +ALTER TABLE runs RENAME COLUMN assessment_id TO scenario_id; diff --git a/internal/db/migrations/013_rename_runs_scenario_id_to_assessment_id.up.sql b/internal/db/migrations/013_rename_runs_scenario_id_to_assessment_id.up.sql new file mode 100644 index 0000000..6a6a9dc --- /dev/null +++ b/internal/db/migrations/013_rename_runs_scenario_id_to_assessment_id.up.sql @@ -0,0 +1,5 @@ +-- The run -> definition FK becomes assessment_id. The runs table name is +-- unchanged; only the column (and its index) are renamed. The FK constraint +-- already references the renamed assessments table. +ALTER TABLE runs RENAME COLUMN scenario_id TO assessment_id; +ALTER INDEX idx_runs_scenario_id RENAME TO idx_runs_assessment_id; diff --git a/internal/db/migrations/014_rename_scenario_results_assertions_to_expectations.down.sql b/internal/db/migrations/014_rename_scenario_results_assertions_to_expectations.down.sql new file mode 100644 index 0000000..8d4aee4 --- /dev/null +++ b/internal/db/migrations/014_rename_scenario_results_assertions_to_expectations.down.sql @@ -0,0 +1 @@ +ALTER TABLE scenario_results RENAME COLUMN expectations TO assertions; diff --git a/internal/db/migrations/014_rename_scenario_results_assertions_to_expectations.up.sql b/internal/db/migrations/014_rename_scenario_results_assertions_to_expectations.up.sql new file mode 100644 index 0000000..7e9a17a --- /dev/null +++ b/internal/db/migrations/014_rename_scenario_results_assertions_to_expectations.up.sql @@ -0,0 +1,3 @@ +-- Drop "assertion" from the vocabulary: the per-expectation outcome column +-- becomes "expectations". The scenario_results table name is unchanged. +ALTER TABLE scenario_results RENAME COLUMN assertions TO expectations; diff --git a/internal/db/migrations/015_rename_retention_appconfig_keys.down.sql b/internal/db/migrations/015_rename_retention_appconfig_keys.down.sql new file mode 100644 index 0000000..158e916 --- /dev/null +++ b/internal/db/migrations/015_rename_retention_appconfig_keys.down.sql @@ -0,0 +1,12 @@ +-- Reverse 015: restore the assessment_(log_)retention_* key names, carrying +-- forward the current values, then remove the run_(log_)retention_* keys. +INSERT INTO app_config (key, value) +SELECT replace(key, 'run_', 'assessment_'), value + FROM app_config + WHERE key IN ('run_log_retention_enabled', 'run_log_retention_days', + 'run_retention_enabled', 'run_retention_days') +ON CONFLICT (key) DO UPDATE SET value = EXCLUDED.value; + +DELETE FROM app_config + WHERE key IN ('run_log_retention_enabled', 'run_log_retention_days', + 'run_retention_enabled', 'run_retention_days'); diff --git a/internal/db/migrations/015_rename_retention_appconfig_keys.up.sql b/internal/db/migrations/015_rename_retention_appconfig_keys.up.sql new file mode 100644 index 0000000..42a8137 --- /dev/null +++ b/internal/db/migrations/015_rename_retention_appconfig_keys.up.sql @@ -0,0 +1,21 @@ +-- Retention keys gate the deletion of runs (executions), so they shorten to +-- run_(log_)retention_*. Carry forward any operator-set values from the former +-- assessment_(log_)retention_* keys, then remove the old keys. +INSERT INTO app_config (key, value) +SELECT replace(key, 'assessment_', 'run_'), value + FROM app_config + WHERE key IN ('assessment_log_retention_enabled', 'assessment_log_retention_days', + 'assessment_retention_enabled', 'assessment_retention_days') +ON CONFLICT (key) DO UPDATE SET value = EXCLUDED.value; + +DELETE FROM app_config + WHERE key IN ('assessment_log_retention_enabled', 'assessment_log_retention_days', + 'assessment_retention_enabled', 'assessment_retention_days'); + +-- Ensure defaults exist for fresh installs (idempotent). Aligned with DefaultAppConfig(). +INSERT INTO app_config (key, value) VALUES + ('run_log_retention_enabled', 'true'::jsonb), + ('run_log_retention_days', '7'::jsonb), + ('run_retention_enabled', 'false'::jsonb), + ('run_retention_days', '30'::jsonb) +ON CONFLICT (key) DO NOTHING; diff --git a/internal/db/migrations/016_add_scenario_errored.down.sql b/internal/db/migrations/016_add_scenario_errored.down.sql new file mode 100644 index 0000000..d9e0aae --- /dev/null +++ b/internal/db/migrations/016_add_scenario_errored.down.sql @@ -0,0 +1 @@ +ALTER TABLE scenario_results DROP COLUMN errored; diff --git a/internal/db/migrations/016_add_scenario_errored.up.sql b/internal/db/migrations/016_add_scenario_errored.up.sql new file mode 100644 index 0000000..527b9f1 --- /dev/null +++ b/internal/db/migrations/016_add_scenario_errored.up.sql @@ -0,0 +1,5 @@ +-- Distinguish execution errors (warmup/detonation/matching infrastructure +-- failures) from clean expectation mismatches. A scenario is "errored" when it +-- failed without producing per-expectation results; the value is computed at +-- write time. Existing rows default to false. +ALTER TABLE scenario_results ADD COLUMN errored boolean NOT NULL DEFAULT false; diff --git a/internal/db/migrations/017_rename_schedules_scenario_id_to_assessment_id.down.sql b/internal/db/migrations/017_rename_schedules_scenario_id_to_assessment_id.down.sql new file mode 100644 index 0000000..99276ba --- /dev/null +++ b/internal/db/migrations/017_rename_schedules_scenario_id_to_assessment_id.down.sql @@ -0,0 +1,2 @@ +ALTER INDEX idx_schedules_assessment_id RENAME TO idx_schedules_scenario_id; +ALTER TABLE schedules RENAME COLUMN assessment_id TO scenario_id; diff --git a/internal/db/migrations/017_rename_schedules_scenario_id_to_assessment_id.up.sql b/internal/db/migrations/017_rename_schedules_scenario_id_to_assessment_id.up.sql new file mode 100644 index 0000000..dc8bedb --- /dev/null +++ b/internal/db/migrations/017_rename_schedules_scenario_id_to_assessment_id.up.sql @@ -0,0 +1,5 @@ +-- The schedule -> definition FK becomes assessment_id, matching the runs table +-- rename in migration 013. The schedules table name is unchanged; only the +-- column (and its index) are renamed. The FK already references assessments. +ALTER TABLE schedules RENAME COLUMN scenario_id TO assessment_id; +ALTER INDEX idx_schedules_scenario_id RENAME TO idx_schedules_assessment_id; diff --git a/internal/db/runs.go b/internal/db/runs.go index d2c7b12..b64846e 100644 --- a/internal/db/runs.go +++ b/internal/db/runs.go @@ -17,7 +17,7 @@ type RunStore interface { Get(ctx context.Context, id uuid.UUID) (*Run, error) List(ctx context.Context, filters ListRunsFilters, limit, offset int) (RunPage, error) // ListExpired returns the IDs of runs created before cutoff whose status is - // not "running" — the assessment-retention sweeper's deletion candidates. + // not "running" — the run-retention sweeper's deletion candidates. ListExpired(ctx context.Context, cutoff time.Time) ([]uuid.UUID, error) Update(ctx context.Context, id uuid.UUID, status string, total, succeeded, failed int, endTime *time.Time) error Delete(ctx context.Context, id uuid.UUID) error @@ -34,42 +34,49 @@ type RunStore interface { // UpdateScenarioIdentity records executor identity mid-run (after detonation), // touching only the identity columns and leaving status/phase untouched. UpdateScenarioIdentity(ctx context.Context, id uuid.UUID, executorName, executorType, executionID, simulationID string) error - // UpdateScenarioAssertions persists partial assertion results mid-run, - // touching only the assertions column and leaving status/phase untouched. - UpdateScenarioAssertions(ctx context.Context, id uuid.UUID, assertionsJSON []byte) error + // UpdateScenarioExpectations persists partial expectation results mid-run, + // touching only the expectations column and leaving status/phase untouched. + UpdateScenarioExpectations(ctx context.Context, id uuid.UUID, expectationsJSON []byte) error CompleteScenarioResult(ctx context.Context, id uuid.UUID, result *ScenarioResult) error IncrementRunCounters(ctx context.Context, id uuid.UUID, successDelta, failDelta int) error - // GetLatestAssertionResults returns the most recent pass/fail for each alert name. - GetLatestAssertionResults(ctx context.Context) ([]LatestAssertionResult, error) + // GetLatestExpectationResults returns the most recent pass/fail for each alert name. + GetLatestExpectationResults(ctx context.Context) ([]LatestExpectationResult, error) } // Run represents a single simrun execution. type Run struct { - ID uuid.UUID `json:"id"` - Status string `json:"status"` - StartTime time.Time `json:"startTime"` - EndTime *time.Time `json:"endTime,omitempty"` - Total int `json:"total"` - Succeeded int `json:"succeeded"` - Failed int `json:"failed"` - ScenarioID *uuid.UUID `json:"scenarioId,omitempty"` - ScenarioName *string `json:"scenarioName,omitempty"` - ScenarioType *string `json:"scenarioType,omitempty"` - ScheduleID *uuid.UUID `json:"scheduleId,omitempty"` - ScheduleName *string `json:"scheduleName,omitempty"` - CreatedBy string `json:"createdBy"` - CreatedAt time.Time `json:"createdAt"` + ID uuid.UUID `json:"id"` + Status string `json:"status"` + StartTime time.Time `json:"startTime"` + EndTime *time.Time `json:"endTime,omitempty"` + Total int `json:"total"` + Succeeded int `json:"succeeded"` + Failed int `json:"failed"` + // Errors counts scenarios that failed during execution (warmup, detonation, + // or matching infrastructure) rather than by missing an expected alert. It is + // a subset of Failed and is derived from scenario_results.errored. + Errors int `json:"errors"` + AssessmentID *uuid.UUID `json:"assessmentId,omitempty"` + AssessmentName *string `json:"assessmentName,omitempty"` + AssessmentType *string `json:"assessmentType,omitempty"` + ScheduleID *uuid.UUID `json:"scheduleId,omitempty"` + ScheduleName *string `json:"scheduleName,omitempty"` + CreatedBy string `json:"createdBy"` + CreatedAt time.Time `json:"createdAt"` } // ScenarioResult represents the result of a single scenario execution. type ScenarioResult struct { - ID uuid.UUID `json:"id"` - RunID uuid.UUID `json:"runId"` - Name string `json:"name"` - Status string `json:"status"` - Phase *string `json:"phase,omitempty"` - IsSuccess *bool `json:"isSuccess"` + ID uuid.UUID `json:"id"` + RunID uuid.UUID `json:"runId"` + Name string `json:"name"` + Status string `json:"status"` + Phase *string `json:"phase,omitempty"` + IsSuccess *bool `json:"isSuccess"` + // Errored is true when the scenario failed during execution rather than by + // missing an expected alert (see Run.Errors). + Errored bool `json:"errored"` ErrorMessage string `json:"errorMessage,omitempty"` DurationSecs float64 `json:"durationSecs"` MatchingDurSecs float64 `json:"matchingDurSecs"` @@ -78,7 +85,7 @@ type ScenarioResult struct { ExecutorType string `json:"executorType"` ExecutionID string `json:"executionId"` SimulationID string `json:"simulationId,omitempty"` - Assertions json.RawMessage `json:"assertions,omitempty"` + Expectations json.RawMessage `json:"expectations,omitempty"` Indicators json.RawMessage `json:"indicators,omitempty"` Metadata json.RawMessage `json:"metadata,omitempty"` CollectedLogPath *string `json:"collectedLogPath,omitempty"` @@ -96,22 +103,22 @@ type RunPage struct { // ListRunsFilters narrows the result set for RunStore.List. Zero values mean // "no constraint on this dimension". // -// Note: filters that reference saved_scenarios columns (Name, Types) silently -// exclude ad-hoc runs whose scenario_id is NULL, because NULL never matches an +// Note: filters that reference assessments columns (Name, Types) silently +// exclude ad-hoc runs whose assessment_id is NULL, because NULL never matches an // equality/LIKE predicate. type ListRunsFilters struct { - // Name is an ILIKE %name% match against saved_scenarios.name. + // Name is an ILIKE %name% match against assessments.name. Name string - // Types restricts saved_scenarios.type to the listed values. + // Types restricts assessments.type to the listed values. Types []string // Since restricts runs to created_at >= Since. Since *time.Time - // ScenarioID restricts runs to the given saved_scenarios.id. - ScenarioID *uuid.UUID + // AssessmentID restricts runs to the given assessments.id. + AssessmentID *uuid.UUID } -// LatestAssertionResult holds the most recent pass/fail for a given alert name. -type LatestAssertionResult struct { +// LatestExpectationResult holds the most recent pass/fail for a given alert name. +type LatestExpectationResult struct { AlertName string `json:"alertName"` Passed bool `json:"passed"` RunID uuid.UUID `json:"runId"` @@ -129,9 +136,9 @@ func NewRunStore(pool *pgxpool.Pool) RunStore { func (s *runStore) Create(ctx context.Context, run *Run) error { _, err := s.pool.Exec(ctx, - `INSERT INTO runs (id, status, start_time, total, succeeded, failed, scenario_id, schedule_id, schedule_name, created_by) + `INSERT INTO runs (id, status, start_time, total, succeeded, failed, assessment_id, schedule_id, schedule_name, created_by) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10)`, - run.ID, run.Status, run.StartTime, run.Total, run.Succeeded, run.Failed, run.ScenarioID, run.ScheduleID, run.ScheduleName, run.CreatedBy, + run.ID, run.Status, run.StartTime, run.Total, run.Succeeded, run.Failed, run.AssessmentID, run.ScheduleID, run.ScheduleName, run.CreatedBy, ) return err } @@ -139,15 +146,16 @@ func (s *runStore) Create(ctx context.Context, run *Run) error { func (s *runStore) Get(ctx context.Context, id uuid.UUID) (*Run, error) { row := s.pool.QueryRow(ctx, `SELECT r.id, r.status, r.start_time, r.end_time, r.total, r.succeeded, r.failed, - r.scenario_id, ss.name, ss.type, + (SELECT COUNT(*) FROM scenario_results sr WHERE sr.run_id = r.id AND sr.errored) AS errors, + r.assessment_id, a.name, a.type, r.schedule_id, r.schedule_name, r.created_by, r.created_at FROM runs r - LEFT JOIN saved_scenarios ss ON r.scenario_id = ss.id + LEFT JOIN assessments a ON r.assessment_id = a.id WHERE r.id = $1`, id, ) var run Run - err := row.Scan(&run.ID, &run.Status, &run.StartTime, &run.EndTime, &run.Total, &run.Succeeded, &run.Failed, - &run.ScenarioID, &run.ScenarioName, &run.ScenarioType, + err := row.Scan(&run.ID, &run.Status, &run.StartTime, &run.EndTime, &run.Total, &run.Succeeded, &run.Failed, &run.Errors, + &run.AssessmentID, &run.AssessmentName, &run.AssessmentType, &run.ScheduleID, &run.ScheduleName, &run.CreatedBy, &run.CreatedAt) if err != nil { return nil, err @@ -159,11 +167,12 @@ func (s *runStore) List(ctx context.Context, filters ListRunsFilters, limit, off where, args := buildRunsWhere(filters) rows, err := s.pool.Query(ctx, `SELECT r.id, r.status, r.start_time, r.end_time, r.total, r.succeeded, r.failed, - r.scenario_id, ss.name, ss.type, + (SELECT COUNT(*) FROM scenario_results sr WHERE sr.run_id = r.id AND sr.errored) AS errors, + r.assessment_id, a.name, a.type, r.schedule_id, r.schedule_name, r.created_by, r.created_at, COUNT(*) OVER() AS total_count FROM runs r - LEFT JOIN saved_scenarios ss ON r.scenario_id = ss.id + LEFT JOIN assessments a ON r.assessment_id = a.id `+where+` ORDER BY r.created_at DESC LIMIT $`+strconv.Itoa(len(args)+1)+` OFFSET $`+strconv.Itoa(len(args)+2), @@ -178,8 +187,8 @@ func (s *runStore) List(ctx context.Context, filters ListRunsFilters, limit, off for rows.Next() { var run Run var total int - if err := rows.Scan(&run.ID, &run.Status, &run.StartTime, &run.EndTime, &run.Total, &run.Succeeded, &run.Failed, - &run.ScenarioID, &run.ScenarioName, &run.ScenarioType, + if err := rows.Scan(&run.ID, &run.Status, &run.StartTime, &run.EndTime, &run.Total, &run.Succeeded, &run.Failed, &run.Errors, + &run.AssessmentID, &run.AssessmentName, &run.AssessmentType, &run.ScheduleID, &run.ScheduleName, &run.CreatedBy, &run.CreatedAt, &total); err != nil { return RunPage{}, err } @@ -192,7 +201,7 @@ func (s *runStore) List(ctx context.Context, filters ListRunsFilters, limit, off if len(page.Runs) == 0 { // COUNT(*) OVER() collapses to no rows when LIMIT/OFFSET yields nothing. // Re-run the same filter as a plain COUNT so the UI can show "of N". - countSQL := `SELECT COUNT(*) FROM runs r LEFT JOIN saved_scenarios ss ON r.scenario_id = ss.id ` + where + countSQL := `SELECT COUNT(*) FROM runs r LEFT JOIN assessments a ON r.assessment_id = a.id ` + where if err := s.pool.QueryRow(ctx, countSQL, args...).Scan(&page.Total); err != nil { return RunPage{}, err } @@ -201,13 +210,13 @@ func (s *runStore) List(ctx context.Context, filters ListRunsFilters, limit, off } // buildRunsWhere returns a WHERE clause (or "") and its positional args for the -// runs+saved_scenarios join. Placeholders are $1..$N in argument order. +// runs+assessments join. Placeholders are $1..$N in argument order. func buildRunsWhere(f ListRunsFilters) (string, []any) { var clauses []string var args []any if f.Name != "" { args = append(args, "%"+f.Name+"%") - clauses = append(clauses, "ss.name ILIKE $"+strconv.Itoa(len(args))) + clauses = append(clauses, "a.name ILIKE $"+strconv.Itoa(len(args))) } if len(f.Types) > 0 { placeholders := make([]string, len(f.Types)) @@ -215,15 +224,15 @@ func buildRunsWhere(f ListRunsFilters) (string, []any) { args = append(args, t) placeholders[i] = "$" + strconv.Itoa(len(args)) } - clauses = append(clauses, "ss.type IN ("+strings.Join(placeholders, ",")+")") + clauses = append(clauses, "a.type IN ("+strings.Join(placeholders, ",")+")") } if f.Since != nil { args = append(args, *f.Since) clauses = append(clauses, "r.created_at >= $"+strconv.Itoa(len(args))) } - if f.ScenarioID != nil { - args = append(args, *f.ScenarioID) - clauses = append(clauses, "r.scenario_id = $"+strconv.Itoa(len(args))) + if f.AssessmentID != nil { + args = append(args, *f.AssessmentID) + clauses = append(clauses, "r.assessment_id = $"+strconv.Itoa(len(args))) } if len(clauses) == 0 { return "", args @@ -267,11 +276,11 @@ func (s *runStore) Delete(ctx context.Context, id uuid.UUID) error { func (s *runStore) AddScenarioResult(ctx context.Context, runID uuid.UUID, result *ScenarioResult) error { _, err := s.pool.Exec(ctx, - `INSERT INTO scenario_results (run_id, name, status, is_success, error_message, duration_secs, matching_dur_secs, time_executed, executor_name, executor_type, execution_id, simulation_id, assertions, indicators, metadata, collected_log_path, collected_doc_count, discovered_alerts) - VALUES ($1, $2, 'completed', $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15, $16, $17)`, - runID, result.Name, result.IsSuccess, result.ErrorMessage, result.DurationSecs, + `INSERT INTO scenario_results (run_id, name, status, is_success, errored, error_message, duration_secs, matching_dur_secs, time_executed, executor_name, executor_type, execution_id, simulation_id, expectations, indicators, metadata, collected_log_path, collected_doc_count, discovered_alerts) + VALUES ($1, $2, 'completed', $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15, $16, $17, $18)`, + runID, result.Name, result.IsSuccess, result.Errored, result.ErrorMessage, result.DurationSecs, result.MatchingDurSecs, result.TimeExecuted, result.ExecutorName, result.ExecutorType, - result.ExecutionID, result.SimulationID, result.Assertions, result.Indicators, result.Metadata, + result.ExecutionID, result.SimulationID, result.Expectations, result.Indicators, result.Metadata, result.CollectedLogPath, result.CollectedDocCount, result.DiscoveredAlerts, ) return err @@ -279,11 +288,11 @@ func (s *runStore) AddScenarioResult(ctx context.Context, runID uuid.UUID, resul func (s *runStore) GetScenarioResults(ctx context.Context, runID uuid.UUID) ([]ScenarioResult, error) { rows, err := s.pool.Query(ctx, - `SELECT id, run_id, name, status, phase, is_success, + `SELECT id, run_id, name, status, phase, is_success, errored, COALESCE(error_message, ''), COALESCE(duration_secs, 0), COALESCE(matching_dur_secs, 0), time_executed, COALESCE(executor_name, ''), COALESCE(executor_type, ''), COALESCE(execution_id, ''), COALESCE(simulation_id, ''), - assertions, indicators, metadata, collected_log_path, COALESCE(collected_doc_count, 0), discovered_alerts, created_at + expectations, indicators, metadata, collected_log_path, COALESCE(collected_doc_count, 0), discovered_alerts, created_at FROM scenario_results WHERE run_id = $1 ORDER BY created_at`, runID, ) @@ -295,9 +304,9 @@ func (s *runStore) GetScenarioResults(ctx context.Context, runID uuid.UUID) ([]S results := []ScenarioResult{} for rows.Next() { var r ScenarioResult - if err := rows.Scan(&r.ID, &r.RunID, &r.Name, &r.Status, &r.Phase, &r.IsSuccess, &r.ErrorMessage, + if err := rows.Scan(&r.ID, &r.RunID, &r.Name, &r.Status, &r.Phase, &r.IsSuccess, &r.Errored, &r.ErrorMessage, &r.DurationSecs, &r.MatchingDurSecs, &r.TimeExecuted, &r.ExecutorName, - &r.ExecutorType, &r.ExecutionID, &r.SimulationID, &r.Assertions, &r.Indicators, &r.Metadata, + &r.ExecutorType, &r.ExecutionID, &r.SimulationID, &r.Expectations, &r.Indicators, &r.Metadata, &r.CollectedLogPath, &r.CollectedDocCount, &r.DiscoveredAlerts, &r.CreatedAt); err != nil { return nil, err } @@ -308,17 +317,17 @@ func (s *runStore) GetScenarioResults(ctx context.Context, runID uuid.UUID) ([]S func (s *runStore) GetScenarioResult(ctx context.Context, id uuid.UUID) (*ScenarioResult, error) { row := s.pool.QueryRow(ctx, - `SELECT id, run_id, name, status, phase, is_success, + `SELECT id, run_id, name, status, phase, is_success, errored, COALESCE(error_message, ''), COALESCE(duration_secs, 0), COALESCE(matching_dur_secs, 0), time_executed, COALESCE(executor_name, ''), COALESCE(executor_type, ''), COALESCE(execution_id, ''), COALESCE(simulation_id, ''), - assertions, indicators, metadata, collected_log_path, COALESCE(collected_doc_count, 0), discovered_alerts, created_at + expectations, indicators, metadata, collected_log_path, COALESCE(collected_doc_count, 0), discovered_alerts, created_at FROM scenario_results WHERE id = $1`, id, ) var r ScenarioResult - err := row.Scan(&r.ID, &r.RunID, &r.Name, &r.Status, &r.Phase, &r.IsSuccess, &r.ErrorMessage, + err := row.Scan(&r.ID, &r.RunID, &r.Name, &r.Status, &r.Phase, &r.IsSuccess, &r.Errored, &r.ErrorMessage, &r.DurationSecs, &r.MatchingDurSecs, &r.TimeExecuted, &r.ExecutorName, - &r.ExecutorType, &r.ExecutionID, &r.SimulationID, &r.Assertions, &r.Indicators, &r.Metadata, + &r.ExecutorType, &r.ExecutionID, &r.SimulationID, &r.Expectations, &r.Indicators, &r.Metadata, &r.CollectedLogPath, &r.CollectedDocCount, &r.DiscoveredAlerts, &r.CreatedAt) if err != nil { return nil, err @@ -351,10 +360,10 @@ func (s *runStore) UpdateScenarioIdentity(ctx context.Context, id uuid.UUID, exe return err } -func (s *runStore) UpdateScenarioAssertions(ctx context.Context, id uuid.UUID, assertionsJSON []byte) error { +func (s *runStore) UpdateScenarioExpectations(ctx context.Context, id uuid.UUID, expectationsJSON []byte) error { _, err := s.pool.Exec(ctx, - `UPDATE scenario_results SET assertions = $2 WHERE id = $1`, - id, assertionsJSON, + `UPDATE scenario_results SET expectations = $2 WHERE id = $1`, + id, expectationsJSON, ) return err } @@ -365,13 +374,14 @@ func (s *runStore) CompleteScenarioResult(ctx context.Context, id uuid.UUID, res status = 'completed', phase = NULL, is_success = $2, error_message = $3, duration_secs = $4, matching_dur_secs = $5, time_executed = $6, executor_name = $7, executor_type = $8, execution_id = $9, - simulation_id = $10, assertions = $11, indicators = $12, metadata = $13, - collected_log_path = $14, collected_doc_count = $15, discovered_alerts = $16 + simulation_id = $10, expectations = $11, indicators = $12, metadata = $13, + collected_log_path = $14, collected_doc_count = $15, discovered_alerts = $16, + errored = $17 WHERE id = $1`, id, result.IsSuccess, result.ErrorMessage, result.DurationSecs, result.MatchingDurSecs, result.TimeExecuted, result.ExecutorName, result.ExecutorType, result.ExecutionID, - result.SimulationID, result.Assertions, result.Indicators, result.Metadata, - result.CollectedLogPath, result.CollectedDocCount, result.DiscoveredAlerts, + result.SimulationID, result.Expectations, result.Indicators, result.Metadata, + result.CollectedLogPath, result.CollectedDocCount, result.DiscoveredAlerts, result.Errored, ) return err } @@ -392,7 +402,7 @@ func (s *runStore) CompleteRun(ctx context.Context, id uuid.UUID, endTime *time. return err } -func (s *runStore) GetLatestAssertionResults(ctx context.Context) ([]LatestAssertionResult, error) { +func (s *runStore) GetLatestExpectationResults(ctx context.Context) ([]LatestExpectationResult, error) { rows, err := s.pool.Query(ctx, ` SELECT DISTINCT ON (a.value->>'alertName') a.value->>'alertName' AS alert_name, @@ -400,9 +410,9 @@ func (s *runStore) GetLatestAssertionResults(ctx context.Context) ([]LatestAsser sr.run_id, sr.created_at FROM scenario_results sr, - jsonb_array_elements(sr.assertions) AS a(value) + jsonb_array_elements(sr.expectations) AS a(value) WHERE sr.status = 'completed' - AND sr.assertions IS NOT NULL + AND sr.expectations IS NOT NULL AND a.value->>'matcherType' = 'Elastic Security alert' ORDER BY a.value->>'alertName', sr.created_at DESC `) @@ -411,9 +421,9 @@ func (s *runStore) GetLatestAssertionResults(ctx context.Context) ([]LatestAsser } defer rows.Close() - var results []LatestAssertionResult + var results []LatestExpectationResult for rows.Next() { - var r LatestAssertionResult + var r LatestExpectationResult if err := rows.Scan(&r.AlertName, &r.Passed, &r.RunID, &r.CreatedAt); err != nil { return nil, err } diff --git a/internal/db/scenarios.go b/internal/db/scenarios.go deleted file mode 100644 index 9190e10..0000000 --- a/internal/db/scenarios.go +++ /dev/null @@ -1,189 +0,0 @@ -package db - -import ( - "context" - "strconv" - "strings" - "time" - - "github.com/google/uuid" - "github.com/jackc/pgx/v5/pgxpool" -) - -// ScenarioStore manages saved scenario YAML persistence. -type ScenarioStore interface { - Save(ctx context.Context, name, scenarioType, yaml, createdBy string) (*SavedScenario, error) - Get(ctx context.Context, id uuid.UUID) (*SavedScenario, error) - // List returns a filtered, paginated slice of scenarios for the UI. - List(ctx context.Context, filters ListScenariosFilters, limit, offset int) (ScenarioPage, error) - // ListAll returns every scenario in updated_at DESC order. For internal - // callers (e.g. coverage maps) that need the full set in one shot. - ListAll(ctx context.Context) ([]SavedScenario, error) - Update(ctx context.Context, id uuid.UUID, name, scenarioType, yaml, updatedBy string) error - Delete(ctx context.Context, id uuid.UUID) error -} - -// SavedScenario represents a saved scenario configuration. -type SavedScenario struct { - ID uuid.UUID `json:"id"` - Name string `json:"name"` - Type string `json:"type"` - YAML string `json:"yaml"` - CreatedBy string `json:"createdBy"` - UpdatedBy string `json:"updatedBy"` - CreatedAt time.Time `json:"createdAt"` - UpdatedAt time.Time `json:"updatedAt"` -} - -// ScenarioPage is a paginated slice of saved scenarios with the total row count. -type ScenarioPage struct { - Scenarios []SavedScenario `json:"scenarios"` - Total int `json:"total"` -} - -// ListScenariosFilters narrows the result set for ScenarioStore.List. Zero -// values mean "no constraint on this dimension". -type ListScenariosFilters struct { - // Name is an ILIKE %name% match against saved_scenarios.name. - Name string - // Types restricts saved_scenarios.type to the listed values. - Types []string - // Since restricts scenarios to updated_at >= Since. - Since *time.Time -} - -type scenarioStore struct { - pool *pgxpool.Pool -} - -// NewScenarioStore creates a new ScenarioStore backed by PostgreSQL. -func NewScenarioStore(pool *pgxpool.Pool) ScenarioStore { - return &scenarioStore{pool: pool} -} - -func (s *scenarioStore) Save(ctx context.Context, name, scenarioType, yaml, createdBy string) (*SavedScenario, error) { - var sc SavedScenario - err := s.pool.QueryRow(ctx, - `INSERT INTO saved_scenarios (name, type, yaml, created_by, updated_by) VALUES ($1, $2, $3, $4, $4) - RETURNING id, name, type, yaml, created_by, updated_by, created_at, updated_at`, - name, scenarioType, yaml, createdBy, - ).Scan(&sc.ID, &sc.Name, &sc.Type, &sc.YAML, &sc.CreatedBy, &sc.UpdatedBy, &sc.CreatedAt, &sc.UpdatedAt) - if err != nil { - return nil, err - } - return &sc, nil -} - -func (s *scenarioStore) Get(ctx context.Context, id uuid.UUID) (*SavedScenario, error) { - var sc SavedScenario - err := s.pool.QueryRow(ctx, - `SELECT id, name, type, yaml, created_by, updated_by, created_at, updated_at FROM saved_scenarios WHERE id = $1`, id, - ).Scan(&sc.ID, &sc.Name, &sc.Type, &sc.YAML, &sc.CreatedBy, &sc.UpdatedBy, &sc.CreatedAt, &sc.UpdatedAt) - if err != nil { - return nil, err - } - return &sc, nil -} - -func (s *scenarioStore) List(ctx context.Context, filters ListScenariosFilters, limit, offset int) (ScenarioPage, error) { - where, args := buildScenariosWhere(filters) - rows, err := s.pool.Query(ctx, - `SELECT id, name, type, yaml, created_by, updated_by, created_at, updated_at, - COUNT(*) OVER() AS total_count - FROM saved_scenarios - `+where+` - ORDER BY updated_at DESC - LIMIT $`+strconv.Itoa(len(args)+1)+` OFFSET $`+strconv.Itoa(len(args)+2), - append(args, limit, offset)..., - ) - if err != nil { - return ScenarioPage{}, err - } - defer rows.Close() - - page := ScenarioPage{Scenarios: []SavedScenario{}} - for rows.Next() { - var sc SavedScenario - var total int - if err := rows.Scan(&sc.ID, &sc.Name, &sc.Type, &sc.YAML, &sc.CreatedBy, &sc.UpdatedBy, &sc.CreatedAt, &sc.UpdatedAt, &total); err != nil { - return ScenarioPage{}, err - } - page.Scenarios = append(page.Scenarios, sc) - page.Total = total - } - if err := rows.Err(); err != nil { - return ScenarioPage{}, err - } - if len(page.Scenarios) == 0 { - // COUNT(*) OVER() collapses to no rows when LIMIT/OFFSET yields nothing. - // Re-run the same filter as a plain COUNT so the UI can show "of N". - countSQL := `SELECT COUNT(*) FROM saved_scenarios ` + where - if err := s.pool.QueryRow(ctx, countSQL, args...).Scan(&page.Total); err != nil { - return ScenarioPage{}, err - } - } - return page, nil -} - -func (s *scenarioStore) ListAll(ctx context.Context) ([]SavedScenario, error) { - rows, err := s.pool.Query(ctx, - `SELECT id, name, type, yaml, created_by, updated_by, created_at, updated_at - FROM saved_scenarios ORDER BY updated_at DESC`, - ) - if err != nil { - return nil, err - } - defer rows.Close() - - scenarios := []SavedScenario{} - for rows.Next() { - var sc SavedScenario - if err := rows.Scan(&sc.ID, &sc.Name, &sc.Type, &sc.YAML, &sc.CreatedBy, &sc.UpdatedBy, &sc.CreatedAt, &sc.UpdatedAt); err != nil { - return nil, err - } - scenarios = append(scenarios, sc) - } - return scenarios, rows.Err() -} - -// buildScenariosWhere returns a WHERE clause (or "") and its positional args -// for saved_scenarios. Placeholders are $1..$N in argument order. -func buildScenariosWhere(f ListScenariosFilters) (string, []any) { - var clauses []string - var args []any - if f.Name != "" { - args = append(args, "%"+f.Name+"%") - clauses = append(clauses, "name ILIKE $"+strconv.Itoa(len(args))) - } - if len(f.Types) > 0 { - placeholders := make([]string, len(f.Types)) - for i, t := range f.Types { - args = append(args, t) - placeholders[i] = "$" + strconv.Itoa(len(args)) - } - clauses = append(clauses, "type IN ("+strings.Join(placeholders, ",")+")") - } - if f.Since != nil { - args = append(args, *f.Since) - clauses = append(clauses, "updated_at >= $"+strconv.Itoa(len(args))) - } - if len(clauses) == 0 { - return "", args - } - return "WHERE " + strings.Join(clauses, " AND "), args -} - -func (s *scenarioStore) Update(ctx context.Context, id uuid.UUID, name, scenarioType, yaml, updatedBy string) error { - _, err := s.pool.Exec(ctx, - `UPDATE saved_scenarios SET name = $2, type = $3, yaml = $4, updated_by = $5, updated_at = NOW() WHERE id = $1`, - id, name, scenarioType, yaml, updatedBy, - ) - return err -} - -func (s *scenarioStore) Delete(ctx context.Context, id uuid.UUID) error { - _, err := s.pool.Exec(ctx, - `DELETE FROM saved_scenarios WHERE id = $1`, id, - ) - return err -} diff --git a/internal/db/schedules.go b/internal/db/schedules.go index 0a3cabd..381e7c8 100644 --- a/internal/db/schedules.go +++ b/internal/db/schedules.go @@ -10,9 +10,9 @@ import ( // ScheduleStore manages schedule persistence. type ScheduleStore interface { - Create(ctx context.Context, scenarioID uuid.UUID, cronExpr string, enabled bool, parallelism int, createdBy string) (*Schedule, error) + Create(ctx context.Context, assessmentID uuid.UUID, cronExpr string, enabled bool, parallelism int, createdBy string) (*Schedule, error) Get(ctx context.Context, id uuid.UUID) (*Schedule, error) - GetByScenarioID(ctx context.Context, scenarioID uuid.UUID) (*Schedule, error) + GetByAssessmentID(ctx context.Context, assessmentID uuid.UUID) (*Schedule, error) List(ctx context.Context) ([]Schedule, error) ListEnabled(ctx context.Context) ([]Schedule, error) Update(ctx context.Context, id uuid.UUID, cronExpr string, enabled bool, parallelism int, updatedBy string) error @@ -20,10 +20,10 @@ type ScheduleStore interface { UpdateLastRun(ctx context.Context, id uuid.UUID, lastRunAt time.Time) error } -// Schedule represents a cron schedule for a saved scenario. +// Schedule represents a cron schedule for an assessment. type Schedule struct { ID uuid.UUID `json:"id"` - ScenarioID uuid.UUID `json:"scenarioId"` + AssessmentID uuid.UUID `json:"assessmentId"` CronExpression string `json:"cronExpression"` Enabled bool `json:"enabled"` Parallelism int `json:"parallelism"` @@ -43,14 +43,14 @@ func NewScheduleStore(pool *pgxpool.Pool) ScheduleStore { return &scheduleStore{pool: pool} } -func (s *scheduleStore) Create(ctx context.Context, scenarioID uuid.UUID, cronExpr string, enabled bool, parallelism int, createdBy string) (*Schedule, error) { +func (s *scheduleStore) Create(ctx context.Context, assessmentID uuid.UUID, cronExpr string, enabled bool, parallelism int, createdBy string) (*Schedule, error) { var sch Schedule err := s.pool.QueryRow(ctx, - `INSERT INTO schedules (scenario_id, cron_expression, enabled, parallelism, created_by, updated_by) + `INSERT INTO schedules (assessment_id, cron_expression, enabled, parallelism, created_by, updated_by) VALUES ($1, $2, $3, $4, $5, $5) - RETURNING id, scenario_id, cron_expression, enabled, parallelism, last_run_at, created_by, updated_by, created_at, updated_at`, - scenarioID, cronExpr, enabled, parallelism, createdBy, - ).Scan(&sch.ID, &sch.ScenarioID, &sch.CronExpression, &sch.Enabled, &sch.Parallelism, &sch.LastRunAt, &sch.CreatedBy, &sch.UpdatedBy, &sch.CreatedAt, &sch.UpdatedAt) + RETURNING id, assessment_id, cron_expression, enabled, parallelism, last_run_at, created_by, updated_by, created_at, updated_at`, + assessmentID, cronExpr, enabled, parallelism, createdBy, + ).Scan(&sch.ID, &sch.AssessmentID, &sch.CronExpression, &sch.Enabled, &sch.Parallelism, &sch.LastRunAt, &sch.CreatedBy, &sch.UpdatedBy, &sch.CreatedAt, &sch.UpdatedAt) if err != nil { return nil, err } @@ -60,21 +60,21 @@ func (s *scheduleStore) Create(ctx context.Context, scenarioID uuid.UUID, cronEx func (s *scheduleStore) Get(ctx context.Context, id uuid.UUID) (*Schedule, error) { var sch Schedule err := s.pool.QueryRow(ctx, - `SELECT id, scenario_id, cron_expression, enabled, parallelism, last_run_at, created_by, updated_by, created_at, updated_at + `SELECT id, assessment_id, cron_expression, enabled, parallelism, last_run_at, created_by, updated_by, created_at, updated_at FROM schedules WHERE id = $1`, id, - ).Scan(&sch.ID, &sch.ScenarioID, &sch.CronExpression, &sch.Enabled, &sch.Parallelism, &sch.LastRunAt, &sch.CreatedBy, &sch.UpdatedBy, &sch.CreatedAt, &sch.UpdatedAt) + ).Scan(&sch.ID, &sch.AssessmentID, &sch.CronExpression, &sch.Enabled, &sch.Parallelism, &sch.LastRunAt, &sch.CreatedBy, &sch.UpdatedBy, &sch.CreatedAt, &sch.UpdatedAt) if err != nil { return nil, err } return &sch, nil } -func (s *scheduleStore) GetByScenarioID(ctx context.Context, scenarioID uuid.UUID) (*Schedule, error) { +func (s *scheduleStore) GetByAssessmentID(ctx context.Context, assessmentID uuid.UUID) (*Schedule, error) { var sch Schedule err := s.pool.QueryRow(ctx, - `SELECT id, scenario_id, cron_expression, enabled, parallelism, last_run_at, created_by, updated_by, created_at, updated_at - FROM schedules WHERE scenario_id = $1`, scenarioID, - ).Scan(&sch.ID, &sch.ScenarioID, &sch.CronExpression, &sch.Enabled, &sch.Parallelism, &sch.LastRunAt, &sch.CreatedBy, &sch.UpdatedBy, &sch.CreatedAt, &sch.UpdatedAt) + `SELECT id, assessment_id, cron_expression, enabled, parallelism, last_run_at, created_by, updated_by, created_at, updated_at + FROM schedules WHERE assessment_id = $1`, assessmentID, + ).Scan(&sch.ID, &sch.AssessmentID, &sch.CronExpression, &sch.Enabled, &sch.Parallelism, &sch.LastRunAt, &sch.CreatedBy, &sch.UpdatedBy, &sch.CreatedAt, &sch.UpdatedAt) if err != nil { return nil, err } @@ -83,7 +83,7 @@ func (s *scheduleStore) GetByScenarioID(ctx context.Context, scenarioID uuid.UUI func (s *scheduleStore) List(ctx context.Context) ([]Schedule, error) { rows, err := s.pool.Query(ctx, - `SELECT id, scenario_id, cron_expression, enabled, parallelism, last_run_at, created_by, updated_by, created_at, updated_at + `SELECT id, assessment_id, cron_expression, enabled, parallelism, last_run_at, created_by, updated_by, created_at, updated_at FROM schedules ORDER BY created_at DESC`, ) if err != nil { @@ -94,7 +94,7 @@ func (s *scheduleStore) List(ctx context.Context) ([]Schedule, error) { schedules := []Schedule{} for rows.Next() { var sch Schedule - if err := rows.Scan(&sch.ID, &sch.ScenarioID, &sch.CronExpression, &sch.Enabled, &sch.Parallelism, &sch.LastRunAt, &sch.CreatedBy, &sch.UpdatedBy, &sch.CreatedAt, &sch.UpdatedAt); err != nil { + if err := rows.Scan(&sch.ID, &sch.AssessmentID, &sch.CronExpression, &sch.Enabled, &sch.Parallelism, &sch.LastRunAt, &sch.CreatedBy, &sch.UpdatedBy, &sch.CreatedAt, &sch.UpdatedAt); err != nil { return nil, err } schedules = append(schedules, sch) @@ -104,7 +104,7 @@ func (s *scheduleStore) List(ctx context.Context) ([]Schedule, error) { func (s *scheduleStore) ListEnabled(ctx context.Context) ([]Schedule, error) { rows, err := s.pool.Query(ctx, - `SELECT id, scenario_id, cron_expression, enabled, parallelism, last_run_at, created_by, updated_by, created_at, updated_at + `SELECT id, assessment_id, cron_expression, enabled, parallelism, last_run_at, created_by, updated_by, created_at, updated_at FROM schedules WHERE enabled = true ORDER BY created_at DESC`, ) if err != nil { @@ -115,7 +115,7 @@ func (s *scheduleStore) ListEnabled(ctx context.Context) ([]Schedule, error) { schedules := []Schedule{} for rows.Next() { var sch Schedule - if err := rows.Scan(&sch.ID, &sch.ScenarioID, &sch.CronExpression, &sch.Enabled, &sch.Parallelism, &sch.LastRunAt, &sch.CreatedBy, &sch.UpdatedBy, &sch.CreatedAt, &sch.UpdatedAt); err != nil { + if err := rows.Scan(&sch.ID, &sch.AssessmentID, &sch.CronExpression, &sch.Enabled, &sch.Parallelism, &sch.LastRunAt, &sch.CreatedBy, &sch.UpdatedBy, &sch.CreatedAt, &sch.UpdatedAt); err != nil { return nil, err } schedules = append(schedules, sch) diff --git a/internal/matchers/datadog/datadog.go b/internal/matchers/datadog/datadog.go index 27b3cf3..175c363 100644 --- a/internal/matchers/datadog/datadog.go +++ b/internal/matchers/datadog/datadog.go @@ -20,12 +20,12 @@ import ( const QueryAllOpenSignals = `@workflow.triage.state:open` -func (m *DatadogAlertGeneratedAssertionBuilder) HasExpectedAlert(indicators []string, logger *logrus.Entry) (bool, error) { - return m.DatadogAlertGeneratedAssertion.HasExpectedAlert(indicators, logger) +func (m *DatadogAlertMatcherBuilder) HasExpectedAlert(indicators []string, logger *logrus.Entry) (bool, error) { + return m.DatadogAlertMatcher.HasExpectedAlert(indicators, logger) } -func (m *DatadogAlertGeneratedAssertionBuilder) Cleanup(indicators []string, logger *logrus.Entry) error { - return m.DatadogAlertGeneratedAssertion.Cleanup(indicators, logger) +func (m *DatadogAlertMatcherBuilder) Cleanup(indicators []string, logger *logrus.Entry) error { + return m.DatadogAlertMatcher.Cleanup(indicators, logger) } const QueryOpenSignalsByAlertNameAndSeverity = `@workflow.triage.state:open @workflow.rule.name:"%s" %s` @@ -93,7 +93,7 @@ func (m *DatadogSecuritySignalsAPIImpl) CloseSignal(id string) error { return nil } -func (m *DatadogAlertGeneratedAssertion) HasExpectedAlert(indicators []string, logger *logrus.Entry) (bool, error) { +func (m *DatadogAlertMatcher) HasExpectedAlert(indicators []string, logger *logrus.Entry) (bool, error) { logger = m.prepareLogger(logger) query := m.buildDatadogSignalQuery() @@ -108,19 +108,19 @@ func (m *DatadogAlertGeneratedAssertion) HasExpectedAlert(indicators []string, l return matchingSignal != nil, nil } -func (m *DatadogAlertGeneratedAssertion) String() string { +func (m *DatadogAlertMatcher) String() string { return fmt.Sprintf("Datadog security signal '%s'", m.AlertFilter.RuleName) } -func (m *DatadogAlertGeneratedAssertion) MatcherName() string { +func (m *DatadogAlertMatcher) MatcherName() string { return "Datadog security signal" } -func (m *DatadogAlertGeneratedAssertion) AlertName() string { +func (m *DatadogAlertMatcher) AlertName() string { return m.AlertFilter.RuleName } -func (m *DatadogAlertGeneratedAssertion) Cleanup(indicators []string, logger *logrus.Entry) error { +func (m *DatadogAlertMatcher) Cleanup(indicators []string, logger *logrus.Entry) error { logger = m.prepareLogger(logger) signals, err := m.searchSignalsWithQuery(QueryAllOpenSignals) @@ -140,7 +140,7 @@ func (m *DatadogAlertGeneratedAssertion) Cleanup(indicators []string, logger *lo } // TODO: Would probably make more sense to retrieve all open signal and iterate instead of doing 2 pass -func (m *DatadogAlertGeneratedAssertion) buildDatadogSignalQuery() string { +func (m *DatadogAlertMatcher) buildDatadogSignalQuery() string { severityQuery := "" if m.AlertFilter.Severity != "" { severityQuery = fmt.Sprintf(QuerySeverity, m.AlertFilter.Severity) + " " @@ -152,7 +152,7 @@ func (m *DatadogAlertGeneratedAssertion) buildDatadogSignalQuery() string { ) } -func (m *DatadogAlertGeneratedAssertion) prepareLogger(logger *logrus.Entry) *logrus.Entry { +func (m *DatadogAlertMatcher) prepareLogger(logger *logrus.Entry) *logrus.Entry { if logger == nil { logger = logrus.NewEntry(logrus.StandardLogger()) } @@ -162,7 +162,7 @@ func (m *DatadogAlertGeneratedAssertion) prepareLogger(logger *logrus.Entry) *lo }) } -func (m *DatadogAlertGeneratedAssertion) searchSignalsWithQuery(query string) ([]datadogV2.SecurityMonitoringSignal, error) { +func (m *DatadogAlertMatcher) searchSignalsWithQuery(query string) ([]datadogV2.SecurityMonitoringSignal, error) { signals, err := m.SignalsAPI.SearchSignals(query) if err != nil { return nil, errors.New("unable to search for Datadog security signal: " + err.Error()) @@ -170,7 +170,7 @@ func (m *DatadogAlertGeneratedAssertion) searchSignalsWithQuery(query string) ([ return signals, nil } -func (m *DatadogAlertGeneratedAssertion) findMatchingSignal(signals []datadogV2.SecurityMonitoringSignal, indicators []string, logger *logrus.Entry) *datadogV2.SecurityMonitoringSignal { +func (m *DatadogAlertMatcher) findMatchingSignal(signals []datadogV2.SecurityMonitoringSignal, indicators []string, logger *logrus.Entry) *datadogV2.SecurityMonitoringSignal { logger.WithField("signal_count", len(signals)).Info("Received signals from Datadog") if len(signals) == 0 { @@ -186,7 +186,7 @@ func (m *DatadogAlertGeneratedAssertion) findMatchingSignal(signals []datadogV2. return nil } -func (m *DatadogAlertGeneratedAssertion) signalMatchesExecution(signal datadogV2.SecurityMonitoringSignal, indicators []string, logger *logrus.Entry) bool { +func (m *DatadogAlertMatcher) signalMatchesExecution(signal datadogV2.SecurityMonitoringSignal, indicators []string, logger *logrus.Entry) bool { buf, _ := json.Marshal(signal.Attributes.Custom) rawSignal := strings.ToLower(string(buf)) diff --git a/internal/matchers/datadog/datadog_test.go b/internal/matchers/datadog/datadog_test.go index 82c2a07..72a986b 100644 --- a/internal/matchers/datadog/datadog_test.go +++ b/internal/matchers/datadog/datadog_test.go @@ -151,7 +151,7 @@ func TestDatadog(t *testing.T) { mockDatadog.On("SearchSignals", expectedQuery).Return(union(signalsMatchingOnlyRuleAndSeverity, signalsMatchingBoth), nil) mockDatadog.On("CloseSignal", mock.AnythingOfType("string")).Return(nil) - matcher := DatadogAlertGeneratedAssertion{ + matcher := DatadogAlertMatcher{ SignalsAPI: mockDatadog, AlertFilter: alertFilter, } @@ -199,7 +199,7 @@ func TestSignalMatchesExecutionCaseInsensitive(t *testing.T) { "resource_id": "/SUBSCRIPTIONS/ABC-123/RESOURCEGROUPS/RG", }} - matcher := DatadogAlertGeneratedAssertion{} + matcher := DatadogAlertMatcher{} // Indicator is lower-case, signal value is upper-case. assert.True(t, matcher.signalMatchesExecution(*signal, []string{"abc-123"}, testLogger)) diff --git a/internal/matchers/datadog/types.go b/internal/matchers/datadog/types.go index 6b00668..3e04e6c 100644 --- a/internal/matchers/datadog/types.go +++ b/internal/matchers/datadog/types.go @@ -15,14 +15,14 @@ type DatadogAlertFilter struct { // There might be other attributes in the future } -type DatadogAlertGeneratedAssertion struct { +type DatadogAlertMatcher struct { SignalsAPI DatadogSecuritySignalsAPI AlertFilter *DatadogAlertFilter } // builder -type DatadogAlertGeneratedAssertionBuilder struct { - DatadogAlertGeneratedAssertion +type DatadogAlertMatcherBuilder struct { + DatadogAlertMatcher } // getEnvWithFallback returns the value of the primary env var, or falls back to legacy @@ -48,8 +48,8 @@ func getDDSite(envVars map[string]string) string { // DatadogSecuritySignal creates a new Datadog security signal matcher. // envVars provides run-specific env vars; pass nil to read from process env (CLI path). -func DatadogSecuritySignal(name string, envVars map[string]string) *DatadogAlertGeneratedAssertionBuilder { - builder := &DatadogAlertGeneratedAssertionBuilder{} +func DatadogSecuritySignal(name string, envVars map[string]string) *DatadogAlertMatcherBuilder { + builder := &DatadogAlertMatcherBuilder{} ddApiKey := getEnvWithFallback(envVars, "SR_DATADOG_API_KEY", "DD_API_KEY") ddAppKey := getEnvWithFallback(envVars, "SR_DATADOG_APP_KEY", "DD_APP_KEY") ctx := context.WithValue(context.Background(), datadog.ContextAPIKeys, map[string]datadog.APIKey{ @@ -70,7 +70,7 @@ func DatadogSecuritySignal(name string, envVars map[string]string) *DatadogAlert return builder } -func (m *DatadogAlertGeneratedAssertionBuilder) WithSeverity(severity string) *DatadogAlertGeneratedAssertionBuilder { +func (m *DatadogAlertMatcherBuilder) WithSeverity(severity string) *DatadogAlertMatcherBuilder { m.AlertFilter.Severity = severity return m } diff --git a/internal/matchers/elastic/elastic.go b/internal/matchers/elastic/elastic.go index ccb10d7..1aefed0 100644 --- a/internal/matchers/elastic/elastic.go +++ b/internal/matchers/elastic/elastic.go @@ -113,7 +113,7 @@ func (m *ElasticSecurityDetectionAlertsAPIImpl) CloseAlert(id string) error { // getAPI returns the API client, lazily initializing it if needed. // This allows matchers to be created without credentials (for lint), // with credential validation deferred until the API is actually used. -func (m *ElasticSecurityAlertGeneratedAssertion) getAPI() (ElasticSecurityDetectionAlertsAPI, error) { +func (m *ElasticSecurityAlertMatcher) getAPI() (ElasticSecurityDetectionAlertsAPI, error) { if m.AlertsAPI != nil { return m.AlertsAPI, nil } @@ -135,7 +135,7 @@ func (m *ElasticSecurityAlertGeneratedAssertion) getAPI() (ElasticSecurityDetect return m.AlertsAPI, nil } -func (m *ElasticSecurityAlertGeneratedAssertion) HasExpectedAlert(indicators []string, logger *logrus.Entry) (bool, error) { +func (m *ElasticSecurityAlertMatcher) HasExpectedAlert(indicators []string, logger *logrus.Entry) (bool, error) { logger = m.prepareLogger(logger) alerts, err := m.searchAndMatch(indicators, logger) @@ -146,19 +146,19 @@ func (m *ElasticSecurityAlertGeneratedAssertion) HasExpectedAlert(indicators []s return alerts != nil, nil } -func (m *ElasticSecurityAlertGeneratedAssertion) String() string { +func (m *ElasticSecurityAlertMatcher) String() string { return fmt.Sprintf("Elastic Security alert '%s'", m.AlertFilter.RuleName) } -func (m *ElasticSecurityAlertGeneratedAssertion) MatcherName() string { +func (m *ElasticSecurityAlertMatcher) MatcherName() string { return "Elastic Security alert" } -func (m *ElasticSecurityAlertGeneratedAssertion) AlertName() string { +func (m *ElasticSecurityAlertMatcher) AlertName() string { return m.AlertFilter.RuleName } -func (m *ElasticSecurityAlertGeneratedAssertion) Cleanup(indicators []string, logger *logrus.Entry) error { +func (m *ElasticSecurityAlertMatcher) Cleanup(indicators []string, logger *logrus.Entry) error { logger = m.prepareLogger(logger) matchingAlerts, err := m.searchAndMatch(indicators, logger) @@ -185,7 +185,7 @@ func (m *ElasticSecurityAlertGeneratedAssertion) Cleanup(indicators []string, lo return nil } -func (m *ElasticSecurityAlertGeneratedAssertion) buildElasticAlertQuery() string { +func (m *ElasticSecurityAlertMatcher) buildElasticAlertQuery() string { type queryStruct struct { Size int `json:"size"` Query map[string]interface{} `json:"query"` @@ -225,17 +225,17 @@ func (m *ElasticSecurityAlertGeneratedAssertion) buildElasticAlertQuery() string return string(queryBytes) } -// sinceValue returns the timestamp filter value for the assertion query. +// sinceValue returns the timestamp filter value for the matcher query. // Uses the configured since time if set, otherwise falls back to "now-10h" // for backward compatibility with CLI usage where start time isn't tracked. -func (m *ElasticSecurityAlertGeneratedAssertion) sinceValue() string { +func (m *ElasticSecurityAlertMatcher) sinceValue() string { if m.since.IsZero() { return "now-15m" } return m.since.UTC().Format(time.RFC3339) } -func (m *ElasticSecurityAlertGeneratedAssertion) prepareLogger(logger *logrus.Entry) *logrus.Entry { +func (m *ElasticSecurityAlertMatcher) prepareLogger(logger *logrus.Entry) *logrus.Entry { if logger == nil { logger = logrus.NewEntry(logrus.StandardLogger()) } @@ -245,7 +245,7 @@ func (m *ElasticSecurityAlertGeneratedAssertion) prepareLogger(logger *logrus.En }) } -func (m *ElasticSecurityAlertGeneratedAssertion) searchAndMatch(indicators []string, logger *logrus.Entry) ([]ElasticSecurityDetectionAlert, error) { +func (m *ElasticSecurityAlertMatcher) searchAndMatch(indicators []string, logger *logrus.Entry) ([]ElasticSecurityDetectionAlert, error) { api, err := m.getAPI() if err != nil { return nil, err @@ -396,7 +396,7 @@ func extractAlertField(alert ElasticSecurityDetectionAlert, field string) (strin return "", false } -func (m *ElasticSecurityAlertGeneratedAssertion) alertMatchesExecution(alert ElasticSecurityDetectionAlert, indicators []string, logger *logrus.Entry) bool { +func (m *ElasticSecurityAlertMatcher) alertMatchesExecution(alert ElasticSecurityDetectionAlert, indicators []string, logger *logrus.Entry) bool { matched := alertMatchesIndicators(alert, indicators) if matched { logger.WithField("alert_id", alert.ID).Debug("Found matching alert based on provided indicators") diff --git a/internal/matchers/elastic/elastic_test.go b/internal/matchers/elastic/elastic_test.go index 56c7d73..d23b7d5 100644 --- a/internal/matchers/elastic/elastic_test.go +++ b/internal/matchers/elastic/elastic_test.go @@ -9,7 +9,7 @@ import ( func TestElasticAlertMatchesExecution(t *testing.T) { // Test the alertMatchesExecution method - assertion := &ElasticSecurityAlertGeneratedAssertion{ + assertion := &ElasticSecurityAlertMatcher{ AlertFilter: ElasticSecurityAlertFilter{ RuleName: "Test Rule", }, @@ -56,7 +56,7 @@ func TestAlertMatchesIndicatorsCaseInsensitive(t *testing.T) { func TestBuildElasticAlertQuery(t *testing.T) { // Test the query building method - assertion := &ElasticSecurityAlertGeneratedAssertion{ + assertion := &ElasticSecurityAlertMatcher{ AlertFilter: ElasticSecurityAlertFilter{ RuleName: "Test Rule", Severity: "high", @@ -70,8 +70,8 @@ func TestBuildElasticAlertQuery(t *testing.T) { assert.Contains(t, query, "kibana.alert.severity") } -func TestElasticAlertGeneratedAssertionString(t *testing.T) { - assertion := &ElasticSecurityAlertGeneratedAssertion{ +func TestElasticSecurityAlertMatcherString(t *testing.T) { + assertion := &ElasticSecurityAlertMatcher{ AlertFilter: ElasticSecurityAlertFilter{ RuleName: "My Test Rule", }, diff --git a/internal/matchers/elastic/types.go b/internal/matchers/elastic/types.go index 18a10fa..c568f25 100644 --- a/internal/matchers/elastic/types.go +++ b/internal/matchers/elastic/types.go @@ -8,8 +8,8 @@ type ElasticSecurityAlertFilter struct { Severity string } -// ElasticAlertGeneratedAssertion implements the AlertGeneratedMatcher interface -type ElasticSecurityAlertGeneratedAssertion struct { +// ElasticSecurityAlertMatcher implements the AlertGeneratedMatcher interface +type ElasticSecurityAlertMatcher struct { AlertsAPI ElasticSecurityDetectionAlertsAPI AlertFilter ElasticSecurityAlertFilter envVars map[string]string // run-specific env vars for credential isolation @@ -20,8 +20,8 @@ type ElasticSecurityAlertGeneratedAssertion struct { // The API client is lazily initialized when first used (in HasExpectedAlert or Cleanup), // allowing lint to validate scenario files without requiring API credentials. // envVars provides run-specific env vars; pass nil to read from process env (CLI path). -func ElasticSecurityAlert(name string, envVars map[string]string) (*ElasticSecurityAlertGeneratedAssertion, error) { - return &ElasticSecurityAlertGeneratedAssertion{ +func ElasticSecurityAlert(name string, envVars map[string]string) (*ElasticSecurityAlertMatcher, error) { + return &ElasticSecurityAlertMatcher{ AlertFilter: ElasticSecurityAlertFilter{RuleName: name}, envVars: envVars, }, nil @@ -29,13 +29,13 @@ func ElasticSecurityAlert(name string, envVars map[string]string) (*ElasticSecur // WithSeverity adds severity filtering to the matcher // Returns self for method chaining -func (m *ElasticSecurityAlertGeneratedAssertion) WithSeverity(severity string) *ElasticSecurityAlertGeneratedAssertion { +func (m *ElasticSecurityAlertMatcher) WithSeverity(severity string) *ElasticSecurityAlertMatcher { // Modify in place - no need to create new objects m.AlertFilter.Severity = severity return m } // SetSince restricts the query to alerts created after the given time. -func (m *ElasticSecurityAlertGeneratedAssertion) SetSince(t time.Time) { +func (m *ElasticSecurityAlertMatcher) SetSince(t time.Time) { m.since = t } diff --git a/internal/parser/main.go b/internal/parser/main.go index 30d9192..ca887a4 100644 --- a/internal/parser/main.go +++ b/internal/parser/main.go @@ -143,16 +143,16 @@ func buildScenario(parsedScenario SimrunSchemaJsonScenariosElem, metadata *Simru } } - // Assertions and timeout + // Expectations and timeout if len(parsedScenario.Expectations) == 0 { - return nil, fmt.Errorf("scenario '%s' has no assertions defined", parsedScenario.Name) + return nil, fmt.Errorf("scenario '%s' has no expectations defined", parsedScenario.Name) } - assertions, err := buildAssertions(parsedScenario.Expectations, opts.EnvVars) + builtMatchers, err := buildMatchers(parsedScenario.Expectations, opts.EnvVars) if err != nil { return nil, err } - scenario.Assertions = assertions + scenario.Matchers = builtMatchers timeout, err := time.ParseDuration(parsedScenario.Expectations[0].Timeout) if err != nil { @@ -328,30 +328,30 @@ func extractTargets(target *SimrunSchemaJsonTargets) map[string]string { return result } -// buildAssertions creates assertion matchers from expectations -func buildAssertions(expectations []SimrunSchemaJsonScenariosElemExpectationsElem, envVars map[string]string) ([]matchers.AlertGeneratedMatcher, error) { - var assertions []matchers.AlertGeneratedMatcher +// buildMatchers creates alert matchers from expectations +func buildMatchers(expectations []SimrunSchemaJsonScenariosElemExpectationsElem, envVars map[string]string) ([]matchers.AlertGeneratedMatcher, error) { + var built []matchers.AlertGeneratedMatcher for _, expectation := range expectations { if datadogMatcher := expectation.DatadogSecuritySignal; datadogMatcher != nil { - assertion := datadog.DatadogSecuritySignal(datadogMatcher.Name, envVars) + matcher := datadog.DatadogSecuritySignal(datadogMatcher.Name, envVars) if severity := datadogMatcher.Severity; severity != nil { - assertion.WithSeverity(*severity) + matcher.WithSeverity(*severity) } - assertions = append(assertions, assertion) + built = append(built, matcher) } if elasticMatcher := expectation.ElasticSecurityAlert; elasticMatcher != nil { - assertion, err := elastic.ElasticSecurityAlert(elasticMatcher.Name, envVars) + matcher, err := elastic.ElasticSecurityAlert(elasticMatcher.Name, envVars) if err != nil { return nil, fmt.Errorf("failed to create Elastic Security alert matcher: %w", err) } if severity := elasticMatcher.Severity; severity != nil { - assertion.WithSeverity(string(*severity)) + matcher.WithSeverity(string(*severity)) } - assertions = append(assertions, assertion) + built = append(built, matcher) } } - return assertions, nil + return built, nil } diff --git a/internal/results/executor.go b/internal/results/executor.go index 9660751..1359da8 100644 --- a/internal/results/executor.go +++ b/internal/results/executor.go @@ -12,15 +12,15 @@ import ( func RunScenariosParallel( scenarios []*runner.Scenario, parallelism int, - callback func(result *ScenarioRunResult), -) []ScenarioRunResult { + callback func(result *runner.ScenarioResult), +) []runner.ScenarioResult { numWorkers := parallelism if numScenarios := len(scenarios); numScenarios < numWorkers { numWorkers = numScenarios } scenarioChan := make(chan *runner.Scenario, len(scenarios)) - resultsChan := make(chan *ScenarioRunResult, len(scenarios)) + resultsChan := make(chan *runner.ScenarioResult, len(scenarios)) var wg sync.WaitGroup @@ -45,7 +45,7 @@ func RunScenariosParallel( close(resultsChan) }() - var allResults []ScenarioRunResult + var allResults []runner.ScenarioResult for result := range resultsChan { callback(result) allResults = append(allResults, *result) @@ -54,70 +54,18 @@ func RunScenariosParallel( return allResults } -func runSingleScenario(scenarios <-chan *runner.Scenario, results chan<- *ScenarioRunResult) { +func runSingleScenario(scenarios <-chan *runner.Scenario, results chan<- *runner.ScenarioResult) { for scenario := range scenarios { testRunner := runner.NewRunner() - testRunner.Scenarios = append(testRunner.Scenarios, scenario) testRunner.Interval = 10 * time.Second start := time.Now() - scenarioResults, runError := testRunner.Run() + result := testRunner.Run(scenario) end := time.Now() - var executorType, executorName, simulationID string - if scenario.Detonator != nil { - executorType = "detonator" - executorName = scenario.Detonator.String() - simulationID = scenario.Detonator.SimulationId() - } else if scenario.Injector != nil { - executorType = "injector" - executorName = scenario.Injector.String() - } else { - executorType = "unknown" - executorName = "unknown" - } - - if len(scenarioResults) > 0 { - results <- &ScenarioRunResult{ - Name: scenario.Name, - ErrorMessage: scenarioResults[0].Error, - Success: scenarioResults[0].Success, - DurationSeconds: end.Sub(start).Seconds(), - MatchingDurationSeconds: scenarioResults[0].MatchingDurationSeconds, - TimeExecuted: start, - ExecutorName: executorName, - ExecutorType: executorType, - ExecutionId: scenarioResults[0].ExecutionId, - SimulationID: simulationID, - Assertions: scenario.Assertions, - FailedAssertions: scenario.FailedAssertions, - Indicators: scenario.Indicators, - Metadata: scenario.Metadata, - CollectedLogPath: scenario.CollectedLogPath, - CollectedDocCount: scenario.CollectedDocCount, - DiscoveredAlerts: scenario.DiscoveredAlerts, - ExploreMode: scenario.ExploreMode, - } - } else { - errorMessage := "Scenario failed to execute" - if runError != nil { - errorMessage = runError.Error() - } - results <- &ScenarioRunResult{ - Name: scenario.Name, - ErrorMessage: errorMessage, - Success: false, - DurationSeconds: end.Sub(start).Seconds(), - MatchingDurationSeconds: 0, - TimeExecuted: start, - ExecutorName: executorName, - ExecutorType: executorType, - ExecutionId: "", - SimulationID: simulationID, - Assertions: scenario.Assertions, - Indicators: scenario.Indicators, - Metadata: scenario.Metadata, - } - } + // The runner populates everything except the wall-clock timing. + result.TimeExecuted = start + result.DurationSeconds = end.Sub(start).Seconds() + results <- &result } } diff --git a/internal/results/types.go b/internal/results/types.go index 288346b..34175b1 100644 --- a/internal/results/types.go +++ b/internal/results/types.go @@ -1,41 +1,7 @@ -// Package results defines the shared run and scenario result types and a -// parallel scenario executor. +// Package results provides a parallel scenario executor over the shared +// runner.ScenarioResult / runner.RunResult result types. package results -import ( - "time" - - "github.com/IBM/simrun/internal/matchers" - "github.com/IBM/simrun/internal/runner" -) - -type ScenarioRunResult struct { - Name string `json:"name"` - Success bool `json:"isSuccess"` - ErrorMessage string `json:"errorMessage"` - DurationSeconds float64 `json:"durationSeconds"` - MatchingDurationSeconds float64 `json:"matchingDurationSeconds"` - TimeExecuted time.Time `json:"timeExecuted"` - ExecutorName string `json:"executorName"` - ExecutorType string `json:"executorType"` - ExecutionId string `json:"executionId"` - SimulationID string `json:"simulationId,omitempty"` - Assertions []matchers.AlertGeneratedMatcher `json:"matchers,omitempty"` - FailedAssertions []matchers.AlertGeneratedMatcher `json:"-"` - Indicators *runner.Indicators `json:"indicators,omitempty"` - Metadata *runner.Metadata `json:"metadata,omitempty"` - CollectedLogPath string `json:"collectedLogPath,omitempty"` - CollectedDocCount int `json:"collectedDocCount,omitempty"` - DiscoveredAlerts []runner.DiscoveredAlert `json:"discoveredAlerts,omitempty"` - ExploreMode bool `json:"exploreMode,omitempty"` -} - -type SimrunRunResult struct { - RunId string `json:"runId"` - StartTime time.Time `json:"startTime"` - EndTime time.Time `json:"endTime"` - TotalScenarios int `json:"totalScenarios"` - SuccessScenarios int `json:"successScenarios"` - FailedScenarios int `json:"failedScenarios"` - Scenarios []ScenarioRunResult `json:"scenarios"` -} +// RunResult and ScenarioResult are defined in the runner package (the single +// in-memory result types shared across runner, executor, and web). This file is +// retained as the package anchor; see executor.go for the fan-out. diff --git a/internal/runner/runner.go b/internal/runner/runner.go index c19fe76..5f8f4fc 100644 --- a/internal/runner/runner.go +++ b/internal/runner/runner.go @@ -4,7 +4,6 @@ package runner import ( "context" - "errors" "fmt" "strconv" "strings" @@ -15,76 +14,77 @@ import ( "github.com/sirupsen/logrus" ) -type ScenarioResult struct { - Name string `json:"name"` - Success bool `json:"success"` - ExecutionId string `json:"executionId"` - Error string `json:"error,omitempty"` - MatchingDurationSeconds float64 `json:"matchingDurationSeconds,omitempty"` - CollectedLogPath string `json:"collectedLogPath,omitempty"` - CollectedDocCount int `json:"collectedDocCount,omitempty"` -} - +// Runner executes exactly one scenario. Fan-out across multiple scenarios is +// the sole responsibility of the parallel executor (see internal/results). type Runner struct { - Scenarios []*Scenario - Interval time.Duration + Interval time.Duration } func NewRunner() *Runner { return &Runner{Interval: 2 * time.Second} } -func (m *Runner) Run() ([]ScenarioResult, error) { - var results []ScenarioResult - failedScenarios := map[string]error{} - - for i := range m.Scenarios { - scenario := m.Scenarios[i] - executionId, matchingDuration, err := m.runScenario(scenario) - - result := ScenarioResult{ - Name: scenario.Name, - ExecutionId: executionId, +// Eventually polls fn at interval until it reports done (true), returns an +// error, or the deadline passes. A zero deadline means "no deadline" (poll +// until done or error). On deadline it returns nil — the caller inspects the +// state fn was mutating to decide success or failure. +func Eventually(fn func() (bool, error), interval time.Duration, deadline time.Time) error { + hasDeadline := !deadline.IsZero() + for { + if hasDeadline && time.Now().After(deadline) { + return nil } - + done, err := fn() if err != nil { - result.Success = false - result.Error = err.Error() - failedScenarios[scenario.Name] = err - } else { - result.Success = true + return err } - - result.MatchingDurationSeconds = matchingDuration - results = append(results, result) - } - - if len(failedScenarios) > 0 { - var errorMessage strings.Builder - errorMessage.WriteString("At least one scenario failed:\n\n") - for scenario, err := range failedScenarios { - errorMessage.WriteString(scenario) - errorMessage.WriteString(" returned: ") - errorMessage.WriteString(err.Error()) - errorMessage.WriteRune('\n') + if done { + return nil } - return results, errors.New(errorMessage.String()) + time.Sleep(interval) } +} - return results, nil +// Run executes the given scenario and returns its populated result. Wall-clock +// timing (TimeExecuted, DurationSeconds) is filled in by the caller. +func (m *Runner) Run(scenario *Scenario) ScenarioResult { + return m.runScenario(scenario) } -func (m *Runner) runScenario(scenario *Scenario) (string, float64, error) { - if scenario.Detonator == nil && scenario.Injector == nil { - return "", 0, fmt.Errorf("scenario must have either a detonator or an injector") +// runScenario executes a single scenario and returns its populated result. +// It does not mutate the scenario input; wall-clock timing (TimeExecuted, +// DurationSeconds) is filled in by the caller. +func (m *Runner) runScenario(scenario *Scenario) ScenarioResult { + result := ScenarioResult{ + Name: scenario.Name, + Matchers: scenario.Matchers, + Indicators: scenario.Indicators, + Metadata: scenario.Metadata, + ExploreMode: scenario.ExploreMode, + } + switch { + case scenario.Detonator != nil: + result.ExecutorType = "detonator" + result.ExecutorName = scenario.Detonator.String() + result.SimulationID = scenario.Detonator.SimulationId() + case scenario.Injector != nil: + result.ExecutorType = "injector" + result.ExecutorName = scenario.Injector.String() + default: + result.ExecutorType = "unknown" + result.ExecutorName = "unknown" + result.ErrorMessage = "scenario must have either a detonator or an injector" + return result } executionOutput, logger, err := m.executeScenario(scenario) + // Preserve the execution_id even on failure so a partially-completed + // detonation (e.g. terraform applied, detonation timed out) stays + // correlatable. Indexing a nil map yields "". + result.ExecutionId = executionOutput["execution_id"] if err != nil { - // Preserve the execution_id even on failure so a partially-completed - // detonation (e.g. terraform applied, detonation timed out) stays - // correlatable. Indexing a nil map yields "". - return executionOutput["execution_id"], 0, err + result.ErrorMessage = err.Error() + return result } indicators := m.buildIndicatorsList(scenario, executionOutput) @@ -92,8 +92,6 @@ func (m *Runner) runScenario(scenario *Scenario) (string, float64, error) { defer m.CleanupScenario(scenario, indicators, logger) start := time.Now() - executionId := executionOutput["execution_id"] - // Add simulation_id to executionOutput for collector to use if scenario.Detonator != nil { executionOutput["simulation_id"] = scenario.Detonator.SimulationId() @@ -101,55 +99,59 @@ func (m *Runner) runScenario(scenario *Scenario) (string, float64, error) { // Surface executor identity now that detonation has resolved execution_id / // simulation_id, so a still-running row shows what is executing and where. - reportIdentity(scenario, executionId) + reportIdentity(scenario, result.ExecutionId) - // If no assertions, no collector, and not explore mode, just return - if !scenario.ExploreMode && len(scenario.Assertions) == 0 && scenario.Collector == nil { - return executionId, 0, nil + // If no matchers, no collector, and not explore mode, just return success. + if !scenario.ExploreMode && len(scenario.Matchers) == 0 && scenario.Collector == nil { + result.Success = true + return result } logger.WithFields(logrus.Fields{ - "execution_id": executionId, + "execution_id": result.ExecutionId, }).Info("Scenario successfully executed") deadline := start.Add(scenario.Timeout) ctx, cancel := context.WithDeadline(context.Background(), deadline) defer cancel() - // Run assertions or explore mode - var assertionsErr error + // Run matchers or explore mode + var matchErr error if scenario.ExploreMode { reportStatus(scenario, "exploring") - assertionsErr = m.runExploreMode(scenario, indicators, logger, start, deadline) + result.DiscoveredAlerts, matchErr = m.runExploreMode(scenario, indicators, logger, start, deadline) } else if isCollectMode(scenario) { - // Collect mode: skip assertions, wait for timeout, then collect + // Collect mode: skip matching, wait for timeout, then collect reportStatus(scenario, "waiting") logger.Infof("Collect mode: waiting %s before collecting logs", scenario.Timeout) <-ctx.Done() - } else if len(scenario.Assertions) > 0 { + } else if len(scenario.Matchers) > 0 { reportStatus(scenario, "matching") - assertionsErr = m.runAssertions(scenario, indicators, logger, start, deadline) + result.UnmetExpectations, matchErr = m.runMatchers(scenario, indicators, logger, start, deadline) } - // Run collection at the end (after assertions complete or timeout) + // Run collection at the end (after matching completes or timeout) if scenario.Collector != nil { reportStatus(scenario, "collecting") - collectedLogPath, collectedDocCount := m.runCollection(ctx, scenario, executionOutput, logger) - scenario.CollectedLogPath = collectedLogPath - scenario.CollectedDocCount = collectedDocCount + result.CollectedLogPath, result.CollectedDocCount = m.runCollection(ctx, scenario, executionOutput, logger) } - matchingDuration := time.Since(start).Seconds() - return executionId, matchingDuration, assertionsErr + result.MatchingDurationSeconds = time.Since(start).Seconds() + if matchErr != nil { + result.ErrorMessage = matchErr.Error() + } else { + result.Success = true + } + return result } // isCollectMode returns true when the scenario has a collector and only -// placeholder assertions generated by the frontend for collect-type scenarios. +// placeholder matchers generated by the frontend for collect-type scenarios. func isCollectMode(scenario *Scenario) bool { if scenario.Collector == nil { return false } - for _, a := range scenario.Assertions { + for _, a := range scenario.Matchers { if !strings.HasSuffix(a.AlertName(), " - collect mode") { return false } @@ -218,95 +220,101 @@ func (m *Runner) executeInjector(scenario *Scenario) (*logrus.Entry, map[string] return logger, executionOutput, err } -func (m *Runner) runAssertions(scenario *Scenario, indicators []string, logger *logrus.Entry, start time.Time, deadline time.Time) error { - // Set start time on Elastic assertions to scope queries to this run - for _, a := range scenario.Assertions { - if ea, ok := a.(*elastic.ElasticSecurityAlertGeneratedAssertion); ok { +// runMatchers polls every matcher until all match or the deadline passes, +// reporting partial progress as each newly matches. It returns the matchers +// that never matched (unmet expectations) and any check error. +func (m *Runner) runMatchers(scenario *Scenario, indicators []string, logger *logrus.Entry, start time.Time, deadline time.Time) ([]matchers.AlertGeneratedMatcher, error) { + // Set start time on Elastic matchers to scope queries to this run + for _, a := range scenario.Matchers { + if ea, ok := a.(*elastic.ElasticSecurityAlertMatcher); ok { ea.SetSince(start) } } - // Build a queue containing all assertions - remainingAssertions := make(chan matchers.AlertGeneratedMatcher, len(scenario.Assertions)) - for i := range scenario.Assertions { - remainingAssertions <- scenario.Assertions[i] + // Build a queue containing all matchers + remaining := make(chan matchers.AlertGeneratedMatcher, len(scenario.Matchers)) + for i := range scenario.Matchers { + remaining <- scenario.Matchers[i] } - logger.Info("Waiting for assertions") - hasDeadline := scenario.Timeout > 0 - - matchedSet := make(map[matchers.AlertGeneratedMatcher]bool, len(scenario.Assertions)) + logger.Info("Waiting for expectations") + matchedSet := make(map[matchers.AlertGeneratedMatcher]bool, len(scenario.Matchers)) - for len(remainingAssertions) > 0 { - if hasDeadline && time.Now().After(deadline) { - logger.WithFields(logrus.Fields{ - "remaining_alerts": len(remainingAssertions), - }).Warn("Timeout exceeded waiting for alerts") - break - } + // No deadline check when Timeout <= 0 (poll until all match). + pollDeadline := time.Time{} + if scenario.Timeout > 0 { + pollDeadline = deadline + } - assertion := <-remainingAssertions - matched, err := m.checkAssertion(assertion, indicators, logger, start) - if err != nil { - return err - } - if !matched { - remainingAssertions <- assertion - time.Sleep(m.Interval) - continue + err := Eventually(func() (bool, error) { + // One pass over the matchers still outstanding; re-queue the misses. + for n := len(remaining); n > 0; n-- { + matcher := <-remaining + matched, err := m.checkMatcher(matcher, indicators, logger, start) + if err != nil { + return false, err + } + if !matched { + remaining <- matcher + continue + } + // Newly matched: report the updated partial state (write-on-change). + matchedSet[matcher] = true + reportExpectations(scenario, matchedSet) } - // Newly matched: persist the updated partial state (write-on-change). - matchedSet[assertion] = true - reportAssertions(scenario, matchedSet) + return len(remaining) == 0, nil + }, m.Interval, pollDeadline) + if err != nil { + return nil, err } - numRemainingAssertions := len(remainingAssertions) - if numRemainingAssertions == 0 { - logger.Info("All assertions passed") - return nil + numRemaining := len(remaining) + if numRemaining == 0 { + logger.Info("All expectations passed") + return nil, nil } - return m.buildAssertionError(scenario, remainingAssertions, numRemainingAssertions, logger) + return m.buildUnmetExpectationsError(scenario, remaining, numRemaining, logger) } -func (m *Runner) checkAssertion(assertion matchers.AlertGeneratedMatcher, indicators []string, logger *logrus.Entry, start time.Time) (bool, error) { - hasAlert, err := assertion.HasExpectedAlert(indicators, logger) +func (m *Runner) checkMatcher(matcher matchers.AlertGeneratedMatcher, indicators []string, logger *logrus.Entry, start time.Time) (bool, error) { + hasAlert, err := matcher.HasExpectedAlert(indicators, logger) if err != nil { return false, err } if hasAlert { logger.WithFields(logrus.Fields{ - "matcher": assertion.MatcherName(), - "alert": assertion.AlertName(), + "matcher": matcher.MatcherName(), + "alert": matcher.AlertName(), "time_seconds": strconv.Itoa(int(time.Since(start).Seconds())), }).Info("Confirmed that the expected signal was created") return true, nil } logger.WithFields(logrus.Fields{ - "matcher": assertion.MatcherName(), - "alert": assertion.AlertName(), - }).Info("Assertion not yet matched, will retry") + "matcher": matcher.MatcherName(), + "alert": matcher.AlertName(), + }).Info("Expectation not yet matched, will retry") return false, nil } -func (m *Runner) buildAssertionError(scenario *Scenario, remainingAssertions chan matchers.AlertGeneratedMatcher, numRemainingAssertions int, logger *logrus.Entry) error { - failedAssertions := make([]string, 0, numRemainingAssertions) - scenario.FailedAssertions = make([]matchers.AlertGeneratedMatcher, 0, numRemainingAssertions) - for i := 0; i < numRemainingAssertions; i++ { - assertion := <-remainingAssertions - failedAssertions = append(failedAssertions, assertion.String()) - scenario.FailedAssertions = append(scenario.FailedAssertions, assertion) +func (m *Runner) buildUnmetExpectationsError(scenario *Scenario, remaining chan matchers.AlertGeneratedMatcher, numRemaining int, logger *logrus.Entry) ([]matchers.AlertGeneratedMatcher, error) { + unmetNames := make([]string, 0, numRemaining) + unmet := make([]matchers.AlertGeneratedMatcher, 0, numRemaining) + for range numRemaining { + matcher := <-remaining + unmetNames = append(unmetNames, matcher.String()) + unmet = append(unmet, matcher) } logger.WithFields(logrus.Fields{ - "scenario": scenario.Name, - "failed_assertions": numRemainingAssertions, - "assertion_details": failedAssertions, - }).Warn("Scenario assertions did not pass") + "scenario": scenario.Name, + "unmet_expectations": numRemaining, + "expectation_details": unmetNames, + }).Warn("Scenario expectations did not pass") - return fmt.Errorf("%d out of %d assertions did not pass: %s", numRemainingAssertions, len(scenario.Assertions), strings.Join(failedAssertions, ", ")) + return unmet, fmt.Errorf("%d out of %d expectations did not pass: %s", numRemaining, len(scenario.Matchers), strings.Join(unmetNames, ", ")) } func (m *Runner) runCollection(ctx context.Context, scenario *Scenario, executionOutput map[string]string, logger *logrus.Entry) (string, int) { @@ -330,15 +338,15 @@ func (m *Runner) runCollection(ctx context.Context, scenario *Scenario, executio return outputPath, collected } -func (m *Runner) runExploreMode(scenario *Scenario, indicators []string, logger *logrus.Entry, start time.Time, deadline time.Time) error { +func (m *Runner) runExploreMode(scenario *Scenario, indicators []string, logger *logrus.Entry, start time.Time, deadline time.Time) ([]DiscoveredAlert, error) { if len(indicators) == 0 { - return fmt.Errorf("explore mode requires at least one indicator to search for") + return nil, fmt.Errorf("explore mode requires at least one indicator to search for") } // Create API client and query once, reuse across all polling iterations api, err := elastic.CreateAPIFromEnvVars(scenario.EnvVars) if err != nil { - return err + return nil, err } query := elastic.BuildExploreQuery(start) @@ -346,17 +354,19 @@ func (m *Runner) runExploreMode(scenario *Scenario, indicators []string, logger // Track unique alerts by ID to avoid duplicates across polls seen := make(map[string]bool) + var discovered []DiscoveredAlert - for time.Now().Before(deadline) { + // Explore always runs to the deadline (never satisfied early), collecting + // every distinct matching alert it sees. + err = Eventually(func() (bool, error) { results, err := elastic.ExploreAlerts(api, query, indicators, logger) if err != nil { - return err + return false, err } - for _, r := range results { if !seen[r.AlertID] { seen[r.AlertID] = true - scenario.DiscoveredAlerts = append(scenario.DiscoveredAlerts, DiscoveredAlert{ + discovered = append(discovered, DiscoveredAlert{ RuleName: r.RuleName, AlertID: r.AlertID, Severity: r.Severity, @@ -368,17 +378,19 @@ func (m *Runner) runExploreMode(scenario *Scenario, indicators []string, logger }).Info("Explore mode: discovered matching alert") } } - - time.Sleep(m.Interval) + return false, nil + }, m.Interval, deadline) + if err != nil { + return discovered, err } - logger.WithField("total_discovered", len(scenario.DiscoveredAlerts)).Info("Explore mode: completed") + logger.WithField("total_discovered", len(discovered)).Info("Explore mode: completed") // Optional cleanup: close discovered alerts - if scenario.CleanupAlerts && len(scenario.DiscoveredAlerts) > 0 { + if scenario.CleanupAlerts && len(discovered) > 0 { reportStatus(scenario, "cleanup") - alertIDs := make([]string, len(scenario.DiscoveredAlerts)) - for i, a := range scenario.DiscoveredAlerts { + alertIDs := make([]string, len(discovered)) + for i, a := range discovered { alertIDs[i] = a.AlertID } if err := elastic.CloseAlerts(api, alertIDs, logger); err != nil { @@ -386,24 +398,24 @@ func (m *Runner) runExploreMode(scenario *Scenario, indicators []string, logger } } - if len(scenario.DiscoveredAlerts) == 0 { - return fmt.Errorf("explore mode: no matching alerts discovered within timeout") + if len(discovered) == 0 { + return discovered, fmt.Errorf("explore mode: no matching alerts discovered within timeout") } - return nil + return discovered, nil } func (m *Runner) CleanupScenario(scenario *Scenario, indicators []string, logger *logrus.Entry) { - if len(scenario.Assertions) == 0 || scenario.ExploreMode { + if len(scenario.Matchers) == 0 || scenario.ExploreMode { return } reportStatus(scenario, "cleanup") - for _, assertion := range scenario.Assertions { - if err := assertion.Cleanup(indicators, logger); err != nil { + for _, matcher := range scenario.Matchers { + if err := matcher.Cleanup(indicators, logger); err != nil { logger.WithFields(logrus.Fields{ - "matcher": assertion.MatcherName(), - "alert": assertion.AlertName(), + "matcher": matcher.MatcherName(), + "alert": matcher.AlertName(), "error": err.Error(), }).Warn("Failed to clean up generated signals") } @@ -438,16 +450,16 @@ func reportIdentity(scenario *Scenario, executionID string) { scenario.IdentityCallback(scenario.Name, identity) } -// reportAssertions emits the current pass/pending state of every assertion. -// Matched assertions carry Passed=true; not-yet-matched ones carry Passed=nil +// reportExpectations emits the current pass/pending state of every expectation. +// Matched expectations carry Passed=true; not-yet-matched ones carry Passed=nil // (pending) — no terminal failure is recorded until completion. -func reportAssertions(scenario *Scenario, matchedSet map[matchers.AlertGeneratedMatcher]bool) { - if scenario.AssertionsCallback == nil { +func reportExpectations(scenario *Scenario, matchedSet map[matchers.AlertGeneratedMatcher]bool) { + if scenario.ExpectationsCallback == nil { return } - results := make([]AssertionResult, 0, len(scenario.Assertions)) - for _, a := range scenario.Assertions { - r := AssertionResult{ + results := make([]ExpectationResult, 0, len(scenario.Matchers)) + for _, a := range scenario.Matchers { + r := ExpectationResult{ MatcherType: a.MatcherName(), AlertName: a.AlertName(), } @@ -457,7 +469,7 @@ func reportAssertions(scenario *Scenario, matchedSet map[matchers.AlertGenerated } results = append(results, r) } - scenario.AssertionsCallback(scenario.Name, results) + scenario.ExpectationsCallback(scenario.Name, results) } func (m *Runner) buildIndicatorsList(scenario *Scenario, detonationOutput map[string]string) []string { diff --git a/internal/runner/runner_test.go b/internal/runner/runner_test.go index 60e1587..2916c05 100644 --- a/internal/runner/runner_test.go +++ b/internal/runner/runner_test.go @@ -2,7 +2,6 @@ package runner import ( "encoding/json" - "errors" "net/http" "net/http/httptest" "testing" @@ -23,13 +22,13 @@ func TestRunnerWorks(t *testing.T) { testCases := []struct { Name string AlertExistsSequence []bool - HasNoAssertion bool + HasNoMatcher bool ExpectError bool }{ {Name: "Alert exists from the beginning", AlertExistsSequence: []bool{true}}, {Name: "Alert doesn't exist then exists", AlertExistsSequence: []bool{false, true}}, {Name: "Alert never exists", AlertExistsSequence: []bool{false}, ExpectError: true}, - {Name: "No assertion", HasNoAssertion: true}, + {Name: "No matcher", HasNoMatcher: true}, } for i := range testCases { @@ -57,53 +56,40 @@ func TestRunnerWorks(t *testing.T) { mockMatcher.On("AlertName").Return("sample alert") mockMatcher.On("Cleanup", []string{"my-uid"}, mock.AnythingOfType("*logrus.Entry")).Return(nil) - var assertions []matchers.AlertGeneratedMatcher - assertions = []matchers.AlertGeneratedMatcher{} - if !testCase.HasNoAssertion { - assertions = []matchers.AlertGeneratedMatcher{mockMatcher} + var matcherList []matchers.AlertGeneratedMatcher + if !testCase.HasNoMatcher { + matcherList = []matchers.AlertGeneratedMatcher{mockMatcher} } - runner := Runner{ - Scenarios: []*Scenario{ - { - Name: "test-scenario", - Detonator: mockDetonator, - Assertions: assertions, - Timeout: 50 * time.Millisecond, - }, - }, - Interval: 1 * time.Millisecond, - } - results, err := runner.Run() + r := Runner{Interval: 1 * time.Millisecond} + result := r.Run(&Scenario{ + Name: "test-scenario", + Detonator: mockDetonator, + Matchers: matcherList, + Timeout: 50 * time.Millisecond, + }) if testCase.ExpectError { - assert.NotNil(t, err) + assert.False(t, result.Success) + assert.NotEmpty(t, result.ErrorMessage) } else { - assert.Nil(t, err) - assert.Len(t, results, 1) - assert.True(t, results[0].Success) + assert.True(t, result.Success) + assert.Empty(t, result.ErrorMessage) } mockDetonator.AssertNumberOfCalls(t, "Detonate", 1) - if !testCase.HasNoAssertion { + if !testCase.HasNoMatcher { mockMatcher.AssertCalled(t, "Cleanup", []string{"my-uid"}, mock.AnythingOfType("*logrus.Entry")) } - }) } - } -func TestRunnerErrorHandling(t *testing.T) { - - mockDetonator := &detonatorMocks.MockDetonator{} - mockDetonator.On("Detonate").Return(map[string]string{"execution_id": "my-uid"}, nil) - mockDetonator.On("String").Return("mock-detonator") - mockDetonator.On("SimulationId").Return("test-simulation") - mockDetonator.On("PackName").Return("") - mockDetonator.On("SetStatusCallback", mock.AnythingOfType("func(string)")).Return() - +// TestRunnerFailedDetonationRetainsExecutionID verifies that a scenario whose +// detonation errors still surfaces the execution_id (so a partial detonation +// stays correlatable) and is reported as a failure. +func TestRunnerFailedDetonationRetainsExecutionID(t *testing.T) { mockFailingDetonator := &detonatorMocks.MockDetonator{} - mockFailingDetonator.On("Detonate").Return(map[string]string{"execution_id": "failed-uid"}, errors.New("foo")) + mockFailingDetonator.On("Detonate").Return(map[string]string{"execution_id": "failed-uid"}, assert.AnError) mockFailingDetonator.On("String").Return("mock-failing-detonator") mockFailingDetonator.On("SimulationId").Return("failing-simulation") mockFailingDetonator.On("PackName").Return("") @@ -113,59 +99,18 @@ func TestRunnerErrorHandling(t *testing.T) { mockMatcher.On("String").Return("sample") mockMatcher.On("MatcherName").Return("sample") mockMatcher.On("AlertName").Return("sample alert") - mockMatcher.On("Cleanup", []string{"my-uid"}, mock.AnythingOfType("*logrus.Entry")).Return(nil) - mockMatcher.On("HasExpectedAlert", []string{"my-uid"}, mock.AnythingOfType("*logrus.Entry")).Return(true, nil) - - runner := Runner{ - Scenarios: []*Scenario{ - { - Name: "test-scenario1", - Detonator: mockDetonator, - Assertions: []matchers.AlertGeneratedMatcher{mockMatcher}, - Timeout: 5 * time.Second, - }, - { - Name: "test-scenario2-error", - Detonator: mockFailingDetonator, - Assertions: []matchers.AlertGeneratedMatcher{mockMatcher}, - Timeout: 5 * time.Second, - }, - { - Name: "test-scenario3", - Detonator: mockDetonator, - Assertions: []matchers.AlertGeneratedMatcher{mockMatcher}, - Timeout: 5 * time.Second, - }, - }, - Interval: 0, - } - results, err := runner.Run() - assert.Error(t, err, "the runner should return an error when a scenario returns an error") - assert.Len(t, results, 3) // Should have results for all scenarios - - // Check that we have both success and failed scenarios - var successCount, failedCount int - for _, result := range results { - if result.Success { - successCount++ - } else { - failedCount++ - } - } - assert.Equal(t, 2, successCount) - assert.Equal(t, 1, failedCount) - - // A failed scenario must still carry its execution_id so the result can be - // correlated with the partial detonation (e.g. terraform that did apply). - for _, result := range results { - if !result.Success { - assert.Equal(t, "failed-uid", result.ExecutionId, - "failed scenario should retain the execution_id from the detonation output") - } - } - // All scenarios should have been detonated, even if one returned an error - mockDetonator.AssertNumberOfCalls(t, "Detonate", 2) + r := Runner{Interval: 0} + result := r.Run(&Scenario{ + Name: "test-scenario-error", + Detonator: mockFailingDetonator, + Matchers: []matchers.AlertGeneratedMatcher{mockMatcher}, + Timeout: 5 * time.Second, + }) + + assert.False(t, result.Success, "a scenario whose detonation errors must fail") + assert.Equal(t, "failed-uid", result.ExecutionId, + "failed scenario should retain the execution_id from the detonation output") mockFailingDetonator.AssertNumberOfCalls(t, "Detonate", 1) } @@ -189,21 +134,15 @@ func TestRunnerWithExecutionUUID(t *testing.T) { mockMatcher.On("AlertName").Return("sample alert") mockMatcher.On("Cleanup", expectedIndicators, mock.AnythingOfType("*logrus.Entry")).Return(nil) - runner := NewRunner() - runner.Scenarios = []*Scenario{ - { - Name: "test-scenario-with-uuid", - Detonator: mockDetonator, - Assertions: []matchers.AlertGeneratedMatcher{mockMatcher}, - Timeout: 5 * time.Second, - }, - } - runner.Interval = 0 - - results, err := runner.Run() - assert.NoError(t, err) - assert.Len(t, results, 1) - assert.True(t, results[0].Success) + r := NewRunner() + r.Interval = 0 + result := r.Run(&Scenario{ + Name: "test-scenario-with-uuid", + Detonator: mockDetonator, + Matchers: []matchers.AlertGeneratedMatcher{mockMatcher}, + Timeout: 5 * time.Second, + }) + assert.True(t, result.Success) // Verify both nanoid and UUID were passed to matcher mockMatcher.AssertCalled(t, "HasExpectedAlert", expectedIndicators, mock.AnythingOfType("*logrus.Entry")) @@ -235,7 +174,8 @@ func TestRunnerExploreModeFailsWhenNoAlertsDiscovered(t *testing.T) { mockDetonator.On("SetStatusCallback", mock.AnythingOfType("func(string)")).Return() mockDetonator.On("SetEnvVars", mock.AnythingOfType("map[string]string")).Return() - scenario := &Scenario{ + r := Runner{Interval: 1 * time.Millisecond} + result := r.Run(&Scenario{ Name: "explore-no-alerts", Detonator: mockDetonator, ExploreMode: true, @@ -244,19 +184,11 @@ func TestRunnerExploreModeFailsWhenNoAlertsDiscovered(t *testing.T) { "SR_KIBANA_URL": ts.URL, "SR_ELASTIC_API_KEY": "test", }, - } - - runner := Runner{ - Scenarios: []*Scenario{scenario}, - Interval: 1 * time.Millisecond, - } + }) - results, err := runner.Run() - assert.Error(t, err, "explore mode with zero discovered alerts should fail") - assert.Len(t, results, 1) - assert.False(t, results[0].Success) - assert.Contains(t, results[0].Error, "no matching alerts discovered") - assert.Empty(t, scenario.DiscoveredAlerts) + assert.False(t, result.Success, "explore mode with zero discovered alerts should fail") + assert.Contains(t, result.ErrorMessage, "no matching alerts discovered") + assert.Empty(t, result.DiscoveredAlerts) } func TestRunnerExploreModeSucceedsWhenAlertsDiscovered(t *testing.T) { @@ -278,7 +210,8 @@ func TestRunnerExploreModeSucceedsWhenAlertsDiscovered(t *testing.T) { mockDetonator.On("SetStatusCallback", mock.AnythingOfType("func(string)")).Return() mockDetonator.On("SetEnvVars", mock.AnythingOfType("map[string]string")).Return() - scenario := &Scenario{ + r := Runner{Interval: 1 * time.Millisecond} + result := r.Run(&Scenario{ Name: "explore-with-alerts", Detonator: mockDetonator, ExploreMode: true, @@ -287,26 +220,18 @@ func TestRunnerExploreModeSucceedsWhenAlertsDiscovered(t *testing.T) { "SR_KIBANA_URL": ts.URL, "SR_ELASTIC_API_KEY": "test", }, - } - - runner := Runner{ - Scenarios: []*Scenario{scenario}, - Interval: 1 * time.Millisecond, - } + }) - results, err := runner.Run() - assert.NoError(t, err) - assert.Len(t, results, 1) - assert.True(t, results[0].Success) - assert.GreaterOrEqual(t, len(scenario.DiscoveredAlerts), 1) - assert.Equal(t, "alert-1", scenario.DiscoveredAlerts[0].AlertID) + assert.True(t, result.Success) + assert.GreaterOrEqual(t, len(result.DiscoveredAlerts), 1) + assert.Equal(t, "alert-1", result.DiscoveredAlerts[0].AlertID) } -// TestRunnerFiresIdentityAndAssertionCallbacks verifies the mid-run hooks that +// TestRunnerFiresIdentityAndExpectationCallbacks verifies the mid-run hooks that // power the live scenario detail view: identity is emitted exactly once after -// detonation, and the assertions callback fires once per newly-matched -// assertion, carrying passed=true for matches and nil (pending) for the rest. -func TestRunnerFiresIdentityAndAssertionCallbacks(t *testing.T) { +// detonation, and the expectations callback fires once per newly-matched +// expectation, carrying passed=true for matches and nil (pending) for the rest. +func TestRunnerFiresIdentityAndExpectationCallbacks(t *testing.T) { mockDetonator := &detonatorMocks.MockDetonator{} mockDetonator.On("Detonate").Return(map[string]string{"execution_id": "exec-123"}, nil) mockDetonator.On("String").Return("mock-detonator") @@ -315,7 +240,7 @@ func TestRunnerFiresIdentityAndAssertionCallbacks(t *testing.T) { mockDetonator.On("SetStatusCallback", mock.AnythingOfType("func(string)")).Return() // matcherA matches on the first poll; matcherB only on the second, so the - // assertions callback must fire twice with distinct partial states. + // expectations callback must fire twice with distinct partial states. matcherA := &matcherMocks.MockAlertGeneratedMatcher{} matcherA.On("HasExpectedAlert", []string{"exec-123"}, mock.AnythingOfType("*logrus.Entry")).Return(true, nil) matcherA.On("MatcherName").Return("Elastic") @@ -332,28 +257,26 @@ func TestRunnerFiresIdentityAndAssertionCallbacks(t *testing.T) { matcherB.On("Cleanup", []string{"exec-123"}, mock.AnythingOfType("*logrus.Entry")).Return(nil) var identities []ScenarioIdentity - var assertionSnapshots [][]AssertionResult + var snapshots [][]ExpectationResult scenario := &Scenario{ - Name: "callback-scenario", - Detonator: mockDetonator, - Assertions: []matchers.AlertGeneratedMatcher{matcherA, matcherB}, - Timeout: 5 * time.Second, + Name: "callback-scenario", + Detonator: mockDetonator, + Matchers: []matchers.AlertGeneratedMatcher{matcherA, matcherB}, + Timeout: 5 * time.Second, IdentityCallback: func(_ string, id ScenarioIdentity) { identities = append(identities, id) }, - AssertionsCallback: func(_ string, results []AssertionResult) { - snap := make([]AssertionResult, len(results)) + ExpectationsCallback: func(_ string, results []ExpectationResult) { + snap := make([]ExpectationResult, len(results)) copy(snap, results) - assertionSnapshots = append(assertionSnapshots, snap) + snapshots = append(snapshots, snap) }, } - runner := Runner{Scenarios: []*Scenario{scenario}, Interval: 1 * time.Millisecond} - results, err := runner.Run() - assert.NoError(t, err) - assert.Len(t, results, 1) - assert.True(t, results[0].Success) + r := Runner{Interval: 1 * time.Millisecond} + result := r.Run(scenario) + assert.True(t, result.Success) // Identity fires exactly once, after detonation, carrying all four fields. assert.Len(t, identities, 1) @@ -364,21 +287,21 @@ func TestRunnerFiresIdentityAndAssertionCallbacks(t *testing.T) { SimulationID: "sim-456", }, identities[0]) - // One callback per newly-matched assertion: A then B → two snapshots. - assert.Len(t, assertionSnapshots, 2) + // One callback per newly-matched expectation: A then B → two snapshots. + assert.Len(t, snapshots, 2) - first := assertionSnapshots[0] - assertAssertionPassed(t, first, "alert-a", boolPtr(true)) - assertAssertionPassed(t, first, "alert-b", nil) // still pending + first := snapshots[0] + assertExpectationPassed(t, first, "alert-a", boolPtr(true)) + assertExpectationPassed(t, first, "alert-b", nil) // still pending - last := assertionSnapshots[len(assertionSnapshots)-1] - assertAssertionPassed(t, last, "alert-a", boolPtr(true)) - assertAssertionPassed(t, last, "alert-b", boolPtr(true)) + last := snapshots[len(snapshots)-1] + assertExpectationPassed(t, last, "alert-a", boolPtr(true)) + assertExpectationPassed(t, last, "alert-b", boolPtr(true)) } func boolPtr(b bool) *bool { return &b } -func assertAssertionPassed(t *testing.T, results []AssertionResult, alertName string, want *bool) { +func assertExpectationPassed(t *testing.T, results []ExpectationResult, alertName string, want *bool) { t.Helper() for _, r := range results { if r.AlertName != alertName { @@ -393,7 +316,7 @@ func assertAssertionPassed(t *testing.T, results []AssertionResult, alertName st } return } - t.Fatalf("assertion %q not found in snapshot", alertName) + t.Fatalf("expectation %q not found in snapshot", alertName) } func TestRunnerWithInjector(t *testing.T) { @@ -410,22 +333,16 @@ func TestRunnerWithInjector(t *testing.T) { mockMatcher.On("AlertName").Return("sample alert") mockMatcher.On("Cleanup", []string{"my-injection-uid"}, mock.AnythingOfType("*logrus.Entry")).Return(nil) - scenario := Scenario{ - Name: "test scenario with injector", - Injector: mockInjector, - Assertions: []matchers.AlertGeneratedMatcher{mockMatcher}, - Timeout: 1 * time.Second, - } - - runner := NewRunner() - runner.Scenarios = []*Scenario{&scenario} - runner.Interval = 10 * time.Millisecond - - results, err := runner.Run() - assert.NoError(t, err, "the runner should not return an error when the injection succeeds") - assert.Len(t, results, 1) - assert.True(t, results[0].Success) - assert.Equal(t, "my-injection-uid", results[0].ExecutionId) + r := NewRunner() + r.Interval = 10 * time.Millisecond + result := r.Run(&Scenario{ + Name: "test scenario with injector", + Injector: mockInjector, + Matchers: []matchers.AlertGeneratedMatcher{mockMatcher}, + Timeout: 1 * time.Second, + }) + assert.True(t, result.Success, "the runner should succeed when the injection succeeds") + assert.Equal(t, "my-injection-uid", result.ExecutionId) mockInjector.AssertNumberOfCalls(t, "Inject", 1) } diff --git a/internal/runner/scenario.go b/internal/runner/scenario.go index 52aaccd..f561ccb 100644 --- a/internal/runner/scenario.go +++ b/internal/runner/scenario.go @@ -17,27 +17,54 @@ type Scenario struct { Injector injectors.Injector Collector collectors.Collector Timeout time.Duration - Assertions []matchers.AlertGeneratedMatcher + Matchers []matchers.AlertGeneratedMatcher Indicators *Indicators Metadata *Metadata StatusCallback func(scenarioName, phase string) // IdentityCallback fires once after detonation, carrying executor identity. IdentityCallback func(scenarioName string, identity ScenarioIdentity) - // AssertionsCallback fires when an assertion newly matches, carrying the - // current pass/pending state of every assertion. - AssertionsCallback func(scenarioName string, results []AssertionResult) - ExploreMode bool // when true, discover all matching alerts instead of asserting specific rules - CleanupAlerts bool // when true in explore mode, close discovered alerts after run - - // Populated by runner after assertion matching completes - FailedAssertions []matchers.AlertGeneratedMatcher + // ExpectationsCallback fires when an expectation newly matches, carrying the + // current pass/pending state of every expectation. + ExpectationsCallback func(scenarioName string, results []ExpectationResult) + ExploreMode bool // when true, discover all matching alerts instead of matching specific rules + CleanupAlerts bool // when true in explore mode, close discovered alerts after run +} - // Populated by runner after explore mode completes - DiscoveredAlerts []DiscoveredAlert +// ScenarioResult is the single in-memory outcome of executing one scenario, +// returned by the runner and consumed by the parallel executor and the web +// layer. The runner populates everything except the wall-clock timing +// (TimeExecuted, DurationSeconds), which the executor records around the call. +// The persistence row (db.ScenarioResult) is a separate column-shaped DTO. +type ScenarioResult struct { + Name string `json:"name"` + Success bool `json:"isSuccess"` + ErrorMessage string `json:"errorMessage"` + DurationSeconds float64 `json:"durationSeconds"` + MatchingDurationSeconds float64 `json:"matchingDurationSeconds"` + TimeExecuted time.Time `json:"timeExecuted"` + ExecutorName string `json:"executorName"` + ExecutorType string `json:"executorType"` + ExecutionId string `json:"executionId"` + SimulationID string `json:"simulationId,omitempty"` + Matchers []matchers.AlertGeneratedMatcher `json:"expectations,omitempty"` + UnmetExpectations []matchers.AlertGeneratedMatcher `json:"-"` + Indicators *Indicators `json:"indicators,omitempty"` + Metadata *Metadata `json:"metadata,omitempty"` + CollectedLogPath string `json:"collectedLogPath,omitempty"` + CollectedDocCount int `json:"collectedDocCount,omitempty"` + DiscoveredAlerts []DiscoveredAlert `json:"discoveredAlerts,omitempty"` + ExploreMode bool `json:"exploreMode,omitempty"` +} - // Populated by runner after collection completes - CollectedLogPath string - CollectedDocCount int +// RunResult is the aggregate outcome of a whole run (one assessment execution). +type RunResult struct { + RunId string `json:"runId"` + StartTime time.Time `json:"startTime"` + EndTime time.Time `json:"endTime"` + TotalScenarios int `json:"totalScenarios"` + SuccessScenarios int `json:"successScenarios"` + FailedScenarios int `json:"failedScenarios"` + Scenarios []ScenarioResult `json:"scenarios"` } // ScenarioIdentity is the executor identity surfaced mid-run, after detonation. @@ -48,9 +75,9 @@ type ScenarioIdentity struct { SimulationID string } -// AssertionResult is the mid-run state of a single assertion. Passed is nil -// while the assertion is still pending (not yet matched). -type AssertionResult struct { +// ExpectationResult is the mid-run state of a single expectation. Passed is nil +// while the expectation is still pending (not yet matched). +type ExpectationResult struct { MatcherType string AlertName string Passed *bool diff --git a/internal/testutil/fakes/fakes.go b/internal/testutil/fakes/fakes.go index 1b6fb2a..23bfea8 100644 --- a/internal/testutil/fakes/fakes.go +++ b/internal/testutil/fakes/fakes.go @@ -20,6 +20,7 @@ import ( "github.com/IBM/simrun/internal/db" "github.com/google/uuid" "github.com/jackc/pgx/v5" + "github.com/jackc/pgx/v5/pgconn" ) // Stores bundles every fake store. Use New() to construct a wired-up bundle. @@ -38,7 +39,7 @@ type Stores struct { func New() *Stores { return &Stores{ Run: &RunStore{runs: map[uuid.UUID]*db.Run{}, results: map[uuid.UUID]*db.ScenarioResult{}}, - Scenario: &ScenarioStore{scenarios: map[uuid.UUID]*db.SavedScenario{}}, + Scenario: &ScenarioStore{scenarios: map[uuid.UUID]*db.Assessment{}}, Pack: &PackStore{packs: map[string]*db.Pack{}}, Secret: &SecretStore{secrets: map[uuid.UUID]*db.SecretGroup{}}, Connector: &ConnectorStore{connectors: map[uuid.UUID]*db.Connector{}}, @@ -96,20 +97,20 @@ func (s *RunStore) List(_ context.Context, filters db.ListRunsFilters, limit, of func matchesRunFilters(r *db.Run, f db.ListRunsFilters) bool { if f.Name != "" { - if r.ScenarioName == nil || !strings.Contains(strings.ToLower(*r.ScenarioName), strings.ToLower(f.Name)) { + if r.AssessmentName == nil || !strings.Contains(strings.ToLower(*r.AssessmentName), strings.ToLower(f.Name)) { return false } } if len(f.Types) > 0 { - if r.ScenarioType == nil || !slices.Contains(f.Types, *r.ScenarioType) { + if r.AssessmentType == nil || !slices.Contains(f.Types, *r.AssessmentType) { return false } } if f.Since != nil && r.CreatedAt.Before(*f.Since) { return false } - if f.ScenarioID != nil { - if r.ScenarioID == nil || *r.ScenarioID != *f.ScenarioID { + if f.AssessmentID != nil { + if r.AssessmentID == nil || *r.AssessmentID != *f.AssessmentID { return false } } @@ -242,14 +243,14 @@ func (s *RunStore) UpdateScenarioIdentity(_ context.Context, id uuid.UUID, execu return nil } -func (s *RunStore) UpdateScenarioAssertions(_ context.Context, id uuid.UUID, assertionsJSON []byte) error { +func (s *RunStore) UpdateScenarioExpectations(_ context.Context, id uuid.UUID, expectationsJSON []byte) error { s.mu.Lock() defer s.mu.Unlock() r, ok := s.results[id] if !ok { return pgx.ErrNoRows } - r.Assertions = assertionsJSON + r.Expectations = expectationsJSON return nil } @@ -277,8 +278,8 @@ func (s *RunStore) IncrementRunCounters(_ context.Context, id uuid.UUID, success return nil } -func (s *RunStore) GetLatestAssertionResults(_ context.Context) ([]db.LatestAssertionResult, error) { - return []db.LatestAssertionResult{}, nil +func (s *RunStore) GetLatestExpectationResults(_ context.Context) ([]db.LatestExpectationResult, error) { + return []db.LatestExpectationResult{}, nil } // All returns a snapshot of all runs (test-only convenience). @@ -296,16 +297,23 @@ func (s *RunStore) All() []db.Run { type ScenarioStore struct { mu sync.Mutex - scenarios map[uuid.UUID]*db.SavedScenario + scenarios map[uuid.UUID]*db.Assessment } -var _ db.ScenarioStore = (*ScenarioStore)(nil) +var _ db.AssessmentStore = (*ScenarioStore)(nil) -func (s *ScenarioStore) Save(_ context.Context, name, scenarioType, yaml, createdBy string) (*db.SavedScenario, error) { +func (s *ScenarioStore) Save(_ context.Context, name, scenarioType, yaml, createdBy string) (*db.Assessment, error) { s.mu.Lock() defer s.mu.Unlock() + // Mirror the production UNIQUE(name) constraint so handlers can map the + // SQLSTATE 23505 violation to a 409. + for _, sc := range s.scenarios { + if sc.Name == name { + return nil, &pgconn.PgError{Code: "23505", ConstraintName: "assessments_name_key"} + } + } now := time.Now() - sc := &db.SavedScenario{ + sc := &db.Assessment{ ID: uuid.New(), Name: name, Type: scenarioType, @@ -320,7 +328,7 @@ func (s *ScenarioStore) Save(_ context.Context, name, scenarioType, yaml, create return &cp, nil } -func (s *ScenarioStore) Get(_ context.Context, id uuid.UUID) (*db.SavedScenario, error) { +func (s *ScenarioStore) Get(_ context.Context, id uuid.UUID) (*db.Assessment, error) { s.mu.Lock() defer s.mu.Unlock() sc, ok := s.scenarios[id] @@ -331,10 +339,22 @@ func (s *ScenarioStore) Get(_ context.Context, id uuid.UUID) (*db.SavedScenario, return &cp, nil } -func (s *ScenarioStore) List(_ context.Context, filters db.ListScenariosFilters, limit, offset int) (db.ScenarioPage, error) { +func (s *ScenarioStore) GetByName(_ context.Context, name string) (*db.Assessment, error) { + s.mu.Lock() + defer s.mu.Unlock() + for _, sc := range s.scenarios { + if sc.Name == name { + cp := *sc + return &cp, nil + } + } + return nil, pgx.ErrNoRows +} + +func (s *ScenarioStore) List(_ context.Context, filters db.ListAssessmentsFilters, limit, offset int) (db.AssessmentPage, error) { s.mu.Lock() defer s.mu.Unlock() - all := make([]db.SavedScenario, 0, len(s.scenarios)) + all := make([]db.Assessment, 0, len(s.scenarios)) for _, sc := range s.scenarios { if !scenarioMatchesFilters(*sc, filters) { continue @@ -344,17 +364,17 @@ func (s *ScenarioStore) List(_ context.Context, filters db.ListScenariosFilters, sort.Slice(all, func(i, j int) bool { return all[i].UpdatedAt.After(all[j].UpdatedAt) }) total := len(all) if offset >= total { - return db.ScenarioPage{Scenarios: []db.SavedScenario{}, Total: total}, nil + return db.AssessmentPage{Assessments: []db.Assessment{}, Total: total}, nil } end := min(offset+limit, total) - page := append([]db.SavedScenario(nil), all[offset:end]...) - return db.ScenarioPage{Scenarios: page, Total: total}, nil + page := append([]db.Assessment(nil), all[offset:end]...) + return db.AssessmentPage{Assessments: page, Total: total}, nil } -func (s *ScenarioStore) ListAll(_ context.Context) ([]db.SavedScenario, error) { +func (s *ScenarioStore) ListAll(_ context.Context) ([]db.Assessment, error) { s.mu.Lock() defer s.mu.Unlock() - out := make([]db.SavedScenario, 0, len(s.scenarios)) + out := make([]db.Assessment, 0, len(s.scenarios)) for _, sc := range s.scenarios { out = append(out, *sc) } @@ -362,7 +382,7 @@ func (s *ScenarioStore) ListAll(_ context.Context) ([]db.SavedScenario, error) { return out, nil } -func scenarioMatchesFilters(sc db.SavedScenario, f db.ListScenariosFilters) bool { +func scenarioMatchesFilters(sc db.Assessment, f db.ListAssessmentsFilters) bool { if f.Name != "" && !strings.Contains(strings.ToLower(sc.Name), strings.ToLower(f.Name)) { return false } @@ -740,28 +760,28 @@ func (s *ConfigStore) GetAppConfig(_ context.Context) (config.AppConfig, error) out.SSHLoggingEnabled = v } } - if raw, ok := s.data["assessment_log_retention_enabled"]; ok { + if raw, ok := s.data["run_log_retention_enabled"]; ok { var v bool if err := json.Unmarshal(raw, &v); err == nil { - out.AssessmentLogRetentionEnabled = v + out.RunLogRetentionEnabled = v } } - if raw, ok := s.data["assessment_log_retention_days"]; ok { + if raw, ok := s.data["run_log_retention_days"]; ok { var v int if err := json.Unmarshal(raw, &v); err == nil && v > 0 { - out.AssessmentLogRetentionDays = v + out.RunLogRetentionDays = v } } - if raw, ok := s.data["assessment_retention_enabled"]; ok { + if raw, ok := s.data["run_retention_enabled"]; ok { var v bool if err := json.Unmarshal(raw, &v); err == nil { - out.AssessmentRetentionEnabled = v + out.RunRetentionEnabled = v } } - if raw, ok := s.data["assessment_retention_days"]; ok { + if raw, ok := s.data["run_retention_days"]; ok { var v int if err := json.Unmarshal(raw, &v); err == nil && v > 0 { - out.AssessmentRetentionDays = v + out.RunRetentionDays = v } } return out, nil @@ -771,14 +791,14 @@ func (s *ConfigStore) UpdateAppConfig(_ context.Context, c config.AppConfig) err s.mu.Lock() defer s.mu.Unlock() for k, v := range map[string]any{ - "parallelism": c.Parallelism, - "terraform_version": c.TerraformVersion, - "pack_logs_enabled": c.PackLogsEnabled, - "ssh_logging_enabled": c.SSHLoggingEnabled, - "assessment_log_retention_enabled": c.AssessmentLogRetentionEnabled, - "assessment_log_retention_days": c.AssessmentLogRetentionDays, - "assessment_retention_enabled": c.AssessmentRetentionEnabled, - "assessment_retention_days": c.AssessmentRetentionDays, + "parallelism": c.Parallelism, + "terraform_version": c.TerraformVersion, + "pack_logs_enabled": c.PackLogsEnabled, + "ssh_logging_enabled": c.SSHLoggingEnabled, + "run_log_retention_enabled": c.RunLogRetentionEnabled, + "run_log_retention_days": c.RunLogRetentionDays, + "run_retention_enabled": c.RunRetentionEnabled, + "run_retention_days": c.RunRetentionDays, } { raw, err := json.Marshal(v) if err != nil { @@ -798,13 +818,13 @@ type ScheduleStore struct { var _ db.ScheduleStore = (*ScheduleStore)(nil) -func (s *ScheduleStore) Create(_ context.Context, scenarioID uuid.UUID, cronExpr string, enabled bool, parallelism int, createdBy string) (*db.Schedule, error) { +func (s *ScheduleStore) Create(_ context.Context, assessmentID uuid.UUID, cronExpr string, enabled bool, parallelism int, createdBy string) (*db.Schedule, error) { s.mu.Lock() defer s.mu.Unlock() now := time.Now() sched := &db.Schedule{ ID: uuid.New(), - ScenarioID: scenarioID, + AssessmentID: assessmentID, CronExpression: cronExpr, Enabled: enabled, Parallelism: parallelism, @@ -829,11 +849,11 @@ func (s *ScheduleStore) Get(_ context.Context, id uuid.UUID) (*db.Schedule, erro return &cp, nil } -func (s *ScheduleStore) GetByScenarioID(_ context.Context, scenarioID uuid.UUID) (*db.Schedule, error) { +func (s *ScheduleStore) GetByAssessmentID(_ context.Context, assessmentID uuid.UUID) (*db.Schedule, error) { s.mu.Lock() defer s.mu.Unlock() for _, sched := range s.schedules { - if sched.ScenarioID == scenarioID { + if sched.AssessmentID == assessmentID { cp := *sched return &cp, nil } diff --git a/internal/testutil/fakes/fakes_test.go b/internal/testutil/fakes/fakes_test.go index 95509c2..935201c 100644 --- a/internal/testutil/fakes/fakes_test.go +++ b/internal/testutil/fakes/fakes_test.go @@ -32,7 +32,7 @@ func TestRunStore_ListExpired(t *testing.T) { assert.NotContains(t, ids, oldRunning, "running run must be skipped even when old") } -// UpdateScenarioIdentity and UpdateScenarioAssertions are the mid-run partial +// UpdateScenarioIdentity and UpdateScenarioExpectations are the mid-run partial // writes behind the live scenario detail view: each must touch only its own // columns and leave the lifecycle status/phase untouched. (Production SQL is // Postgres-bound and verified manually per task 5.3; this pins the contract the @@ -57,18 +57,18 @@ func TestRunStore_PartialScenarioUpdates(t *testing.T) { assert.Equal(t, "detonator", got.ExecutorType) assert.Equal(t, "exec-9", got.ExecutionID) assert.Equal(t, "sim-9", got.SimulationID) - assert.Nil(t, got.Assertions, "identity write must not populate assertions") + assert.Nil(t, got.Expectations, "identity write must not populate assertions") // Assertions write: only the assertions column changes; everything else stays. partial := []byte(`[{"matcherType":"Elastic","alertName":"a","passed":true},{"matcherType":"Elastic","alertName":"b"}]`) - require.NoError(t, s.UpdateScenarioAssertions(ctx, id, partial)) + require.NoError(t, s.UpdateScenarioExpectations(ctx, id, partial)) got, err = s.GetScenarioResult(ctx, id) require.NoError(t, err) assert.Equal(t, "running", got.Status, "assertions write must not change status") require.NotNil(t, got.Phase) assert.Equal(t, "matching", *got.Phase, "assertions write must not change phase") assert.Equal(t, "exec-9", got.ExecutionID, "assertions write must not touch identity") - assert.JSONEq(t, string(partial), string(got.Assertions)) + assert.JSONEq(t, string(partial), string(got.Expectations)) } func mustCreateRun(t *testing.T, ctx context.Context, s *RunStore, status string, createdAt time.Time) uuid.UUID { diff --git a/internal/web/api_config_test.go b/internal/web/api_config_test.go index 4104af9..c2ffb92 100644 --- a/internal/web/api_config_test.go +++ b/internal/web/api_config_test.go @@ -13,7 +13,7 @@ import ( // Retention day fields are floored at 1 so a config write cannot configure // immediate deletion of run logs or assessments. func TestHandleUpdateConfig_RetentionDaysRejectsZero(t *testing.T) { - for _, key := range []string{"assessment_log_retention_days", "assessment_retention_days"} { + for _, key := range []string{"run_log_retention_days", "run_retention_days"} { t.Run(key, func(t *testing.T) { ts := testserver.New(t) @@ -39,7 +39,7 @@ func TestHandleUpdateConfig_RetentionDaysPersistsValid(t *testing.T) { ts := testserver.New(t) resp := ts.Put(t, "/api/config", web.UpdateConfigRequest{ - Key: "assessment_retention_days", + Key: "run_retention_days", Value: []byte(`14`), }) defer resp.Body.Close() @@ -47,5 +47,5 @@ func TestHandleUpdateConfig_RetentionDaysPersistsValid(t *testing.T) { cfg, err := ts.Stores.Config.GetAppConfig(t.Context()) require.NoError(t, err) - assert.Equal(t, 14, cfg.AssessmentRetentionDays) + assert.Equal(t, 14, cfg.RunRetentionDays) } diff --git a/internal/web/api_runs_test.go b/internal/web/api_runs_test.go index 1f9b07c..03c2ecb 100644 --- a/internal/web/api_runs_test.go +++ b/internal/web/api_runs_test.go @@ -130,12 +130,12 @@ func TestHandleListRuns_Filters(t *testing.T) { n, tp := name, typ now := time.Now().Add(-age) require.NoError(t, ts.Stores.Run.Create(ctx, &db.Run{ - ID: id, - Status: "completed", - StartTime: now, - CreatedAt: now, - ScenarioName: &n, - ScenarioType: &tp, + ID: id, + Status: "completed", + StartTime: now, + CreatedAt: now, + AssessmentName: &n, + AssessmentType: &tp, })) return id } @@ -211,12 +211,12 @@ func TestHandleListRuns_FiltersExcludeUnlinkedRuns(t *testing.T) { name, typ := "linked-scenario", "standard" linkedID := uuid.New() require.NoError(t, ts.Stores.Run.Create(ctx, &db.Run{ - ID: linkedID, - Status: "completed", - StartTime: time.Now(), - CreatedAt: time.Now(), - ScenarioName: &name, - ScenarioType: &typ, + ID: linkedID, + Status: "completed", + StartTime: time.Now(), + CreatedAt: time.Now(), + AssessmentName: &name, + AssessmentType: &typ, })) // Unfiltered: both visible. @@ -263,8 +263,8 @@ func TestHandleListRuns_FilterValidation(t *testing.T) { defer resp3.Body.Close() assert.Equal(t, http.StatusBadRequest, resp3.StatusCode) - // Invalid scenario_id rejected. - resp4 := ts.Get(t, "/api/runs?scenario_id=not-a-uuid") + // Invalid assessment_id rejected. + resp4 := ts.Get(t, "/api/runs?assessment_id=not-a-uuid") defer resp4.Body.Close() assert.Equal(t, http.StatusBadRequest, resp4.StatusCode) } @@ -278,19 +278,19 @@ func TestHandleListRuns_FilterByScenarioID(t *testing.T) { want := uuid.New() require.NoError(t, ts.Stores.Run.Create(ctx, &db.Run{ - ID: want, - Status: "completed", - StartTime: time.Now(), - CreatedAt: time.Now(), - ScenarioID: &scenarioA, + ID: want, + Status: "completed", + StartTime: time.Now(), + CreatedAt: time.Now(), + AssessmentID: &scenarioA, })) // Other scenario — must be filtered out. require.NoError(t, ts.Stores.Run.Create(ctx, &db.Run{ - ID: uuid.New(), - Status: "completed", - StartTime: time.Now(), - CreatedAt: time.Now(), - ScenarioID: &scenarioB, + ID: uuid.New(), + Status: "completed", + StartTime: time.Now(), + CreatedAt: time.Now(), + AssessmentID: &scenarioB, })) // Ad-hoc run with no scenario — must also be filtered out. require.NoError(t, ts.Stores.Run.Create(ctx, &db.Run{ @@ -300,7 +300,7 @@ func TestHandleListRuns_FilterByScenarioID(t *testing.T) { CreatedAt: time.Now(), })) - resp := ts.Get(t, "/api/runs?scenario_id="+scenarioA.String()) + resp := ts.Get(t, "/api/runs?assessment_id="+scenarioA.String()) defer resp.Body.Close() require.Equal(t, http.StatusOK, resp.StatusCode) diff --git a/internal/web/api_scenarios_test.go b/internal/web/api_scenarios_test.go index 382b731..d1805de 100644 --- a/internal/web/api_scenarios_test.go +++ b/internal/web/api_scenarios_test.go @@ -14,10 +14,10 @@ import ( ) type scenarioListResponse struct { - Scenarios []db.SavedScenario `json:"scenarios"` - Total int `json:"total"` - Page int `json:"page"` - PerPage int `json:"perPage"` + Assessments []db.Assessment `json:"assessments"` + Total int `json:"total"` + Page int `json:"page"` + PerPage int `json:"perPage"` } const sampleYAML = `scenarios: @@ -34,7 +34,7 @@ const sampleYAML = `scenarios: func TestHandleLint_ValidYAML(t *testing.T) { ts := testserver.New(t) - resp := ts.Post(t, "/api/scenarios/lint", web.LintRequest{YAML: sampleYAML}) + resp := ts.Post(t, "/api/assessments/lint", web.LintRequest{YAML: sampleYAML}) defer resp.Body.Close() require.Equal(t, http.StatusOK, resp.StatusCode) @@ -50,7 +50,7 @@ func TestHandleLint_ValidYAML(t *testing.T) { func TestHandleLint_InvalidYAML(t *testing.T) { ts := testserver.New(t) - resp := ts.Post(t, "/api/scenarios/lint", web.LintRequest{YAML: "scenarios: [oops"}) + resp := ts.Post(t, "/api/assessments/lint", web.LintRequest{YAML: "scenarios: [oops"}) defer resp.Body.Close() require.Equal(t, http.StatusOK, resp.StatusCode) @@ -64,53 +64,53 @@ func TestScenarioCRUD(t *testing.T) { ts := testserver.New(t) // Create - resp := ts.Post(t, "/api/scenarios", web.SaveScenarioRequest{ + resp := ts.Post(t, "/api/assessments", web.SaveAssessmentRequest{ Name: "my scenario", YAML: sampleYAML, }) require.Equal(t, http.StatusCreated, resp.StatusCode) - var saved db.SavedScenario + var saved db.Assessment testserver.DecodeJSON(t, resp, &saved) assert.Equal(t, "my scenario", saved.Name) assert.Equal(t, web.ScenarioTypeStandard, saved.Type, "should default to standard") id := saved.ID // List - resp = ts.Get(t, "/api/scenarios") + resp = ts.Get(t, "/api/assessments") require.Equal(t, http.StatusOK, resp.StatusCode) var list scenarioListResponse testserver.DecodeJSON(t, resp, &list) - assert.Len(t, list.Scenarios, 1) + assert.Len(t, list.Assessments, 1) assert.Equal(t, 1, list.Total) assert.Equal(t, 1, list.Page) assert.Equal(t, 50, list.PerPage) // Get - resp = ts.Get(t, "/api/scenarios/"+id.String()) + resp = ts.Get(t, "/api/assessments/"+id.String()) require.Equal(t, http.StatusOK, resp.StatusCode) - var got db.SavedScenario + var got db.Assessment testserver.DecodeJSON(t, resp, &got) assert.Equal(t, "my scenario", got.Name) // Update - resp = ts.Put(t, "/api/scenarios/"+id.String(), web.SaveScenarioRequest{ + resp = ts.Put(t, "/api/assessments/"+id.String(), web.SaveAssessmentRequest{ Name: "renamed", YAML: sampleYAML, }) require.Equal(t, http.StatusNoContent, resp.StatusCode) resp.Body.Close() - resp = ts.Get(t, "/api/scenarios/"+id.String()) + resp = ts.Get(t, "/api/assessments/"+id.String()) require.Equal(t, http.StatusOK, resp.StatusCode) testserver.DecodeJSON(t, resp, &got) assert.Equal(t, "renamed", got.Name) // Delete - resp = ts.Delete(t, "/api/scenarios/"+id.String()) + resp = ts.Delete(t, "/api/assessments/"+id.String()) require.Equal(t, http.StatusNoContent, resp.StatusCode) resp.Body.Close() - resp = ts.Get(t, "/api/scenarios/"+id.String()) + resp = ts.Get(t, "/api/assessments/"+id.String()) assert.Equal(t, http.StatusNotFound, resp.StatusCode) resp.Body.Close() } @@ -118,7 +118,7 @@ func TestScenarioCRUD(t *testing.T) { func TestHandleSaveScenario_RejectsInvalidType(t *testing.T) { ts := testserver.New(t) - resp := ts.Post(t, "/api/scenarios", web.SaveScenarioRequest{ + resp := ts.Post(t, "/api/assessments", web.SaveAssessmentRequest{ Name: "x", Type: "garbage", YAML: sampleYAML, @@ -131,7 +131,7 @@ func TestHandleSaveScenario_RejectsInvalidType(t *testing.T) { func TestHandleGetScenario_BadID(t *testing.T) { ts := testserver.New(t) - resp := ts.Get(t, "/api/scenarios/not-a-uuid") + resp := ts.Get(t, "/api/assessments/not-a-uuid") defer resp.Body.Close() assert.Equal(t, http.StatusBadRequest, resp.StatusCode) } @@ -139,7 +139,7 @@ func TestHandleGetScenario_BadID(t *testing.T) { func TestHandleGetScenario_NotFound(t *testing.T) { ts := testserver.New(t) - resp := ts.Get(t, "/api/scenarios/"+uuid.New().String()) + resp := ts.Get(t, "/api/assessments/"+uuid.New().String()) defer resp.Body.Close() assert.Equal(t, http.StatusNotFound, resp.StatusCode) } @@ -148,17 +148,17 @@ func TestHandleRun_StartsRun(t *testing.T) { ts := testserver.New(t) // Save a scenario first. - resp := ts.Post(t, "/api/scenarios", web.SaveScenarioRequest{ + resp := ts.Post(t, "/api/assessments", web.SaveAssessmentRequest{ Name: "to run", YAML: sampleYAML, }) require.Equal(t, http.StatusCreated, resp.StatusCode) - var saved db.SavedScenario + var saved db.Assessment testserver.DecodeJSON(t, resp, &saved) // Run it. - resp = ts.Post(t, "/api/scenarios/run", web.RunRequest{ - ScenarioID: saved.ID.String(), + resp = ts.Post(t, "/api/runs", web.RunRequest{ + AssessmentID: saved.ID.String(), }) defer resp.Body.Close() require.Equal(t, http.StatusAccepted, resp.StatusCode) @@ -183,7 +183,7 @@ func TestHandleRun_StartsRun(t *testing.T) { func TestHandleRun_BadScenarioID(t *testing.T) { ts := testserver.New(t) - resp := ts.Post(t, "/api/scenarios/run", web.RunRequest{ScenarioID: "not-a-uuid"}) + resp := ts.Post(t, "/api/runs", web.RunRequest{AssessmentID: "not-a-uuid"}) defer resp.Body.Close() assert.Equal(t, http.StatusBadRequest, resp.StatusCode) } @@ -191,22 +191,22 @@ func TestHandleRun_BadScenarioID(t *testing.T) { func TestHandleRun_MissingScenario(t *testing.T) { ts := testserver.New(t) - resp := ts.Post(t, "/api/scenarios/run", web.RunRequest{ScenarioID: uuid.New().String()}) + resp := ts.Post(t, "/api/runs", web.RunRequest{AssessmentID: uuid.New().String()}) defer resp.Body.Close() assert.Equal(t, http.StatusBadRequest, resp.StatusCode) - assert.Contains(t, testserver.ReadBody(t, resp), "scenario not found") + assert.Contains(t, testserver.ReadBody(t, resp), "assessment not found") } func TestHandleListScenarios_Empty(t *testing.T) { ts := testserver.New(t) - resp := ts.Get(t, "/api/scenarios") + resp := ts.Get(t, "/api/assessments") defer resp.Body.Close() require.Equal(t, http.StatusOK, resp.StatusCode) var got scenarioListResponse testserver.DecodeJSON(t, resp, &got) - assert.Empty(t, got.Scenarios) + assert.Empty(t, got.Assessments) assert.Equal(t, 0, got.Total) assert.Equal(t, 1, got.Page) assert.Equal(t, 50, got.PerPage) @@ -223,39 +223,39 @@ func TestHandleListScenarios_Pagination(t *testing.T) { } // Page 1, per_page=2 — newest two. - resp := ts.Get(t, "/api/scenarios?page=1&per_page=2") + resp := ts.Get(t, "/api/assessments?page=1&per_page=2") defer resp.Body.Close() var got scenarioListResponse testserver.DecodeJSON(t, resp, &got) - require.Len(t, got.Scenarios, 2) + require.Len(t, got.Assessments, 2) assert.Equal(t, 5, got.Total) assert.Equal(t, 1, got.Page) assert.Equal(t, 2, got.PerPage) - assert.Equal(t, "scen-e", got.Scenarios[0].Name) - assert.Equal(t, "scen-d", got.Scenarios[1].Name) + assert.Equal(t, "scen-e", got.Assessments[0].Name) + assert.Equal(t, "scen-d", got.Assessments[1].Name) // Page 3, per_page=2 — only "scen-a" left. - resp2 := ts.Get(t, "/api/scenarios?page=3&per_page=2") + resp2 := ts.Get(t, "/api/assessments?page=3&per_page=2") defer resp2.Body.Close() var got2 scenarioListResponse testserver.DecodeJSON(t, resp2, &got2) - require.Len(t, got2.Scenarios, 1) + require.Len(t, got2.Assessments, 1) assert.Equal(t, 5, got2.Total) - assert.Equal(t, "scen-a", got2.Scenarios[0].Name) + assert.Equal(t, "scen-a", got2.Assessments[0].Name) // Page beyond range — empty slice, total still reported. - resp3 := ts.Get(t, "/api/scenarios?page=10&per_page=2") + resp3 := ts.Get(t, "/api/assessments?page=10&per_page=2") defer resp3.Body.Close() var got3 scenarioListResponse testserver.DecodeJSON(t, resp3, &got3) - assert.Empty(t, got3.Scenarios) + assert.Empty(t, got3.Assessments) assert.Equal(t, 5, got3.Total) } func TestHandleListScenarios_PerPageClamped(t *testing.T) { ts := testserver.New(t) - resp := ts.Get(t, "/api/scenarios?per_page=500") + resp := ts.Get(t, "/api/assessments?per_page=500") defer resp.Body.Close() var got scenarioListResponse testserver.DecodeJSON(t, resp, &got) @@ -281,16 +281,16 @@ func TestHandleListScenarios_Filters(t *testing.T) { ts.Stores.Scenario.SetUpdatedAt(old.ID, time.Now().Add(-72*time.Hour)) t.Run("name ILIKE", func(t *testing.T) { - resp := ts.Get(t, "/api/scenarios?name=brute") + resp := ts.Get(t, "/api/assessments?name=brute") defer resp.Body.Close() var got scenarioListResponse testserver.DecodeJSON(t, resp, &got) - require.Len(t, got.Scenarios, 1) - assert.Equal(t, "login bruteforce", got.Scenarios[0].Name) + require.Len(t, got.Assessments, 1) + assert.Equal(t, "login bruteforce", got.Assessments[0].Name) }) t.Run("multi type", func(t *testing.T) { - resp := ts.Get(t, "/api/scenarios?type=standard&type=explore") + resp := ts.Get(t, "/api/assessments?type=standard&type=explore") defer resp.Body.Close() var got scenarioListResponse testserver.DecodeJSON(t, resp, &got) @@ -298,7 +298,7 @@ func TestHandleListScenarios_Filters(t *testing.T) { }) t.Run("since window", func(t *testing.T) { - resp := ts.Get(t, "/api/scenarios?since=24h") + resp := ts.Get(t, "/api/assessments?since=24h") defer resp.Body.Close() var got scenarioListResponse testserver.DecodeJSON(t, resp, &got) @@ -306,27 +306,59 @@ func TestHandleListScenarios_Filters(t *testing.T) { }) t.Run("combined filters", func(t *testing.T) { - resp := ts.Get(t, "/api/scenarios?name=ransom&type=explore&since=24h") + resp := ts.Get(t, "/api/assessments?name=ransom&type=explore&since=24h") defer resp.Body.Close() var got scenarioListResponse testserver.DecodeJSON(t, resp, &got) - require.Len(t, got.Scenarios, 1) - assert.Equal(t, "exfil ransom", got.Scenarios[0].Name) + require.Len(t, got.Assessments, 1) + assert.Equal(t, "exfil ransom", got.Assessments[0].Name) }) } +func TestHandleGetAssessmentByName(t *testing.T) { + ts := testserver.New(t) + + resp := ts.Post(t, "/api/assessments", web.SaveAssessmentRequest{ + Name: "aws-privesc", + YAML: sampleYAML, + }) + require.Equal(t, http.StatusCreated, resp.StatusCode) + resp.Body.Close() + + resp = ts.Get(t, "/api/assessments/by-name/aws-privesc") + defer resp.Body.Close() + require.Equal(t, http.StatusOK, resp.StatusCode) + var got db.Assessment + testserver.DecodeJSON(t, resp, &got) + assert.Equal(t, "aws-privesc", got.Name) + assert.Equal(t, sampleYAML, got.YAML, "by-name response includes the raw yaml field") +} + +func TestHandleSaveAssessment_DuplicateNameReturns409(t *testing.T) { + ts := testserver.New(t) + + first := ts.Post(t, "/api/assessments", web.SaveAssessmentRequest{Name: "dupe", YAML: sampleYAML}) + require.Equal(t, http.StatusCreated, first.StatusCode) + first.Body.Close() + + second := ts.Post(t, "/api/assessments", web.SaveAssessmentRequest{Name: "dupe", YAML: sampleYAML}) + defer second.Body.Close() + assert.Equal(t, http.StatusConflict, second.StatusCode) + assert.Contains(t, testserver.ReadBody(t, second), "already exists") +} + func TestHandleListScenarios_RejectsBadFilters(t *testing.T) { ts := testserver.New(t) - resp := ts.Get(t, "/api/scenarios?type=bogus") + resp := ts.Get(t, "/api/assessments?type=bogus") defer resp.Body.Close() assert.Equal(t, http.StatusBadRequest, resp.StatusCode) - resp2 := ts.Get(t, "/api/scenarios?since=not-a-duration") + resp2 := ts.Get(t, "/api/assessments?since=not-a-duration") defer resp2.Body.Close() assert.Equal(t, http.StatusBadRequest, resp2.StatusCode) - resp3 := ts.Get(t, "/api/scenarios?since=0s") + resp3 := ts.Get(t, "/api/assessments?since=0s") defer resp3.Body.Close() assert.Equal(t, http.StatusBadRequest, resp3.StatusCode) } diff --git a/internal/web/api_schedules_test.go b/internal/web/api_schedules_test.go index d5f9d82..d6e13cd 100644 --- a/internal/web/api_schedules_test.go +++ b/internal/web/api_schedules_test.go @@ -12,15 +12,15 @@ import ( "github.com/stretchr/testify/require" ) -func saveScenario(t *testing.T, ts *testserver.TS) db.SavedScenario { +func saveScenario(t *testing.T, ts *testserver.TS) db.Assessment { t.Helper() - resp := ts.Post(t, "/api/scenarios", web.SaveScenarioRequest{ + resp := ts.Post(t, "/api/assessments", web.SaveAssessmentRequest{ Name: "scheduled scenario", YAML: sampleYAML, }) defer resp.Body.Close() require.Equal(t, http.StatusCreated, resp.StatusCode) - var saved db.SavedScenario + var saved db.Assessment testserver.DecodeJSON(t, resp, &saved) return saved } @@ -31,7 +31,7 @@ func TestScheduleCRUD(t *testing.T) { // Create resp := ts.Post(t, "/api/schedules", web.CreateScheduleRequest{ - ScenarioID: scenario.ID.String(), + AssessmentID: scenario.ID.String(), CronExpression: "0 * * * *", Enabled: true, Parallelism: 5, @@ -39,7 +39,7 @@ func TestScheduleCRUD(t *testing.T) { require.Equal(t, http.StatusCreated, resp.StatusCode) var sched db.Schedule testserver.DecodeJSON(t, resp, &sched) - assert.Equal(t, scenario.ID, sched.ScenarioID) + assert.Equal(t, scenario.ID, sched.AssessmentID) assert.Equal(t, "0 * * * *", sched.CronExpression) assert.Equal(t, 5, sched.Parallelism) @@ -55,7 +55,7 @@ func TestScheduleCRUD(t *testing.T) { require.Equal(t, http.StatusOK, resp.StatusCode) // GetByScenario - resp = ts.Get(t, "/api/scenarios/"+scenario.ID.String()+"/schedule") + resp = ts.Get(t, "/api/assessments/"+scenario.ID.String()+"/schedule") require.Equal(t, http.StatusOK, resp.StatusCode) // Update @@ -86,7 +86,7 @@ func TestHandleCreateSchedule_RejectsInvalidCron(t *testing.T) { scenario := saveScenario(t, ts) resp := ts.Post(t, "/api/schedules", web.CreateScheduleRequest{ - ScenarioID: scenario.ID.String(), + AssessmentID: scenario.ID.String(), CronExpression: "garbage", }) defer resp.Body.Close() @@ -98,7 +98,7 @@ func TestHandleCreateSchedule_MissingScenario(t *testing.T) { ts := testserver.New(t) resp := ts.Post(t, "/api/schedules", web.CreateScheduleRequest{ - ScenarioID: uuid.New().String(), + AssessmentID: uuid.New().String(), CronExpression: "0 * * * *", }) defer resp.Body.Close() diff --git a/internal/web/connector_handlers.go b/internal/web/connector_handlers.go index 9cfa258..50d63c9 100644 --- a/internal/web/connector_handlers.go +++ b/internal/web/connector_handlers.go @@ -20,21 +20,21 @@ import ( // ConnectorHandlers provides REST handlers for connector management. type ConnectorHandlers struct { - connectorStore db.ConnectorStore - secretStore db.SecretStore - scenarioStore db.ScenarioStore - runStore db.RunStore - credResolver *credentials.Resolver + connectorStore db.ConnectorStore + secretStore db.SecretStore + assessmentStore db.AssessmentStore + runStore db.RunStore + credResolver *credentials.Resolver } // NewConnectorHandlers creates a new ConnectorHandlers instance. -func NewConnectorHandlers(connectorStore db.ConnectorStore, secretStore db.SecretStore, scenarioStore db.ScenarioStore, runStore db.RunStore, credResolver *credentials.Resolver) *ConnectorHandlers { +func NewConnectorHandlers(connectorStore db.ConnectorStore, secretStore db.SecretStore, assessmentStore db.AssessmentStore, runStore db.RunStore, credResolver *credentials.Resolver) *ConnectorHandlers { return &ConnectorHandlers{ - connectorStore: connectorStore, - secretStore: secretStore, - scenarioStore: scenarioStore, - runStore: runStore, - credResolver: credResolver, + connectorStore: connectorStore, + secretStore: secretStore, + assessmentStore: assessmentStore, + runStore: runStore, + credResolver: credResolver, } } diff --git a/internal/web/coverage.go b/internal/web/coverage.go index 57de48a..c7fc93c 100644 --- a/internal/web/coverage.go +++ b/internal/web/coverage.go @@ -83,7 +83,7 @@ type scenarioYAMLElasticAlert struct { // buildRuleNameToScenariosMap parses saved scenarios' YAML and returns a map // from Elastic Security rule name to the scenarios that cover that rule. -func buildRuleNameToScenariosMap(scenarios []db.SavedScenario) map[string][]CoverageScenario { +func buildRuleNameToScenariosMap(scenarios []db.Assessment) map[string][]CoverageScenario { result := make(map[string][]CoverageScenario) for _, saved := range scenarios { @@ -123,12 +123,12 @@ func buildRuleNameToScenariosMap(scenarios []db.SavedScenario) map[string][]Cove } // buildCoverageResponse joins Elastic rules with scenario coverage data and -// the latest assertion results to produce the full coverage response. -func buildCoverageResponse(rules []elastic.RuleSummary, scenarioMap map[string][]CoverageScenario, assertionResults []db.LatestAssertionResult) CoverageResponse { - // Build assertion lookup by alert name. - assertionByName := make(map[string]db.LatestAssertionResult, len(assertionResults)) - for _, ar := range assertionResults { - assertionByName[ar.AlertName] = ar +// the latest expectation results to produce the full coverage response. +func buildCoverageResponse(rules []elastic.RuleSummary, scenarioMap map[string][]CoverageScenario, expectationResults []db.LatestExpectationResult) CoverageResponse { + // Build expectation lookup by alert name. + expectationByName := make(map[string]db.LatestExpectationResult, len(expectationResults)) + for _, ar := range expectationResults { + expectationByName[ar.AlertName] = ar } entries := make([]RuleCoverageEntry, 0, len(rules)) @@ -148,8 +148,8 @@ func buildCoverageResponse(rules []elastic.RuleSummary, scenarioMap map[string][ Scenarios: scenarios, } - // Attach the most recent assertion result if available. - if ar, ok := assertionByName[rule.Name]; ok { + // Attach the most recent expectation result if available. + if ar, ok := expectationByName[rule.Name]; ok { entry.LastResult = &CoverageLastResult{ Passed: ar.Passed, RunID: ar.RunID.String(), diff --git a/internal/web/coverage_test.go b/internal/web/coverage_test.go index 12f6744..b5a0019 100644 --- a/internal/web/coverage_test.go +++ b/internal/web/coverage_test.go @@ -14,7 +14,7 @@ func TestBuildRuleNameToScenariosMap(t *testing.T) { scenario1ID := uuid.New() scenario2ID := uuid.New() - scenarios := []db.SavedScenario{ + scenarios := []db.Assessment{ { ID: scenario1ID, Name: "AWS IAM Brute Force Test", @@ -65,7 +65,7 @@ func TestBuildRuleNameToScenariosMap(t *testing.T) { } func TestBuildRuleNameToScenariosMap_InvalidYAML(t *testing.T) { - scenarios := []db.SavedScenario{ + scenarios := []db.Assessment{ { ID: uuid.New(), Name: "Bad Scenario", @@ -110,7 +110,7 @@ func TestBuildCoverageResponse(t *testing.T) { runID := uuid.New() now := time.Now() - assertionResults := []db.LatestAssertionResult{ + assertionResults := []db.LatestExpectationResult{ { AlertName: "Covered Rule", Passed: true, diff --git a/internal/web/elastic_rules_handlers.go b/internal/web/elastic_rules_handlers.go index ba516bc..f8677f4 100644 --- a/internal/web/elastic_rules_handlers.go +++ b/internal/web/elastic_rules_handlers.go @@ -170,7 +170,7 @@ func (h *ConnectorHandlers) HandleRuleCoverage(w http.ResponseWriter, r *http.Re } // List saved scenarios - scenarios, err := h.scenarioStore.ListAll(ctx) + scenarios, err := h.assessmentStore.ListAll(ctx) if err != nil { writeError(w, http.StatusInternalServerError, err.Error()) return @@ -179,15 +179,15 @@ func (h *ConnectorHandlers) HandleRuleCoverage(w http.ResponseWriter, r *http.Re // Build rule-name-to-scenarios mapping scenarioMap := buildRuleNameToScenariosMap(scenarios) - // Get latest assertion results - assertionResults, err := h.runStore.GetLatestAssertionResults(ctx) + // Get latest expectation results + expectationResults, err := h.runStore.GetLatestExpectationResults(ctx) if err != nil { writeError(w, http.StatusInternalServerError, err.Error()) return } // Build and return response - response := buildCoverageResponse(rulesResp.Data, scenarioMap, assertionResults) + response := buildCoverageResponse(rulesResp.Data, scenarioMap, expectationResults) writeJSON(w, http.StatusOK, response) } diff --git a/internal/web/handlers.go b/internal/web/handlers.go index 919d635..cfdce09 100644 --- a/internal/web/handlers.go +++ b/internal/web/handlers.go @@ -2,6 +2,7 @@ package web import ( "encoding/json" + "errors" "fmt" "io" "net/http" @@ -15,12 +16,13 @@ import ( "github.com/IBM/simrun/internal/version" "github.com/go-chi/chi/v5" "github.com/google/uuid" + "github.com/jackc/pgx/v5/pgconn" ) // Handlers provides REST handlers for scenarios, runs, config, and version. type Handlers struct { scenarioService *ScenarioService - scenarioStore db.ScenarioStore + assessmentStore db.AssessmentStore runStore db.RunStore configStore db.ConfigStore scheduler *Scheduler @@ -28,10 +30,10 @@ type Handlers struct { } // NewHandlers creates a new Handlers instance. -func NewHandlers(ss *ScenarioService, scenarioStore db.ScenarioStore, runStore db.RunStore, configStore db.ConfigStore, scheduler *Scheduler, dataDir string) *Handlers { +func NewHandlers(ss *ScenarioService, assessmentStore db.AssessmentStore, runStore db.RunStore, configStore db.ConfigStore, scheduler *Scheduler, dataDir string) *Handlers { return &Handlers{ scenarioService: ss, - scenarioStore: scenarioStore, + assessmentStore: assessmentStore, runStore: runStore, configStore: configStore, scheduler: scheduler, @@ -39,7 +41,7 @@ func NewHandlers(ss *ScenarioService, scenarioStore db.ScenarioStore, runStore d } } -// HandleLint handles POST /api/scenarios/lint +// HandleLint handles POST /api/assessments/lint func (h *Handlers) HandleLint(w http.ResponseWriter, r *http.Request) { var req LintRequest if err := json.NewDecoder(r.Body).Decode(&req); err != nil { @@ -56,7 +58,8 @@ func (h *Handlers) HandleLint(w http.ResponseWriter, r *http.Request) { writeJSON(w, http.StatusOK, resp) } -// HandleRun handles POST /api/scenarios/run +// HandleRun handles POST /api/runs. It starts a run of the saved assessment +// referenced by {assessmentId} and returns the new runId. func (h *Handlers) HandleRun(w http.ResponseWriter, r *http.Request) { var req RunRequest if err := json.NewDecoder(r.Body).Decode(&req); err != nil { @@ -64,9 +67,9 @@ func (h *Handlers) HandleRun(w http.ResponseWriter, r *http.Request) { return } - scenarioID, err := uuid.Parse(req.ScenarioID) + assessmentID, err := uuid.Parse(req.AssessmentID) if err != nil { - writeError(w, http.StatusBadRequest, "invalid scenarioId") + writeError(w, http.StatusBadRequest, "invalid assessmentId") return } @@ -79,7 +82,7 @@ func (h *Handlers) HandleRun(w http.ResponseWriter, r *http.Request) { } } - runID, err := h.scenarioService.Run(r.Context(), scenarioID, &RunOptions{ + runID, err := h.scenarioService.Run(r.Context(), assessmentID, &RunOptions{ Parallelism: req.Parallelism, CreatedBy: getUserEmail(r), ExploreMode: req.ExploreMode, @@ -94,45 +97,45 @@ func (h *Handlers) HandleRun(w http.ResponseWriter, r *http.Request) { writeJSON(w, http.StatusAccepted, RunResponse{RunID: runID}) } -// HandleListScenarios handles GET /api/scenarios. +// HandleListAssessments handles GET /api/assessments. // Pagination: page (default 1), per_page (default 50, clamped to [1, 100]). -// Filters: name (ILIKE %name% on scenario name), type (repeatable — +// Filters: name (ILIKE %name% on assessment name), type (repeatable — // e.g. ?type=standard&type=explore), since (Go duration like "24h" — returns -// scenarios updated in that window). -func (h *Handlers) HandleListScenarios(w http.ResponseWriter, r *http.Request) { +// assessments updated in that window). +func (h *Handlers) HandleListAssessments(w http.ResponseWriter, r *http.Request) { page, perPage := parsePagination(r, 50, 100) - filters, err := parseScenarioFilters(r) + filters, err := parseAssessmentFilters(r) if err != nil { writeError(w, http.StatusBadRequest, err.Error()) return } - res, err := h.scenarioStore.List(r.Context(), filters, perPage, (page-1)*perPage) + res, err := h.assessmentStore.List(r.Context(), filters, perPage, (page-1)*perPage) if err != nil { writeError(w, http.StatusInternalServerError, err.Error()) return } writeJSON(w, http.StatusOK, map[string]any{ - "scenarios": res.Scenarios, - "total": res.Total, - "page": page, - "perPage": perPage, + "assessments": res.Assessments, + "total": res.Total, + "page": page, + "perPage": perPage, }) } -// parseScenarioFilters extracts filter query params for HandleListScenarios. -func parseScenarioFilters(r *http.Request) (db.ListScenariosFilters, error) { +// parseAssessmentFilters extracts filter query params for HandleListAssessments. +func parseAssessmentFilters(r *http.Request) (db.ListAssessmentsFilters, error) { q := r.URL.Query() - f := db.ListScenariosFilters{Name: q.Get("name")} + f := db.ListAssessmentsFilters{Name: q.Get("name")} for _, t := range q["type"] { if !validScenarioTypes[t] { - return db.ListScenariosFilters{}, fmt.Errorf("invalid type %q (allowed: standard, explore, collect)", t) + return db.ListAssessmentsFilters{}, fmt.Errorf("invalid type %q (allowed: standard, explore, collect)", t) } f.Types = append(f.Types, t) } if s := q.Get("since"); s != "" { d, err := time.ParseDuration(s) if err != nil || d <= 0 { - return db.ListScenariosFilters{}, fmt.Errorf("invalid since %q (expected Go duration like '24h')", s) + return db.ListAssessmentsFilters{}, fmt.Errorf("invalid since %q (expected Go duration like '24h')", s) } t := time.Now().Add(-d) f.Since = &t @@ -140,22 +143,26 @@ func parseScenarioFilters(r *http.Request) (db.ListScenariosFilters, error) { return f, nil } -// HandleSaveScenario handles POST /api/scenarios -func (h *Handlers) HandleSaveScenario(w http.ResponseWriter, r *http.Request) { - var req SaveScenarioRequest +// HandleSaveAssessment handles POST /api/assessments. A duplicate name returns 409. +func (h *Handlers) HandleSaveAssessment(w http.ResponseWriter, r *http.Request) { + var req SaveAssessmentRequest if err := json.NewDecoder(r.Body).Decode(&req); err != nil { writeError(w, http.StatusBadRequest, "invalid request body") return } - scenarioType, err := normalizeScenarioType(req.Type) + assessmentType, err := normalizeScenarioType(req.Type) if err != nil { writeError(w, http.StatusBadRequest, err.Error()) return } - saved, err := h.scenarioStore.Save(r.Context(), req.Name, scenarioType, req.YAML, getUserEmail(r)) + saved, err := h.assessmentStore.Save(r.Context(), req.Name, assessmentType, req.YAML, getUserEmail(r)) if err != nil { + if isUniqueViolation(err) { + writeError(w, http.StatusConflict, fmt.Sprintf("an assessment named %q already exists", req.Name)) + return + } writeError(w, http.StatusInternalServerError, err.Error()) return } @@ -163,44 +170,66 @@ func (h *Handlers) HandleSaveScenario(w http.ResponseWriter, r *http.Request) { writeJSON(w, http.StatusCreated, saved) } -// HandleGetScenario handles GET /api/scenarios/{id} -func (h *Handlers) HandleGetScenario(w http.ResponseWriter, r *http.Request) { +// HandleGetAssessment handles GET /api/assessments/{id} +func (h *Handlers) HandleGetAssessment(w http.ResponseWriter, r *http.Request) { id, err := uuid.Parse(chi.URLParam(r, "id")) if err != nil { - writeError(w, http.StatusBadRequest, "invalid scenario ID") + writeError(w, http.StatusBadRequest, "invalid assessment ID") + return + } + + assessment, err := h.assessmentStore.Get(r.Context(), id) + if err != nil { + writeError(w, http.StatusNotFound, "assessment not found") + return + } + + writeJSON(w, http.StatusOK, assessment) +} + +// HandleGetAssessmentByName handles GET /api/assessments/by-name/{name}. The +// returned JSON includes the raw yaml field. +func (h *Handlers) HandleGetAssessmentByName(w http.ResponseWriter, r *http.Request) { + name := chi.URLParam(r, "name") + if name == "" { + writeError(w, http.StatusBadRequest, "assessment name is required") return } - scenario, err := h.scenarioStore.Get(r.Context(), id) + assessment, err := h.assessmentStore.GetByName(r.Context(), name) if err != nil { - writeError(w, http.StatusNotFound, "scenario not found") + writeError(w, http.StatusNotFound, "assessment not found") return } - writeJSON(w, http.StatusOK, scenario) + writeJSON(w, http.StatusOK, assessment) } -// HandleUpdateScenario handles PUT /api/scenarios/{id} -func (h *Handlers) HandleUpdateScenario(w http.ResponseWriter, r *http.Request) { +// HandleUpdateAssessment handles PUT /api/assessments/{id} +func (h *Handlers) HandleUpdateAssessment(w http.ResponseWriter, r *http.Request) { id, err := uuid.Parse(chi.URLParam(r, "id")) if err != nil { - writeError(w, http.StatusBadRequest, "invalid scenario ID") + writeError(w, http.StatusBadRequest, "invalid assessment ID") return } - var req SaveScenarioRequest + var req SaveAssessmentRequest if err := json.NewDecoder(r.Body).Decode(&req); err != nil { writeError(w, http.StatusBadRequest, "invalid request body") return } - scenarioType, err := normalizeScenarioType(req.Type) + assessmentType, err := normalizeScenarioType(req.Type) if err != nil { writeError(w, http.StatusBadRequest, err.Error()) return } - if err := h.scenarioStore.Update(r.Context(), id, req.Name, scenarioType, req.YAML, getUserEmail(r)); err != nil { + if err := h.assessmentStore.Update(r.Context(), id, req.Name, assessmentType, req.YAML, getUserEmail(r)); err != nil { + if isUniqueViolation(err) { + writeError(w, http.StatusConflict, fmt.Sprintf("an assessment named %q already exists", req.Name)) + return + } writeError(w, http.StatusInternalServerError, err.Error()) return } @@ -208,15 +237,15 @@ func (h *Handlers) HandleUpdateScenario(w http.ResponseWriter, r *http.Request) w.WriteHeader(http.StatusNoContent) } -// HandleDeleteScenario handles DELETE /api/scenarios/{id} -func (h *Handlers) HandleDeleteScenario(w http.ResponseWriter, r *http.Request) { +// HandleDeleteAssessment handles DELETE /api/assessments/{id} +func (h *Handlers) HandleDeleteAssessment(w http.ResponseWriter, r *http.Request) { id, err := uuid.Parse(chi.URLParam(r, "id")) if err != nil { - writeError(w, http.StatusBadRequest, "invalid scenario ID") + writeError(w, http.StatusBadRequest, "invalid assessment ID") return } - if err := h.scenarioStore.Delete(r.Context(), id); err != nil { + if err := h.assessmentStore.Delete(r.Context(), id); err != nil { writeError(w, http.StatusInternalServerError, err.Error()) return } @@ -273,16 +302,39 @@ func parseRunFilters(r *http.Request) (db.ListRunsFilters, error) { t := time.Now().Add(-d) f.Since = &t } - if s := q.Get("scenario_id"); s != "" { + if s := q.Get("assessment_id"); s != "" { id, err := uuid.Parse(s) if err != nil { - return db.ListRunsFilters{}, fmt.Errorf("invalid scenario_id %q", s) + return db.ListRunsFilters{}, fmt.Errorf("invalid assessment_id %q", s) } - f.ScenarioID = &id + f.AssessmentID = &id } return f, nil } +// HandleListAssessmentRuns handles GET /api/assessments/{id}/runs — the runs of +// a single assessment, most recent first. Read-only; runs are created at +// POST /api/runs. +func (h *Handlers) HandleListAssessmentRuns(w http.ResponseWriter, r *http.Request) { + id, err := uuid.Parse(chi.URLParam(r, "id")) + if err != nil { + writeError(w, http.StatusBadRequest, "invalid assessment ID") + return + } + page, perPage := parsePagination(r, 50, 100) + res, err := h.runStore.List(r.Context(), db.ListRunsFilters{AssessmentID: &id}, perPage, (page-1)*perPage) + if err != nil { + writeError(w, http.StatusInternalServerError, err.Error()) + return + } + writeJSON(w, http.StatusOK, map[string]any{ + "runs": res.Runs, + "total": res.Total, + "page": page, + "perPage": perPage, + }) +} + // parsePagination reads `page` and `per_page` query params, applying defaults and clamps. func parsePagination(r *http.Request, defaultPerPage, maxPerPage int) (page, perPage int) { page = 1 @@ -429,7 +481,7 @@ func (h *Handlers) HandleUpdateConfig(w http.ResponseWriter, r *http.Request) { // Retention day fields must be >= 1 so they cannot be set to a value that // deletes data immediately. Other keys keep the permissive key/value behavior. - if req.Key == "assessment_log_retention_days" || req.Key == "assessment_retention_days" { + if req.Key == "run_log_retention_days" || req.Key == "run_retention_days" { var days int if err := json.Unmarshal(req.Value, &days); err != nil || days < 1 { writeError(w, http.StatusBadRequest, req.Key+" must be at least 1") @@ -455,6 +507,13 @@ func (h *Handlers) HandleVersion(w http.ResponseWriter, r *http.Request) { }) } +// isUniqueViolation reports whether err is a Postgres unique-constraint +// violation (SQLSTATE 23505), used to map duplicate assessment names to 409. +func isUniqueViolation(err error) bool { + var pgErr *pgconn.PgError + return errors.As(err, &pgErr) && pgErr.Code == "23505" +} + // normalizeScenarioType validates and defaults the scenario type. func normalizeScenarioType(typ string) (string, error) { if typ == "" { diff --git a/internal/web/retention_test.go b/internal/web/retention_test.go index 8052353..5f9e9cc 100644 --- a/internal/web/retention_test.go +++ b/internal/web/retention_test.go @@ -86,7 +86,7 @@ func TestSweepAssessments_PurgesAgedSkipsRunningAndRecent(t *testing.T) { recentID, recentJSONL, _ := makeRun(t, ctx, store, dataDir, "completed", 1) runningID, runningJSONL, _ := makeRun(t, ctx, store, dataDir, "running", 40) - web.SweepAssessments(ctx, store, dataDir, true, 30) + web.SweepRuns(ctx, store, dataDir, true, 30) // Aged completed run: everything gone. _, err := store.Get(ctx, oldID) @@ -115,7 +115,7 @@ func TestSweepAssessments_DisabledIsNoOp(t *testing.T) { id, jsonl, ndjson := makeRun(t, ctx, store, dataDir, "completed", 40) - web.SweepAssessments(ctx, store, dataDir, false, 30) + web.SweepRuns(ctx, store, dataDir, false, 30) _, err := store.Get(ctx, id) assert.NoError(t, err, "disabled sweeper must keep the run") @@ -168,7 +168,7 @@ func TestSweepAssessments_RemovesTerraformDirs(t *testing.T) { dir1 := seedTerraformDir(t, dataDir, e1) dir2 := seedTerraformDir(t, dataDir, e2) - web.SweepAssessments(ctx, store, dataDir, true, 30) + web.SweepRuns(ctx, store, dataDir, true, 30) _, err := store.Get(ctx, id) assert.Error(t, err, "aged run row should be deleted") @@ -191,7 +191,7 @@ func TestSweepAssessments_SkipsUnsafeExecutionID(t *testing.T) { safeDir := seedTerraformDir(t, dataDir, safe) base := filepath.Join(dataDir, "terraform") - web.SweepAssessments(ctx, store, dataDir, true, 30) + web.SweepRuns(ctx, store, dataDir, true, 30) _, err := store.Get(ctx, id) assert.Error(t, err, "aged run row should still be deleted") @@ -210,7 +210,7 @@ func TestSweepAssessments_MissingTerraformDirIsBestEffort(t *testing.T) { // No terraform dir is seeded for this execution id. id := makeAgedRunWithExecutions(t, ctx, store, dataDir, uuid.NewString()) - web.SweepAssessments(ctx, store, dataDir, true, 30) + web.SweepRuns(ctx, store, dataDir, true, 30) _, err := store.Get(ctx, id) assert.Error(t, err, "delete should succeed even though the Terraform dir was missing") diff --git a/internal/web/run_deletion.go b/internal/web/run_deletion.go index af5b680..778e7d3 100644 --- a/internal/web/run_deletion.go +++ b/internal/web/run_deletion.go @@ -87,12 +87,12 @@ func deleteRunWithArtifacts(ctx context.Context, runStore db.RunStore, dataDir s return nil } -// SweepAssessments deletes whole runs (row + scenario_results + JSONL log + +// SweepRuns deletes whole runs (row + scenario_results + JSONL log + // collected .ndjson artifacts) whose created_at is older than days. It is a // no-op when enabled is false. Runs still in the "running" status are excluded // by ListExpired, so an actively-writing run is never purged. A per-run delete // failure is logged and the sweep continues with the remaining runs. -func SweepAssessments(ctx context.Context, runStore db.RunStore, dataDir string, enabled bool, days int) { +func SweepRuns(ctx context.Context, runStore db.RunStore, dataDir string, enabled bool, days int) { if !enabled { return } @@ -100,13 +100,13 @@ func SweepAssessments(ctx context.Context, runStore db.RunStore, dataDir string, cutoff := time.Now().AddDate(0, 0, -days) ids, err := runStore.ListExpired(ctx, cutoff) if err != nil { - logrus.WithError(err).Warn("assessment sweep: failed to list expired runs") + logrus.WithError(err).Warn("run retention sweep: failed to list expired runs") return } for _, id := range ids { if err := deleteRunWithArtifacts(ctx, runStore, dataDir, id); err != nil { - logrus.WithError(err).WithField("run_id", id).Warn("assessment sweep: failed to delete expired run") + logrus.WithError(err).WithField("run_id", id).Warn("run retention sweep: failed to delete expired run") } } } diff --git a/internal/web/scenario_export.go b/internal/web/scenario_export.go index 33ebd13..0c5b27e 100644 --- a/internal/web/scenario_export.go +++ b/internal/web/scenario_export.go @@ -8,7 +8,7 @@ import ( "github.com/IBM/simrun/internal/credentials" "github.com/IBM/simrun/internal/db" - "github.com/IBM/simrun/internal/results" + "github.com/IBM/simrun/internal/runner" "github.com/elastic/go-elasticsearch/v9" "github.com/google/uuid" log "github.com/sirupsen/logrus" @@ -32,7 +32,7 @@ func NewResultExporter(connectorStore db.ConnectorStore, creds *credentials.Reso } // Export iterates enabled connectors and dispatches to the matching backend. -func (e *ResultExporter) Export(ctx context.Context, runID uuid.UUID, scenarioResults []results.ScenarioRunResult) { +func (e *ResultExporter) Export(ctx context.Context, runID uuid.UUID, scenarioResults []runner.ScenarioResult) { if e.connectorStore == nil || len(scenarioResults) == 0 { return } @@ -57,7 +57,7 @@ func (e *ResultExporter) Export(ctx context.Context, runID uuid.UUID, scenarioRe // exportToElastic indexes results into the configured datastream on an enabled // Elastic connector. No-op if export is disabled on the connector. -func (e *ResultExporter) exportToElastic(ctx context.Context, connector *db.Connector, runID uuid.UUID, scenarioResults []results.ScenarioRunResult) { +func (e *ResultExporter) exportToElastic(ctx context.Context, connector *db.Connector, runID uuid.UUID, scenarioResults []runner.ScenarioResult) { var cfg ElasticConnectorConfig if err := json.Unmarshal(connector.Config, &cfg); err != nil { return @@ -106,12 +106,12 @@ func (e *ResultExporter) exportToElastic(ctx context.Context, connector *db.Conn // indexResults indexes each scenario result as a document in Elasticsearch. // It continues on individual failures and returns the count of successfully indexed documents. -func (e *ResultExporter) indexResults(ctx context.Context, client *elasticsearch.Client, indexName, runID string, scenarioResults []results.ScenarioRunResult) int { +func (e *ResultExporter) indexResults(ctx context.Context, client *elasticsearch.Client, indexName, runID string, scenarioResults []runner.ScenarioResult) int { indexed := 0 for _, scenario := range scenarioResults { - var assertions []map[string]interface{} - for _, a := range scenario.Assertions { - assertions = append(assertions, map[string]interface{}{ + var expectations []map[string]interface{} + for _, a := range scenario.Matchers { + expectations = append(expectations, map[string]interface{}{ "matcher_type": a.MatcherName(), "alert_name": a.AlertName(), }) @@ -128,7 +128,7 @@ func (e *ResultExporter) indexResults(ctx context.Context, client *elasticsearch "executor": scenario.ExecutorName, "executor_type": scenario.ExecutorType, "execution_id": scenario.ExecutionId, - "assertions": assertions, + "expectations": expectations, } if scenario.Metadata != nil { diff --git a/internal/web/scenario_results.go b/internal/web/scenario_results.go index 4eef163..b04645b 100644 --- a/internal/web/scenario_results.go +++ b/internal/web/scenario_results.go @@ -5,36 +5,35 @@ import ( "github.com/IBM/simrun/internal/db" "github.com/IBM/simrun/internal/matchers" - "github.com/IBM/simrun/internal/results" "github.com/IBM/simrun/internal/runner" "github.com/google/uuid" log "github.com/sirupsen/logrus" ) -// assertionDTO is the per-matcher status row persisted under -// `scenario_results.assertions`. Field names are wire-format. -type assertionDTO struct { +// expectationDTO is the per-matcher status row persisted under +// `scenario_results.expectations`. Field names are wire-format. +type expectationDTO struct { MatcherType string `json:"matcherType"` AlertName string `json:"alertName"` Passed bool `json:"passed"` } -// partialAssertionDTO is the mid-run counterpart of assertionDTO: a pending -// (not-yet-matched) assertion omits `passed` so the frontend renders it muted -// rather than as a failure. The terminal write uses assertionDTO (passed always +// partialExpectationDTO is the mid-run counterpart of expectationDTO: a pending +// (not-yet-matched) expectation omits `passed` so the frontend renders it muted +// rather than as a failure. The terminal write uses expectationDTO (passed always // present) instead. -type partialAssertionDTO struct { +type partialExpectationDTO struct { MatcherType string `json:"matcherType"` AlertName string `json:"alertName"` Passed *bool `json:"passed,omitempty"` } -// buildPartialAssertionsJSON marshals the runner's mid-run assertion state into -// the same wire shape as buildScenarioResultRow, preserving pending vs passed. -func buildPartialAssertionsJSON(results []runner.AssertionResult) ([]byte, error) { - dtos := make([]partialAssertionDTO, 0, len(results)) +// buildPartialExpectationsJSON marshals the runner's mid-run expectation state +// into the same wire shape as buildScenarioResultRow, preserving pending vs passed. +func buildPartialExpectationsJSON(results []runner.ExpectationResult) ([]byte, error) { + dtos := make([]partialExpectationDTO, 0, len(results)) for _, r := range results { - dtos = append(dtos, partialAssertionDTO{ + dtos = append(dtos, partialExpectationDTO{ MatcherType: r.MatcherType, AlertName: r.AlertName, Passed: r.Passed, @@ -44,35 +43,35 @@ func buildPartialAssertionsJSON(results []runner.AssertionResult) ([]byte, error } // buildScenarioResultRow projects an in-memory scenario result into the -// `scenario_results` row shape: marshals assertions/indicators/metadata/ +// `scenario_results` row shape: marshals expectations/indicators/metadata/ // discovered-alerts and copies scalar fields. Marshaling errors are logged // and the offending JSON field is left nil so the row still persists. -func buildScenarioResultRow(runID uuid.UUID, result *results.ScenarioRunResult) *db.ScenarioResult { - // Build a set of failed assertions for quick lookup. - // If the scenario failed but FailedAssertions is nil (e.g., error during - // assertion check), fall back to marking all assertions as failed. - failedSet := make(map[matchers.AlertGeneratedMatcher]struct{}, len(result.FailedAssertions)) - for _, fa := range result.FailedAssertions { +func buildScenarioResultRow(runID uuid.UUID, result *runner.ScenarioResult) *db.ScenarioResult { + // Build a set of unmet expectations for quick lookup. + // If the scenario failed but UnmetExpectations is nil (e.g., error during + // matching), fall back to marking all expectations as failed. + failedSet := make(map[matchers.AlertGeneratedMatcher]struct{}, len(result.UnmetExpectations)) + for _, fa := range result.UnmetExpectations { failedSet[fa] = struct{}{} } - hasPerMatcherResults := result.Success || result.FailedAssertions != nil - var assertionDTOs []assertionDTO - for _, a := range result.Assertions { + hasPerMatcherResults := result.Success || result.UnmetExpectations != nil + var expectationDTOs []expectationDTO + for _, a := range result.Matchers { passed := result.Success if hasPerMatcherResults { _, failed := failedSet[a] passed = !failed } - assertionDTOs = append(assertionDTOs, assertionDTO{ + expectationDTOs = append(expectationDTOs, expectationDTO{ MatcherType: a.MatcherName(), AlertName: a.AlertName(), Passed: passed, }) } - assertionsJSON, err := json.Marshal(assertionDTOs) + expectationsJSON, err := json.Marshal(expectationDTOs) if err != nil { - log.WithError(err).Warn("Failed to marshal assertions") - assertionsJSON = nil + log.WithError(err).Warn("Failed to marshal expectations") + expectationsJSON = nil } indicatorsJSON, err := json.Marshal(result.Indicators) if err != nil { @@ -100,10 +99,15 @@ func buildScenarioResultRow(runID uuid.UUID, result *results.ScenarioRunResult) } isSuccess := result.Success + // A scenario errored (vs. cleanly missing an expectation) when it failed + // without producing per-expectation results: warmup/detonation failures and + // matching-infrastructure errors leave UnmetExpectations nil. + errored := !result.Success && result.UnmetExpectations == nil return &db.ScenarioResult{ RunID: runID, Name: result.Name, IsSuccess: &isSuccess, + Errored: errored, ErrorMessage: result.ErrorMessage, DurationSecs: result.DurationSeconds, MatchingDurSecs: result.MatchingDurationSeconds, @@ -112,7 +116,7 @@ func buildScenarioResultRow(runID uuid.UUID, result *results.ScenarioRunResult) ExecutorType: result.ExecutorType, ExecutionID: result.ExecutionId, SimulationID: result.SimulationID, - Assertions: assertionsJSON, + Expectations: expectationsJSON, Indicators: indicatorsJSON, Metadata: metadataJSON, CollectedLogPath: collectedLogPath, diff --git a/internal/web/scenario_results_test.go b/internal/web/scenario_results_test.go index 27ad74d..863a8f7 100644 --- a/internal/web/scenario_results_test.go +++ b/internal/web/scenario_results_test.go @@ -6,7 +6,6 @@ import ( "time" "github.com/IBM/simrun/internal/matchers" - "github.com/IBM/simrun/internal/results" "github.com/IBM/simrun/internal/runner" "github.com/google/uuid" "github.com/sirupsen/logrus" @@ -34,7 +33,7 @@ func TestBuildScenarioResultRow_SuccessWithAssertions(t *testing.T) { runID := uuid.New() a1 := stubMatcher{matcher: "elastic", alert: "Suspicious AWS API Call"} a2 := stubMatcher{matcher: "datadog", alert: "Privilege Escalation"} - res := &results.ScenarioRunResult{ + res := &runner.ScenarioResult{ Name: "scenario-a", Success: true, DurationSeconds: 1.5, @@ -43,7 +42,7 @@ func TestBuildScenarioResultRow_SuccessWithAssertions(t *testing.T) { ExecutorName: "simrun", ExecutorType: "detonator", ExecutionId: "exec-1", - Assertions: []matchers.AlertGeneratedMatcher{a1, a2}, + Matchers: []matchers.AlertGeneratedMatcher{a1, a2}, } row := buildScenarioResultRow(runID, res) @@ -52,9 +51,10 @@ func TestBuildScenarioResultRow_SuccessWithAssertions(t *testing.T) { assert.Equal(t, "scenario-a", row.Name) require.NotNil(t, row.IsSuccess) assert.True(t, *row.IsSuccess) + assert.False(t, row.Errored, "a successful scenario never counts as an execution error") - var got []assertionDTO - require.NoError(t, json.Unmarshal(row.Assertions, &got)) + var got []expectationDTO + require.NoError(t, json.Unmarshal(row.Expectations, &got)) require.Len(t, got, 2) for _, d := range got { assert.True(t, d.Passed, "all assertions pass on a successful run: %+v", d) @@ -69,12 +69,12 @@ func TestBuildScenarioResultRow_FailureWithNilFailedAssertions(t *testing.T) { // success=true" — that would be misleading). runID := uuid.New() a := stubMatcher{matcher: "elastic", alert: "Whatever"} - res := &results.ScenarioRunResult{ - Name: "scenario-b", - Success: false, - ErrorMessage: "detonate timeout", - Assertions: []matchers.AlertGeneratedMatcher{a}, - FailedAssertions: nil, + res := &runner.ScenarioResult{ + Name: "scenario-b", + Success: false, + ErrorMessage: "detonate timeout", + Matchers: []matchers.AlertGeneratedMatcher{a}, + UnmetExpectations: nil, } row := buildScenarioResultRow(runID, res) @@ -82,17 +82,40 @@ func TestBuildScenarioResultRow_FailureWithNilFailedAssertions(t *testing.T) { require.NotNil(t, row.IsSuccess) assert.False(t, *row.IsSuccess) assert.Equal(t, "detonate timeout", row.ErrorMessage) + assert.True(t, row.Errored, "a failure with no per-expectation results is an execution error") - var got []assertionDTO - require.NoError(t, json.Unmarshal(row.Assertions, &got)) + var got []expectationDTO + require.NoError(t, json.Unmarshal(row.Expectations, &got)) require.Len(t, got, 1) assert.False(t, got[0].Passed, "fallback branch marks all assertions as failed when FailedAssertions is nil") } +func TestBuildScenarioResultRow_MatchingFailureIsNotErrored(t *testing.T) { + // A scenario that ran but missed an expected alert is a clean expectation + // mismatch, not an execution error: UnmetExpectations is populated, so it must + // not inflate the run's error count (which drives the warning vs. completed + // status icon in the UI). + runID := uuid.New() + a := stubMatcher{matcher: "elastic", alert: "Expected Alert"} + res := &runner.ScenarioResult{ + Name: "scenario-c", + Success: false, + ErrorMessage: "1 out of 1 expectations did not pass: elastic/Expected Alert", + Matchers: []matchers.AlertGeneratedMatcher{a}, + UnmetExpectations: []matchers.AlertGeneratedMatcher{a}, + } + + row := buildScenarioResultRow(runID, res) + require.NotNil(t, row) + require.NotNil(t, row.IsSuccess) + assert.False(t, *row.IsSuccess) + assert.False(t, row.Errored, "an unmet expectation is a matching failure, not an execution error") +} + func TestBuildScenarioResultRow_ExploreModeIncludesDiscoveredAlerts(t *testing.T) { // Explore mode emits DiscoveredAlerts; the column is empty in non-explore runs. runID := uuid.New() - res := &results.ScenarioRunResult{ + res := &runner.ScenarioResult{ Name: "scenario-c", Success: true, ExploreMode: true, diff --git a/internal/web/scenarios.go b/internal/web/scenarios.go index da12a50..9fc55a7 100644 --- a/internal/web/scenarios.go +++ b/internal/web/scenarios.go @@ -18,29 +18,29 @@ import ( // ScenarioService handles scenario parsing, execution, and result persistence. type ScenarioService struct { - runStore db.RunStore - scenarioStore db.ScenarioStore - packStore db.PackStore - configStore db.ConfigStore - creds *credentials.Resolver - exporter *ResultExporter - hub *Hub - runLogRegistry *RunLogRegistry - dataDir string + runStore db.RunStore + assessmentStore db.AssessmentStore + packStore db.PackStore + configStore db.ConfigStore + creds *credentials.Resolver + exporter *ResultExporter + hub *Hub + runLogRegistry *RunLogRegistry + dataDir string } // NewScenarioService creates a new ScenarioService. -func NewScenarioService(runStore db.RunStore, scenarioStore db.ScenarioStore, packStore db.PackStore, configStore db.ConfigStore, creds *credentials.Resolver, exporter *ResultExporter, hub *Hub, runLogRegistry *RunLogRegistry, dataDir string) *ScenarioService { +func NewScenarioService(runStore db.RunStore, assessmentStore db.AssessmentStore, packStore db.PackStore, configStore db.ConfigStore, creds *credentials.Resolver, exporter *ResultExporter, hub *Hub, runLogRegistry *RunLogRegistry, dataDir string) *ScenarioService { return &ScenarioService{ - runStore: runStore, - scenarioStore: scenarioStore, - packStore: packStore, - configStore: configStore, - creds: creds, - exporter: exporter, - hub: hub, - runLogRegistry: runLogRegistry, - dataDir: dataDir, + runStore: runStore, + assessmentStore: assessmentStore, + packStore: packStore, + configStore: configStore, + creds: creds, + exporter: exporter, + hub: hub, + runLogRegistry: runLogRegistry, + dataDir: dataDir, } } @@ -103,7 +103,7 @@ func (s *ScenarioService) Lint(yamlContent []byte) (*LintResponse, error) { Name: sc.Name, ExecutorType: executorType, ExecutorName: executorName, - Assertions: len(sc.Assertions), + Expectations: len(sc.Matchers), }) } @@ -124,12 +124,12 @@ type RunOptions struct { Timeout time.Duration // global timeout override; 0 means use per-scenario YAML timeout } -// Run starts async scenario execution. Returns the runId immediately. -// It fetches the scenario YAML from the database using the provided scenarioID. -func (s *ScenarioService) Run(ctx context.Context, scenarioID uuid.UUID, opts *RunOptions) (string, error) { - savedScenario, err := s.scenarioStore.Get(ctx, scenarioID) +// Run starts async execution of a saved assessment. Returns the runId +// immediately. It fetches the assessment YAML using the provided assessmentID. +func (s *ScenarioService) Run(ctx context.Context, assessmentID uuid.UUID, opts *RunOptions) (string, error) { + assessment, err := s.assessmentStore.Get(ctx, assessmentID) if err != nil { - return "", fmt.Errorf("scenario not found: %w", err) + return "", fmt.Errorf("assessment not found: %w", err) } appCfg := config.DefaultAppConfig() @@ -175,7 +175,7 @@ func (s *ScenarioService) Run(ctx context.Context, scenarioID uuid.UUID, opts *R PackLogsEnabled: appCfg.PackLogsEnabled, } - parseResult, err := parser.ParseWithOptions([]byte(savedScenario.YAML), parseOpts) + parseResult, err := parser.ParseWithOptions([]byte(assessment.YAML), parseOpts) if err != nil { return "", err } @@ -198,7 +198,7 @@ func (s *ScenarioService) Run(ctx context.Context, scenarioID uuid.UUID, opts *R Status: "running", StartTime: now, Total: len(scenarios), - ScenarioID: &scenarioID, + AssessmentID: &assessmentID, ScheduleID: opts.ScheduleID, ScheduleName: opts.ScheduleName, CreatedBy: opts.CreatedBy, @@ -243,18 +243,18 @@ func (s *ScenarioService) Run(ctx context.Context, scenarioID uuid.UUID, opts *R } } } - sc.AssertionsCallback = func(scenarioName string, assertions []runner.AssertionResult) { + sc.ExpectationsCallback = func(scenarioName string, expectations []runner.ExpectationResult) { dbID, ok := scenarioDBIDs[scenarioName] if !ok { return } - assertionsJSON, err := buildPartialAssertionsJSON(assertions) + expectationsJSON, err := buildPartialExpectationsJSON(expectations) if err != nil { - log.WithField("scenario", scenarioName).WithError(err).Warn("Failed to marshal partial assertions") + log.WithField("scenario", scenarioName).WithError(err).Warn("Failed to marshal partial expectations") return } - if err := s.runStore.UpdateScenarioAssertions(context.Background(), dbID, assertionsJSON); err != nil { - log.WithField("scenario", scenarioName).WithError(err).Warn("Failed to update scenario assertions") + if err := s.runStore.UpdateScenarioExpectations(context.Background(), dbID, expectationsJSON); err != nil { + log.WithField("scenario", scenarioName).WithError(err).Warn("Failed to update scenario expectations") } } } @@ -279,7 +279,7 @@ func (s *ScenarioService) Run(ctx context.Context, scenarioID uuid.UUID, opts *R } }() - allResults := results.RunScenariosParallel(scenarios, parallelism, func(result *results.ScenarioRunResult) { + allResults := results.RunScenariosParallel(scenarios, parallelism, func(result *runner.ScenarioResult) { // Update run counters incrementally successDelta, failDelta := 0, 0 if result.Success { diff --git a/internal/web/schedule_handler.go b/internal/web/schedule_handler.go index 2d546de..68db2e9 100644 --- a/internal/web/schedule_handler.go +++ b/internal/web/schedule_handler.go @@ -13,17 +13,17 @@ import ( // ScheduleHandlers provides REST handlers for schedule management. type ScheduleHandlers struct { - scheduleStore db.ScheduleStore - scenarioStore db.ScenarioStore - scheduler *Scheduler + scheduleStore db.ScheduleStore + assessmentStore db.AssessmentStore + scheduler *Scheduler } // NewScheduleHandlers creates a new ScheduleHandlers instance. -func NewScheduleHandlers(scheduleStore db.ScheduleStore, scenarioStore db.ScenarioStore, scheduler *Scheduler) *ScheduleHandlers { +func NewScheduleHandlers(scheduleStore db.ScheduleStore, assessmentStore db.AssessmentStore, scheduler *Scheduler) *ScheduleHandlers { return &ScheduleHandlers{ - scheduleStore: scheduleStore, - scenarioStore: scenarioStore, - scheduler: scheduler, + scheduleStore: scheduleStore, + assessmentStore: assessmentStore, + scheduler: scheduler, } } @@ -54,15 +54,15 @@ func (h *ScheduleHandlers) HandleGetSchedule(w http.ResponseWriter, r *http.Requ writeJSON(w, http.StatusOK, schedule) } -// HandleGetScheduleByScenario handles GET /api/scenarios/{scenarioId}/schedule -func (h *ScheduleHandlers) HandleGetScheduleByScenario(w http.ResponseWriter, r *http.Request) { - scenarioID, err := uuid.Parse(chi.URLParam(r, "scenarioId")) +// HandleGetScheduleByAssessment handles GET /api/assessments/{id}/schedule +func (h *ScheduleHandlers) HandleGetScheduleByAssessment(w http.ResponseWriter, r *http.Request) { + assessmentID, err := uuid.Parse(chi.URLParam(r, "id")) if err != nil { - writeError(w, http.StatusBadRequest, "invalid scenario ID") + writeError(w, http.StatusBadRequest, "invalid assessment ID") return } - schedule, err := h.scheduleStore.GetByScenarioID(r.Context(), scenarioID) + schedule, err := h.scheduleStore.GetByAssessmentID(r.Context(), assessmentID) if err != nil { writeError(w, http.StatusNotFound, "schedule not found") return @@ -79,14 +79,14 @@ func (h *ScheduleHandlers) HandleCreateSchedule(w http.ResponseWriter, r *http.R return } - scenarioID, err := uuid.Parse(req.ScenarioID) + assessmentID, err := uuid.Parse(req.AssessmentID) if err != nil { - writeError(w, http.StatusBadRequest, "invalid scenario ID") + writeError(w, http.StatusBadRequest, "invalid assessment ID") return } - if _, err := h.scenarioStore.Get(r.Context(), scenarioID); err != nil { - writeError(w, http.StatusNotFound, "scenario not found") + if _, err := h.assessmentStore.Get(r.Context(), assessmentID); err != nil { + writeError(w, http.StatusNotFound, "assessment not found") return } @@ -100,10 +100,10 @@ func (h *ScheduleHandlers) HandleCreateSchedule(w http.ResponseWriter, r *http.R parallelism = 10 } - schedule, err := h.scheduleStore.Create(r.Context(), scenarioID, req.CronExpression, req.Enabled, parallelism, getUserEmail(r)) + schedule, err := h.scheduleStore.Create(r.Context(), assessmentID, req.CronExpression, req.Enabled, parallelism, getUserEmail(r)) if err != nil { if strings.Contains(err.Error(), "unique") || strings.Contains(err.Error(), "duplicate") { - writeError(w, http.StatusConflict, "schedule already exists for this scenario") + writeError(w, http.StatusConflict, "schedule already exists for this assessment") return } writeError(w, http.StatusInternalServerError, err.Error()) diff --git a/internal/web/scheduler.go b/internal/web/scheduler.go index 0e1169b..b1b3a8c 100644 --- a/internal/web/scheduler.go +++ b/internal/web/scheduler.go @@ -14,7 +14,7 @@ import ( // Scheduler manages cron-based scenario execution. type Scheduler struct { scheduleStore db.ScheduleStore - scenarioStore db.ScenarioStore + assessmentStore db.AssessmentStore scenarioService *ScenarioService cron *cron.Cron mu sync.Mutex @@ -23,12 +23,12 @@ type Scheduler struct { } // NewScheduler creates a new Scheduler. -func NewScheduler(scheduleStore db.ScheduleStore, scenarioStore db.ScenarioStore, scenarioService *ScenarioService) *Scheduler { +func NewScheduler(scheduleStore db.ScheduleStore, assessmentStore db.AssessmentStore, scenarioService *ScenarioService) *Scheduler { ctx, cancel := context.WithCancel(context.Background()) return &Scheduler{ scheduleStore: scheduleStore, - scenarioStore: scenarioStore, + assessmentStore: assessmentStore, scenarioService: scenarioService, cron: cron.New(), ctx: ctx, @@ -98,22 +98,22 @@ func (s *Scheduler) loadSchedules() error { // addSchedule registers a single schedule as a cron job. func (s *Scheduler) addSchedule(schedule db.Schedule) error { scheduleID := schedule.ID - scenarioID := schedule.ScenarioID + assessmentID := schedule.AssessmentID _, err := s.cron.AddFunc(schedule.CronExpression, func() { - s.executeSchedule(scheduleID, scenarioID) + s.executeSchedule(scheduleID, assessmentID) }) return err } // executeSchedule runs when a cron job fires. -func (s *Scheduler) executeSchedule(scheduleID, scenarioID uuid.UUID) { +func (s *Scheduler) executeSchedule(scheduleID, assessmentID uuid.UUID) { logger := log.WithFields(log.Fields{ - "scheduleId": scheduleID, - "scenarioId": scenarioID, + "scheduleId": scheduleID, + "assessmentId": assessmentID, }) - logger.Info("Executing scheduled scenario") + logger.Info("Executing scheduled assessment") // Use context.Background() so in-progress runs are not cancelled when the scheduler stops. ctx := context.Background() @@ -124,22 +124,22 @@ func (s *Scheduler) executeSchedule(scheduleID, scenarioID uuid.UUID) { return } - scenario, err := s.scenarioStore.Get(ctx, scenarioID) + assessment, err := s.assessmentStore.Get(ctx, assessmentID) if err != nil { - logger.WithError(err).Error("Failed to load scenario for scheduled run") + logger.WithError(err).Error("Failed to load assessment for scheduled run") return } - scheduleName := scenario.Name + " (scheduled)" + scheduleName := assessment.Name + " (scheduled)" - _, err = s.scenarioService.Run(ctx, scenarioID, &RunOptions{ + _, err = s.scenarioService.Run(ctx, assessmentID, &RunOptions{ Parallelism: schedule.Parallelism, ScheduleID: &scheduleID, ScheduleName: &scheduleName, CreatedBy: "system", }) if err != nil { - logger.WithError(err).Error("Failed to execute scheduled scenario") + logger.WithError(err).Error("Failed to execute scheduled assessment") return } diff --git a/internal/web/server.go b/internal/web/server.go index 0506100..16f016d 100644 --- a/internal/web/server.go +++ b/internal/web/server.go @@ -91,15 +91,16 @@ func (s *Server) setupRoutes(handlers *Handlers, packHandlers *PackHandlers, sec r.Use(auth.RequireAuth(s.sessionStore)) } - // Scenarios - r.Post("/scenarios/lint", handlers.HandleLint) - r.Post("/scenarios/run", handlers.HandleRun) - r.Get("/scenarios", handlers.HandleListScenarios) - r.Post("/scenarios", handlers.HandleSaveScenario) - r.Get("/scenarios/{id}", handlers.HandleGetScenario) - r.Put("/scenarios/{id}", handlers.HandleUpdateScenario) - r.Delete("/scenarios/{id}", handlers.HandleDeleteScenario) - r.Get("/scenarios/{scenarioId}/schedule", scheduleHandlers.HandleGetScheduleByScenario) + // Assessments (saved definitions; GitHub-Actions "workflow") + r.Post("/assessments/lint", handlers.HandleLint) + r.Get("/assessments", handlers.HandleListAssessments) + r.Post("/assessments", handlers.HandleSaveAssessment) + r.Get("/assessments/by-name/{name}", handlers.HandleGetAssessmentByName) + r.Get("/assessments/{id}", handlers.HandleGetAssessment) + r.Put("/assessments/{id}", handlers.HandleUpdateAssessment) + r.Delete("/assessments/{id}", handlers.HandleDeleteAssessment) + r.Get("/assessments/{id}/runs", handlers.HandleListAssessmentRuns) + r.Get("/assessments/{id}/schedule", scheduleHandlers.HandleGetScheduleByAssessment) // Schedules r.Get("/schedules", scheduleHandlers.HandleListSchedules) @@ -108,7 +109,8 @@ func (s *Server) setupRoutes(handlers *Handlers, packHandlers *PackHandlers, sec r.Put("/schedules/{id}", scheduleHandlers.HandleUpdateSchedule) r.Delete("/schedules/{id}", scheduleHandlers.HandleDeleteSchedule) - // Runs + // Runs (executions; created here, read/deleted by id) + r.Post("/runs", handlers.HandleRun) r.Get("/runs", handlers.HandleListRuns) r.Get("/runs/{runId}", handlers.HandleGetRun) r.Delete("/runs/{runId}", handlers.HandleDeleteRun) diff --git a/internal/web/types.go b/internal/web/types.go index 7c28a4f..8a05086 100644 --- a/internal/web/types.go +++ b/internal/web/types.go @@ -29,14 +29,14 @@ type LintRequest struct { } type RunRequest struct { - ScenarioID string `json:"scenarioId"` + AssessmentID string `json:"assessmentId"` Parallelism int `json:"parallelism,omitempty"` ExploreMode bool `json:"exploreMode,omitempty"` CleanupAlerts bool `json:"cleanupAlerts,omitempty"` Timeout string `json:"timeout,omitempty"` } -type SaveScenarioRequest struct { +type SaveAssessmentRequest struct { Name string `json:"name"` Type string `json:"type,omitempty"` YAML string `json:"yaml"` @@ -93,7 +93,7 @@ type SecretGroupResponse struct { // Schedule request types type CreateScheduleRequest struct { - ScenarioID string `json:"scenarioId"` + AssessmentID string `json:"assessmentId"` CronExpression string `json:"cronExpression"` Enabled bool `json:"enabled"` Parallelism int `json:"parallelism,omitempty"` @@ -117,7 +117,7 @@ type LintedScenario struct { Name string `json:"name"` ExecutorType string `json:"executorType"` ExecutorName string `json:"executorName"` - Assertions int `json:"assertions"` + Expectations int `json:"expectations"` } type RunResponse struct { diff --git a/openspec/changes/archive/2026-06-24-assessment-vocabulary-refactor/.openspec.yaml b/openspec/changes/archive/2026-06-24-assessment-vocabulary-refactor/.openspec.yaml new file mode 100644 index 0000000..a4ac4d7 --- /dev/null +++ b/openspec/changes/archive/2026-06-24-assessment-vocabulary-refactor/.openspec.yaml @@ -0,0 +1,2 @@ +schema: spec-driven +created: 2026-06-23 diff --git a/openspec/changes/archive/2026-06-24-assessment-vocabulary-refactor/design.md b/openspec/changes/archive/2026-06-24-assessment-vocabulary-refactor/design.md new file mode 100644 index 0000000..a602256 --- /dev/null +++ b/openspec/changes/archive/2026-06-24-assessment-vocabulary-refactor/design.md @@ -0,0 +1,114 @@ +## Context + +simrun is architecturally a test runner for detection rules, but its vocabulary obscures that and collides across layers. A brainstorm established two intertwined problems and one strong analogy: + +1. **Vocabulary** — "scenario" names both the container and the case; "assertion"/"matcher"/"expectation" are three words for two concepts; the execution entity is `run` in the backend but shipped as "assessment" in the UI. +2. **Duplicated runner machinery** — `Runner.Run()` carries a vestigial multi-scenario loop, three modes are encoded three different ways, the poll loops are duplicated, and three near-identical result types coexist. +3. **The GitHub Actions analogy** — simrun's domain hierarchy maps almost exactly onto Actions, and the bottom two levels are already simrun's unchangeable YAML keys. + +Constraints: +- **User-authored YAML is immovable** (`scenarios:`, `expectations:`, `targets:` stay). +- **The component interfaces are good** (`Detonator`/`Injector`/`Collector`/`Matcher`) — out of scope. +- **The `runs` table should not be renamed.** An earlier draft renamed it to `assessment_runs`; that jammed the parent's name into the child and read badly. GitHub keeps the child as plain `run` and lets the URL namespace disambiguate. + +## Goals / Non-Goals + +**Goals:** +- A GitHub-Actions-style resource model: `Assessment` → `Run` → `Scenario` → `Expectation`. +- Keep `scenario` and `expectation` as the case/check nouns everywhere so code and YAML agree. +- Keep the `runs` and `scenario_results` tables in place; disambiguate runs by URL nesting, not by renaming. +- Collapse duplicated runner machinery: one fan-out layer, one `Mode`, one poll primitive, one result type. +- Backward-compatible config migration (no operator-set retention values lost). + +**Non-Goals:** +- Changing user-authored YAML keys. +- Touching the component interfaces or the worker-pool concurrency model. +- Adding dependent scenario chains / DAG orchestration — explicitly deferred (see Open Questions). +- Renaming `scenario` → `case` (rejected: would re-create a code-vs-YAML mismatch). + +## Decisions + +### Decision 1: Model the resource hierarchy on GitHub Actions +``` + GitHub Actions simrun + ────────────── ────── + Workflow (.yml) ↔ Assessment (saved definition, addressable by name) + Workflow Run ↔ Run (execution; table stays `runs`) + Job (parallel) ↔ Scenario (parallel worker-pool unit) — YAML key + Step ↔ Expectation (checked by a Matcher) — YAML key +``` +The bottom two levels need no renaming. Only the top two get names: `Assessment` (definition) and `Run` (execution). + +- **Why `Run` stays `Run`** (not `AssessmentRun`): GitHub addresses runs as `/actions/runs/{id}` and `/actions/workflows/{id}/runs` — the namespace disambiguates, so the child noun stays short. simrun mirrors this. Bonus: the `runs` table and retention keys (`run_retention_*`) stay clean, and the migration shrinks. +- **Run routing:** a Run is a top-level resource. Create is singular at `POST /api/runs` (body `{assessmentId, …opts}`) — *not* duplicated as a nested create — so there is one way to mint a run. Reads may live in multiple places (GitHub-style): `GET /api/runs`, `GET /api/runs/{id}`, and the nested read collection `GET /api/assessments/{id}/runs` (which internally applies `ListRunsFilters` with `assessment_id`). Writes singular, reads plural. + +### Decision 2: `Assessment` is the definition, addressable by unique name +The saved definition becomes `Assessment`. Because each assessment serializes to `.yaml` (GitHub addresses workflows by filename), `name` becomes UNIQUE and serves as the slug: `GET /api/assessments/{name}` returns the JSON, which already includes the raw `yaml` field. (A separate `.yaml` raw endpoint was considered and dropped — it would only re-serve a string already present in the JSON; Rule 2.) + +- **Why "Assessment"** over Plan/Playbook/Suite: it is the security-domain-native word ("a security assessment" *is* a defined set of attack scenarios). `Plan` is disqualified by the `terraform plan` collision pervasive in this codebase. `Playbook` connotes an ordered procedure, contradicting the parallel/independent scenario model. `Suite` is accurate but flavorless and smuggles back a "test" prefix. + +### Decision 3: Keep `scenario` and `expectation`; drop "assertion" +The two YAML-locked words map onto GitHub's job/step and stay. Following the Gomega split (`Expect(x).To(Equal(y))`): **Expectation** = the declared check (YAML key) *and* its outcome, **Matcher** = the pluggable mechanism that performs the check. "Assertion" sat redundantly between them and collided with both; it is removed. +- `scenario.Assertions` → `scenario.Matchers`; `FailedAssertions` → `UnmetExpectations`. +- `ElasticSecurityAlertGeneratedAssertion` → `ElasticSecurityAlertMatcher`. +- `AssertionResult` / `LatestAssertionResult` → `ExpectationResult`. +- "Assertion" is currently spread across **three** wire/storage identifiers that must all converge on `expectations`: the `scenario_results.assertions` JSONB column, the run-result JSON key `matchers` (`results/types.go`), and the lint JSON count key `assertions` (`web/types.go`). "Matcher" is retained only for the mechanism (the `AlertGeneratedMatcher` interface and its impls), not for results. + +### Decision 4: One fan-out layer, one result type, no input mutation +`Runner` executes exactly one scenario and **returns** a consolidated `ScenarioResult`; it no longer mutates the `Scenario` input. The worker pool in `internal/results` is the sole fan-out. The two near-identical **in-memory** result types (`runner.ScenarioResult`, `results.ScenarioRunResult`) collapse into one `ScenarioResult`; `SimrunRunResult` → `RunResult`. The persistence row `db.ScenarioResult` (raw-JSON columns + DB-only fields like `Status`/`Phase`/`CreatedAt`) is **kept as a separate DTO** so `db` stays decoupled from `runner`/`matchers`; the existing `web.buildScenarioResultRow` remains the single marshal boundary. (Folding the row DTO into the in-memory type would invert the layering — `db` importing the domain — and force one struct to carry fields half its callers ignore; rejected per Rule 2/Rule 6.) + +### Decision 5: Extract `Eventually(fn, interval, deadline)` +One poll-until-satisfied primitive shared by the assert and explore paths. + +### Deferred: mode unification (separate change) +A clean-up of how execution mode is selected — today three disconnected mechanisms: the `type` column (storage/list-filter only, never read at run time), the per-run `exploreMode` bool, and the `" - collect mode"` expectation-name suffix — into one authoritative `Mode` enum is **out of scope here**. Verified that `type` does not drive runtime behavior today, so unifying it is a *behavior change* (it would wire `type` to execution and require migrating existing collect scenarios). It is tracked as its own change. This change leaves `exploreMode`/`cleanupAlerts` and the collect-mode suffix untouched. + +### Canonical rename map + +| Concept | Old | New | +|---|---|---| +| Saved definition | `SavedScenario` / `saved_scenarios` / `/api/scenarios` | `Assessment` / `assessments` / `/api/assessments` | +| Definition address | by UUID | by unique `name` | +| Execution | `Run` / `runs` / `/api/runs` | **unchanged** (`/api/runs` + `/api/assessments/{id}/runs`) | +| Run ingress | `POST /api/scenarios/run` body `{scenarioId}` | `POST /api/runs` body `{assessmentId}` | +| Runs of one assessment | (none) | `GET /api/assessments/{id}/runs` (nested read) | +| Run→definition FK | `runs.scenario_id` | `runs.assessment_id` | +| Per-expectation outcome column | `scenario_results.assertions` (JSONB) | `scenario_results.expectations` | +| Run-result JSON key | `matchers` | `expectations` | +| Lint JSON count key | `assertions` | `expectations` | +| Per-case outcome (in-memory ×2) | `runner.ScenarioResult`, `results.ScenarioRunResult` | one in-memory `ScenarioResult` (`db.ScenarioResult` row DTO kept separate) | +| Run aggregate | `SimrunRunResult` | `RunResult` | +| Per-expectation outcome | `AssertionResult` / `LatestAssertionResult` | `ExpectationResult` | +| Runtime checks | `scenario.Assertions []AlertGeneratedMatcher` | `scenario.Matchers` | +| Failed checks | `FailedAssertions` | `UnmetExpectations` | +| Concrete matcher struct | `ElasticSecurityAlertGeneratedAssertion` | `ElasticSecurityAlertMatcher` | +| Retention keys | `assessment_(log_)retention_*` | `run_(log_)retention_*` | +| Mode encoding (`ExploreMode`/`CleanupAlerts`/suffix) | — | **unchanged** (deferred to a separate change) | +| Case noun (UNCHANGED) | `Scenario`, `scenario_results`, YAML `scenarios:` | unchanged | +| Declared check (UNCHANGED) | `Expectation`, YAML `expectations:` | unchanged | + +## Risks / Trade-offs + +- **Large breaking API surface in one change** → Stage by tranche (Migration Plan); the rename is mechanical and compiler-enforced in Go (wide but shallow). +- **Frontend/backend drift during rollout** → Land the API path/key renames together as one coordinated tranche, not piecemeal. +- **Retention key migration could drop operator values** → Migration copies old key values to new names before dropping old keys; covered by a spec scenario. +- **Unique `name` constraint breaks existing duplicate-named definitions** → Migration must detect collisions and disambiguate (e.g., suffix) before adding the UNIQUE index; surface any renames in migration logs (fail loud). +- **Spec deltas cover contract-changing requirements, not every incidental "run" mention** → Unchanged-behavior requirements (WebSocket keep-alive, backpressure, export) keep their semantics; their cosmetic wording is handled by the mechanical sweep in tasks. + +## Migration Plan + +Staged tranches, each independently shippable: + +- **Tranche A (zero user impact):** consolidate the three result types into one `ScenarioResult` (+ `RunResult`); runner returns instead of mutating; drop "assertion" → `Matcher`/`UnmetExpectations`/`ExpectationResult`; extract `Eventually`. Pure Go internal, behavior-preserving refactor. +- **Tranche B (lean runner):** collapse the vestigial `Runner.Run()` multi-scenario loop so `Runner` is single-scenario; the worker pool is the only fan-out. +- **Tranche C (the rename, coordinated):** DB migration `saved_scenarios` → `assessments` (+ UNIQUE `name`, with collision disambiguation), FK `runs.scenario_id` → `assessment_id`, column `scenario_results.assertions` → `expectations`; API `/api/scenarios*` → `/api/assessments*`, addressable by name; run ingress `POST /api/runs` body `{assessmentId}` plus nested read `GET /api/assessments/{id}/runs`; retention config-key rename to `run_(log_)retention_*` with value-carrying backfill; frontend surfaces **Assessments** (library) and **Runs** (history) and updates API-client paths/keys — landed together. + +Rollback: tranches A/B are behavior-preserving refactors revertable via VCS. Tranche C's DB migrations ship with `down` migrations restoring the prior table/column/key names. + +## Open Questions + +- Slug form for `name`: raw string or slugified (lowercase, hyphenated)? Default: store `name` verbatim, require uniqueness, URL-encode in paths. Revisit if names with spaces/slashes prove awkward. +- Keep a temporary alias for `POST /api/scenarios/run` during transition? Default: replace; no known external API consumers. +- Inline-YAML (unsaved) runs do not exist today (every run references a saved definition). If ad-hoc runs are ever wanted, that is a *new feature* (`POST /api/runs` with a YAML body) — deliberately out of scope here so this stays a rename. +- `RunResponse` stays `{runId}` — renaming to `{id}` is cosmetic churn with frontend coupling and is not adopted. +- Dependent scenario chains (scenario B consumes scenario A's output) — the one place that would justify a DAG/workflow engine. Deferred to a separate proposal. diff --git a/openspec/changes/archive/2026-06-24-assessment-vocabulary-refactor/proposal.md b/openspec/changes/archive/2026-06-24-assessment-vocabulary-refactor/proposal.md new file mode 100644 index 0000000..f8c5931 --- /dev/null +++ b/openspec/changes/archive/2026-06-24-assessment-vocabulary-refactor/proposal.md @@ -0,0 +1,49 @@ +## Why + +simrun is, architecturally, a test runner for detections — but its vocabulary doesn't say so, and the words it uses collide across layers. "Scenario" names both the saved file *and* each case inside it ("a scenario of scenarios"). "Assertion", "matcher", and "expectation" are three words for two concepts. The execution entity is `run` in the backend but already shipped as "assessment" in the UI, so the two halves of the product disagree. The runner also carries a vestigial multi-scenario loop and three near-identical result types. + +The fix is to model the domain on **GitHub Actions**, whose 4-level hierarchy maps almost exactly onto simrun — and whose bottom two levels are already simrun's (unchangeable) YAML keys. + +## What Changes + +Adopt a GitHub-Actions-style resource model and one consistent vocabulary end to end: + +``` + Assessment (definition — the saved YAML; GitHub "workflow") + └── Run (execution; GitHub "workflow run") + └── Scenario (parallel unit; GitHub "job") — YAML key, unchanged + └── Expectation (check via Matcher; GitHub "step") — YAML key, unchanged +``` + +- **BREAKING** The saved definition (`SavedScenario` / `saved_scenarios` / `/api/scenarios`) becomes **Assessment** (`assessments` / `/api/assessments`), addressable by its unique `name` (e.g. `GET /api/assessments/aws-privesc`) — mirroring GitHub addressing a workflow by filename. `name` becomes unique; the raw YAML is already returned in the JSON `yaml` field. +- The execution entity stays **Run** (`runs` table unchanged), GitHub-style: created at `POST /api/runs`, item at `GET/DELETE /api/runs/{id}`, all runs at `GET /api/runs`, and a nested read collection `GET /api/assessments/{id}/runs` for one assessment's history. Create is singular (`POST /api/runs`); reads may be nested. No `assessment_run`. +- **BREAKING** Run ingress moves from `POST /api/scenarios/run` body `{scenarioId}` to `POST /api/runs` body `{assessmentId}` — same shape, renamed path and field. Runs always reference a saved assessment; there is no inline-YAML run path today. +- **BREAKING** The run→definition FK `runs.scenario_id` becomes `runs.assessment_id`. +- The per-case noun stays **Scenario** at every layer (the YAML key `scenarios:` is unchanged, so code and YAML finally agree). +- **BREAKING** Drop **assertion** from the vocabulary. Keep **Expectation** (declared check = the YAML key, and the *outcome*) and **Matcher** (the *mechanism*). This spans three identifiers today: DB column `scenario_results.assertions` → `expectations`; run-result JSON key `matchers` → `expectations`; lint JSON count key `assertions` → `expectations`. Also `AssertionResult`/`LatestAssertionResult` → `ExpectationResult`; concrete `…Assertion` structs → `…Matcher`; `scenario.Assertions` → `scenario.Matchers`. +- **BREAKING** Retention config keys gate runs, so they shorten to `run_retention_*` / `run_log_retention_*` (from `assessment_retention_*` / `assessment_log_retention_*`), with operator-set values carried forward. + +**Lean-runner cleanup (internal, behavior-preserving):** +- Remove the vestigial multi-scenario loop in `Runner.Run()`; `Runner` executes exactly one scenario and the worker pool is the sole fan-out. +- Extract one `Eventually(fn, interval, deadline)` poll primitive shared by the assert and explore paths. +- Consolidate the two near-identical in-memory result types (`runner.ScenarioResult`, `results.ScenarioRunResult`) into one `ScenarioResult`; the persistence row `db.ScenarioResult` stays a separate column-shaped DTO so `db` stays decoupled from the domain. Stop mutating the `Scenario` input to carry outputs. `SimrunRunResult` → `RunResult`. + +> **Out of scope (deferred to a separate change):** unifying the three disconnected mode mechanisms (`type` column, per-run `exploreMode` bool, `" - collect mode"` suffix) into one authoritative `Mode` enum. That is a behavior change (it would wire `type` to execution for the first time and require migrating existing collect scenarios), so it is tracked separately. This change leaves `exploreMode`/`cleanupAlerts` and the collect-mode suffix exactly as they are. + +## Capabilities + +### New Capabilities + + +### Modified Capabilities +- `runs`: the `runs` table is retained, but the run→definition FK becomes `assessment_id`, the composite GET joins `assessments`, per-expectation outcomes rename `AssertionResult` → `ExpectationResult` (DB column `assertions` → `expectations`, result JSON key `matchers` → `expectations`), and the runner contract is clarified (single-scenario execution, one consolidated `ScenarioResult`). +- `scenarios`: the **saved definition** becomes **Assessment** (`saved_scenarios` → `assessments`, `/api/scenarios*` → `/api/assessments*`, addressable by unique name), and run ingress moves to `POST /api/runs` body `{assessmentId}` with a nested read `GET /api/assessments/{id}/runs`. The per-case "scenario" vocabulary and the YAML `scenarios:`/`expectations:` schema are unchanged. +- `assessment-retention`: retention config keys rename to `run_retention_*` / `run_log_retention_*` (they gate `runs`), with prior operator values migrated. + +## Impact + +- **DB**: migrations renaming `saved_scenarios` → `assessments` (+ unique `name`), FK `runs.scenario_id` → `assessment_id`, column `scenario_results.assertions` → `expectations`; `AppConfig` retention key rename with value-carrying backfill. The `runs` and `scenario_results` table names are unchanged (only the `assertions` column renames). +- **API/WS**: `/api/scenarios*` → `/api/assessments*`; run ingress at `POST /api/runs` body `{assessmentId}`; nested read `GET /api/assessments/{id}/runs`; result JSON key `matchers` → `expectations`; coordinated frontend update. +- **Go packages**: `internal/runner`, `internal/results`, `internal/db`, `internal/web`, `internal/parser`, `internal/matchers/*` (type/field renames + result consolidation + lean-runner refactor). +- **Frontend**: `web/frontend` — surface **Assessments** (the library of saved definitions) and **Runs** / **Run History** (executions); update API-client paths and keys. +- **Not changed**: user-authored YAML (`scenarios:`, `expectations:`, `targets:` keys), the component interfaces (`Detonator`/`Injector`/`Collector`/`Matcher`), the worker-pool executor, and the `runs` / `scenario_results` tables. diff --git a/openspec/changes/archive/2026-06-24-assessment-vocabulary-refactor/specs/assessment-retention/spec.md b/openspec/changes/archive/2026-06-24-assessment-vocabulary-refactor/specs/assessment-retention/spec.md new file mode 100644 index 0000000..cd6f075 --- /dev/null +++ b/openspec/changes/archive/2026-06-24-assessment-vocabulary-refactor/specs/assessment-retention/spec.md @@ -0,0 +1,97 @@ +## MODIFIED Requirements + +### Requirement: Retention Settings In AppConfig +The system SHALL extend `AppConfig` with retention settings persisted in the +`app_config` table and served by `GET /api/config` / `PUT /api/config`: +`run_log_retention_enabled` (bool), `run_log_retention_days` (int), +`run_retention_enabled` (bool), and `run_retention_days` (int). These keys gate +the deletion of **runs** (executions) and their logs; they are renamed from the +former `assessment_log_retention_*` / `assessment_retention_*` keys (which the +new vocabulary makes misleading, since "assessment" now denotes the definition). +Defaults SHALL be `run_log_retention_enabled = true`, `run_log_retention_days = 7`, +`run_retention_enabled = false`, `run_retention_days = 30`, backfilled by a +migration that also carries forward any operator-set values from the previous key +names. + +#### Scenario: Defaults when unset +- **WHEN** no `app_config` row exists for the retention keys +- **THEN** `GET /api/config` returns `run_log_retention_enabled = true`, `run_log_retention_days = 7`, `run_retention_enabled = false`, and `run_retention_days = 30` + +#### Scenario: Prior values migrated +- **WHEN** the database held `assessment_retention_days = 14` under the old key name before this change +- **THEN** after migration `run_retention_days = 14` and the old key is removed + +#### Scenario: Admin updates retention +- **WHEN** a client sends `PUT /api/config` with `run_retention_enabled = true` and `run_retention_days = 14` +- **THEN** both values are persisted and returned by a subsequent `GET /api/config` + +### Requirement: Retention Settings Validation +The system SHALL reject `PUT /api/config` with HTTP 400 when +`run_log_retention_days` or `run_retention_days` is less than 1, so retention +cannot be set to a value that deletes data immediately. + +#### Scenario: Zero log retention rejected +- **WHEN** a client sends `PUT /api/config` with `run_log_retention_days = 0` +- **THEN** the response is HTTP 400 and the stored value is unchanged + +#### Scenario: Zero run retention rejected +- **WHEN** a client sends `PUT /api/config` with `run_retention_days = 0` +- **THEN** the response is HTTP 400 and the stored value is unchanged + +### Requirement: Log-Retention Sweeper +The system SHALL run a background sweeper that periodically scans +`/run-logs/` and deletes any `.jsonl` file whose last modification +time is older than `run_log_retention_days`. The sweeper SHALL run once at +startup and then on a fixed 1-hour interval, SHALL re-read `AppConfig` each tick, +and SHALL be a no-op when `run_log_retention_enabled = false`. Deleting a log file +SHALL NOT delete or modify the corresponding `runs` row. + +#### Scenario: Old log swept +- **WHEN** a run's JSONL file was last modified longer ago than `run_log_retention_days` and log retention is enabled +- **THEN** the sweeper deletes the file and leaves the `runs` row intact + +#### Scenario: Log retention disabled +- **WHEN** `run_log_retention_enabled = false` +- **THEN** the log sweeper deletes no files regardless of age + +### Requirement: Assessment-Retention Sweeper +The system SHALL run a background sweeper that periodically deletes whole runs +whose `created_at` is older than `run_retention_days`. For each expired run the +system SHALL delete the `runs` row (cascading to `scenario_results`), the run's +JSONL log file, and every collected `.ndjson` file referenced by that run's +`scenario_results.collected_log_path`. The sweeper SHALL run once at startup and +then on a fixed 1-hour interval, SHALL re-read `AppConfig` each tick, SHALL be a +no-op when `run_retention_enabled = false`, and SHALL skip runs whose `status` is +still `running`. + +#### Scenario: Old run purged +- **WHEN** a completed run's `created_at` is older than `run_retention_days` and run retention is enabled +- **THEN** the `runs` row, its `scenario_results`, its JSONL log file, and all of its collected `.ndjson` files are deleted + +#### Scenario: Run retention disabled +- **WHEN** `run_retention_enabled = false` +- **THEN** the sweeper deletes no runs regardless of age + +#### Scenario: Running run skipped +- **WHEN** a run older than `run_retention_days` still has `status = "running"` +- **THEN** the sweeper does not delete it + +### Requirement: Swept Logs Surface As Empty +The system SHALL serve `GET /api/runs/{id}/logs` with HTTP 200 and body `[]` when +the run's JSONL file has been swept by log retention, reusing the existing +missing-file behavior so an expired-log run is indistinguishable from one that +never logged. + +#### Scenario: Logs requested after sweep +- **WHEN** a client GETs logs for a run whose JSONL file was deleted by the log sweeper +- **THEN** the response is HTTP 200 with body `[]` + +### Requirement: Configure Retention From Assessments Page +The system SHALL present a "Run retention" control on the runs page that opens a +dialog for editing `run_log_retention_enabled`, `run_log_retention_days`, +`run_retention_enabled`, and `run_retention_days`, and SHALL persist changes via +`PUT /api/config`. + +#### Scenario: Open and save +- **WHEN** an admin opens the dialog, enables run retention with a 14-day window, and saves +- **THEN** the new values are sent to `PUT /api/config` and reflected on the page after save diff --git a/openspec/changes/archive/2026-06-24-assessment-vocabulary-refactor/specs/runs/spec.md b/openspec/changes/archive/2026-06-24-assessment-vocabulary-refactor/specs/runs/spec.md new file mode 100644 index 0000000..dbc4509 --- /dev/null +++ b/openspec/changes/archive/2026-06-24-assessment-vocabulary-refactor/specs/runs/spec.md @@ -0,0 +1,138 @@ +## MODIFIED Requirements + +### Requirement: Run Record Resource +The system SHALL persist runs in the `runs` table with fields: `id` (UUID), +`status`, `start_time`, `end_time` (nullable), `total`, `succeeded`, `failed`, +`assessment_id` (nullable FK to `assessments`), `schedule_id` (nullable FK), +`schedule_name` (nullable), `created_by`, `created_at`. A **run** is one +execution of an **assessment** (the saved definition). The `runs` table name is +unchanged; only the definition FK is renamed from `scenario_id` to +`assessment_id`. + +#### Scenario: Run created on start +- **WHEN** a run is started +- **THEN** a row is inserted into `runs` with `status = "running"`, `start_time = now()`, `total` set to the number of scenarios to execute, and `succeeded = failed = 0` + +### Requirement: Scenario Result Lifecycle +The system SHALL track each scenario as a row in `scenario_results` with +`status` transitioning `pending` → `running` → `completed`, and `phase` +populated during `running` (e.g., `"warmup"`, `"detonating"`, `"matching"`, +`"collecting"`, `"cleanup"`, `"queued"`). + +The system SHALL populate the row's executor identity — `executor_name`, +`executor_type`, `execution_id`, and `simulation_id` — as soon as detonation +returns these values, while the row is still `running`. When a detonator does +not produce a `simulation_id`, that field SHALL remain empty without blocking +the other identity fields. + +While in the `matching` phase, the system SHALL persist per-expectation results +incrementally as the matcher resolves them, so the row's `expectations` reflect +the current passed/pending state before the scenario completes. An expectation +not yet matched SHALL be represented as not-yet-passed (no terminal failure is +recorded until completion). Each per-expectation outcome is an `ExpectationResult` +(formerly `AssertionResult`). + +These incremental writes SHALL NOT alter the terminal completion write: when a +scenario completes, `status` becomes `completed`, `phase` is cleared, and the +final `is_success`, `expectations`, durations, and `discovered_alerts` are written. + +#### Scenario: Phase transitions +- **WHEN** a scenario enters the matching phase +- **THEN** its `scenario_results` row has `status = "running"` and `phase = "matching"` + +#### Scenario: Expectation progress exposed during matching +- **WHEN** a scenario expecting 3 expectations has matched 2 of them and is still matching +- **THEN** the scenario's `expectations` in `GET /api/runs/{id}` show 2 passed and 1 not-yet-passed while `status = "running"` + +#### Scenario: Completion write unchanged +- **WHEN** a scenario finishes after its identity and partial expectations were written mid-run +- **THEN** the final row has `status = "completed"`, `phase = null`, and `is_success` plus the full `expectations` and `discovered_alerts` set + +### Requirement: Get Run Returns Composite Object +The system SHALL respond to `GET /api/runs/{id}` with the envelope +`{run, scenarios}` where `run` includes a LEFT JOIN of `assessments.name` and +`.type`, and `scenarios` is the list of `scenario_results` rows for the run. + +#### Scenario: Assessment still exists +- **WHEN** a client GETs a run whose `assessment_id` references an existing assessment +- **THEN** `run.assessmentName` and `run.assessmentType` are populated + +#### Scenario: Assessment deleted +- **WHEN** the run's source assessment has been deleted +- **THEN** `run.assessmentName` and `run.assessmentType` are null but the run is still returned 200 + +### Requirement: List Runs Unpaginated +The system SHALL return all runs from `GET /api/runs` ordered by `created_at` +descending, and SHALL serve the runs of a single assessment via the nested +collection `GET /api/assessments/{id}/runs` in the same order (the nested handler +applies the existing `ListRunsFilters` with `assessment_id` set). Runs are +created only at `POST /api/runs`; the nested route is read-only. **Note:** there +is no pagination today; flagged as a known scaling gap. + +#### Scenario: List ordering +- **WHEN** a client requests `/api/runs` +- **THEN** the most recent run is first + +#### Scenario: Runs scoped to one assessment +- **WHEN** a client requests `/api/assessments/{id}/runs` +- **THEN** only runs whose `assessment_id` equals `{id}` are returned, most recent first + +### Requirement: Delete Run Cascades to Results and Log File +The system SHALL delete the `runs` row on `DELETE /api/runs/{id}`, cascade-delete +all `scenario_results` rows via the FK, and best-effort remove the run's on-disk +artifacts: the run's JSONL log file, every collected `.ndjson` file referenced by +the run's `scenario_results.collected_log_path`, and, for each scenario result +with a non-empty `execution_id`, the run's Terraform working directory at +`/terraform//`. The system SHALL skip Terraform-directory +removal for any `execution_id` that does not resolve to a direct child of +`/terraform/`. Failure to remove any on-disk artifact SHALL be logged +and SHALL NOT fail the request. + +#### Scenario: Successful delete +- **WHEN** a client deletes a run with 3 results +- **THEN** the `runs` row, all 3 `scenario_results` rows, and the JSONL log file are removed + +#### Scenario: Unsafe execution id skipped +- **WHEN** a scenario result has a blank `execution_id` or one containing a path separator +- **THEN** no Terraform directory is removed for that result and the `/terraform/` base directory is left intact + +### Requirement: Get Run Logs +The system SHALL respond to `GET /api/runs/{id}/logs` by reading the JSONL file +from disk and returning the array of entries. A missing file SHALL produce an +empty array `[]`, not an error. + +#### Scenario: Missing log file +- **WHEN** a run's log file does not exist on disk +- **THEN** the response is HTTP 200 with body `[]` + +### Requirement: Schedule Attribution on Triggered Runs +The system SHALL set `schedule_id` and `schedule_name` on runs created by the +in-process scheduler, and `created_by = "system"`. Manual runs SHALL have +`schedule_id = null`. + +#### Scenario: Scheduled run +- **WHEN** the scheduler fires for an assessment +- **THEN** the new `runs` row has `schedule_id` set and `created_by = "system"` + +## ADDED Requirements + +### Requirement: Single-Scenario Runner With Consolidated Result +The runner SHALL execute exactly one scenario per `Runner` instance and return a +single consolidated `ScenarioResult` value rather than mutating the scenario +input to carry outputs. Fan-out across multiple scenarios SHALL be the sole +responsibility of the parallel executor (worker pool). There SHALL be exactly one +in-memory `ScenarioResult` type shared across the runner, executor, and web +layers (replacing the two former near-identical in-memory types +`runner.ScenarioResult` and `results.ScenarioRunResult`), and one `RunResult` +aggregate (formerly `SimrunRunResult`). The persistence layer keeps its own +column-shaped row DTO (`db.ScenarioResult`, with `json.RawMessage` fields and +DB-only columns) so `db` stays decoupled from the domain packages; the single +marshal boundary that projects the in-memory result onto the row DTO is retained. + +#### Scenario: Runner returns a result +- **WHEN** the runner finishes executing a scenario +- **THEN** it returns a populated `ScenarioResult` and the scenario input struct is not used as the output carrier + +#### Scenario: Fan-out lives in the executor +- **WHEN** a run contains N scenarios +- **THEN** the parallel executor schedules N single-scenario runner executions and there is no second multi-scenario loop inside `Runner` diff --git a/openspec/changes/archive/2026-06-24-assessment-vocabulary-refactor/specs/scenarios/spec.md b/openspec/changes/archive/2026-06-24-assessment-vocabulary-refactor/specs/scenarios/spec.md new file mode 100644 index 0000000..11a7db8 --- /dev/null +++ b/openspec/changes/archive/2026-06-24-assessment-vocabulary-refactor/specs/scenarios/spec.md @@ -0,0 +1,74 @@ +## MODIFIED Requirements + +### Requirement: Assessment Resource +The system SHALL persist saved assessment definitions in the `assessments` table +with fields `id` (UUID), `name` (UNIQUE), `yaml` (raw text), `type` +(`standard` | `explore` | `collect`, default `standard`), `created_by`, +`updated_by`, `created_at`, `updated_at`. An **assessment** is the saved +definition; its `yaml` body contains a `scenarios:` array of individual scenarios +(the per-case vocabulary and YAML schema are unchanged). Because an assessment +serializes to `.yaml`, `name` is unique and serves as a human-addressable +slug. + +#### Scenario: Create with default type +- **WHEN** a client posts an assessment without specifying `type` +- **THEN** the inserted row has `type = "standard"` + +#### Scenario: Duplicate name rejected +- **WHEN** a client posts an assessment whose `name` already exists +- **THEN** the response is HTTP 409 and no row is inserted + +#### Scenario: Invalid type rejected +- **WHEN** a client posts an assessment with `type: "fast"` +- **THEN** the response is HTTP 400 with `"type must be 'standard', 'explore', or 'collect'"` + +### Requirement: List Assessments +The system SHALL serve `GET /api/assessments` as a paginated, filterable list +ordered by `updated_at DESC`, returning `{assessments, total, page, perPage}` +where `assessments` is the page slice (possibly empty, never `null`). The system +SHALL additionally serve a single assessment by its unique name via +`GET /api/assessments/{name}`, returning its JSON (which already includes the +raw `yaml` field). Query parameters (`page`, `per_page`, `name`, `type`, `since`) +behave as before against `assessments.name`/`.type`. + +#### Scenario: Most-recently-updated first +- **WHEN** a client requests `/api/assessments` with no parameters +- **THEN** the response is HTTP 200 with `{assessments, total, page: 1, perPage: 50}` ordered with the most recently updated assessment first + +#### Scenario: Fetch by name +- **WHEN** a client requests `/api/assessments/aws-privesc` +- **THEN** the response is the JSON for the assessment named `aws-privesc`, including its raw `yaml` field + +#### Scenario: Invalid type rejected +- **WHEN** a client requests `/api/assessments?type=bogus` +- **THEN** the response is HTTP 400 and no rows are returned + +### Requirement: Run Endpoint Is Asynchronous +The system SHALL start a run via `POST /api/runs` with `{assessmentId}` in the +body, performing pre-flight setup (load AppConfig, packs, assessment; build run +env; parse YAML) and starting the run in a detached goroutine. The response SHALL +return HTTP 202 with `{runId: ""}` once the `runs` row is inserted. A Run is +a top-level resource: it is created at `POST /api/runs` and read/deleted at +`/api/runs/{id}`. (This replaces `POST /api/scenarios/run` body `{scenarioId}` — +the same shape with a renamed path and field. Runs always reference a saved +assessment; there is no inline-YAML run path today.) + +#### Scenario: Run a saved assessment +- **WHEN** a client posts `{assessmentId}` to `/api/runs` for an existing assessment +- **THEN** the response is HTTP 202 with a `runId` +- **AND** a new `runs` row exists with `status = "running"` and `assessment_id` set to the posted id + +#### Scenario: Pre-flight failure +- **WHEN** the request references a saved assessment that does not exist +- **THEN** the response is HTTP 400 and no `runs` row is created + +## RENAMED Requirements + +- FROM: `### Requirement: Saved Scenario Resource` +- TO: `### Requirement: Assessment Resource` + +- FROM: `### Requirement: List Saved Scenarios` +- TO: `### Requirement: List Assessments` + +- FROM: `### Requirement: Delete Scenario Cascades and Reloads Scheduler` +- TO: `### Requirement: Delete Assessment Cascades and Reloads Scheduler` diff --git a/openspec/changes/archive/2026-06-24-assessment-vocabulary-refactor/tasks.md b/openspec/changes/archive/2026-06-24-assessment-vocabulary-refactor/tasks.md new file mode 100644 index 0000000..42a2c39 --- /dev/null +++ b/openspec/changes/archive/2026-06-24-assessment-vocabulary-refactor/tasks.md @@ -0,0 +1,37 @@ +## 1. Tranche A — Internal refactor (zero user impact) + +- [x] 1.1 Extract `Eventually(fn func() (bool, error), interval, deadline) error` and reimplement `runAssertions` and `runExploreMode` polling on top of it (behavior-preserving) +- [x] 1.2 Rename matcher vocabulary: `scenario.Assertions` → `Matchers`, `FailedAssertions` → `UnmetExpectations`; concrete `*Assertion` structs (e.g. `ElasticSecurityAlertGeneratedAssertion`) → `*Matcher` +- [x] 1.3 Rename `AssertionResult`/`LatestAssertionResult` → `ExpectationResult` across `internal/db`, `internal/web`; converge the three result identifiers on `expectations` — run-result JSON key `matchers` (`results/types.go`) → `expectations`, lint count key `assertions` (`web/types.go`) → `expectations` (DB column handled in 3.x) +- [x] 1.4 Consolidate the two in-memory types `runner.ScenarioResult` and `results.ScenarioRunResult` into one `ScenarioResult` (persistence row `db.ScenarioResult` kept as a separate column-shaped DTO so `db` stays decoupled); rename `SimrunRunResult` → `RunResult` +- [x] 1.5 Make `runScenario` return the populated `ScenarioResult` instead of mutating the `Scenario` input; update the executor to consume the returned value +- [x] 1.6 Update unit tests in `internal/runner`, `internal/results`, and matchers to the new names/shapes; `go build ./...` and `go test ./...` green + +> Note: unifying mode selection (`type`/`exploreMode`/`" - collect mode"` suffix → one authoritative `Mode` enum) is a behavior change handled in a separate change; `ExploreMode`/`CleanupAlerts` and the collect-mode suffix are left untouched here. + +## 2. Tranche B — Lean runner (single-scenario) + +- [x] 2.1 Remove `Runner.Scenarios []` and the multi-scenario `for` loop + `failedScenarios` aggregate-error builder; make `Runner` execute exactly one scenario +- [x] 2.2 Confirm `RunScenariosParallel` is the sole fan-out path; adjust `runSingleScenario` to call the single-scenario runner and map its `ScenarioResult` +- [x] 2.3 Update/retire tests that exercised the multi-scenario loop; `go test ./...` green + +## 3. Tranche C — Rename to the GitHub-Actions model (coordinated) + +- [x] 3.1 DB migration: rename `saved_scenarios` → `assessments`; add a `UNIQUE` constraint on `assessments.name`, disambiguating any pre-existing duplicate names first and logging each rename +- [x] 3.2 DB migration: rename FK column `runs.scenario_id` → `runs.assessment_id` (preserve the FK/index) +- [x] 3.3 DB migration: rename column `scenario_results.assertions` (JSONB) → `expectations`; add matching `down` migration +- [x] 3.4 DB migration: rename retention `AppConfig` keys `assessment_(log_)retention_*` → `run_(log_)retention_*`, carrying forward operator-set values, then remove the old keys; add matching `down` migrations +- [x] 3.5 Rename Go types/stores: `SavedScenario` → `Assessment`, `ScenarioStore` saved-definition methods, `RunStore` FK field, config keys in `AppConfig`/`DefaultAppConfig()` +- [x] 3.6 API: move `/api/scenarios*` → `/api/assessments*`; add `GET /api/assessments/{name}` (JSON, includes raw `yaml` field); enforce unique-name 409 on create +- [x] 3.7 API: move run ingress to `POST /api/runs` body `{assessmentId}` (replacing `POST /api/scenarios/run` body `{scenarioId}`); add nested read `GET /api/assessments/{id}/runs` (delegates to `ListRunsFilters` with `assessment_id`); update `GET /api/runs/{id}` composite to join `assessments` and return `assessmentName`/`assessmentType` (response stays `{runId}`) +- [x] 3.8 API: retention handlers read/write the new `run_(log_)retention_*` keys; validation messages updated +- [x] 3.9 Frontend: rename routes/components/state from scenario-library → **Assessments** and run-history → **Runs**; update API-client paths and JSON keys (result `matchers`/lint `assertions` → `expectations`, assessment by name) +- [x] 3.10 Frontend: relabel the retention dialog to "Run retention" and bind to the new config keys + +## 4. Verification + +- [x] 4.1 `mise run build` (frontend + server) succeeds; `mise run lint` clean +- [x] 4.2 `go test ./...` green; add/adjust handler tests for the new endpoints (run a saved assessment, fetch assessment by name, duplicate-name 409) +- [~] 4.3 Migration round-trip — **up verified on a live operator DB** (schema_migrations=15 clean; `assessments` + UNIQUE `name`, `runs.assessment_id`, `scenario_results.expectations`; retention keys carried operator-set values `run_log_retention_days=3`/`run_retention_days=60` forward, no `assessment_*` keys left). `.down.sql` counterparts ship but the down round-trip was **not** run against the live DB to avoid disrupting real data +- [x] 4.4 Manual smoke — **verified on live system**: created/listed assessments, ran one via `POST /api/runs {assessmentId}`, `runs` rows carry `assessment_id`, `GET /api/runs/{id}` returns `assessmentName`/`assessmentType` + `scenarios[].expectations`, run appears under `GET /api/assessments/{id}/runs`, duplicate name → 409, `/api/rules/coverage` 200 (jsonb `expectations` query OK) +- [x] 4.5 Confirm no `assertion`/`SavedScenario`/`assessment_run`/old-retention-key identifiers remain (grep), and YAML `scenarios:`/`expectations:` parsing is unchanged diff --git a/openspec/specs/assessment-retention/spec.md b/openspec/specs/assessment-retention/spec.md index 54bbaae..78444a8 100644 --- a/openspec/specs/assessment-retention/spec.md +++ b/openspec/specs/assessment-retention/spec.md @@ -14,83 +14,82 @@ page. ### Requirement: Retention Settings In AppConfig The system SHALL extend `AppConfig` with retention settings persisted in the `app_config` table and served by `GET /api/config` / `PUT /api/config`: -`assessment_log_retention_enabled` (bool), `assessment_log_retention_days` (int), -`assessment_retention_enabled` (bool), and `assessment_retention_days` (int). -Defaults SHALL be `assessment_log_retention_enabled = true`, -`assessment_log_retention_days = 7`, `assessment_retention_enabled = false`, -`assessment_retention_days = 30`, backfilled by a migration aligned with -`DefaultAppConfig()`. +`run_log_retention_enabled` (bool), `run_log_retention_days` (int), +`run_retention_enabled` (bool), and `run_retention_days` (int). These keys gate +the deletion of **runs** (executions) and their logs; they are renamed from the +former `assessment_log_retention_*` / `assessment_retention_*` keys (which the +new vocabulary makes misleading, since "assessment" now denotes the definition). +Defaults SHALL be `run_log_retention_enabled = true`, `run_log_retention_days = 7`, +`run_retention_enabled = false`, `run_retention_days = 30`, backfilled by a +migration that also carries forward any operator-set values from the previous key +names. #### Scenario: Defaults when unset - **WHEN** no `app_config` row exists for the retention keys -- **THEN** `GET /api/config` returns `assessment_log_retention_enabled = true`, `assessment_log_retention_days = 7`, `assessment_retention_enabled = false`, and `assessment_retention_days = 30` +- **THEN** `GET /api/config` returns `run_log_retention_enabled = true`, `run_log_retention_days = 7`, `run_retention_enabled = false`, and `run_retention_days = 30` + +#### Scenario: Prior values migrated +- **WHEN** the database held `assessment_retention_days = 14` under the old key name before this change +- **THEN** after migration `run_retention_days = 14` and the old key is removed #### Scenario: Admin updates retention -- **WHEN** a client sends `PUT /api/config` with `assessment_retention_enabled = true` and `assessment_retention_days = 14` +- **WHEN** a client sends `PUT /api/config` with `run_retention_enabled = true` and `run_retention_days = 14` - **THEN** both values are persisted and returned by a subsequent `GET /api/config` ### Requirement: Retention Settings Validation The system SHALL reject `PUT /api/config` with HTTP 400 when -`assessment_log_retention_days` or `assessment_retention_days` is less than 1, so -retention cannot be set to a value that deletes data immediately. +`run_log_retention_days` or `run_retention_days` is less than 1, so retention +cannot be set to a value that deletes data immediately. #### Scenario: Zero log retention rejected -- **WHEN** a client sends `PUT /api/config` with `assessment_log_retention_days = 0` +- **WHEN** a client sends `PUT /api/config` with `run_log_retention_days = 0` - **THEN** the response is HTTP 400 and the stored value is unchanged -#### Scenario: Zero assessment retention rejected -- **WHEN** a client sends `PUT /api/config` with `assessment_retention_days = 0` +#### Scenario: Zero run retention rejected +- **WHEN** a client sends `PUT /api/config` with `run_retention_days = 0` - **THEN** the response is HTTP 400 and the stored value is unchanged ### Requirement: Log-Retention Sweeper The system SHALL run a background sweeper that periodically scans -`/run-logs/` and deletes any `.jsonl` file whose last -modification time is older than `assessment_log_retention_days`. The sweeper SHALL run -once at startup and then on a fixed 1-hour interval, SHALL re-read `AppConfig` -each tick, and SHALL be a no-op when `assessment_log_retention_enabled = false`. -Deleting a log file SHALL NOT delete or modify the corresponding `runs` row. +`/run-logs/` and deletes any `.jsonl` file whose last modification +time is older than `run_log_retention_days`. The sweeper SHALL run once at +startup and then on a fixed 1-hour interval, SHALL re-read `AppConfig` each tick, +and SHALL be a no-op when `run_log_retention_enabled = false`. Deleting a log file +SHALL NOT delete or modify the corresponding `runs` row. #### Scenario: Old log swept -- **WHEN** a run's JSONL file was last modified longer ago than `assessment_log_retention_days` and log retention is enabled +- **WHEN** a run's JSONL file was last modified longer ago than `run_log_retention_days` and log retention is enabled - **THEN** the sweeper deletes the file and leaves the `runs` row intact -#### Scenario: Recent log retained -- **WHEN** a run's JSONL file is newer than `assessment_log_retention_days` -- **THEN** the sweeper leaves the file in place - #### Scenario: Log retention disabled -- **WHEN** `assessment_log_retention_enabled = false` +- **WHEN** `run_log_retention_enabled = false` - **THEN** the log sweeper deletes no files regardless of age ### Requirement: Assessment-Retention Sweeper The system SHALL run a background sweeper that periodically deletes whole runs -whose `created_at` is older than `assessment_retention_days`. For each expired -run the system SHALL delete the `runs` row (cascading to `scenario_results`), -the run's JSONL log file, and every collected `.ndjson` file referenced by that -run's `scenario_results.collected_log_path`. The sweeper SHALL run once at -startup and then on a fixed 1-hour interval, SHALL re-read `AppConfig` each -tick, SHALL be a no-op when `assessment_retention_enabled = false`, and SHALL -skip runs whose `status` is still `running`. - -#### Scenario: Old assessment purged -- **WHEN** a completed run's `created_at` is older than `assessment_retention_days` and assessment retention is enabled +whose `created_at` is older than `run_retention_days`. For each expired run the +system SHALL delete the `runs` row (cascading to `scenario_results`), the run's +JSONL log file, and every collected `.ndjson` file referenced by that run's +`scenario_results.collected_log_path`. The sweeper SHALL run once at startup and +then on a fixed 1-hour interval, SHALL re-read `AppConfig` each tick, SHALL be a +no-op when `run_retention_enabled = false`, and SHALL skip runs whose `status` is +still `running`. + +#### Scenario: Old run purged +- **WHEN** a completed run's `created_at` is older than `run_retention_days` and run retention is enabled - **THEN** the `runs` row, its `scenario_results`, its JSONL log file, and all of its collected `.ndjson` files are deleted -#### Scenario: Recent assessment retained -- **WHEN** a run's `created_at` is newer than `assessment_retention_days` -- **THEN** the sweeper leaves the run and all its artifacts in place +#### Scenario: Run retention disabled +- **WHEN** `run_retention_enabled = false` +- **THEN** the sweeper deletes no runs regardless of age -#### Scenario: Assessment retention disabled -- **WHEN** `assessment_retention_enabled = false` -- **THEN** the assessment sweeper deletes no runs regardless of age - -#### Scenario: Running assessment skipped -- **WHEN** a run older than `assessment_retention_days` still has `status = "running"` +#### Scenario: Running run skipped +- **WHEN** a run older than `run_retention_days` still has `status = "running"` - **THEN** the sweeper does not delete it ### Requirement: Swept Logs Surface As Empty -The system SHALL serve `GET /api/runs/{runId}/logs` with HTTP 200 and body `[]` -when the run's JSONL file has been swept by log retention, reusing the existing +The system SHALL serve `GET /api/runs/{id}/logs` with HTTP 200 and body `[]` when +the run's JSONL file has been swept by log retention, reusing the existing missing-file behavior so an expired-log run is indistinguishable from one that never logged. @@ -99,15 +98,11 @@ never logged. - **THEN** the response is HTTP 200 with body `[]` ### Requirement: Configure Retention From Assessments Page -The system SHALL present an "Assessment retention" control on the assessments -page that opens a dialog for editing `assessment_log_retention_enabled`, -`assessment_log_retention_days`, `assessment_retention_enabled`, and -`assessment_retention_days`, and SHALL persist changes via `PUT /api/config`. +The system SHALL present a "Run retention" control on the runs page that opens a +dialog for editing `run_log_retention_enabled`, `run_log_retention_days`, +`run_retention_enabled`, and `run_retention_days`, and SHALL persist changes via +`PUT /api/config`. #### Scenario: Open and save -- **WHEN** an admin opens the dialog, enables assessment retention with a 14-day window, and saves +- **WHEN** an admin opens the dialog, enables run retention with a 14-day window, and saves - **THEN** the new values are sent to `PUT /api/config` and reflected on the page after save - -#### Scenario: Invalid input surfaced -- **WHEN** an admin enters a retention period below 1 and saves -- **THEN** the API returns HTTP 400 and the dialog surfaces the error without losing the entered values diff --git a/openspec/specs/runs/spec.md b/openspec/specs/runs/spec.md index 7a6b802..1ade11e 100644 --- a/openspec/specs/runs/spec.md +++ b/openspec/specs/runs/spec.md @@ -12,13 +12,16 @@ enabled Elastic connector has `export_enabled = true`. ### Requirement: Run Record Resource The system SHALL persist runs in the `runs` table with fields: `id` (UUID), -`status`, `start_time`, `end_time` (nullable), `total`, `succeeded`, -`failed`, `schedule_id` (nullable FK), `schedule_name` (nullable), -`created_by`, `created_at`. +`status`, `start_time`, `end_time` (nullable), `total`, `succeeded`, `failed`, +`assessment_id` (nullable FK to `assessments`), `schedule_id` (nullable FK), +`schedule_name` (nullable), `created_by`, `created_at`. A **run** is one +execution of an **assessment** (the saved definition). The `runs` table name is +unchanged; only the definition FK is renamed from `scenario_id` to +`assessment_id`. #### Scenario: Run created on start -- **WHEN** a scenario run is started -- **THEN** a row is inserted with `status = "running"`, `start_time = now()`, `total` set to the number of scenarios to execute, and `succeeded = failed = 0` +- **WHEN** a run is started +- **THEN** a row is inserted into `runs` with `status = "running"`, `start_time = now()`, `total` set to the number of scenarios to execute, and `succeeded = failed = 0` ### Requirement: Run Has Two Lifecycle States The system SHALL only set run `status` to `"running"` (on creation) or @@ -40,36 +43,32 @@ populated during `running` (e.g., `"warmup"`, `"detonating"`, `"matching"`, The system SHALL populate the row's executor identity — `executor_name`, `executor_type`, `execution_id`, and `simulation_id` — as soon as detonation -returns these values, while the row is still `running`, rather than only at -completion. When a detonator does not produce a `simulation_id`, that field -SHALL remain empty without blocking the other identity fields. +returns these values, while the row is still `running`. When a detonator does +not produce a `simulation_id`, that field SHALL remain empty without blocking +the other identity fields. -While in the `matching` phase, the system SHALL persist per-assertion results -incrementally as the matcher resolves them, so the row's `assertions` reflect -the current passed/pending state before the scenario completes. An assertion +While in the `matching` phase, the system SHALL persist per-expectation results +incrementally as the matcher resolves them, so the row's `expectations` reflect +the current passed/pending state before the scenario completes. An expectation not yet matched SHALL be represented as not-yet-passed (no terminal failure is -recorded until completion). +recorded until completion). Each per-expectation outcome is an `ExpectationResult` +(formerly `AssertionResult`). These incremental writes SHALL NOT alter the terminal completion write: when a scenario completes, `status` becomes `completed`, `phase` is cleared, and the -final `is_success`, `assertions`, durations, and `discovered_alerts` are -written as they are today. +final `is_success`, `expectations`, durations, and `discovered_alerts` are written. #### Scenario: Phase transitions - **WHEN** a scenario enters the matching phase - **THEN** its `scenario_results` row has `status = "running"` and `phase = "matching"` -#### Scenario: Executor identity exposed during run -- **WHEN** a scenario has detonated and is in the `matching` phase -- **THEN** `GET /api/runs/{runId}` returns that scenario with `status = "running"` and non-empty `executor_name`, `executor_type`, and `execution_id` - -#### Scenario: Assertion progress exposed during matching -- **WHEN** a scenario expecting 3 assertions has matched 2 of them and is still matching -- **THEN** the scenario's `assertions` in `GET /api/runs/{runId}` show 2 passed and 1 not-yet-passed while `status = "running"` +#### Scenario: Expectation progress exposed during matching +- **WHEN** a scenario expecting 3 expectations has matched 2 of them and is still matching +- **THEN** the scenario's `expectations` in `GET /api/runs/{id}` show 2 passed and 1 not-yet-passed while `status = "running"` #### Scenario: Completion write unchanged -- **WHEN** a scenario finishes after its identity and partial assertions were written mid-run -- **THEN** the final row has `status = "completed"`, `phase = null`, and `is_success` plus the full `assertions` and `discovered_alerts` set, with no stale `running`-phase values +- **WHEN** a scenario finishes after its identity and partial expectations were written mid-run +- **THEN** the final row has `status = "completed"`, `phase = null`, and `is_success` plus the full `expectations` and `discovered_alerts` set ### Requirement: Atomic Counter Increments The system SHALL update run counters with atomic SQL increments @@ -81,54 +80,49 @@ replacement, so concurrent scenario completions do not lose updates. - **THEN** the final `succeeded` value is exactly 3 ### Requirement: Get Run Returns Composite Object -The system SHALL respond to `GET /api/runs/{runId}` with the envelope -`{run, scenarios}` where `run` includes a LEFT JOIN of -`saved_scenarios.name` and `.type`, and `scenarios` is the list of -`scenario_results` rows for the run. +The system SHALL respond to `GET /api/runs/{id}` with the envelope +`{run, scenarios}` where `run` includes a LEFT JOIN of `assessments.name` and +`.type`, and `scenarios` is the list of `scenario_results` rows for the run. -#### Scenario: Saved scenario still exists -- **WHEN** a client GETs a run whose `scenario_id` references an existing scenario -- **THEN** `run.scenarioName` and `run.scenarioType` are populated +#### Scenario: Assessment still exists +- **WHEN** a client GETs a run whose `assessment_id` references an existing assessment +- **THEN** `run.assessmentName` and `run.assessmentType` are populated -#### Scenario: Saved scenario deleted -- **WHEN** the run's source scenario has been deleted -- **THEN** `run.scenarioName` and `run.scenarioType` are null but the run is still returned 200 +#### Scenario: Assessment deleted +- **WHEN** the run's source assessment has been deleted +- **THEN** `run.assessmentName` and `run.assessmentType` are null but the run is still returned 200 ### Requirement: List Runs Unpaginated -The system SHALL return all runs from `GET /api/runs` ordered by -`created_at` descending. **Note:** there is no pagination today; flagged -as a known scaling gap. +The system SHALL return all runs from `GET /api/runs` ordered by `created_at` +descending, and SHALL serve the runs of a single assessment via the nested +collection `GET /api/assessments/{id}/runs` in the same order (the nested handler +applies the existing `ListRunsFilters` with `assessment_id` set). Runs are +created only at `POST /api/runs`; the nested route is read-only. **Note:** there +is no pagination today; flagged as a known scaling gap. #### Scenario: List ordering - **WHEN** a client requests `/api/runs` - **THEN** the most recent run is first +#### Scenario: Runs scoped to one assessment +- **WHEN** a client requests `/api/assessments/{id}/runs` +- **THEN** only runs whose `assessment_id` equals `{id}` are returned, most recent first + ### Requirement: Delete Run Cascades to Results and Log File -The system SHALL delete the `runs` row on `DELETE /api/runs/{runId}`, -cascade-delete all `scenario_results` rows via the FK, and best-effort -remove the run's on-disk artifacts: the run's JSONL log file, every collected -`.ndjson` file referenced by the run's `scenario_results.collected_log_path`, -and, for each scenario result with a non-empty `execution_id`, the run's -Terraform working directory at `/terraform//`. The -system SHALL skip Terraform-directory removal for any `execution_id` that does -not resolve to a direct child of `/terraform/` — including blank/whitespace -ids, ids containing a path separator, and `.`/`..` — so cleanup can never escape or -remove the `/terraform/` base directory. Failure to remove any on-disk -artifact (log file, collected `.ndjson`, or Terraform directory) SHALL be logged +The system SHALL delete the `runs` row on `DELETE /api/runs/{id}`, cascade-delete +all `scenario_results` rows via the FK, and best-effort remove the run's on-disk +artifacts: the run's JSONL log file, every collected `.ndjson` file referenced by +the run's `scenario_results.collected_log_path`, and, for each scenario result +with a non-empty `execution_id`, the run's Terraform working directory at +`/terraform//`. The system SHALL skip Terraform-directory +removal for any `execution_id` that does not resolve to a direct child of +`/terraform/`. Failure to remove any on-disk artifact SHALL be logged and SHALL NOT fail the request. #### Scenario: Successful delete - **WHEN** a client deletes a run with 3 results - **THEN** the `runs` row, all 3 `scenario_results` rows, and the JSONL log file are removed -#### Scenario: Terraform directories removed -- **WHEN** a client deletes a run whose scenario results have execution IDs `E1` and `E2` -- **THEN** the directories `/terraform/E1/` and `/terraform/E2/` are removed along with the row and log file - -#### Scenario: Missing Terraform directory does not fail delete -- **WHEN** a run is deleted but its `/terraform//` directory is already gone -- **THEN** the delete succeeds and the missing directory is ignored - #### Scenario: Unsafe execution id skipped - **WHEN** a scenario result has a blank `execution_id` or one containing a path separator - **THEN** no Terraform directory is removed for that result and the `/terraform/` base directory is left intact @@ -143,9 +137,9 @@ The system SHALL write run log entries as one JSON object per line to - **THEN** a single line is appended to the run's JSONL file containing the timestamp, level, message, and structured fields ### Requirement: Get Run Logs -The system SHALL respond to `GET /api/runs/{runId}/logs` by reading the -JSONL file from disk and returning the array of entries. A missing file -SHALL produce an empty array `[]`, not an error. +The system SHALL respond to `GET /api/runs/{id}/logs` by reading the JSONL file +from disk and returning the array of entries. A missing file SHALL produce an +empty array `[]`, not an error. #### Scenario: Missing log file - **WHEN** a run's log file does not exist on disk @@ -220,10 +214,31 @@ with `` from the connector config (default `"asp.results"`). - **THEN** no export occurs and the run completion is unaffected ### Requirement: Schedule Attribution on Triggered Runs -The system SHALL set `schedule_id` and `schedule_name` on runs created by -the in-process scheduler, and `created_by = "system"`. Manual runs SHALL -have `schedule_id = null`. +The system SHALL set `schedule_id` and `schedule_name` on runs created by the +in-process scheduler, and `created_by = "system"`. Manual runs SHALL have +`schedule_id = null`. #### Scenario: Scheduled run -- **WHEN** the scheduler fires for a saved scenario +- **WHEN** the scheduler fires for an assessment - **THEN** the new `runs` row has `schedule_id` set and `created_by = "system"` + +### Requirement: Single-Scenario Runner With Consolidated Result +The runner SHALL execute exactly one scenario per `Runner` instance and return a +single consolidated `ScenarioResult` value rather than mutating the scenario +input to carry outputs. Fan-out across multiple scenarios SHALL be the sole +responsibility of the parallel executor (worker pool). There SHALL be exactly one +in-memory `ScenarioResult` type shared across the runner, executor, and web +layers (replacing the two former near-identical in-memory types +`runner.ScenarioResult` and `results.ScenarioRunResult`), and one `RunResult` +aggregate (formerly `SimrunRunResult`). The persistence layer keeps its own +column-shaped row DTO (`db.ScenarioResult`, with `json.RawMessage` fields and +DB-only columns) so `db` stays decoupled from the domain packages; the single +marshal boundary that projects the in-memory result onto the row DTO is retained. + +#### Scenario: Runner returns a result +- **WHEN** the runner finishes executing a scenario +- **THEN** it returns a populated `ScenarioResult` and the scenario input struct is not used as the output carrier + +#### Scenario: Fan-out lives in the executor +- **WHEN** a run contains N scenarios +- **THEN** the parallel executor schedules N single-scenario runner executions and there is no second multi-scenario loop inside `Runner` diff --git a/openspec/specs/scenarios/spec.md b/openspec/specs/scenarios/spec.md index fc07a25..d755d13 100644 --- a/openspec/specs/scenarios/spec.md +++ b/openspec/specs/scenarios/spec.md @@ -11,17 +11,26 @@ over the WebSocket. ## Requirements -### Requirement: Saved Scenario Resource -The system SHALL persist saved scenarios in `saved_scenarios` with fields -`id` (UUID), `name`, `yaml` (raw text), `type` (`standard` | `explore` | `collect`, -default `standard`), `created_by`, `updated_by`, `created_at`, `updated_at`. +### Requirement: Assessment Resource +The system SHALL persist saved assessment definitions in the `assessments` table +with fields `id` (UUID), `name` (UNIQUE), `yaml` (raw text), `type` +(`standard` | `explore` | `collect`, default `standard`), `created_by`, +`updated_by`, `created_at`, `updated_at`. An **assessment** is the saved +definition; its `yaml` body contains a `scenarios:` array of individual scenarios +(the per-case vocabulary and YAML schema are unchanged). Because an assessment +serializes to `.yaml`, `name` is unique and serves as a human-addressable +slug. #### Scenario: Create with default type -- **WHEN** a client posts a scenario without specifying `type` +- **WHEN** a client posts an assessment without specifying `type` - **THEN** the inserted row has `type = "standard"` +#### Scenario: Duplicate name rejected +- **WHEN** a client posts an assessment whose `name` already exists +- **THEN** the response is HTTP 409 and no row is inserted + #### Scenario: Invalid type rejected -- **WHEN** a client posts a scenario with `type: "fast"` +- **WHEN** a client posts an assessment with `type: "fast"` - **THEN** the response is HTTP 400 with `"type must be 'standard', 'explore', or 'collect'"` ### Requirement: YAML Stored Verbatim @@ -46,65 +55,27 @@ and return either `{valid: true, scenarios: [...]}` or `{valid: false, error}`. - **WHEN** a client posts YAML that the parser rejects (e.g., unknown pack) - **THEN** the response is `{valid: false, error: ""}` -### Requirement: List Saved Scenarios -The system SHALL serve `GET /api/scenarios` as a paginated, filterable -list ordered by `updated_at DESC`. The response SHALL be a JSON object -`{scenarios, total, page, perPage}` where `scenarios` is the page slice -(possibly empty array, never `null`) and `total` is the row count after -filters but before `LIMIT/OFFSET`. - -Query parameters: -- `page` (integer, default `1`, must be `>= 1`). -- `per_page` (integer, default `50`, clamped to `[1, 100]`). -- `name` (string, optional) — case-insensitive substring match against - `saved_scenarios.name` (`ILIKE %name%`). -- `type` (string, repeatable) — restricts `saved_scenarios.type` to - the listed values. Allowed values: `standard`, `explore`, `collect`. - An unrecognized value SHALL return HTTP 400. -- `since` (Go duration string, optional, e.g. `24h`, `168h`) — - restricts results to `updated_at >= now() - since`. A malformed or - non-positive duration SHALL return HTTP 400. +### Requirement: List Assessments +The system SHALL serve `GET /api/assessments` as a paginated, filterable list +ordered by `updated_at DESC`, returning `{assessments, total, page, perPage}` +where `assessments` is the page slice (possibly empty, never `null`). The system +SHALL additionally serve a single assessment by its unique name via +`GET /api/assessments/by-name/{name}`, returning its JSON (which already includes the +raw `yaml` field). Query parameters (`page`, `per_page`, `name`, `type`, `since`) +behave as before against `assessments.name`/`.type`. #### Scenario: Most-recently-updated first -- **WHEN** a client requests `/api/scenarios` with no parameters -- **THEN** the response is HTTP 200 with `{scenarios, total, page: 1, perPage: 50}` and `scenarios` is ordered with the most recently updated scenario first - -#### Scenario: Pagination slice -- **WHEN** a client requests `/api/scenarios?page=2&per_page=25` and there are 60 saved scenarios matching no filters -- **THEN** `total = 60`, `page = 2`, `perPage = 25`, and `scenarios.length` is 25 (rows 26–50 in `updated_at DESC` order) - -#### Scenario: Empty page beyond range -- **WHEN** a client requests `page=99` on a table with 10 rows -- **THEN** the response is HTTP 200 with `scenarios: []` and `total: 10` +- **WHEN** a client requests `/api/assessments` with no parameters +- **THEN** the response is HTTP 200 with `{assessments, total, page: 1, perPage: 50}` ordered with the most recently updated assessment first -#### Scenario: Name substring filter -- **WHEN** a client requests `/api/scenarios?name=login` -- **THEN** only scenarios whose `name` contains `"login"` (case-insensitive) are returned, and `total` reflects the filtered count - -#### Scenario: Multi-type filter -- **WHEN** a client requests `/api/scenarios?type=standard&type=explore` -- **THEN** the response includes only scenarios whose `type` is `standard` or `explore` +#### Scenario: Fetch by name +- **WHEN** a client requests `/api/assessments/by-name/aws-privesc` +- **THEN** the response is the JSON for the assessment named `aws-privesc`, including its raw `yaml` field #### Scenario: Invalid type rejected -- **WHEN** a client requests `/api/scenarios?type=bogus` +- **WHEN** a client requests `/api/assessments?type=bogus` - **THEN** the response is HTTP 400 and no rows are returned -#### Scenario: Since window filter -- **WHEN** a client requests `/api/scenarios?since=24h` -- **THEN** the response includes only scenarios with `updated_at >= now() - 24h` - -#### Scenario: Malformed since rejected -- **WHEN** a client requests `/api/scenarios?since=abc` -- **THEN** the response is HTTP 400 - -#### Scenario: Combined filters -- **WHEN** a client requests `/api/scenarios?name=ssh&type=explore&since=168h&page=1&per_page=25` -- **THEN** results are scenarios whose name ILIKE `%ssh%` AND type is `explore` AND `updated_at` is within the past week, paginated to the first 25 in `updated_at DESC` order, with `total` reflecting all matches - -#### Scenario: per_page clamped to maximum -- **WHEN** a client requests `/api/scenarios?per_page=500` -- **THEN** `perPage` in the response is `100` and at most 100 rows are returned - ### Requirement: Update Without Re-Validation The system SHALL accept `PUT /api/scenarios/{id}` updates that replace `name`, `type`, and `yaml` atomically. The new YAML SHALL NOT be re-linted @@ -115,7 +86,7 @@ run time. - **WHEN** a client PUTs a scenario with malformed YAML - **THEN** the response is 204 (or success) and the row is updated -### Requirement: Delete Scenario Cascades and Reloads Scheduler +### Requirement: Delete Assessment Cascades and Reloads Scheduler The system SHALL delete the scenario row on `DELETE /api/scenarios/{id}`, which cascades to remove any associated schedule via the FK constraint. The handler SHALL trigger an in-process scheduler reload after the DB write @@ -126,19 +97,22 @@ to evict orphaned cron entries. - **THEN** the schedule row is removed and the scheduler no longer fires for that scenario ### Requirement: Run Endpoint Is Asynchronous -The system SHALL accept `POST /api/scenarios/run` with a body identifying -either a saved scenario by ID or inline YAML, perform pre-flight setup -(load AppConfig, packs, scenarios; build run env; parse YAML), and start the -run in a detached goroutine. The response SHALL return HTTP 202 with -`{runId: ""}` once the `runs` row is inserted. - -#### Scenario: Successful run start -- **WHEN** a client posts a valid run request for an existing scenario +The system SHALL start a run via `POST /api/runs` with `{assessmentId}` in the +body, performing pre-flight setup (load AppConfig, packs, assessment; build run +env; parse YAML) and starting the run in a detached goroutine. The response SHALL +return HTTP 202 with `{runId: ""}` once the `runs` row is inserted. A Run is +a top-level resource: it is created at `POST /api/runs` and read/deleted at +`/api/runs/{id}`. (This replaces `POST /api/scenarios/run` body `{scenarioId}` — +the same shape with a renamed path and field. Runs always reference a saved +assessment; there is no inline-YAML run path today.) + +#### Scenario: Run a saved assessment +- **WHEN** a client posts `{assessmentId}` to `/api/runs` for an existing assessment - **THEN** the response is HTTP 202 with a `runId` -- **AND** a new `runs` row exists with `status = "running"` +- **AND** a new `runs` row exists with `status = "running"` and `assessment_id` set to the posted id #### Scenario: Pre-flight failure -- **WHEN** the request references a saved scenario that does not exist +- **WHEN** the request references a saved assessment that does not exist - **THEN** the response is HTTP 400 and no `runs` row is created ### Requirement: Run Environment Construction diff --git a/openspec/specs/schedules/spec.md b/openspec/specs/schedules/spec.md index f792b11..fe97b1f 100644 --- a/openspec/specs/schedules/spec.md +++ b/openspec/specs/schedules/spec.md @@ -10,16 +10,16 @@ and use standard Unix cron expressions. ### Requirement: Schedule Resource The system SHALL persist schedules in the `schedules` table with fields: -`id` (UUID), `scenario_id` (UUID, unique, FK to `saved_scenarios.id`, +`id` (UUID), `assessment_id` (UUID, unique, FK to `assessments.id`, `ON DELETE CASCADE`), `cron_expression`, `enabled` (default true), `parallelism`, `last_run_at` (nullable), `created_at`, `updated_at`. -#### Scenario: One schedule per scenario -- **WHEN** a scenario already has a schedule and a client posts a second `POST /api/schedules` for the same `scenario_id` -- **THEN** the response is HTTP 409 with `"schedule already exists for this scenario"` +#### Scenario: One schedule per assessment +- **WHEN** an assessment already has a schedule and a client posts a second `POST /api/schedules` for the same `assessmentId` +- **THEN** the response is HTTP 409 with `"schedule already exists for this assessment"` -#### Scenario: Cascade on scenario delete -- **WHEN** a saved scenario is deleted +#### Scenario: Cascade on assessment delete +- **WHEN** an assessment is deleted - **THEN** its schedule row is removed atomically by the database ### Requirement: Cron Expression Format @@ -106,16 +106,16 @@ implementation. - **WHEN** a schedule is created with `parallelism: 0` - **THEN** the persisted row has `parallelism = 10` -### Requirement: Get Schedule by Scenario -`GET /api/scenarios/{scenarioId}/schedule` SHALL return the schedule row for -the given scenario, or HTTP 404 if no schedule exists. +### Requirement: Get Schedule by Assessment +`GET /api/assessments/{id}/schedule` SHALL return the schedule row for +the given assessment, or HTTP 404 if no schedule exists. #### Scenario: Schedule exists -- **WHEN** scenario `S` has a schedule and a client requests `/api/scenarios//schedule` +- **WHEN** assessment `A` has a schedule and a client requests `/api/assessments//schedule` - **THEN** the response is 200 with the schedule object #### Scenario: No schedule -- **WHEN** scenario `S` has no schedule +- **WHEN** assessment `A` has no schedule - **THEN** the response is HTTP 404 ### Requirement: Listing Schedules diff --git a/web/frontend/src/lib/api/client.ts b/web/frontend/src/lib/api/client.ts index 1ccedd9..b9142c8 100644 --- a/web/frontend/src/lib/api/client.ts +++ b/web/frontend/src/lib/api/client.ts @@ -1,12 +1,12 @@ import type { Run, - SavedScenario, + Assessment, Pack, PackManifest, LintResponse, RunResponse, RunListResponse, - ScenarioListResponse, + AssessmentListResponse, AppConfig, VersionInfo, SecretGroup, @@ -56,25 +56,25 @@ async function request(path: string, options?: RequestInit): Promise { return res.json(); } -// Scenarios - lint & run -export async function lintScenario(yaml: string): Promise { - return request('/scenarios/lint', { method: 'POST', body: JSON.stringify({ yaml }) }); +// Assessments - lint & run +export async function lintAssessment(yaml: string): Promise { + return request('/assessments/lint', { method: 'POST', body: JSON.stringify({ yaml }) }); } -export async function runScenario( - scenarioId: string, +export async function runAssessment( + assessmentId: string, parallelism?: number, exploreMode?: boolean, cleanupAlerts?: boolean, timeout?: string ): Promise { - return request('/scenarios/run', { + return request('/runs', { method: 'POST', - body: JSON.stringify({ scenarioId, parallelism, exploreMode, cleanupAlerts, timeout }) + body: JSON.stringify({ assessmentId, parallelism, exploreMode, cleanupAlerts, timeout }) }); } -// Saved scenarios CRUD +// Assessments CRUD export interface ScenarioFilters { name?: string; types?: string[]; @@ -82,43 +82,50 @@ export interface ScenarioFilters { since?: string; } -export async function listScenarios( +export async function listAssessments( page = 1, perPage = 50, filters: ScenarioFilters = {} -): Promise { +): Promise { const qs = new URLSearchParams(); qs.set('page', String(page)); qs.set('per_page', String(perPage)); if (filters.name) qs.set('name', filters.name); if (filters.since) qs.set('since', filters.since); for (const t of filters.types ?? []) qs.append('type', t); - return request(`/scenarios?${qs.toString()}`); + return request(`/assessments?${qs.toString()}`); } -export async function getScenario(id: string): Promise { - return request(`/scenarios/${id}`); +export async function getAssessment(id: string): Promise { + return request(`/assessments/${id}`); } -export async function saveScenario( +export async function getAssessmentByName(name: string): Promise { + return request(`/assessments/by-name/${encodeURIComponent(name)}`); +} + +export async function saveAssessment( name: string, yaml: string, type?: string -): Promise { - return request('/scenarios', { method: 'POST', body: JSON.stringify({ name, type, yaml }) }); +): Promise { + return request('/assessments', { method: 'POST', body: JSON.stringify({ name, type, yaml }) }); } -export async function updateScenario( +export async function updateAssessment( id: string, name: string, yaml: string, type?: string ): Promise { - return request(`/scenarios/${id}`, { method: 'PUT', body: JSON.stringify({ name, type, yaml }) }); + return request(`/assessments/${id}`, { + method: 'PUT', + body: JSON.stringify({ name, type, yaml }) + }); } -export async function deleteScenario(id: string): Promise { - return request(`/scenarios/${id}`, { method: 'DELETE' }); +export async function deleteAssessment(id: string): Promise { + return request(`/assessments/${id}`, { method: 'DELETE' }); } // Runs @@ -127,7 +134,7 @@ export interface RunFilters { types?: string[]; // Go-style duration: "24h", "168h", etc. Empty/undefined = no constraint. since?: string; - scenarioId?: string; + assessmentId?: string; } export async function listRuns( @@ -140,11 +147,22 @@ export async function listRuns( qs.set('per_page', String(perPage)); if (filters.name) qs.set('name', filters.name); if (filters.since) qs.set('since', filters.since); - if (filters.scenarioId) qs.set('scenario_id', filters.scenarioId); + if (filters.assessmentId) qs.set('assessment_id', filters.assessmentId); for (const t of filters.types ?? []) qs.append('type', t); return request(`/runs?${qs.toString()}`); } +export async function listAssessmentRuns( + assessmentId: string, + page = 1, + perPage = 50 +): Promise { + const qs = new URLSearchParams(); + qs.set('page', String(page)); + qs.set('per_page', String(perPage)); + return request(`/assessments/${assessmentId}/runs?${qs.toString()}`); +} + export async function getRun(runId: string): Promise { const data = await request<{ run: Run; scenarios: Run['scenarioResults'] }>(`/runs/${runId}`); return { ...data.run, scenarioResults: data.scenarios ?? [] }; @@ -278,23 +296,23 @@ export async function listSchedules(): Promise { return request('/schedules'); } -export async function getScheduleByScenario(scenarioId: string): Promise { +export async function getScheduleByAssessment(assessmentId: string): Promise { try { - return await request(`/scenarios/${scenarioId}/schedule`); + return await request(`/assessments/${assessmentId}/schedule`); } catch { return null; } } export async function createSchedule( - scenarioId: string, + assessmentId: string, cronExpression: string, enabled: boolean, parallelism: number ): Promise { return request('/schedules', { method: 'POST', - body: JSON.stringify({ scenarioId, cronExpression, enabled, parallelism }) + body: JSON.stringify({ assessmentId, cronExpression, enabled, parallelism }) }); } diff --git a/web/frontend/src/lib/components/LiveLog.svelte b/web/frontend/src/lib/components/LiveLog.svelte deleted file mode 100644 index 51b83f5..0000000 --- a/web/frontend/src/lib/components/LiveLog.svelte +++ /dev/null @@ -1,44 +0,0 @@ - - -
- {#each logs as log} -
- [{log.level.toUpperCase().padEnd(5)}] - [{log.scenarioName}] - {log.message} -
- {/each} - {#if logs.length === 0} -
Waiting for logs...
- {/if} -
diff --git a/web/frontend/src/lib/components/NewAssessmentDialog.svelte b/web/frontend/src/lib/components/NewAssessmentDialog.svelte index 1a80aaa..26e92be 100644 --- a/web/frontend/src/lib/components/NewAssessmentDialog.svelte +++ b/web/frontend/src/lib/components/NewAssessmentDialog.svelte @@ -8,8 +8,8 @@ import { Input } from '$lib/components/ui/input/index.js'; import { Label } from '$lib/components/ui/label/index.js'; import { Badge } from '$lib/components/ui/badge/index.js'; - import { listScenarios, runScenario } from '$lib/api/client'; - import type { SavedScenario } from '$lib/types'; + import { listAssessments, runAssessment } from '$lib/api/client'; + import type { Assessment } from '$lib/types'; import { scenarioTypeVariant } from '$lib/utils/format'; import FileIcon from '@lucide/svelte/icons/file'; @@ -21,35 +21,35 @@ let loading = $state(true); let error = $state(''); - let selectedScenarioId = $state(''); + let selectedAssessmentId = $state(''); let parallelism = $state(5); let timeout = $state('10m'); let running = $state(false); - let dialogScenarios = $state([]); + let dialogAssessments = $state([]); - let selectedScenarioType = $derived( - dialogScenarios.find((s) => s.id === selectedScenarioId)?.type || 'standard' + let selectedAssessmentType = $derived( + dialogAssessments.find((s) => s.id === selectedAssessmentId)?.type || 'standard' ); let needsTimeout = $derived( - selectedScenarioType === 'explore' || selectedScenarioType === 'collect' + selectedAssessmentType === 'explore' || selectedAssessmentType === 'collect' ); $effect(() => { if (open) { loading = true; error = ''; - selectedScenarioId = ''; + selectedAssessmentId = ''; running = false; timeout = '10m'; - // Fetch first 100 most-recently-updated. A picker for >100 scenarios + // Fetch first 100 most-recently-updated. A picker for >100 assessments // needs a real search affordance — out of scope here. - listScenarios(1, 100, {}) + listAssessments(1, 100, {}) .then((page) => { - dialogScenarios = page.scenarios; + dialogAssessments = page.assessments; }) .catch((e) => { - error = e instanceof Error ? e.message : 'Failed to load scenarios'; + error = e instanceof Error ? e.message : 'Failed to load assessments'; }) .finally(() => { loading = false; @@ -58,7 +58,7 @@ }); async function handleRun() { - if (!selectedScenarioId) return; + if (!selectedAssessmentId) return; if (needsTimeout && timeout && !/^\d+[smh]$/.test(timeout)) { error = 'Invalid timeout format. Use a duration like 10m, 30s, or 1h.'; return; @@ -66,28 +66,28 @@ running = true; error = ''; try { - const isExplore = selectedScenarioType === 'explore'; - const result = await runScenario( - selectedScenarioId, + const isExplore = selectedAssessmentType === 'explore'; + const result = await runAssessment( + selectedAssessmentId, parallelism, isExplore, false, needsTimeout ? timeout : undefined ); open = false; - goto(`/assessments/${result.runId}`); + goto(`/runs/${result.runId}`); } catch (e) { - error = e instanceof Error ? e.message : 'Failed to start assessment'; + error = e instanceof Error ? e.message : 'Failed to start run'; running = false; } } - + New Assessment - Select a saved scenario and start a new assessment. + Select an assessment and start a new run. {#if error} @@ -97,24 +97,24 @@ {/if} {#if loading} -

Loading scenarios...

- {:else if dialogScenarios.length === 0} +

Loading assessments...

+ {:else if dialogAssessments.length === 0} - No saved scenarios + No saved assessments Create a scenario first, then come back to start an assessment.Create an assessment first, then come back to start a run. Go to Assessments @@ -130,28 +130,28 @@ - {#each dialogScenarios as scenario} + {#each dialogAssessments as assessment} (selectedScenarioId = scenario.id)} + class="cursor-pointer {selectedAssessmentId === assessment.id ? 'bg-muted' : ''}" + onclick={() => (selectedAssessmentId = assessment.id)} > (selectedScenarioId = scenario.id)} + name="assessment" + value={assessment.id} + checked={selectedAssessmentId === assessment.id} + onchange={() => (selectedAssessmentId = assessment.id)} class="h-4 w-4" /> - {scenario.name} + {assessment.name} - {scenario.type || 'standard'}{assessment.type || 'standard'} - {new Date(scenario.updatedAt).toLocaleDateString()} + {new Date(assessment.updatedAt).toLocaleDateString()} {/each} @@ -183,7 +183,7 @@ {#if needsTimeout}

- {#if selectedScenarioType === 'explore'} + {#if selectedAssessmentType === 'explore'} Explore mode: searches all alerts for indicators instead of matching specific rules. Waits for the full timeout to discover all triggered alerts. {:else} @@ -196,7 +196,7 @@

-
diff --git a/web/frontend/src/lib/components/NewScenarioDialog.svelte b/web/frontend/src/lib/components/NewAssessmentTypeDialog.svelte similarity index 91% rename from web/frontend/src/lib/components/NewScenarioDialog.svelte rename to web/frontend/src/lib/components/NewAssessmentTypeDialog.svelte index 1aaff9e..1700e04 100644 --- a/web/frontend/src/lib/components/NewScenarioDialog.svelte +++ b/web/frontend/src/lib/components/NewAssessmentTypeDialog.svelte @@ -44,16 +44,16 @@ function pick(t: ScenarioType) { open = false; - goto(`/scenarios/new?type=${t}`); + goto(`/assessments/new?type=${t}`); } - + - New Scenario + New Assessment - Choose a scenario type. The type is fixed once created. + Choose an assessment type. The type is fixed once created.
diff --git a/web/frontend/src/lib/components/PackParametersDialog.svelte b/web/frontend/src/lib/components/PackParametersDialog.svelte index d941622..94acc0b 100644 --- a/web/frontend/src/lib/components/PackParametersDialog.svelte +++ b/web/frontend/src/lib/components/PackParametersDialog.svelte @@ -113,7 +113,7 @@ - + Pack Parameters diff --git a/web/frontend/src/lib/components/PackSimulationsSheet.svelte b/web/frontend/src/lib/components/PackSimulationsSheet.svelte index 309d7dc..67b89b8 100644 --- a/web/frontend/src/lib/components/PackSimulationsSheet.svelte +++ b/web/frontend/src/lib/components/PackSimulationsSheet.svelte @@ -124,7 +124,7 @@ - + {#if manifest}
{#if selectedDetail?.kind === 'simulation'} @@ -189,7 +189,7 @@
- +
@@ -198,7 +198,7 @@ > Description -
+

{selectedSimulation.description || 'No description available.'}

@@ -300,7 +300,7 @@ Parameters Schema
{JSON.stringify(
+										class="rounded-md border border-border bg-muted/30 p-3 text-xs font-mono overflow-x-auto whitespace-pre-wrap max-h-64 overflow-y-auto">{JSON.stringify(
 											selectedSimulation.params_schema,
 											null,
 											2
@@ -369,7 +369,7 @@
 						
- +
@@ -378,7 +378,7 @@ > Description -
+

{selectedTemplate.description || 'No description available.'}

@@ -455,7 +455,7 @@
- +
{#if filteredSimulations.length === 0 && filteredTemplates.length === 0}

diff --git a/web/frontend/src/lib/components/RecentScenariosSection.svelte b/web/frontend/src/lib/components/RecentAssessmentsSection.svelte similarity index 62% rename from web/frontend/src/lib/components/RecentScenariosSection.svelte rename to web/frontend/src/lib/components/RecentAssessmentsSection.svelte index 02f90d4..c0ab044 100644 --- a/web/frontend/src/lib/components/RecentScenariosSection.svelte +++ b/web/frontend/src/lib/components/RecentAssessmentsSection.svelte @@ -1,46 +1,46 @@ - Recent Scenarios + Recent Assessments - {#if scenarios.length === 0} -

No scenarios yet.

+ {#if assessments.length === 0} +

No assessments yet.

{:else}
- {#each scenarios as scenario} + {#each assessments as assessment}
- {scenario.name} + {assessment.name} - {scenario.type || 'standard'} + {assessment.type || 'standard'}
- {#if scenario.createdBy && scenario.createdBy !== 'anonymous'} + {#if assessment.createdBy && assessment.createdBy !== 'anonymous'} - {formatUserEmail(scenario.createdBy)} + {formatUserEmail(assessment.createdBy)} - {scenario.createdBy} + {assessment.createdBy} {/if} - {new Date(scenario.createdAt).toLocaleDateString()} + {new Date(assessment.createdAt).toLocaleDateString()}
diff --git a/web/frontend/src/lib/components/RecentRunsTable.svelte b/web/frontend/src/lib/components/RecentRunsTable.svelte index 03a95ed..5b3ec6d 100644 --- a/web/frontend/src/lib/components/RecentRunsTable.svelte +++ b/web/frontend/src/lib/components/RecentRunsTable.svelte @@ -10,12 +10,12 @@ - Recent Assessments + Recent Runs {#if runs.length === 0}

- No assessments yet. Start an assessment from the Scenarios page. + No runs yet. Start a run from the Assessments page.

{:else}
@@ -23,7 +23,7 @@ diff --git a/web/frontend/src/lib/components/RetentionDialog.svelte b/web/frontend/src/lib/components/RetentionDialog.svelte index d16b5e1..559ed01 100644 --- a/web/frontend/src/lib/components/RetentionDialog.svelte +++ b/web/frontend/src/lib/components/RetentionDialog.svelte @@ -28,17 +28,17 @@ // Form state, seeded from current config each time the dialog opens. let logEnabled = $state(true); let logDays = $state(7); - let assessmentEnabled = $state(false); - let assessmentDays = $state(30); + let runEnabled = $state(false); + let runDays = $state(30); let saving = $state(false); let error = $state(''); $effect(() => { if (open) { - logEnabled = boolOf(config['assessment_log_retention_enabled'], true); - logDays = numOf(config['assessment_log_retention_days'], 7); - assessmentEnabled = boolOf(config['assessment_retention_enabled'], false); - assessmentDays = numOf(config['assessment_retention_days'], 30); + logEnabled = boolOf(config['run_log_retention_enabled'], true); + logDays = numOf(config['run_log_retention_days'], 7); + runEnabled = boolOf(config['run_retention_enabled'], false); + runDays = numOf(config['run_retention_days'], 30); error = ''; } }); @@ -47,11 +47,11 @@ error = ''; const logDaysInt = Math.trunc(Number(logDays)); - const assessmentDaysInt = Math.trunc(Number(assessmentDays)); + const runDaysInt = Math.trunc(Number(runDays)); // Validate up front to avoid a partial save (keys are PUT one at a time). // Only check a section's days when its toggle is on. - if ((logEnabled && logDaysInt < 1) || (assessmentEnabled && assessmentDaysInt < 1)) { + if ((logEnabled && logDaysInt < 1) || (runEnabled && runDaysInt < 1)) { error = 'Retention periods must be at least 1 day.'; return; } @@ -63,14 +63,12 @@ // currently stored days value so an empty/invalid greyed-out field is // never PUT (the backend would reject it with HTTP 400). const next: Record = { - assessment_log_retention_enabled: logEnabled, - assessment_log_retention_days: logEnabled + run_log_retention_enabled: logEnabled, + run_log_retention_days: logEnabled ? logDaysInt - : numOf(config['assessment_log_retention_days'], 7), - assessment_retention_enabled: assessmentEnabled, - assessment_retention_days: assessmentEnabled - ? assessmentDaysInt - : numOf(config['assessment_retention_days'], 30) + : numOf(config['run_log_retention_days'], 7), + run_retention_enabled: runEnabled, + run_retention_days: runEnabled ? runDaysInt : numOf(config['run_retention_days'], 30) }; const changed = Object.entries(next).filter(([key, value]) => value !== config[key]); @@ -92,12 +90,11 @@ - + - Assessment retention + Run retention - Control how long run logs and whole assessments are kept before they are deleted - automatically. + Control how long run logs and whole runs are kept before they are deleted automatically. @@ -118,29 +115,29 @@ disabled={!logEnabled} />

- Verbose per-run logs older than this are removed; the assessment record is kept. + Verbose per-run logs older than this are removed; the run record is kept.

- - + +
- +

- Whole assessments older than this — results and collected logs included — are - permanently deleted. + Whole runs older than this — results and collected logs included — are permanently + deleted.

diff --git a/web/frontend/src/lib/components/RunCard.svelte b/web/frontend/src/lib/components/RunCard.svelte deleted file mode 100644 index a398ce4..0000000 --- a/web/frontend/src/lib/components/RunCard.svelte +++ /dev/null @@ -1,34 +0,0 @@ - - -
- - -
- {run.id.slice(0, 8)} - {run.status} -
-
- -
-
- {run.total} total - {run.succeeded} passed - {#if run.failed > 0} - {run.failed} failed - {/if} -
- {formatDuration(run.startTime, run.endTime)} -
-
- {formatTime(run.startTime)} -
-
-
-
diff --git a/web/frontend/src/lib/components/ScenarioEditor.svelte b/web/frontend/src/lib/components/ScenarioEditor.svelte index b51c57a..0a32c23 100644 --- a/web/frontend/src/lib/components/ScenarioEditor.svelte +++ b/web/frontend/src/lib/components/ScenarioEditor.svelte @@ -29,7 +29,7 @@ listConnectors, listElasticRulesAuto, getPackManifest, - lintScenario + lintAssessment } from '$lib/api/client'; import { scenarioTypeVariant } from '$lib/utils/format'; import type { Pack, PackManifest, ElasticRule, Connector, ScenarioType } from '$lib/types'; @@ -260,7 +260,7 @@ } } const yaml = currentYaml(); - const lint = await lintScenario(yaml).catch((e) => ({ + const lint = await lintAssessment(yaml).catch((e) => ({ valid: false, error: e instanceof Error ? e.message : 'Lint failed' })); @@ -276,7 +276,7 @@ await onsave(name.trim(), yaml, opts); isDirty = false; if (!opts.run) { - toast.success(mode === 'create' ? 'Scenario created' : 'Scenario saved'); + toast.success(mode === 'create' ? 'Assessment created' : 'Assessment saved'); } } catch (e) { error = e instanceof Error ? e.message : 'Save failed'; @@ -313,7 +313,7 @@ >

- {mode === 'create' ? 'New Scenario' : 'Edit Scenario'} + {mode === 'create' ? 'New Assessment' : 'Edit Assessment'}

{type} @@ -373,7 +373,7 @@ 0 - ? [...new Set(result.assertions.map((a) => a.matcherType))] + result?.expectations && result.expectations.length > 0 + ? [...new Set(result.expectations.map((a) => a.matcherType))] : [] ); - let assertionCounts = $derived.by(() => { - if (!result?.assertions || result.assertions.length === 0) return null; - const passed = result.assertions.filter((a) => a.passed === true).length; - return { passed, total: result.assertions.length }; + let expectationCounts = $derived.by(() => { + if (!result?.expectations || result.expectations.length === 0) return null; + const passed = result.expectations.filter((a) => a.passed === true).length; + return { passed, total: result.expectations.length }; }); const phaseLabels: Record = { @@ -43,14 +43,14 @@ ); - -{#snippet assertionBar()} - {#if assertionCounts} - {#if assertionCounts.total <= 8} +{#snippet expectationBar()} + {#if expectationCounts} + {#if expectationCounts.total <= 8} {/if} - {#if result.assertions && result.assertions.length > 0} + {#if result.expectations && result.expectations.length > 0}
- Assertions + Expectations
- {#each result.assertions as assertion} + {#each result.expectations as expectation}
- {assertion.passed ? 'PASS' : 'MISSED'} + {expectation.passed ? 'PASS' : 'MISSED'} [{assertion.matcherType}][{expectation.matcherType}] - {assertion.alertName} + {expectation.alertName}
{/each}
diff --git a/web/frontend/src/lib/components/ScheduleDialog.svelte b/web/frontend/src/lib/components/ScheduleDialog.svelte index 82b2e52..74c20d2 100644 --- a/web/frontend/src/lib/components/ScheduleDialog.svelte +++ b/web/frontend/src/lib/components/ScheduleDialog.svelte @@ -6,23 +6,23 @@ import { Switch } from '$lib/components/ui/switch/index.js'; import { Input } from '$lib/components/ui/input/index.js'; import { Label } from '$lib/components/ui/label/index.js'; - import type { SavedScenario, Schedule } from '$lib/types'; + import type { Assessment, Schedule } from '$lib/types'; import { createSchedule, updateSchedule, deleteSchedule, - getScheduleByScenario + getScheduleByAssessment } from '$lib/api/client'; import { describeCronExpression, validateCronExpression, cronPresets } from '$lib/utils/cron'; let { open = $bindable(), - scenario, + assessment, onclose, onsuccess }: { open: boolean; - scenario: SavedScenario; + assessment: Assessment; onclose: () => void; onsuccess: () => void; } = $props(); @@ -37,7 +37,7 @@ let parallelism = $state(10); $effect(() => { - if (open && scenario) { + if (open && assessment) { loadSchedule(); } }); @@ -46,7 +46,7 @@ loading = true; error = ''; try { - const schedule = await getScheduleByScenario(scenario.id); + const schedule = await getScheduleByAssessment(assessment.id); if (schedule) { existingSchedule = schedule; cronExpression = schedule.cronExpression; @@ -78,7 +78,7 @@ if (existingSchedule) { await updateSchedule(existingSchedule.id, cronExpression, enabled, parallelism); } else { - await createSchedule(scenario.id, cronExpression, enabled, parallelism); + await createSchedule(assessment.id, cronExpression, enabled, parallelism); } onsuccess(); onclose(); @@ -116,13 +116,13 @@ - + {existingSchedule ? 'Edit Schedule' : 'Create Schedule'} - Schedule "{scenario.name}" to run automatically. + Schedule "{assessment.name}" to run automatically. diff --git a/web/frontend/src/lib/components/SectionCards.svelte b/web/frontend/src/lib/components/SectionCards.svelte index eaec253..98c609a 100644 --- a/web/frontend/src/lib/components/SectionCards.svelte +++ b/web/frontend/src/lib/components/SectionCards.svelte @@ -12,12 +12,12 @@ totalRuns, ruleCoveragePercent, activeRuns, - savedScenarios + savedAssessments }: { totalRuns: number; ruleCoveragePercent: number; activeRuns: number; - savedScenarios: number; + savedAssessments: number; } = $props(); const rateIsGood = $derived(ruleCoveragePercent >= 50); @@ -65,7 +65,7 @@ Coverage needs attention {/if}
-
Rules covered by saved scenarios
+
Rules covered by saved assessments
@@ -75,14 +75,14 @@
- Total Assessments + Total Runs
{totalRuns} -
All completed and active assessments
+
All completed and active runs
@@ -98,14 +98,14 @@ class="h-4 w-4 {activeRuns > 0 ? 'text-primary' : 'text-muted-foreground'}" />
- Active Assessments + Active Runs
{activeRuns} -
Assessments currently in progress
+
Runs currently in progress
@@ -115,14 +115,14 @@
- Saved Scenarios + Assessments
- {savedScenarios} + {savedAssessments} -
Configured scenario definitions
+
Configured assessment definitions
diff --git a/web/frontend/src/lib/components/Sidebar.svelte b/web/frontend/src/lib/components/Sidebar.svelte index fe9dcbc..76f250e 100644 --- a/web/frontend/src/lib/components/Sidebar.svelte +++ b/web/frontend/src/lib/components/Sidebar.svelte @@ -25,8 +25,8 @@ const navItems: { href: string; label: string; icon: Component }[] = [ { href: '/', label: 'Dashboard', icon: LayoutDashboardIcon }, - { href: '/assessments', label: 'Assessments', icon: PlayIcon }, - { href: '/scenarios', label: 'Scenarios', icon: FileTextIcon }, + { href: '/runs', label: 'Runs', icon: PlayIcon }, + { href: '/assessments', label: 'Assessments', icon: FileTextIcon }, { href: '/packs', label: 'Packs', icon: PackageIcon }, { href: '/rules/coverage', label: 'Rule Coverage', icon: ShieldCheckIcon }, @@ -97,7 +97,7 @@ {/snippet} - {#if item.href === '/assessments' && $activeRuns.length > 0} + {#if item.href === '/runs' && $activeRuns.length > 0} {$activeRuns.length} diff --git a/web/frontend/src/lib/components/SimulationDetailSheet.svelte b/web/frontend/src/lib/components/SimulationDetailSheet.svelte deleted file mode 100644 index 067b3b9..0000000 --- a/web/frontend/src/lib/components/SimulationDetailSheet.svelte +++ /dev/null @@ -1,237 +0,0 @@ - - - - - {#if simulation} -
- -
-
-
- -
-
- - {simulation.name} - -

- {simulation.id} -

-
-
- - -
- - - {simulation.scope.toUpperCase()} - - {#if simulation.isSlow} - - - Slow - - {/if} - {#if simulation.params_schema && Object.keys(simulation.params_schema).length > 0} - - - Parameterized - - {/if} - {#if simulation.terraform} - - - Terraform - - {/if} -
-
- - - -
- -
-

- Description -

-
-

- {simulation.description || 'No description available.'} -

-
-
- - - - -
-

- MITRE ATT&CK Mapping -

- - {#if simulation.mitre?.tactics?.length > 0} -
-
- - Tactics -
-
- {#each simulation.mitre.tactics as tacticId} - - - - {tacticId} - - - - -

{getTacticName(tacticId)}

-
-
- {/each} -
-
- {/if} - - {#if simulation.mitre?.techniques?.length > 0} -
-
- - Techniques -
-
- {#each simulation.mitre.techniques as techId} - - - - {techId} - - - - -

{getTechniqueName(techId)}

-
-
- {/each} -
-
- {/if} - - {#if !simulation.mitre?.tactics?.length && !simulation.mitre?.techniques?.length} -

No MITRE mapping defined.

- {/if} -
- - - - -
-

- Pack -

-

{packName}

-
- - - {#if simulation.params_schema && Object.keys(simulation.params_schema).length > 0} - -
-

- Parameters Schema -

-
{JSON.stringify(
-										simulation.params_schema,
-										null,
-										2
-									)}
-
- {/if} -
-
- - -
-
- {simulation.scope}.{simulation.id.split('.').pop()} - {#if simulation.isSlow} - - - Slow execution - - {/if} -
-
-
- {/if} -
-
diff --git a/web/frontend/src/lib/components/connectors/ConnectorCreateDialog.svelte b/web/frontend/src/lib/components/connectors/ConnectorCreateDialog.svelte index 6c067a5..24e166a 100644 --- a/web/frontend/src/lib/components/connectors/ConnectorCreateDialog.svelte +++ b/web/frontend/src/lib/components/connectors/ConnectorCreateDialog.svelte @@ -182,7 +182,7 @@ if (!o) resetForm(); }} > - + {#if step === 'type'} New Connector diff --git a/web/frontend/src/lib/components/connectors/ConnectorEditDialog.svelte b/web/frontend/src/lib/components/connectors/ConnectorEditDialog.svelte index 3f7b52f..3969962 100644 --- a/web/frontend/src/lib/components/connectors/ConnectorEditDialog.svelte +++ b/web/frontend/src/lib/components/connectors/ConnectorEditDialog.svelte @@ -132,7 +132,7 @@ } }} > - + Edit Connector Update connector settings and credentials. diff --git a/web/frontend/src/lib/stores/assessments.ts b/web/frontend/src/lib/stores/assessments.ts new file mode 100644 index 0000000..8a21aa5 --- /dev/null +++ b/web/frontend/src/lib/stores/assessments.ts @@ -0,0 +1,19 @@ +import { writable } from 'svelte/store'; +import type { Assessment } from '$lib/types'; +import { listAssessments, type ScenarioFilters } from '$lib/api/client'; + +// Holds the current page of assessments for the list view. Other consumers +// that need a different slice (e.g. dashboard widgets, picker dialogs) should +// call `listAssessments()` directly with their own page size rather than read +// this store. +export const assessments = writable([]); + +export async function loadAssessmentPage( + page = 1, + perPage = 50, + filters: ScenarioFilters = {} +) { + const data = await listAssessments(page, perPage, filters); + assessments.set(data.assessments); + return data; +} diff --git a/web/frontend/src/lib/stores/runs.ts b/web/frontend/src/lib/stores/runs.ts index 862731c..836c8db 100644 --- a/web/frontend/src/lib/stores/runs.ts +++ b/web/frontend/src/lib/stores/runs.ts @@ -5,8 +5,8 @@ import { listRuns, getRun } from '$lib/api/client'; export const runs = writable([]); export const currentRun = writable(null); -// Loads page 1 of recent runs into the global store. The assessments list -// page maintains its own paginated state and uses listRuns() directly. +// Loads page 1 of recent runs into the global store. The runs list page +// maintains its own paginated state and uses listRuns() directly. export async function loadRuns() { const data = await listRuns(1, 50); runs.set(data.runs); diff --git a/web/frontend/src/lib/stores/scenarios.ts b/web/frontend/src/lib/stores/scenarios.ts deleted file mode 100644 index 47ed49d..0000000 --- a/web/frontend/src/lib/stores/scenarios.ts +++ /dev/null @@ -1,19 +0,0 @@ -import { writable } from 'svelte/store'; -import type { SavedScenario } from '$lib/types'; -import { listScenarios, type ScenarioFilters } from '$lib/api/client'; - -// Holds the current page of saved scenarios for the list view. Other -// consumers that need a different slice (e.g. dashboard widgets, picker -// dialogs) should call `listScenarios()` directly with their own page -// size rather than read this store. -export const scenarios = writable([]); - -export async function loadScenarioPage( - page = 1, - perPage = 50, - filters: ScenarioFilters = {} -) { - const data = await listScenarios(page, perPage, filters); - scenarios.set(data.scenarios); - return data; -} diff --git a/web/frontend/src/lib/types/index.ts b/web/frontend/src/lib/types/index.ts index 3530445..b07b6e6 100644 --- a/web/frontend/src/lib/types/index.ts +++ b/web/frontend/src/lib/types/index.ts @@ -7,9 +7,12 @@ export interface Run { total: number; succeeded: number; failed: number; - scenarioId?: string; - scenarioName?: string; - scenarioType?: 'standard' | 'explore' | 'collect'; + // Scenarios that failed during execution (warmup/detonation/matching + // infrastructure) rather than by missing an expected alert. Subset of `failed`. + errors: number; + assessmentId?: string; + assessmentName?: string; + assessmentType?: 'standard' | 'explore' | 'collect'; scheduleId?: string; scheduleName?: string; createdBy: string; @@ -32,7 +35,7 @@ export interface ScenarioResult { executorType: string; executionId: string; simulationId: string; - assertions: AssertionInfo[] | null; + expectations: ExpectationInfo[] | null; indicators: Indicators | null; metadata: ScenarioMetadata | null; collectedLogPath?: string; @@ -47,7 +50,7 @@ export interface DiscoveredAlert { severity?: string; } -export interface AssertionInfo { +export interface ExpectationInfo { matcherType: string; alertName: string; passed?: boolean; @@ -63,10 +66,10 @@ export interface ScenarioMetadata { description: string; } -// Saved scenario types +// Assessment (saved definition) types export type ScenarioType = 'standard' | 'explore' | 'collect'; -export interface SavedScenario { +export interface Assessment { id: string; name: string; type: ScenarioType; @@ -145,7 +148,7 @@ export interface SecretEntryInput { // Schedule types export interface Schedule { id: string; - scenarioId: string; + assessmentId: string; cronExpression: string; enabled: boolean; parallelism: number; @@ -167,7 +170,7 @@ export interface LintedScenario { name: string; executorType: string; executorName: string; - assertions: number; + expectations: number; } // Run response @@ -183,9 +186,9 @@ export interface RunListResponse { perPage: number; } -// Paginated saved-scenarios response -export interface ScenarioListResponse { - scenarios: SavedScenario[]; +// Paginated assessments response +export interface AssessmentListResponse { + assessments: Assessment[]; total: number; page: number; perPage: number; diff --git a/web/frontend/src/routes/+page.svelte b/web/frontend/src/routes/+page.svelte index 080a817..c158732 100644 --- a/web/frontend/src/routes/+page.svelte +++ b/web/frontend/src/routes/+page.svelte @@ -4,32 +4,32 @@ import { Skeleton } from '$lib/components/ui/skeleton/index.js'; import SectionCards from '$lib/components/SectionCards.svelte'; import RecentRunsTable from '$lib/components/RecentRunsTable.svelte'; - import RecentScenariosSection from '$lib/components/RecentScenariosSection.svelte'; + import RecentAssessmentsSection from '$lib/components/RecentAssessmentsSection.svelte'; import RecentPacksSection from '$lib/components/RecentPacksSection.svelte'; import { runs, activeRuns, loadRuns } from '$lib/stores/runs'; - import { loadScenarioPage } from '$lib/stores/scenarios'; + import { loadAssessmentPage } from '$lib/stores/assessments'; import { packs, loadPacks } from '$lib/stores/packs'; import { getRuleCoverage } from '$lib/api/client'; - import type { SavedScenario } from '$lib/types'; + import type { Assessment } from '$lib/types'; let loading = $state(true); let error = $state(''); let ruleCoveragePercent = $state(0); - let recentScenarios = $state([]); - let savedScenariosTotal = $state(0); + let recentAssessments = $state([]); + let savedAssessmentsTotal = $state(0); const recentRuns = $derived($runs.slice(0, 5)); const recentPacks = $derived($packs.slice(0, 5)); onMount(async () => { try { - const [, scenarioPage] = await Promise.all([ + const [, assessmentPage] = await Promise.all([ loadRuns(), - loadScenarioPage(1, 5, {}), + loadAssessmentPage(1, 5, {}), loadPacks() ]); - recentScenarios = scenarioPage.scenarios; - savedScenariosTotal = scenarioPage.total; + recentAssessments = assessmentPage.assessments; + savedAssessmentsTotal = assessmentPage.total; // Load rule coverage (non-blocking, may fail if no elastic connector) try { @@ -74,12 +74,12 @@ totalRuns={$runs.length} {ruleCoveragePercent} activeRuns={$activeRuns.length} - savedScenarios={savedScenariosTotal} + savedAssessments={savedAssessmentsTotal} />
- +
{/if} diff --git a/web/frontend/src/routes/assessments/+page.svelte b/web/frontend/src/routes/assessments/+page.svelte index 4580ce1..ff3ee37 100644 --- a/web/frontend/src/routes/assessments/+page.svelte +++ b/web/frontend/src/routes/assessments/+page.svelte @@ -1,39 +1,40 @@
-
-

Assessment History

- {#if runningCount > 0} - - - {runningCount} running - - {/if} -
-
- - -
+

Assessments

+
- +
- + - {#each Array(5) as _, i} + {#each Array(4) as _} {/each}
- {:else if pageRuns.length === 0} + {:else if pageAssessments.length === 0} - + - {hasActiveFilters ? 'No matching assessments' : 'No assessments yet'} + {hasActiveFilters ? 'No matching assessments' : 'No saved assessments'} {hasActiveFilters ? 'Try clearing or adjusting your filters.' - : 'Execute scenario files to see assessment history here.'} + : 'Save assessment YAML files here for quick access.'} @@ -419,10 +421,7 @@ Clear filters {:else} - + {/if} @@ -431,103 +430,118 @@ - ID - Status - Scenario + Name Type - Results - Started By - Started - Duration - + Schedule + Created + Updated + - {#each pageRuns as run (run.id)} - {@const pending = Math.max(0, run.total - run.succeeded - run.failed)} + {#each pageAssessments as assessment} goto(`/assessments/${run.id}`)} + class="group cursor-pointer hover:bg-accent/50 transition-colors" + onclick={() => { + if (renameId !== assessment.id) goto(`/assessments/${assessment.id}`); + }} > - - {run.id.slice(0, 8)} + + {#if renameId === assessment.id} +
e.stopPropagation()} + role="presentation" + > + handleRenameKeydown(e, assessment)} + onblur={() => commitRename(assessment)} + disabled={renameSaving} + class="h-7 max-w-xs" + /> + {#if renameSaving} + + {/if} +
+ {:else} +
+ {assessment.name} + +
+ {/if}
- - {#if run.status === 'running'} - - {/if} - {run.status} + + {assessment.type || 'standard'} - - {run.scenarioName || '--'} - - - {#if run.scenarioType} - {run.scenarioType} - {:else} - -- - {/if} - - {#if run.total > 0} -
-
-
-
- {#if pending > 0} -
- {/if} -
- - {run.succeeded}/{run.total} - -
+ {#if scheduleMap.has(assessment.id)} + {@const schedule = scheduleMap.get(assessment.id)} + + {schedule?.enabled ? 'Scheduled' : 'Disabled'} + {:else} - -- + None {/if}
- - - {formatUserEmail(run.createdBy)} - - {run.createdBy} - +
+ {new Date(assessment.createdAt).toLocaleDateString()} + {#if assessment.createdBy && assessment.createdBy !== 'anonymous'} + + + by {formatUserEmail(assessment.createdBy)} + + {assessment.createdBy} + + {/if} +
- - - {formatRelativeTime(run.startTime)} - - {formatTime(run.startTime)} - +
+ {new Date(assessment.updatedAt).toLocaleDateString()} + {#if assessment.updatedBy && assessment.updatedBy !== 'anonymous'} + + + by {formatUserEmail(assessment.updatedBy)} + + {assessment.updatedBy} + + {/if} +
- {formatDuration(run.startTime, run.endTime)} - +
+ + +
{/each} @@ -598,23 +612,13 @@ {/if}
- - - - Delete Assessment - Are you sure you want to delete assessment "{deleteTarget?.id.slice(0, 8)}"? This will - permanently remove the assessment, all scenario results, and log files. This action cannot - be undone. + Are you sure you want to delete "{deleteTarget?.name}"? This action cannot be undone.
@@ -625,3 +629,19 @@
+ + +{#if scheduleTarget} + { + scheduleDialogOpen = false; + scheduleTarget = null; + }} + onsuccess={loadScheduleMap} + /> +{/if} + + + diff --git a/web/frontend/src/routes/scenarios/[id]/+page.svelte b/web/frontend/src/routes/assessments/[id]/+page.svelte similarity index 89% rename from web/frontend/src/routes/scenarios/[id]/+page.svelte rename to web/frontend/src/routes/assessments/[id]/+page.svelte index 42ccb05..64ab0df 100644 --- a/web/frontend/src/routes/scenarios/[id]/+page.svelte +++ b/web/frontend/src/routes/assessments/[id]/+page.svelte @@ -24,22 +24,22 @@ import XCircleIcon from '@lucide/svelte/icons/x-circle'; import CircleDashedIcon from '@lucide/svelte/icons/circle-dashed'; import { - getScenario, - runScenario, - deleteScenario, - getScheduleByScenario, + getAssessment, + runAssessment, + deleteAssessment, + getScheduleByAssessment, listRuns } from '$lib/api/client'; import { scenarioTypeVariant, formatUserEmail, formatDuration } from '$lib/utils/format'; import { describeCronExpression } from '$lib/utils/cron'; import { parseScenarioYAML } from '$lib/utils/yaml-parser'; - import type { SavedScenario, Schedule, Run } from '$lib/types'; + import type { Assessment, Schedule, Run } from '$lib/types'; let id = $derived($page.params.id!); let loading = $state(true); let loadError = $state(''); - let scenario = $state(null); + let assessment = $state(null); let schedule = $state(null); let runs = $state([]); let runsLoading = $state(true); @@ -59,17 +59,17 @@ onMount(async () => { nowTimer = setInterval(() => (now = Date.now()), 30_000); try { - scenario = await getScenario(id); - schedule = await getScheduleByScenario(id); + assessment = await getAssessment(id); + schedule = await getScheduleByAssessment(id); } catch (e) { - loadError = e instanceof Error ? e.message : 'Failed to load scenario'; + loadError = e instanceof Error ? e.message : 'Failed to load assessment'; loading = false; return; } loading = false; try { - const all = await listRuns(1, 100, { scenarioId: id }); + const all = await listRuns(1, 100, { assessmentId: id }); runs = all.runs; } catch { runs = []; @@ -82,7 +82,7 @@ if (nowTimer) clearInterval(nowTimer); }); - let parsed = $derived(scenario ? parseScenarioYAML(scenario.yaml) : null); + let parsed = $derived(assessment ? parseScenarioYAML(assessment.yaml) : null); let parsedScenarios = $derived(parsed?.scenarios ?? []); let parsedTarget = $derived(parsed?.target); let builderSupported = $derived(parsed?.builderSupported !== false); @@ -109,8 +109,8 @@ let recentRuns = $derived(runs.slice(0, 8)); - let scenarioYamlBytes = $derived(scenario ? new Blob([scenario.yaml]).size : 0); - let scenarioYamlLines = $derived(scenario ? scenario.yaml.split('\n').length : 0); + let scenarioYamlBytes = $derived(assessment ? new Blob([assessment.yaml]).size : 0); + let scenarioYamlLines = $derived(assessment ? assessment.yaml.split('\n').length : 0); let nextRunAt = $derived.by(() => { if (!schedule || !schedule.enabled) return null; @@ -200,12 +200,12 @@ } async function handleRun() { - if (!scenario) return; + if (!assessment) return; actionError = ''; running = true; try { - const resp = await runScenario(scenario.id); - await goto(`/assessments/${resp.runId}`); + const resp = await runAssessment(assessment.id); + await goto(`/runs/${resp.runId}`); } catch (e) { actionError = e instanceof Error ? e.message : 'Run failed'; } finally { @@ -214,12 +214,12 @@ } async function handleDelete() { - if (!scenario) return; + if (!assessment) return; actionError = ''; deleting = true; try { - await deleteScenario(scenario.id); - await goto('/scenarios'); + await deleteAssessment(assessment.id); + await goto('/assessments'); } catch (e) { actionError = e instanceof Error ? e.message : 'Delete failed'; deleting = false; @@ -227,13 +227,13 @@ } async function reloadSchedule() { - schedule = await getScheduleByScenario(id); + schedule = await getScheduleByAssessment(id); } async function copyYaml() { - if (!scenario) return; + if (!assessment) return; try { - await navigator.clipboard.writeText(scenario.yaml); + await navigator.clipboard.writeText(assessment.yaml); copied = true; setTimeout(() => (copied = false), 1400); } catch { @@ -261,23 +261,23 @@
-{:else if loadError || !scenario} +{:else if loadError || !assessment}
- {loadError || 'Scenario not found'} + {loadError || 'Assessment not found'} - +
{:else}
- Scenarios + Assessments - {scenario.name} + {assessment.name} @@ -285,23 +285,23 @@
-

{scenario.name}

- - {scenario.type || 'standard'} +

{assessment.name}

+ + {assessment.type || 'standard'}

- Created {new Date(scenario.createdAt).toLocaleDateString()} - {#if scenario.createdBy && scenario.createdBy !== 'anonymous'} + Created {new Date(assessment.createdAt).toLocaleDateString()} + {#if assessment.createdBy && assessment.createdBy !== 'anonymous'} by - {formatUserEmail(scenario.createdBy)} + {formatUserEmail(assessment.createdBy)} - {scenario.createdBy} + {assessment.createdBy} {/if} - · Updated {new Date(scenario.updatedAt).toLocaleDateString()} + · Updated {new Date(assessment.updatedAt).toLocaleDateString()}

@@ -319,7 +319,7 @@ {schedule ? 'Schedule' : 'Set schedule'} - @@ -420,7 +420,7 @@
- This scenario uses a detonator type that isn't introspected. Review the source below. + This assessment uses a detonator type that isn't introspected. Review the source below.
{/if} @@ -498,7 +498,7 @@
No schedule
- Run this scenario automatically on a recurring cadence. + Run this assessment automatically on a recurring cadence.
Set schedule → @@ -516,7 +516,7 @@ {#if runs.length > recentRuns.length} View all → @@ -544,7 +544,7 @@ {@const tone = runStatusTone(r)}
@@ -631,7 +631,7 @@ (scheduleDialogOpen = false)} onsuccess={async () => { scheduleDialogOpen = false; @@ -642,9 +642,9 @@ - Delete Scenario + Delete Assessment - Are you sure you want to delete "{scenario.name}"? This action cannot be undone. + Are you sure you want to delete "{assessment.name}"? This action cannot be undone.
diff --git a/web/frontend/src/routes/scenarios/[id]/edit/+page.svelte b/web/frontend/src/routes/assessments/[id]/edit/+page.svelte similarity index 77% rename from web/frontend/src/routes/scenarios/[id]/edit/+page.svelte rename to web/frontend/src/routes/assessments/[id]/edit/+page.svelte index 8dd10d0..695d12b 100644 --- a/web/frontend/src/routes/scenarios/[id]/edit/+page.svelte +++ b/web/frontend/src/routes/assessments/[id]/edit/+page.svelte @@ -7,17 +7,17 @@ import { Button } from '$lib/components/ui/button/index.js'; import ScenarioEditor from '$lib/components/ScenarioEditor.svelte'; import ScheduleDialog from '$lib/components/ScheduleDialog.svelte'; - import { getScenario, updateScenario, runScenario } from '$lib/api/client'; + import { getAssessment, updateAssessment, runAssessment } from '$lib/api/client'; import { parseScenarioYAML } from '$lib/utils/yaml-parser'; import { createEmptyTarget } from '$lib/utils/yaml-generator'; import type { FormScenario, FormTarget } from '$lib/utils/yaml-generator'; - import type { SavedScenario } from '$lib/types'; + import type { Assessment } from '$lib/types'; let id = $derived($page.params.id!); let loading = $state(true); let loadError = $state(''); - let scenario = $state(null); + let assessment = $state(null); let initialScenarios = $state([]); let initialTarget = $state(createEmptyTarget()); let initialYaml = $state(''); @@ -27,10 +27,10 @@ onMount(async () => { try { - const s = await getScenario(id); - scenario = s; - initialYaml = s.yaml; - const parseResult = parseScenarioYAML(s.yaml); + const a = await getAssessment(id); + assessment = a; + initialYaml = a.yaml; + const parseResult = parseScenarioYAML(a.yaml); if (parseResult.success && parseResult.builderSupported) { initialScenarios = parseResult.scenarios || []; initialTarget = parseResult.target || createEmptyTarget(); @@ -39,23 +39,23 @@ builderSupported = false; } } catch (e) { - loadError = e instanceof Error ? e.message : 'Failed to load scenario'; + loadError = e instanceof Error ? e.message : 'Failed to load assessment'; } finally { loading = false; } }); async function handleSave(name: string, yaml: string, opts: { run?: boolean }) { - if (!scenario) return; - await updateScenario(scenario.id, name, yaml, scenario.type); + if (!assessment) return; + await updateAssessment(assessment.id, name, yaml, assessment.type); if (opts.run) { - const resp = await runScenario(scenario.id); - await goto(`/assessments/${resp.runId}`); + const resp = await runAssessment(assessment.id); + await goto(`/runs/${resp.runId}`); } } function handleCancel() { - goto('/scenarios'); + goto('/assessments'); } function openSchedule() { @@ -76,14 +76,14 @@ {loadError}
- +
-{:else if scenario} +{:else if assessment} (scheduleDialogOpen = false)} onsuccess={() => (scheduleDialogOpen = false)} /> diff --git a/web/frontend/src/routes/scenarios/new/+page.svelte b/web/frontend/src/routes/assessments/new/+page.svelte similarity index 70% rename from web/frontend/src/routes/scenarios/new/+page.svelte rename to web/frontend/src/routes/assessments/new/+page.svelte index 4ae8c32..ba8740e 100644 --- a/web/frontend/src/routes/scenarios/new/+page.svelte +++ b/web/frontend/src/routes/assessments/new/+page.svelte @@ -3,7 +3,7 @@ import { goto } from '$app/navigation'; import { page } from '$app/stores'; import ScenarioEditor from '$lib/components/ScenarioEditor.svelte'; - import { saveScenario, runScenario } from '$lib/api/client'; + import { saveAssessment, runAssessment } from '$lib/api/client'; import type { ScenarioType } from '$lib/types'; const validTypes: ScenarioType[] = ['standard', 'explore', 'collect']; @@ -15,22 +15,22 @@ onMount(() => { if (!selected) { - goto('/scenarios?new=1', { replaceState: true }); + goto('/assessments?new=1', { replaceState: true }); } }); async function handleSave(name: string, yaml: string, opts: { run?: boolean }) { - const saved = await saveScenario(name, yaml, selected ?? 'standard'); + const saved = await saveAssessment(name, yaml, selected ?? 'standard'); if (opts.run) { - const resp = await runScenario(saved.id); - await goto(`/assessments/${resp.runId}`); + const resp = await runAssessment(saved.id); + await goto(`/runs/${resp.runId}`); } else { - await goto(`/scenarios/${saved.id}/edit`); + await goto(`/assessments/${saved.id}/edit`); } } function handleCancel() { - goto('/scenarios'); + goto('/assessments'); } diff --git a/web/frontend/src/routes/rules/coverage/+page.svelte b/web/frontend/src/routes/rules/coverage/+page.svelte index 2ab1d5c..1a35235 100644 --- a/web/frontend/src/routes/rules/coverage/+page.svelte +++ b/web/frontend/src/routes/rules/coverage/+page.svelte @@ -453,7 +453,7 @@ {#if rule.lastResult}
e.stopPropagation()} > @@ -486,7 +486,7 @@
{scenario.scenarioName} diff --git a/web/frontend/src/routes/scenarios/+page.svelte b/web/frontend/src/routes/runs/+page.svelte similarity index 56% rename from web/frontend/src/routes/scenarios/+page.svelte rename to web/frontend/src/routes/runs/+page.svelte index ce78532..eba0ea2 100644 --- a/web/frontend/src/routes/scenarios/+page.svelte +++ b/web/frontend/src/routes/runs/+page.svelte @@ -1,40 +1,39 @@
-

Scenarios

- +
+

Runs

+ {#if runningCount > 0} + + + {runningCount} running + + {/if} +
+
+ + +
- +
- + - {#each Array(4) as _} + {#each Array(5) as _, i} {/each}
- {:else if pageScenarios.length === 0} + {:else if pageRuns.length === 0} - + - {hasActiveFilters ? 'No matching scenarios' : 'No saved scenarios'} + {hasActiveFilters ? 'No matching runs' : 'No runs yet'} {hasActiveFilters ? 'Try clearing or adjusting your filters.' - : 'Save scenario YAML files here for quick access.'} + : 'Run an assessment to see run history here.'} @@ -421,7 +419,10 @@ Clear filters {:else} - + {/if} @@ -430,118 +431,123 @@ - Name - Type - Schedule - Created - Updated - + Assessment + Results + Duration + - {#each pageScenarios as scenario} + {#each pageRuns as run (run.id)} + {@const pending = Math.max(0, run.total - run.succeeded - run.failed)} + {@const hasErrors = run.status !== 'running' && run.errors > 0} + {@const statusLabel = + run.status === 'running' + ? 'Running' + : hasErrors + ? `${run.errors} execution error${run.errors === 1 ? '' : 's'}` + : 'Completed'} { - if (renameId !== scenario.id) goto(`/scenarios/${scenario.id}`); - }} + class="cursor-pointer hover:bg-accent/50 transition-colors" + onclick={() => goto(`/runs/${run.id}`)} > - - {#if renameId === scenario.id} -
e.stopPropagation()} - role="presentation" - > - handleRenameKeydown(e, scenario)} - onblur={() => commitRename(scenario)} - disabled={renameSaving} - class="h-7 max-w-xs" - /> - {#if renameSaving} - - {/if} -
- {:else} -
- {scenario.name} - + {run.id.slice(0, 8)} + {#if run.assessmentType} + + {run.assessmentType} + {/if} + + + + {formatUserEmail(run.createdBy)} + + {run.createdBy} + + + + + {formatRelativeTime(run.startTime)} + + {formatTime(run.startTime)} + +
- {/if} - - - - {scenario.type || 'standard'} - +
- {#if scheduleMap.has(scenario.id)} - {@const schedule = scheduleMap.get(scenario.id)} - - {schedule?.enabled ? 'Scheduled' : 'Disabled'} - + {#if run.total > 0} +
+
+
+
+ {#if pending > 0} +
+ {/if} +
+ + {run.succeeded}/{run.total} + +
{:else} - None + -- {/if}
- -
- {new Date(scenario.createdAt).toLocaleDateString()} - {#if scenario.createdBy && scenario.createdBy !== 'anonymous'} - - - by {formatUserEmail(scenario.createdBy)} - - {scenario.createdBy} - - {/if} -
+ + + + {formatDuration(run.startTime, run.endTime)} + -
- {new Date(scenario.updatedAt).toLocaleDateString()} - {#if scenario.updatedBy && scenario.updatedBy !== 'anonymous'} - - - by {formatUserEmail(scenario.updatedBy)} - - {scenario.updatedBy} - - {/if} -
-
- -
- - -
+
{/each} @@ -612,13 +618,22 @@ {/if}
+ + + + - Delete Scenario + Delete Run - Are you sure you want to delete "{deleteTarget?.name}"? This action cannot be undone. + Are you sure you want to delete run "{deleteTarget?.id.slice(0, 8)}"? This will permanently + remove the run, all scenario results, and log files. This action cannot be undone.
@@ -629,19 +644,3 @@
- - -{#if scheduleTarget} - { - scheduleDialogOpen = false; - scheduleTarget = null; - }} - onsuccess={loadScheduleMap} - /> -{/if} - - - diff --git a/web/frontend/src/routes/assessments/[runId]/+page.svelte b/web/frontend/src/routes/runs/[id]/+page.svelte similarity index 90% rename from web/frontend/src/routes/assessments/[runId]/+page.svelte rename to web/frontend/src/routes/runs/[id]/+page.svelte index 09bb3dc..3f2d268 100644 --- a/web/frontend/src/routes/assessments/[runId]/+page.svelte +++ b/web/frontend/src/routes/runs/[id]/+page.svelte @@ -13,7 +13,7 @@ import RunLog from '$lib/components/RunLog.svelte'; import { currentRun, loadRun } from '$lib/stores/runs'; import { websocket } from '$lib/stores/websocket'; - import { getRunLogs, getScenario, deleteRun } from '$lib/api/client'; + import { getRunLogs, getAssessment, deleteRun } from '$lib/api/client'; import { ScenarioTracker } from '$lib/stores/scenario-tracker.svelte'; import { statusVariant, formatDuration, formatUserEmail } from '$lib/utils/format'; import type { WSMessage, RunLogEntry } from '$lib/types'; @@ -31,9 +31,9 @@ let deleteDialogOpen = $state(false); let deleting = $state(false); let pollTimer: ReturnType | null = null; - let scenarioName = $state(null); + let assessmentName = $state(null); - const runId = $derived($page.params.runId!); + const runId = $derived($page.params.id!); const trackerSucceeded = $derived( Object.values(tracker.entries).filter((e) => e.status === 'completed' && e.result?.isSuccess) @@ -79,7 +79,7 @@ const id = runId; loading = true; error = ''; - scenarioName = null; + assessmentName = null; tracker.reset(); stopPolling(); @@ -92,19 +92,19 @@ if ($currentRun?.scenarioResults) { tracker.setScenarios($currentRun.scenarioResults); } - if ($currentRun?.scenarioId) { + if ($currentRun?.assessmentId) { try { - const scenario = await getScenario($currentRun.scenarioId); - scenarioName = scenario.name; + const assessment = await getAssessment($currentRun.assessmentId); + assessmentName = assessment.name; } catch { - // Scenario may have been deleted + // Assessment may have been deleted } } if ($currentRun?.status === 'running') { pollTimer = setInterval(pollRun, 5000); } } catch (e) { - error = e instanceof Error ? e.message : 'Failed to load assessment'; + error = e instanceof Error ? e.message : 'Failed to load run'; } finally { loading = false; } @@ -128,7 +128,7 @@ error = ''; try { await deleteRun(runId); - goto('/assessments'); + goto('/runs'); } catch (e) { error = e instanceof Error ? e.message : 'Delete failed'; deleting = false; @@ -160,7 +160,7 @@ - Assessments + Runs @@ -201,15 +201,15 @@

{$currentRun.id.slice(0, 8)}

{$currentRun.status} - {#if $currentRun.scenarioType} + {#if $currentRun.assessmentType} - {$currentRun.scenarioType} + {$currentRun.assessmentType} {/if}
@@ -217,18 +217,18 @@
- {#if $currentRun.scenarioId} + {#if $currentRun.assessmentId} - {#if scenarioName} + {#if assessmentName} - {scenarioName} + {assessmentName} {:else} - {$currentRun.scenarioId.slice(0, 8)} + {$currentRun.assessmentId.slice(0, 8)} {/if} {/if} @@ -343,10 +343,10 @@ - Delete Assessment + Delete Run - Are you sure you want to delete this assessment? This will permanently remove the - assessment, all scenario results, and log files. This action cannot be undone. + Are you sure you want to delete this run? This will permanently remove the run, all scenario + results, and log files. This action cannot be undone.
diff --git a/web/frontend/src/routes/secrets/+page.svelte b/web/frontend/src/routes/secrets/+page.svelte index f5813d9..9361002 100644 --- a/web/frontend/src/routes/secrets/+page.svelte +++ b/web/frontend/src/routes/secrets/+page.svelte @@ -154,7 +154,7 @@ {/snippet} - + New Secret Group - + Edit Secret Group